diff --git a/ge/graph/build/graph_builder.cc b/ge/graph/build/graph_builder.cc index 74b884de..ecb6ceed 100644 --- a/ge/graph/build/graph_builder.cc +++ b/ge/graph/build/graph_builder.cc @@ -77,6 +77,8 @@ Status HandleSubgraphNode(NodePtr &src_node, OutDataAnchorPtr &src_out_anchor) { Status HandleSubgraphDataNode(NodePtr &src_node, OutDataAnchorPtr &src_out_anchor) { uint32_t index = 0; if (!AttrUtils::GetInt(src_node->GetOpDesc(), ATTR_NAME_PARENT_NODE_INDEX, index)) { + REPORT_INNER_ERROR("E19999", "get attr:%s failed from node:%s when HandleSubgraphDataNode", + ATTR_NAME_PARENT_NODE_INDEX.c_str(), src_node->GetName().c_str()); GELOGE(FAILED, "Get attr ATTR_NAME_PARENT_NODE_INDEX failed, node:%s.", src_node->GetName().c_str()); return FAILED; } @@ -109,6 +111,8 @@ Status GraphBuilder::CalcOpParam(const ge::ComputeGraphPtr &graph) { GE_CHECK_NOTNULL(graph); auto instance_ptr = ge::GELib::GetInstance(); if (instance_ptr == nullptr || !instance_ptr->InitFlag()) { + REPORT_INNER_ERROR("E19999", "check gelib instance null when CalcOpParam for graph:%s", + graph->GetName().c_str()); GELOGE(GE_CLI_GE_NOT_INITIALIZED, "GraphBuilder: GE is not initialized"); return GE_CLI_GE_NOT_INITIALIZED; } @@ -121,6 +125,8 @@ Status GraphBuilder::CalcOpParam(const ge::ComputeGraphPtr &graph) { (void)instance_ptr->DNNEngineManagerObj().GetDNNEngineName(node_ptr); kernel_lib_name = node_ptr->GetOpDesc()->GetOpKernelLibName(); if (kernel_lib_name.empty()) { + REPORT_INNER_ERROR("E19999", "op kernel lib is empty in node:%s(%s) when CalcOpParam", + node_ptr->GetName().c_str(), node_ptr->GetType().c_str()); GELOGE(INTERNAL_ERROR, "Get node:%s(%s) kernel lib failed.", node_ptr->GetName().c_str(), node_ptr->GetType().c_str()); return INTERNAL_ERROR; @@ -129,12 +135,16 @@ Status GraphBuilder::CalcOpParam(const ge::ComputeGraphPtr &graph) { auto ret = SetInputSize(node_ptr); if (ret != SUCCESS) { + REPORT_CALL_ERROR("E19999", "Set node:%s(%s) inputDesc size failed when CalcOpParam", + node_ptr->GetName().c_str(), node_ptr->GetType().c_str()); GELOGE(ret, "Set node inputDesc size failed, node name is %s", node_ptr->GetName().c_str()); return ret; } ret = OpsKernelBuilderManager::Instance().CalcOpRunningParam(*node_ptr); if (ret != SUCCESS) { + REPORT_CALL_ERROR("E19999", "Call Calculate op:%s(%s) running param failed", + node_ptr->GetName().c_str(), node_ptr->GetType().c_str()); GELOGE(ret, "Calculate op running param failed, node name is %s", node_ptr->GetName().c_str()); return ret; } @@ -191,6 +201,7 @@ Status GraphBuilder::UpdateParentNodeOutputSize(const ge::ComputeGraphPtr &graph Status GraphBuilder::Build(ComputeGraphPtr &comp_graph, GeRootModelPtr &ge_root_model_ptr, uint64_t session_id) { if (comp_graph == nullptr) { + REPORT_INNER_ERROR("E19999", "check compute_graph nullptr when BuildGraph, session_id:%lu", session_id); GELOGE(GE_GRAPH_PARAM_NULLPTR, "Graph build comp_graph is null."); return GE_GRAPH_PARAM_NULLPTR; } @@ -302,6 +313,8 @@ Status GraphBuilder::SetConstantInputOffset(ComputeGraphPtr &comp_graph) { std::vector weights = OpDescUtils::MutableWeights(peer_node); if (weights.empty()) { + REPORT_INNER_ERROR("E19999", "check weights size of node %s(%s) is empty when SetConstantInputOffset", + node->GetName().c_str(), node->GetType().c_str()); GELOGE(FAILED, "weights size of node %s is empty", node->GetName().c_str()); return FAILED; } @@ -393,6 +406,7 @@ static Status InsertMemcpyNode(const ComputeGraphPtr &graph, const OutDataAnchor .Build(); (void)AttrUtils::SetBool(op_desc, ATTR_NO_NEED_CONSTANT_FOLDING, false); if (GraphUtils::InsertNodeAfter(out_anchor, in_anchors, graph->AddNode(op_desc)) != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Insert IDENTITY node %s after %s failed", name.c_str(), in_node->GetName().c_str()); GELOGE(FAILED, "Insert IDENTITY node %s after %s failed.", name.c_str(), in_node->GetName().c_str()); return FAILED; } @@ -423,6 +437,8 @@ static Status GenerateTaskForConstant(const std::shared_ptr &graph GELOGD("Insert MemcpyAsync node between %s and %s.", in_node->GetName().c_str(), node->GetName().c_str()); std::string name = node->GetName() + "_input_" + std::to_string(in_data_anchor->GetIdx()) + "_Memcpy"; if (InsertMemcpyNode(graph, peer_out_anchor, {in_data_anchor}, name) != SUCCESS) { + REPORT_CALL_ERROR("E19999", "Insert memcpy between %s and %s failed when GenerateTaskForConstant", + in_node->GetName().c_str(), node->GetName().c_str()); GELOGE(FAILED, "Insert memcpy between %s and %s failed.", in_node->GetName().c_str(), node->GetName().c_str()); return FAILED; @@ -470,6 +486,8 @@ Status GraphBuilder::MarkFpBpProfilingTaskAttr(ComputeGraphPtr &com_graph) { GELOGI("The all reduce node of dynamic graph is %s, idx %u", op_desc->GetName().c_str(), node_index); (void)ge::AttrUtils::SetBool(op_desc, ATTR_NAME_INSERT_BP_PROFILILNG_TASK, true); GE_IF_BOOL_EXEC(TypeUtils::CheckUint64MulOverflow(i, kProfilingArStep), + REPORT_INNER_ERROR("E19999", "Multiply result is out of range when calc profiling ar log id " + "for node:%s(%s)", op_desc->GetName().c_str(), op_desc->GetType().c_str()); GELOGE(FAILED, "Multiply result is out of range."); return FAILED); int64_t log_id = i * kProfilingArStep + kProfilingArStartLogid; @@ -549,16 +567,19 @@ Status GraphBuilder::GetTaskInfo(const ge::ModelBuilder &builder, const ModelPtr int64_t memory_size = 0; if (!AttrUtils::GetInt(model_ptr, ATTR_MODEL_MEMORY_SIZE, memory_size)) { + REPORT_INNER_ERROR("E19999", "Get Attr:%s fail in model", ATTR_MODEL_MEMORY_SIZE.c_str()); GELOGE(INTERNAL_ERROR, "Get memory size fail."); return INTERNAL_ERROR; } int64_t p2p_memory_size = 0; if (!AttrUtils::GetInt(model_ptr, ATTR_MODEL_P2P_MEMORY_SIZE, p2p_memory_size)) { + REPORT_INNER_ERROR("E19999", "Get Attr:%s fail in model", ATTR_MODEL_P2P_MEMORY_SIZE.c_str()); GELOGE(INTERNAL_ERROR, "Get p2p memory size fail."); return INTERNAL_ERROR; } int64_t weight_size = 0; if (!AttrUtils::GetInt(model_ptr, ATTR_MODEL_WEIGHT_SIZE, weight_size)) { + REPORT_INNER_ERROR("E19999", "Get Attr:%s fail in model", ATTR_MODEL_WEIGHT_SIZE.c_str()); GELOGE(INTERNAL_ERROR, "Get weight memory size fail."); return INTERNAL_ERROR; } @@ -668,6 +689,7 @@ Status GraphBuilder::SetInputSize(const ge::NodePtr &node_ptr) { Status GraphBuilder::UpdateDataInputSize(const ge::NodePtr &node_ptr) { const auto &op_desc = node_ptr->GetOpDesc(); if (op_desc == nullptr) { + REPORT_INNER_ERROR("E19999", "check op_desc is nullptr when UpdateDataInputSize"); GELOGE(FAILED, "Op desc is nullptr."); return FAILED; } @@ -685,6 +707,8 @@ Status GraphBuilder::UpdateDataInputSize(const ge::NodePtr &node_ptr) { int64_t real_dim_size = 0; ge::graphStatus graph_status = TensorUtils::GetTensorSizeInBytes(output_desc, real_dim_size); if (graph_status != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Get tensor size in bytes failed for op:%s(%s) index:0 when UpdateDataInputSize", + op_desc->GetName().c_str(), op_desc->GetType().c_str()); GELOGE(FAILED, "Get tensor size in bytes failed."); return FAILED; } @@ -692,6 +716,8 @@ Status GraphBuilder::UpdateDataInputSize(const ge::NodePtr &node_ptr) { ge::GeTensorDesc input_desc = op_desc->GetInputDesc(0); ge::TensorUtils::SetSize(input_desc, real_dim_size); if (op_desc->UpdateInputDesc(0, input_desc) != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Update input desc size failed for op:%s(%s) index:0 when UpdateDataInputSize", + op_desc->GetName().c_str(), op_desc->GetType().c_str()); GELOGE(FAILED, "Update input desc size failed."); return FAILED; } @@ -720,6 +746,9 @@ Status GraphBuilder::CalcDynShapeRootGraphDataSize(const ge::OpDescPtr &op_desc) int64_t real_dim_size = 0; ge::graphStatus graph_status = TensorUtils::GetTensorSizeInBytes(output_desc, real_dim_size); if (graph_status != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Get tensor size in bytes failed for op:%s(%s) index:0 " + "when CalcDynShapeRootGraphDataSize", + op_desc->GetName().c_str(), op_desc->GetType().c_str()); GELOGE(FAILED, "Get tensor size in bytes failed."); return FAILED; } @@ -727,6 +756,9 @@ Status GraphBuilder::CalcDynShapeRootGraphDataSize(const ge::OpDescPtr &op_desc) ge::TensorUtils::SetSize(output_desc, real_dim_size); GELOGI("Update dynamic shape graph data output size to [%ld].", real_dim_size); if (op_desc->UpdateOutputDesc(0, output_desc) != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Update output desc size failed for op:%s(%s) index:0 " + "when CalcDynShapeRootGraphDataSize", + op_desc->GetName().c_str(), op_desc->GetType().c_str()); GELOGE(FAILED, "Update dynamic shape graph data output desc size failed."); return FAILED; } @@ -744,6 +776,8 @@ Status GraphBuilder::SecondPartition(ge::ComputeGraphPtr &comp_graph) { GE_CHK_STATUS_RET(ret, "Graph partition Failed."); const auto &graph_2_subgraphlist = graph_partitioner_.GetSubGraphMap(); if (graph_2_subgraphlist.find(comp_graph) == graph_2_subgraphlist.end()) { + REPORT_INNER_ERROR("E19999", "find subgraphlis in graph:%s failed when SecondPartition", + comp_graph->GetName().c_str()); GELOGE(FAILED, "Find subgraph failed."); return FAILED; } @@ -772,6 +806,9 @@ Status GraphBuilder::AddOutputMemTypeForNode(const NodePtr &node) { mem_type); if (!AttrUtils::SetInt(src_desc->MutableOutputDesc(src_out_anchor->GetIdx()), ATTR_OUTPUT_MEMORY_TYPE, mem_type)) { + REPORT_INNER_ERROR("E19999", "Set Attr:%s for node:%s(%s) out_index:%u failed when AddOutputMemTypeForNode", + ATTR_OUTPUT_MEMORY_TYPE.c_str(), src_desc->GetName().c_str(), src_desc->GetType().c_str(), + src_out_anchor->GetIdx()); GELOGE(INTERNAL_ERROR, "Set out_memory_type attr for [%s:%d] failed.", src_desc->GetName().c_str(), src_out_anchor->GetIdx()); return INTERNAL_ERROR; diff --git a/ge/graph/build/label_allocator.cc b/ge/graph/build/label_allocator.cc index 28d0e084..3ab39838 100644 --- a/ge/graph/build/label_allocator.cc +++ b/ge/graph/build/label_allocator.cc @@ -28,6 +28,7 @@ LabelAllocator::LabelAllocator(const ComputeGraphPtr &graph) : compute_graph_(gr Status LabelAllocator::AssignFunctionalLabels() { if (compute_graph_ == nullptr) { + REPORT_INNER_ERROR("E19999", "check param compute_graph nullptr when AssignFunctionalLabels"); GELOGE(INTERNAL_ERROR, "ComputeGraph not set, Assign labels failed."); return INTERNAL_ERROR; } @@ -46,11 +47,15 @@ Status LabelAllocator::AssignFunctionalLabels() { for (auto node : functional_nodes) { LabelMakerPtr maker = LabelMakerFactory::Instance().Create(node->GetType(), compute_graph_, node); if (maker == nullptr) { + REPORT_CALL_ERROR("E19999", "Check Node:%s(%s) label maker not registed when AssignFunctionalLabels", + node->GetName().c_str(), node->GetType().c_str()); GELOGE(INTERNAL_ERROR, "Node: %s label maker not registed.", node->GetType().c_str()); return INTERNAL_ERROR; } if (maker->Run(label_index) != SUCCESS) { + REPORT_CALL_ERROR("E19999", "Node:%s(%s) run label maker failed when AssignFunctionalLabels", + node->GetName().c_str(), node->GetType().c_str()); GELOGE(INTERNAL_ERROR, "Node: %s run label maker failed.", node->GetType().c_str()); return INTERNAL_ERROR; } @@ -63,6 +68,7 @@ Status LabelAllocator::AssignFunctionalLabels() { bool LabelAllocator::CollectFunctionalNode(ComputeGraphPtr &graph, std::set &functional_nodes) { if (graph == nullptr) { + REPORT_INNER_ERROR("E19999", "check param compute_graph nullptr when CollectFunctionalNode"); GELOGE(INTERNAL_ERROR, "Sub ComputeGraph is null."); return false; } @@ -74,12 +80,16 @@ bool LabelAllocator::CollectFunctionalNode(ComputeGraphPtr &graph, std::setGetParentNode(); if (func_node == nullptr) { + REPORT_INNER_ERROR("E19999", "Parent node not set in node:%s(%s), graph:%s", + func_node->GetName().c_str(), func_node->GetType().c_str(), graph->GetName().c_str()); GELOGE(INTERNAL_ERROR, "Parent functional node not set: %s.", graph->GetName().c_str()); return false; } ComputeGraphPtr owner_graph = func_node->GetOwnerComputeGraph(); if (owner_graph == nullptr) { + REPORT_INNER_ERROR("E19999", "ComputeGraph owner not set in node:%s(%s), graph:%s", + func_node->GetName().c_str(), func_node->GetType().c_str(), graph->GetName().c_str()); GELOGE(INTERNAL_ERROR, "ComputeGraph owner not set: %s.", func_node->GetName().c_str()); return false; } diff --git a/ge/graph/build/logical_stream_allocator.cc b/ge/graph/build/logical_stream_allocator.cc index 72fbdf33..837b9454 100644 --- a/ge/graph/build/logical_stream_allocator.cc +++ b/ge/graph/build/logical_stream_allocator.cc @@ -320,6 +320,8 @@ Status SingleStreamPass::Run(ComputeGraphPtr graph, const vector &s if (!HasAssignedStream(*subgraph)) { const string &stream_label = subgraph->subgraph_info.GetStreamLabel(); if (!stream_label.empty()) { + REPORT_INNER_ERROR("E19999", "Stream labels are not supported in SingleStream mode " + "(subgraph: %s, stream label: %s)", subgraph->name.c_str(), stream_label.c_str()); GELOGE(INTERNAL_ERROR, "Stream labels are not supported (subgraph: %s, stream label: %s).", subgraph->name.c_str(), stream_label.c_str()); return INTERNAL_ERROR; @@ -337,6 +339,8 @@ Status NodeStreamUpdatePass::Run(ComputeGraphPtr graph, const vectorengine_conf.id; if (!IsEngineSkip(*subgraph) && !HasAssignedStream(*subgraph)) { + REPORT_INNER_ERROR("E19999", "Subgraph %s has not yet been assigned a stream (engine: %s) " + " when run NodeStreamUpdatePass", subgraph->name.c_str(), engine_name.c_str()); GELOGE(INTERNAL_ERROR, "Subgraph %s has not yet been assigned a stream (engine: %s).", subgraph->name.c_str(), engine_name.c_str()); return INTERNAL_ERROR; @@ -636,6 +640,8 @@ Status LogicalStreamAllocator::DoAssign(const ComputeGraphPtr &graph, const Grap auto iter = subgraph_map.find(graph); if (iter == subgraph_map.end()) { + REPORT_INNER_ERROR("E19999", "Graph %s not found in subgraph_map when do logical stream assign ", + graph->GetName().c_str()); GELOGE(FAILED, "Graph %s not found.", graph->GetName().c_str()); return FAILED; } @@ -675,6 +681,8 @@ Status LogicalStreamAllocator::ConvertSubgraphs(const vector &s const string &engine_name = subgraph_info->GetEngineName(); auto engine_conf_iter = engine_confs.find(engine_name); if ((engine_conf_iter == engine_confs.end()) || (engine_conf_iter->second == nullptr)) { + REPORT_INNER_ERROR("E19999", "Engine conf of subgraph %s not found (engine name: %s) when ConvertSubgraphs", + subgraph_name.c_str(), engine_name.c_str()); GELOGE(INTERNAL_ERROR, "Engine conf of subgraph %s not found (engine name: %s).", subgraph_name.c_str(), engine_name.c_str()); @@ -722,6 +730,7 @@ Status LogicalStreamAllocator::RunPasses(const ComputeGraphPtr &graph, const vec } else if (status == NOT_CHANGED) { GELOGD("[Show][Status]Stream pass %s return NOT_CHANGED.", pass->GetName().c_str()); } else { + REPORT_CALL_ERROR("E19999", "Stream pass %s run failed.", pass->GetName().c_str()); GELOGE(status, "Stream pass %s failed.", pass->GetName().c_str()); return status; } diff --git a/ge/graph/build/memory/graph_mem_assigner.cc b/ge/graph/build/memory/graph_mem_assigner.cc index 9d5b9d08..ea48afd8 100755 --- a/ge/graph/build/memory/graph_mem_assigner.cc +++ b/ge/graph/build/memory/graph_mem_assigner.cc @@ -1215,6 +1215,7 @@ Status GraphMemoryAssigner::CheckOffset() { std::map anchor_to_symbol; std::map> symbol_to_anchors; if (GraphUtils::GetRefMapping(compute_graph_, symbol_to_anchors, anchor_to_symbol) != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Get ref-mapping for graph %s failed", compute_graph_->GetName().c_str()); GELOGE(FAILED, "[Get][RefMapping]fail for graph %s", compute_graph_->GetName().c_str()); return FAILED; } diff --git a/ge/graph/build/memory/hybrid_mem_assigner.cc b/ge/graph/build/memory/hybrid_mem_assigner.cc index 462e190a..4ea52d9d 100755 --- a/ge/graph/build/memory/hybrid_mem_assigner.cc +++ b/ge/graph/build/memory/hybrid_mem_assigner.cc @@ -42,6 +42,7 @@ Status HybridMemAssigner::AssignMemory(std::unique_ptr &block_ Status HybridMemAssigner::Assign() { if (GraphUtils::GetRefMapping(compute_graph_, symbol_to_anchors_, anchor_to_symbol_) != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Get ref-mapping for graph %s failed", compute_graph_->GetName().c_str()); GELOGE(FAILED, "Get ref-mapping for graph %s failed.", compute_graph_->GetName().c_str()); return FAILED; } diff --git a/ge/graph/build/memory/var_mem_assign_util.cc b/ge/graph/build/memory/var_mem_assign_util.cc index f910d2e2..d7b442ae 100755 --- a/ge/graph/build/memory/var_mem_assign_util.cc +++ b/ge/graph/build/memory/var_mem_assign_util.cc @@ -53,6 +53,8 @@ Status VarMemAssignUtil::AssignStaticMemory2Node(ge::ComputeGraphPtr &compute_gr GE_IF_BOOL_EXEC(ge::AttrUtils::GetStr(n->GetOpDesc(), REF_VAR_SRC_VAR_NAME, ref_var_src_var_name), continue); string node_name = n->GetName(); GE_IF_BOOL_EXEC(n->GetOpDesc()->GetAllOutputsDesc().empty(), + REPORT_INNER_ERROR("E19999", "check node:%s has no OutputDesc when AssignStaticMemory2Node", + n->GetName().c_str()); GELOGE(FAILED, "node:%s has no OutputDesc.", n->GetName().c_str()); return FAILED); ge::ConstGeTensorDescPtr tensor_desc = n->GetOpDesc()->GetOutputDescPtr(0); @@ -116,6 +118,8 @@ Status VarMemAssignUtil::SetOutVariableAttr(const ge::NodePtr &node, const ge::N GE_CHECK_NOTNULL(node->GetOpDesc()); output_list = node->GetOpDesc()->GetOutputOffset(); if (output_list.empty()) { + REPORT_INNER_ERROR("E19999", "check node:%s output_offset_list is empty when SetOutVariableAttr", + node->GetName().c_str()); GELOGE(PARAM_INVALID, "Output_list is empty"); return PARAM_INVALID; } @@ -126,7 +130,12 @@ Status VarMemAssignUtil::SetOutVariableAttr(const ge::NodePtr &node, const ge::N VarManager::Instance(session_id)->GetVarAddr(var_node->GetName(), var_tensor_desc, &dev_ptr, memory_type)); int out_list_size = static_cast(output_list.size()); - GE_CHK_BOOL_RET_STATUS(index < out_list_size, FAILED, "index %d >= output_list.size() %d", index, out_list_size); + if (index < out_list_size) { + REPORT_INNER_ERROR("E19999", "param index:%d >= output_list.size() %d in node %s, " + "check invalid when SetOutVariableAttr", index, out_list_size, node->GetName().c_str()); + GELOGE(FAILED, "index %d >= output_list.size() %d", index, out_list_size); + return FAILED; + } output_list[index] = static_cast(reinterpret_cast(dev_ptr)); GELOGI("Assign node outputOffset[index] is: %ld", output_list[index]); @@ -168,9 +177,13 @@ Status VarMemAssignUtil::DealBroadCastNode(uint32_t graph_id, const ge::NodePtr auto broad_cast_index = static_cast(broad_cast_info.idx); auto input_tensor_desc_ptr_vistor = op_desc->GetAllInputsDescPtr(); - GE_CHK_BOOL_RET_STATUS(input_tensor_desc_ptr_vistor.size() > broad_cast_index, FAILED, - "Get broadcast op %s input tensor desc size [%zu] < idx [%d]", node->GetName().c_str(), - input_tensor_desc_ptr_vistor.size(), broad_cast_info.idx); + if (input_tensor_desc_ptr_vistor.size() > broad_cast_index) { + REPORT_INNER_ERROR("E19999", "Get broadcast op %s input tensor desc size [%zu] < idx [%d]", + node->GetName().c_str(), input_tensor_desc_ptr_vistor.size(), broad_cast_info.idx); + GELOGE(FAILED, "Get broadcast op %s input tensor desc size [%zu] < idx [%d]", node->GetName().c_str(), + input_tensor_desc_ptr_vistor.size(), broad_cast_info.idx); + return FAILED; + } const ge::GeTensorDescPtr input_tensor_desc = input_tensor_desc_ptr_vistor.at(static_cast(broad_cast_info.idx)); int64_t input_size = 0; diff --git a/ge/graph/build/model_builder.cc b/ge/graph/build/model_builder.cc index 1a14374d..29ef637f 100755 --- a/ge/graph/build/model_builder.cc +++ b/ge/graph/build/model_builder.cc @@ -116,11 +116,15 @@ Status ModelBuilder::CalcOutputSize(const ge::NodePtr &n) { int64_t size_temp = 0; graphStatus graph_status = TensorUtils::GetTensorMemorySizeInBytes(desc_temp, size_temp); if (graph_status != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Get tensor size in bytes failed for op:%s(%s) index:%u when CalcOutputSize", + node_op_desc->GetName().c_str(), node_op_desc->GetType().c_str(), index); GELOGE(graph_status, "GetTensorMemorySizeInBytes failed!"); return FAILED; } TensorUtils::SetSize(desc_temp, size_temp); if (node_op_desc->UpdateOutputDesc(index, desc_temp) != SUCCESS) { + REPORT_CALL_ERROR("E19999", "Update Output desc size failed for op:%s(%s) index:%u when CalcOutputSize", + node_op_desc->GetName().c_str(), node_op_desc->GetType().c_str(), index); GELOGE(FAILED, "UpdateOutputDesc failed."); return FAILED; } @@ -207,11 +211,15 @@ Status ModelBuilder::AdjustConstWeightSize(const ge::NodePtr &node, size_t &mem_ if (node->GetType() == CONSTANT) { vector weights = OpDescUtils::MutableWeights(node); if (weights.empty()) { + REPORT_INNER_ERROR("E19999", "Check weights size of node %s(%s) is empty when AdjustConstWeightSize", + node->GetName().c_str(), node->GetType().c_str()); GELOGE(FAILED, "weights size of node %s is empty", node->GetName().c_str()); return FAILED; } GeTensorPtr weight = weights[0]; if (weight == nullptr) { + REPORT_INNER_ERROR("E19999", "Check weight of node %s(%s) is nullptr when AdjustConstWeightSize", + node->GetName().c_str(), node->GetType().c_str()); GELOGE(FAILED, "weights[0] is null."); return FAILED; } @@ -353,6 +361,9 @@ Status ModelBuilder::AdjustInputTensorFlag() { auto input_desc = owner_node_op_desc->GetInputDesc(in_anchors->GetIdx()); ge::TensorUtils::SetInputTensor(input_desc, true); if (owner_node_op_desc->UpdateInputDesc(in_anchors->GetIdx(), input_desc) != SUCCESS) { + REPORT_CALL_ERROR("E19999", "Update Input desc size failed for op:%s(%s) index:%u when %s", + owner_node_op_desc->GetName().c_str(), owner_node_op_desc->GetType().c_str(), + in_anchors->GetIdx(), __FUNCTION__); GELOGE(FAILED, "UpdateOutputDesc failed."); return FAILED; } @@ -381,33 +392,51 @@ Status ModelBuilder::BuildModelDef(ge::Model &model) { max_mem_offset_ = mem_type_to_mem_offset_[RT_MEMORY_HBM]; GE_CHK_BOOL_EXEC(ge::AttrUtils::SetInt(&model, ATTR_MODEL_MEMORY_SIZE, max_mem_offset_), + REPORT_INNER_ERROR("E19999", "Set Attr:%s in model failed when %s", + ATTR_MODEL_MEMORY_SIZE.c_str(), __FUNCTION__); GELOGE(FAILED, "SetInt of ATTR_MODEL_MEMORY_SIZE failed."); return FAILED); if (mem_type_to_mem_offset_.find(RT_MEMORY_P2P_DDR) != mem_type_to_mem_offset_.end()) { p2p_mem_offset_ = mem_type_to_mem_offset_[RT_MEMORY_P2P_DDR]; } GE_CHK_BOOL_EXEC(ge::AttrUtils::SetInt(&model, ATTR_MODEL_P2P_MEMORY_SIZE, p2p_mem_offset_), + REPORT_INNER_ERROR("E19999", "Set Attr:%s in model failed when %s", + ATTR_MODEL_P2P_MEMORY_SIZE.c_str(), __FUNCTION__); GELOGE(FAILED, "SetInt of ATTR_MODEL_P2P_MEMORY_SIZE failed."); return FAILED); GE_CHK_BOOL_EXEC(ge::AttrUtils::SetInt(&model, ATTR_MODEL_WEIGHT_SIZE, weight_offset_), + REPORT_INNER_ERROR("E19999", "Set Attr:%s in model failed when %s", + ATTR_MODEL_WEIGHT_SIZE.c_str(), __FUNCTION__); GELOGE(FAILED, "SetInt of ATTR_MODEL_WEIGHT_SIZE failed."); return FAILED); GE_CHK_BOOL_EXEC(ge::AttrUtils::SetInt(&model, ATTR_MODEL_STREAM_NUM, stream_num_), + REPORT_INNER_ERROR("E19999", "Set Attr:%s in model failed when %s", + ATTR_MODEL_STREAM_NUM.c_str(), __FUNCTION__); GELOGE(FAILED, "SetInt of ATTR_MODEL_STREAM_NUM failed."); return FAILED); GE_CHK_BOOL_EXEC(ge::AttrUtils::SetInt(&model, ATTR_MODEL_EVENT_NUM, event_num_), + REPORT_INNER_ERROR("E19999", "Set Attr:%s in model failed when %s", + ATTR_MODEL_EVENT_NUM.c_str(), __FUNCTION__); GELOGE(FAILED, "SetInt of ATTR_MODEL_EVENT_NUM failed."); return FAILED); GE_CHK_BOOL_EXEC(ge::AttrUtils::SetListInt(&model, ATTR_MODEL_HUGE_STREAM_LIST, huge_streams_), + REPORT_INNER_ERROR("E19999", "Set Attr:%s in model failed when %s", + ATTR_MODEL_HUGE_STREAM_LIST.c_str(), __FUNCTION__); GELOGE(FAILED, "SetInt of ATTR_MODEL_HUGE_STREAM_LIST failed."); return FAILED); GE_CHK_BOOL_EXEC(ge::AttrUtils::SetInt(&model, ATTR_MODEL_LABEL_NUM, label_num_), + REPORT_INNER_ERROR("E19999", "Set Attr:%s in model failed when %s", + ATTR_MODEL_LABEL_NUM.c_str(), __FUNCTION__); GELOGE(FAILED, "SetInt of ATTR_MODEL_LABEL_NUM failed."); return FAILED); GE_CHK_BOOL_EXEC(ge::AttrUtils::SetInt(&model, ATTR_MODEL_ZERO_COPY_MEMORY_SIZE, zero_copy_mem_size_), + REPORT_INNER_ERROR("E19999", "Set Attr:%s in model failed when %s", + ATTR_MODEL_ZERO_COPY_MEMORY_SIZE.c_str(), __FUNCTION__); GELOGE(FAILED, "SetInt of ATTR_MODEL_ZERO_COPY_MEMORY_SIZE failed."); return FAILED); GE_CHK_BOOL_EXEC(ge::AttrUtils::SetListStr(&model, ATTR_MODEL_OUT_NODES_NAME, GetLocalOmgContext().net_out_nodes), + REPORT_INNER_ERROR("E19999", "Set Attr:%s in model failed when %s", + ATTR_MODEL_OUT_NODES_NAME.c_str(), __FUNCTION__); GELOGE(FAILED, "SetListStr of ATTR_MODEL_OUT_NODES_NAME failed."); return FAILED); GELOGI("For model, max_mem_offset_: %zu, p2p_mem_size: %zu, zero_copy_mem_size_: %zu", max_mem_offset_, @@ -415,6 +444,8 @@ Status ModelBuilder::BuildModelDef(ge::Model &model) { string fp_ceiling_mode; if (ge::GetContext().GetOption("ge.fpCeilingMode", fp_ceiling_mode) == SUCCESS) { if (!ge::AttrUtils::SetStr(&model, ATTR_FP_CEILING_MODE, fp_ceiling_mode)) { + REPORT_INNER_ERROR("E19999", "Set Attr:%s in model failed when %s", + ATTR_FP_CEILING_MODE.c_str(), __FUNCTION__); GELOGE(FAILED, "Failed to set attr ATTR_FP_CEILING_MODE"); return FAILED; } @@ -429,22 +460,30 @@ Status ModelBuilder::BuildModelDef(ge::Model &model) { int64_t core_type = (ge_core_type == kVectorCore) ? 1 : 0; GELOGI("core_type: %ld", core_type); if (!ge::AttrUtils::SetInt(&model, ATTR_MODEL_CORE_TYPE, core_type)) { + REPORT_INNER_ERROR("E19999", "Set Attr:%s in model failed when %s", + ATTR_MODEL_CORE_TYPE.c_str(), __FUNCTION__); GELOGE(FAILED, "SetInt of ATTR_CORE_TYPE failed."); } InitL1FusionOption(); GE_CHK_BOOL_EXEC(ge::AttrUtils::SetBool(&model, ATTR_NAME_SWITCH_FOR_L1_FUSION, is_l1_fusion_enable_), + REPORT_INNER_ERROR("E19999", "Set Attr:%s in model failed when %s", + ATTR_NAME_SWITCH_FOR_L1_FUSION.c_str(), __FUNCTION__); GELOGE(FAILED, "SetBool of ATTR_NAME_SWITCH_FOR_L1_FUSION failed."); return FAILED); const DumpProperties &dump_properties = DumpManager::GetInstance().GetDumpProperties(session_id_); bool is_op_debug = dump_properties.IsOpDebugOpen(); if (is_op_debug) { if (!ge::AttrUtils::SetBool(&model, ATTR_OP_DEBUG_FLAG, is_op_debug)) { + REPORT_INNER_ERROR("E19999", "Set Attr:%s in model failed when %s", + ATTR_OP_DEBUG_FLAG.c_str(), __FUNCTION__); GELOGE(FAILED, "SetBool of ATTR_OP_DEBUG_FLAG failed."); return FAILED; } uint32_t op_debug_mode = dump_properties.GetOpDebugMode(); GELOGI("Get op debug mode:%d", op_debug_mode); if (!ge::AttrUtils::SetInt(&model, ATTR_OP_DEBUG_MODE, op_debug_mode)) { + REPORT_INNER_ERROR("E19999", "Set Attr:%s in model failed when %s", + ATTR_OP_DEBUG_MODE.c_str(), __FUNCTION__); GELOGE(FAILED, "SetBool of ATTR_OP_DEBUG_MODE failed."); return FAILED; } @@ -516,6 +555,8 @@ Status ModelBuilder::MergeWeights() { // If MutableTensor failed, weight is nullptr. (void)ge::AttrUtils::MutableTensor(op_desc, ATTR_NAME_WEIGHTS, weight); if (weight == nullptr) { + REPORT_INNER_ERROR("E19999", "Can't get const weight in op:%s(%s) when %s", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "Can't get const op weight, name: %s", node->GetName().c_str()); return FAILED; } @@ -538,8 +579,15 @@ Status ModelBuilder::MergeWeights() { continue; } if (weight_data.data() != nullptr) { - GE_IF_BOOL_EXEC(base_addr == nullptr, GELOGE(FAILED, "Base addr is nullptr."); return FAILED); + GE_IF_BOOL_EXEC(base_addr == nullptr, + REPORT_INNER_ERROR("E19999", "Check weight in op:%s(%s) is nullptr when %s", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), __FUNCTION__); + GELOGE(FAILED, "Base addr is nullptr."); + return FAILED); if (weight_offset_ - offset < weight_data.size()) { + REPORT_INNER_ERROR("E19999", "left weight size not enough for op:%s(%s) left_size:%zu, weight_size:%zu when %s", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), + weight_offset_ - offset, weight_data.size(), __FUNCTION__); GELOGE(FAILED, "left weight size not enough. left_size:%lu, weight_size:%lu", weight_offset_ - offset, weight_data.size()); return FAILED; @@ -551,6 +599,9 @@ Status ModelBuilder::MergeWeights() { auto err = memcpy_s(reinterpret_cast(dst_ptr), SECUREC_MEM_MAX_LEN, reinterpret_cast(src_ptr), SECUREC_MEM_MAX_LEN); if (err != EOK) { + REPORT_CALL_ERROR("E19999", "mem copy failed. errret:%u, " + "dst_ptr:%lx, dst_size:%lu, src_ptr:%lx, src_size:%lu, when %s", + err, dst_ptr, SECUREC_MEM_MAX_LEN, src_ptr, SECUREC_MEM_MAX_LEN, __FUNCTION__); GELOGE(FAILED, "mem copy failed. errret:%u, " "dst_ptr:%lx, dst_size:%lu, src_ptr:%lx, src_size:%lu", err, dst_ptr, SECUREC_MEM_MAX_LEN, src_ptr, SECUREC_MEM_MAX_LEN); @@ -562,6 +613,9 @@ Status ModelBuilder::MergeWeights() { } auto err = memcpy_s(reinterpret_cast(dst_ptr), left_size, reinterpret_cast(src_ptr), left_size); if (err != EOK) { + REPORT_CALL_ERROR("E19999", "mem copy failed. errret:%u, " + "dst_ptr:%lx, dst_size:%lu, src_ptr:%lx, src_size:%lu, when %s", + err, dst_ptr, SECUREC_MEM_MAX_LEN, src_ptr, SECUREC_MEM_MAX_LEN, __FUNCTION__); GELOGE(FAILED, "mem copy failed. errret:%u, " "dst_ptr:%lx, dst_size:%lu, src_ptr:%lx, src_size:%lu", err, dst_ptr, SECUREC_MEM_MAX_LEN, src_ptr, SECUREC_MEM_MAX_LEN); @@ -602,6 +656,8 @@ Status ModelBuilder::SaveDataToModel(ge::Model &model, ge::GeModel &ge_model) { } GE_IF_BOOL_EXEC(tbe_kernel == nullptr, continue); if (tbe_name_set.count(tbe_kernel->GetName()) > 0) { + REPORT_INNER_ERROR("E19999", "tbe_kernel name %s can't be the same, judge for op:%s(%s), when %s", + tbe_kernel->GetName().c_str(), n->GetName().c_str(), n->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "tbe_kernel name %s can't be the same", tbe_kernel->GetName().c_str()); return FAILED; } @@ -618,6 +674,8 @@ Status ModelBuilder::SaveDataToModel(ge::Model &model, ge::GeModel &ge_model) { node_op_desc->TryGetExtAttr(ge::OP_EXTATTR_CUSTAICPU_KERNEL, CustAICPUKernelPtr()); GE_IF_BOOL_EXEC(cust_aicpu_kernel == nullptr, continue); if (aicpu_name_set.count(cust_aicpu_kernel->GetName()) > 0) { + REPORT_INNER_ERROR("E19999", "aicpu_kernel name %s can't be the same, judge for op:%s(%s), when %s", + cust_aicpu_kernel->GetName().c_str(), n->GetName().c_str(), n->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "aicpu_kernel name %s can't be the same", cust_aicpu_kernel->GetName().c_str()); return FAILED; } @@ -640,6 +698,7 @@ Status ModelBuilder::SaveDataToModel(ge::Model &model, ge::GeModel &ge_model) { // Add task GeAttrValue::BYTES task_def_bytes; if (!AttrUtils::GetZeroCopyBytes(model, MODEL_ATTR_TASKS, task_def_bytes)) { + REPORT_CALL_ERROR("E19999", "Get attr:%s in model fail when %s", MODEL_ATTR_TASKS.c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Get zero copy bytes fail."); return INTERNAL_ERROR; } @@ -675,6 +734,7 @@ void ModelBuilder::SetModelVersion(ge::Model &model) { Status ModelBuilder::PreBuildModel() { if ((compute_graph_ == nullptr) || !(compute_graph_->IsValid())) { + REPORT_INNER_ERROR("E19999", "Check compute_graph no valid when %s", __FUNCTION__); GELOGE(FAILED, "Graph_ is not valid."); return FAILED; } @@ -754,6 +814,7 @@ Status ModelBuilder::CompileSingleOp() { // Create ge instance std::shared_ptr instance = ge::GELib::GetInstance(); if ((instance == nullptr) || !instance->InitFlag()) { + REPORT_INNER_ERROR("E19999", "Check GELib instance not init before when %s", __FUNCTION__); GELOGE(ge::GE_CLI_GE_NOT_INITIALIZED, "CompileSingleOp failed."); return ge::GE_CLI_GE_NOT_INITIALIZED; } @@ -775,6 +836,8 @@ Status ModelBuilder::CompileSingleOp() { (void)instance->DNNEngineManagerObj().GetDNNEngineName(node); kernel_lib_name = op_desc->GetOpKernelLibName(); if (kernel_lib_name.empty()) { + REPORT_INNER_ERROR("E19999", "Check kernel lib name empty of op:%s(%s) when %s", + node->GetName().c_str(), node->GetType().c_str(), __FUNCTION__); GELOGE(ge::INTERNAL_ERROR, "Get node:%s(%s) kernel lib failed.", node->GetName().c_str(), node->GetType().c_str()); return ge::INTERNAL_ERROR; @@ -785,6 +848,8 @@ Status ModelBuilder::CompileSingleOp() { if (kernel_info != nullptr) { node_vector_map[kernel_lib_name].emplace_back(node); } else { + REPORT_INNER_ERROR("E19999", "Get ops kernel info store failed for op:%s(%s), op_kernel_name:%s, when %s", + node->GetName().c_str(), node->GetType().c_str(), kernel_lib_name.c_str(), __FUNCTION__); GELOGE(ge::GE_GRAPH_PARAM_NULLPTR, "Get op %s ops kernel info store failed", node->GetName().c_str()); return ge::GE_GRAPH_PARAM_NULLPTR; } @@ -800,6 +865,8 @@ Status ModelBuilder::CompileSingleOp() { GELOGI("[GEPERFTRACE] The node size of compile op of %s is %zu", kernel_lib_name.c_str(), node_vector.size()); GE_TIMESTAMP_ADD(BatchCompileOp); if (ret != ge::SUCCESS) { + REPORT_CALL_ERROR("E19999", "Batch compile op failed, kernel lib name, node size:%u, when %s", + node_vector.size(), __FUNCTION__); GELOGE(ret, "Compile op failed, kernel lib name is %s", kernel_lib_name.c_str()); return ret; } diff --git a/ge/graph/build/run_context.cc b/ge/graph/build/run_context.cc index ba328840..100d5aee 100644 --- a/ge/graph/build/run_context.cc +++ b/ge/graph/build/run_context.cc @@ -27,15 +27,21 @@ Status RunContextUtil::InitMemInfo(uint8_t *data_mem_base, uint64_t data_mem_siz std::map mem_type_to_data_mem_size, uint8_t *weight_mem_base, uint64_t weight_mem_size) { if ((data_mem_size > 0) && (data_mem_base == nullptr)) { + REPORT_INNER_ERROR("E19999", "InitMemInfo param data_mem_base is null but data_mem_size = %lu", data_mem_size); GELOGE(PARAM_INVALID, "InitMemInfo param data_mem_base is null but data_mem_size = %lu.", data_mem_size); return PARAM_INVALID; } if ((weight_mem_size > 0) && (weight_mem_base == nullptr)) { + REPORT_INNER_ERROR("E19999", "InitMemInfo param weight_mem_base is null but weight_mem_size = %lu", + weight_mem_size); GELOGE(PARAM_INVALID, "InitMemInfo param weight_mem_base is null but weight_mem_size = %lu.", weight_mem_size); return PARAM_INVALID; } if (mem_type_to_data_mem_base.empty() || mem_type_to_data_mem_size.empty() || mem_type_to_data_mem_base.size() != mem_type_to_data_mem_size.size()) { + REPORT_INNER_ERROR("E19999", "InitMemInfo param mem_type_to_data_mem_base size[%zu] " + "is not equal to the size of mem_type_to_data_mem_size[%zu].", + mem_type_to_data_mem_base.size(), mem_type_to_data_mem_size.size()); GELOGE(PARAM_INVALID, "InitMemInfo param mem_type_to_data_mem_base size[%zu] is not equal to the size of " "mem_type_to_data_mem_size[%zu].", @@ -55,6 +61,7 @@ Status RunContextUtil::CreateRtModelResources(uint32_t stream_num, uint32_t even // Create rt model rtError_t rt_ret = rtModelCreate(&rt_model_, 0); if (rt_ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "call rtModelCreate fail, ret:%d, when %s", static_cast(rt_ret), __FUNCTION__); GELOGE(RT_FAILED, "rtModelCreate failed. rt_ret = %d", static_cast(rt_ret)); return RT_FAILED; } @@ -64,6 +71,8 @@ Status RunContextUtil::CreateRtModelResources(uint32_t stream_num, uint32_t even rtStream_t stream = nullptr; rt_ret = rtStreamCreate(&stream, 0); if (rt_ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "call rtStreamCreate fail, ret:%d, index:%u, when %s", + static_cast(rt_ret), i, __FUNCTION__); GELOGE(RT_FAILED, "rtStreamCreate failed. rt_ret = %d, index = %u", static_cast(rt_ret), i); return RT_FAILED; } @@ -71,6 +80,8 @@ Status RunContextUtil::CreateRtModelResources(uint32_t stream_num, uint32_t even rt_ret = rtModelBindStream(rt_model_, stream, 0); if (rt_ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "call rtModelBindStream fail, ret:%d, index:%u, when %s", + static_cast(rt_ret), i, __FUNCTION__); GELOGE(RT_FAILED, "Bind stream and model failed. rt_ret = %d, index = %u", static_cast(rt_ret), i); return RT_FAILED; } @@ -81,6 +92,8 @@ Status RunContextUtil::CreateRtModelResources(uint32_t stream_num, uint32_t even rtEvent_t event = nullptr; rt_ret = rtEventCreate(&event); if (rt_ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "call rtEventCreate fail, ret:%d, index:%u, when %s", + static_cast(rt_ret), i, __FUNCTION__); GELOGE(RT_FAILED, "rtEventCreate failed. rt_ret = %d, index = %u", static_cast(rt_ret), i); return RT_FAILED; } @@ -92,6 +105,8 @@ Status RunContextUtil::CreateRtModelResources(uint32_t stream_num, uint32_t even rtLabel_t label = nullptr; rt_ret = rtLabelCreateV2(&label, rt_model_); if (rt_ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "call rtLabelCreateV2 fail, ret:%d, index:%u, when %s", + static_cast(rt_ret), i, __FUNCTION__); GELOGE(RT_FAILED, "rtLabelCreate failed. rt_ret = %d, index = %u", static_cast(rt_ret), i); return RT_FAILED; } @@ -143,12 +158,15 @@ Status RunContextUtil::CreateRunContext(Model &model, const ComputeGraphPtr &gra GELOGD("Begin to Create RunContext, session_id = %lu", session_id); // check params if (graph == nullptr) { + REPORT_INNER_ERROR("E19999", "Check param graph nullptr, session_id:%lu, when %s", session_id, __FUNCTION__); GELOGE(PARAM_INVALID, "CreateRunContext param graph is null. session_id=%lu", session_id); return PARAM_INVALID; } uint32_t stream_num = 0; if (!AttrUtils::GetInt(&model, ATTR_MODEL_STREAM_NUM, stream_num)) { + REPORT_INNER_ERROR("Get Attr:%s fail for model, session_id:%lu, when %s", + ATTR_MODEL_STREAM_NUM.c_str(), session_id, __FUNCTION__); GELOGE(INTERNAL_ERROR, "Get stream_num attr from model_def failed. session_id=%lu", session_id); return INTERNAL_ERROR; } @@ -156,6 +174,8 @@ Status RunContextUtil::CreateRunContext(Model &model, const ComputeGraphPtr &gra uint32_t event_num = 0; if (!AttrUtils::GetInt(&model, ATTR_MODEL_EVENT_NUM, event_num)) { + REPORT_INNER_ERROR("Get Attr:%s fail for model, session_id:%lu, when %s", + ATTR_MODEL_EVENT_NUM.c_str(), session_id, __FUNCTION__); GELOGE(INTERNAL_ERROR, "Get event_num attr from model failed. session_id=%lu", session_id); return INTERNAL_ERROR; } @@ -163,6 +183,8 @@ Status RunContextUtil::CreateRunContext(Model &model, const ComputeGraphPtr &gra uint32_t label_num = 0; if (!AttrUtils::GetInt(&model, ATTR_MODEL_LABEL_NUM, label_num)) { + REPORT_INNER_ERROR("Get Attr:%s fail for model, session_id:%lu, when %s", + ATTR_MODEL_LABEL_NUM.c_str(), session_id, __FUNCTION__); GELOGE(INTERNAL_ERROR, "Get label_num attr from model failed. session_id=%lu", session_id); return INTERNAL_ERROR; } diff --git a/ge/graph/build/stream_allocator.cc b/ge/graph/build/stream_allocator.cc index bd7cf7d1..aa75d1c0 100644 --- a/ge/graph/build/stream_allocator.cc +++ b/ge/graph/build/stream_allocator.cc @@ -76,6 +76,7 @@ Status StreamAllocator::AssignLogicalStreams(const std::map &m auto gelib = GELib::GetInstance(); if (gelib == nullptr) { + REPORT_INNER_ERROR("E19999", "Check GELib instance nullptr when %s", __FUNCTION__); GELOGE(FAILED, "Get GELib instance failed."); return FAILED; } @@ -184,6 +185,8 @@ Status StreamAllocator::AssignSingleStream() { } if (stream_num_ > 1) { + REPORT_INNER_ERROR("E19999", "The number of ts streams is %ld, only one is supported when %s", + stream_num_, __FUNCTION__); GELOGE(FAILED, "The number of ts streams is %ld, only one is supported.", stream_num_); return FAILED; } @@ -257,6 +260,9 @@ Status StreamAllocator::SetActiveStreamsByLabel() { } } GE_CHK_BOOL_EXEC(AttrUtils::SetListInt(node->GetOpDesc(), ATTR_NAME_ACTIVE_STREAM_LIST, activated_stream_list), + REPORT_INNER_ERROR("E19999", "Set Attr:%s for op:%s(%s) failed when %s", + ATTR_NAME_ACTIVE_STREAM_LIST.c_str(), + node->GetName().c_str(), node->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "SetListInt failed."); return FAILED); } @@ -307,6 +313,9 @@ Status StreamAllocator::SetActiveStreamsForSubgraphs() { } if (!AttrUtils::SetListInt(first_active_node->GetOpDesc(), ATTR_NAME_ACTIVE_STREAM_LIST, active_streams)) { + REPORT_INNER_ERROR("E19999", "Set Attr:%s for op:%s(%s) failed when %s", + ATTR_NAME_ACTIVE_STREAM_LIST.c_str(), + first_active_node->GetName().c_str(), first_active_node->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "Set active streams for node %s failed.", first_active_node->GetName().c_str()); return FAILED; } @@ -376,6 +385,8 @@ Status StreamAllocator::InsertOneEventInTwoNodes(const NodePtr &cur_node, const } if (next_stream_id == kInvalidStream) { + REPORT_INNER_ERROR("E19999", "Stream id of next_node %s(%s) should not be %ld when %s", + next_node->GetName().c_str(), next_node->GetType().c_str(), kInvalidStream, __FUNCTION__); GELOGE(FAILED, "Stream id of next_node %s should not be %ld", next_node->GetName().c_str(), kInvalidStream); return FAILED; } @@ -589,8 +600,14 @@ Status StreamAllocator::OptimizeByStreamActivate() { // -> stream(streamSwitch) -> stream(streamActivate) -> stream(stream true or false) // No need to insert an event between node in stream(normal) and node in stream(stream true or false) bool StreamAllocator::IsRecvNodeActivatedBySendNode(const NodePtr &send_node_ptr, const NodePtr &recv_node_ptr) const { - GE_CHECK_NOTNULL_EXEC(send_node_ptr->GetOpDesc(), GELOGE(FAILED, "op desc is nullptr"); return false); - GE_CHECK_NOTNULL_EXEC(recv_node_ptr->GetOpDesc(), GELOGE(FAILED, "op desc is nullptr"); return false); + GE_CHECK_NOTNULL_EXEC(send_node_ptr->GetOpDesc(), + REPORT_INNER_ERROR("E19999", "Check param send_node_ptr nullptr when %s", __FUNCTION__); + GELOGE(FAILED, "op desc is nullptr"); + return false); + GE_CHECK_NOTNULL_EXEC(recv_node_ptr->GetOpDesc(), + REPORT_INNER_ERROR("E19999", "Check param recv_node_ptr nullptr when %s", __FUNCTION__); + GELOGE(FAILED, "op desc is nullptr"); + return false); auto cur_stream_id = send_node_ptr->GetOpDesc()->GetStreamId(); if (AttrUtils::HasAttr(recv_node_ptr->GetOpDesc(), ATTR_NAME_STREAM_LABEL)) { // find streamActivate node @@ -714,6 +731,8 @@ Status StreamAllocator::SplitStreams(vector> &split_streams) { continue; } if (stream_id > last_stream_id) { + REPORT_INNER_ERROR("E19999", "streamid(%ld) > last_stream_id(%ld), check invalid when %s", + stream_id, last_stream_id, __FUNCTION__); GELOGE(FAILED, "SplitStreams:streamid(%ld) > last_stream_id(%ld)", stream_id, last_stream_id); return FAILED; } @@ -727,6 +746,8 @@ Status StreamAllocator::SplitStreams(vector> &split_streams) { stream_continuous_2_node_num_map[continuous_stream_label]++; // return error if (stream_continuous_2_node_num_map[continuous_stream_label] > max_node_num_one_stream) { + REPORT_INNER_ERROR("E19999", "Check node[%s] stream_id[%ld] continuous stream label[%s] unsatisfied when %s", + op_desc->GetName().c_str(), stream_id, continuous_stream_label.c_str(), __FUNCTION__); GELOGE(FAILED, "SplitStreams:node[%s] stream_id[%ld] continuous stream label[%s] unsatisfied ", op_desc->GetName().c_str(), stream_id, continuous_stream_label.c_str()); return FAILED; @@ -881,6 +902,8 @@ Status StreamAllocator::UpdateActiveStreamsForSwitchNode(NodePtr &switch_node) { GE_CHECK_NOTNULL(op_desc); if (!AttrUtils::SetListInt(op_desc, ATTR_NAME_ACTIVE_STREAM_LIST, stream_ids)) { + REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s) when %s", ATTR_NAME_ACTIVE_STREAM_LIST.c_str(), + op_desc->GetName().c_str(), op_desc->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "SetListInt failed."); return FAILED; } @@ -895,6 +918,8 @@ Status StreamAllocator::InsertActiveNodesAfterSwitch(NodePtr &switch_node, vecto vector ori_active_label_list; if (!AttrUtils::GetListStr(switch_desc, ATTR_NAME_ACTIVE_LABEL_LIST, ori_active_label_list) || ori_active_label_list.empty()) { + REPORT_INNER_ERROR("E19999", "Get Attr:%s fail for op:%s(%s) when %s", ATTR_NAME_ACTIVE_LABEL_LIST.c_str(), + switch_node->GetName().c_str(), switch_node->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Get active label list of switch %s failed.", switch_node->GetName().c_str()); return INTERNAL_ERROR; } @@ -918,6 +943,8 @@ Status StreamAllocator::InsertActiveNodesAfterSwitch(NodePtr &switch_node, vecto for (auto &active_node : added_active_nodes) { GE_CHECK_NOTNULL(switch_node->GetOutControlAnchor()); if (switch_node->GetOutControlAnchor()->LinkTo(active_node->GetInControlAnchor()) != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Link from %s to %s failed when %s", + switch_node->GetName().c_str(), active_node->GetName().c_str(), __FUNCTION__); GELOGE(FAILED, "Link %s to %s failed.", switch_node->GetName().c_str(), active_node->GetName().c_str()); return FAILED; } @@ -933,6 +960,8 @@ Status StreamAllocator::UpdateActiveStreamsForActiveNode(const vector new_active_streams = active_streams; for (uint32_t logical_stream : active_streams) { if (static_cast(logical_stream) >= split_streams.size()) { + REPORT_INNER_ERROR("E19999", "Check logical stream:%u is out of range:%zu when %s", + logical_stream, split_streams.size(), __FUNCTION__); GELOGE(FAILED, "logical stream is out of range."); return FAILED; } @@ -951,6 +980,8 @@ Status StreamAllocator::UpdateActiveStreamsForActiveNode(const vectorGetOpDesc(), ATTR_NAME_ACTIVE_STREAM_LIST, new_active_streams)) { + REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s) when %s", ATTR_NAME_ACTIVE_STREAM_LIST.c_str(), + node->GetName().c_str(), node->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "Set active streams for node %s failed.", node->GetName().c_str()); return FAILED; } @@ -991,6 +1022,8 @@ Status StreamAllocator::UpdateActiveStreamsForSubgraphs() const { new_active_streams.emplace(static_cast(new_split_stream)); active_streams.assign(new_active_streams.begin(), new_active_streams.end()); if (!AttrUtils::SetListInt(active_op, ATTR_NAME_ACTIVE_STREAM_LIST, active_streams)) { + REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s) when %s", ATTR_NAME_ACTIVE_STREAM_LIST.c_str(), + active_op->GetName().c_str(), active_op->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "Set active streams for node %s failed.", active_node->GetName().c_str()); return FAILED; } @@ -1059,6 +1092,8 @@ Status StreamAllocator::SetActiveStreamsForLoop() { NodePtr pre_switch_node = FindSwitchNodeBeforeLoopActiveNode(node); if (pre_switch_node == nullptr) { + REPORT_INNER_ERROR("E19999", "Find switch node before loop active node %s fail when %s", + node->GetName().c_str(), __FUNCTION__); GELOGE(FAILED, "find switch node before loop active node %s failed", node->GetName().c_str()); return FAILED; } @@ -1066,6 +1101,9 @@ Status StreamAllocator::SetActiveStreamsForLoop() { if (!AttrUtils::GetListStr(node->GetOpDesc(), ATTR_NAME_ACTIVE_LABEL_LIST, activated_label_list) || activated_label_list.empty()) { GE_CHK_BOOL_EXEC(AttrUtils::SetListInt(node->GetOpDesc(), ATTR_NAME_ACTIVE_STREAM_LIST, loop_active_streams), + REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s) when %s", + ATTR_NAME_ACTIVE_STREAM_LIST.c_str(), + node->GetName().c_str(), node->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "SetListInt failed."); return FAILED); for (const auto &stream_id : loop_active_streams) { @@ -1112,6 +1150,8 @@ Status StreamAllocator::CheckStreamActived() const { uint32_t stream_id = static_cast(node->GetOpDesc()->GetStreamId()); auto iter = find(active_streams.begin(), active_streams.end(), stream_id); if (iter != active_streams.end()) { + REPORT_INNER_ERROR("E19999", "Node:%s(%s) cannot active its own stream %u, check invalid when %s", + node->GetName().c_str(), node->GetType().c_str(), stream_id, __FUNCTION__); GELOGE(FAILED, "Node %s cannot active its own stream %u.", node->GetName().c_str(), stream_id); return FAILED; } @@ -1139,6 +1179,7 @@ Status StreamAllocator::RefreshContinuousEvents() { for (size_t i = 0; i < send_events.size(); i++) { auto find_it = old_to_new_events.find(send_events[i]); if (find_it == old_to_new_events.end()) { + REPORT_INNER_ERROR("E19999", "Check invalid send event %u when %s", send_events[i], __FUNCTION__); GELOGE(FAILED, "RefreshContinuousEvents: invalid send event %u", send_events[i]); return FAILED; } @@ -1152,6 +1193,7 @@ Status StreamAllocator::RefreshContinuousEvents() { for (size_t i = 0; i < recv_events.size(); i++) { auto find_it = old_to_new_events.find(recv_events[i]); if (find_it == old_to_new_events.end()) { + REPORT_INNER_ERROR("E19999", "Check invalid recv event %u when %s", recv_events[i], __FUNCTION__); GELOGE(FAILED, "RefreshContinuousEvents: invalid recv event %u", recv_events[i]); return FAILED; } @@ -1180,7 +1222,11 @@ Status StreamAllocator::InsertSyncEventNodes() { int64_t temp_stream_id = node->GetOpDesc()->GetStreamId(); op_desc_ptr->SetStreamId(temp_stream_id); - GE_CHK_BOOL_EXEC(AttrUtils::SetInt(op_desc_ptr, RECV_ATTR_EVENT_ID, event_id), GELOGE(FAILED, "SetInt failed."); + GE_CHK_BOOL_EXEC(AttrUtils::SetInt(op_desc_ptr, RECV_ATTR_EVENT_ID, event_id), + REPORT_INNER_ERROR("E19999", "Set Attr:%s for op:%s(%s) failed, event_id:%u, when %s", + RECV_ATTR_EVENT_ID.c_str(), + node->GetName().c_str(), node->GetType().c_str(), __FUNCTION__); + GELOGE(FAILED, "SetInt failed."); return FAILED); (void)AttrUtils::SetListStr(op_desc_ptr, ATTR_NAME_DATA_DUMP_ORIGIN_OP_NAMES, std::move(std::vector())); @@ -1189,6 +1235,8 @@ Status StreamAllocator::InsertSyncEventNodes() { GE_CHECK_NOTNULL(recv_node->GetOutControlAnchor()); Status status = GraphUtils::AddEdge(recv_node->GetOutControlAnchor(), node->GetInControlAnchor()); if (status != SUCCESS) { + REPORT_INNER_ERROR("E19999", "Add edge from node %s to node %s failed when %s", + recv_node->GetName().c_str(), node->GetName().c_str(), __FUNCTION__); GELOGE(status, "Add edge for node %s and node %s failed.", recv_node->GetName().c_str(), node->GetName().c_str()); return status; @@ -1217,6 +1265,8 @@ Status StreamAllocator::InsertSyncEventNodes() { GE_CHECK_NOTNULL(send_node->GetInControlAnchor()); Status status = GraphUtils::AddEdge(node->GetOutControlAnchor(), send_node->GetInControlAnchor()); if (status != SUCCESS) { + REPORT_INNER_ERROR("E19999", "Add edge from node %s to node %s failed when %s", + node->GetName().c_str(), send_node->GetName().c_str(), __FUNCTION__); GELOGE(status, "Add edge for node %s and node %s failed.", node->GetName().c_str(), send_node->GetName().c_str()); return status; @@ -1228,6 +1278,8 @@ Status StreamAllocator::InsertSyncEventNodes() { Status status = whole_graph_->InsertGraphEvents(); if (status != SUCCESS) { + REPORT_CALL_ERROR("E19999", "Insert Graph Events fail, graph:%s, when %s", + whole_graph_->GetName().c_str(), __FUNCTION__); GELOGE(status, "Graph ReorderEventNodes failed"); return status; } @@ -1274,6 +1326,8 @@ Status StreamAllocator::GetMaxStreamAndTask(bool huge_stream, uint32_t &max_stre } rtError_t ret = rtGetMaxStreamAndTask(stream_type, &max_stream_count, &max_task_count); if (ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "call rtGetMaxStreamAndTask fail, ret:%d, stream_type:%u, when %s", + static_cast(ret), stream_type, __FUNCTION__); GELOGE(FAILED, "Get max stream and task count by rts failed."); return FAILED; } @@ -1416,6 +1470,7 @@ Status StreamAllocator::AddActiveNodes(NodePtr &switch_node, const vectorGetOutControlAnchor()); if (switch_node->GetOutControlAnchor()->Unlink(node->GetInControlAnchor()) != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("Unlink %s to %s failed when %s", + switch_node->GetName().c_str(), node->GetName().c_str(), __FUNCTION__); GELOGE(FAILED, "Unlink %s to %s failed.", switch_node->GetName().c_str(), node->GetName().c_str()); return FAILED; } GE_CHECK_NOTNULL(active_node->GetOutControlAnchor()); if (active_node->GetOutControlAnchor()->LinkTo(node->GetInControlAnchor()) != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("Link %s to %s failed when %s", + active_node->GetName().c_str(), node->GetName().c_str(), __FUNCTION__); GELOGE(FAILED, "Link %s to %s failed.", active_node->GetName().c_str(), node->GetName().c_str()); return FAILED; } @@ -1477,12 +1536,15 @@ Status StreamAllocator::AddActiveNodes(NodePtr &switch_node, const vector &streams = labeled_streams_[active_label]; vector active_streams(streams.begin(), streams.end()); if (!AttrUtils::SetListInt(active_node->GetOpDesc(), ATTR_NAME_ACTIVE_STREAM_LIST, active_streams)) { + REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s) when %s", ATTR_NAME_ACTIVE_STREAM_LIST.c_str(), + active_node->GetName().c_str(), active_node->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "SetListInt of %s failed.", ATTR_NAME_ACTIVE_STREAM_LIST.c_str()); return FAILED; } diff --git a/ge/graph/build/stream_graph_optimizer.cc b/ge/graph/build/stream_graph_optimizer.cc index 05049818..15ca58fb 100644 --- a/ge/graph/build/stream_graph_optimizer.cc +++ b/ge/graph/build/stream_graph_optimizer.cc @@ -14,6 +14,9 @@ * limitations under the License. */ #include "stream_graph_optimizer.h" + +#include + #include "common/util.h" #include "framework/common/debug/ge_log.h" #include "graph/utils/node_utils.h" @@ -122,6 +125,9 @@ Status StreamGraphOptimizer::OptimizeStreamedSubGraph(const ComputeGraphPtr &com GE_CHECK_NOTNULL(op_desc); int64_t stream_id = op_desc->GetStreamId(); if (static_cast(stream_id) >= run_context.graphStreamList.size()) { + REPORT_INNER_ERROR("E19999", "Check stream_id:%ld in op:%s(%s) is bigger than run_context.graphStreamList.size():%zu " + "when %s", stream_id, op_desc->GetName().c_str(), + op_desc->GetType().c_str(), run_context.graphStreamList.size(), __FUNCTION__); GELOGE(FAILED, "stream_id %ld is bigger than run_context.graphStreamList.size() %zu", stream_id, run_context.graphStreamList.size()); return FAILED; @@ -135,6 +141,9 @@ Status StreamGraphOptimizer::OptimizeStreamedSubGraph(const ComputeGraphPtr &com for (auto iter = graph_optimizers.begin(); iter != graph_optimizers.end(); ++iter) { GE_CHECK_NOTNULL(*iter); Status ret = (*iter)->OptimizeStreamGraph(*subgraph, run_context); + REPORT_CALL_ERROR("E19999", "Call optimize streamed subgraph failed, subgraph: %s, engine_name: %s, graph " + "Optimizer num: %zu, ret: %u", subgraph->GetName().c_str(), engine_name.c_str(), + graph_optimizers.size(), ret); if (ret != SUCCESS) { GELOGE( ret, diff --git a/ge/graph/build/task_generator.cc b/ge/graph/build/task_generator.cc index 4eda4020..9e5e83c2 100755 --- a/ge/graph/build/task_generator.cc +++ b/ge/graph/build/task_generator.cc @@ -69,6 +69,7 @@ Status TaskGenerator::GetTaskInfo(Model &model, ComputeGraphPtr &graph, uint64_t GELOGD("Begin to Get TaskInfo. session_id=%lu", session_id); // Check params if (graph == nullptr) { + REPORT_INNER_ERROR("E19999", "Check param graph is null, session_id:%lu, when %s", session_id, __FUNCTION__); GELOGE(PARAM_INVALID, "GetTaskInfo param graph is null. session_id=%lu", session_id); return PARAM_INVALID; } @@ -93,6 +94,8 @@ Status TaskGenerator::GetTaskInfo(Model &model, ComputeGraphPtr &graph, uint64_t op_name.push_back(iter.second); } GE_CHK_BOOL_EXEC(ge::AttrUtils::SetListStr(model, ATTR_MODEL_TASK_INDEX_OP_NAME, op_name), + REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for model:%s when %s", + ATTR_MODEL_TASK_INDEX_OP_NAME.c_str(), model.GetName().c_str(), __FUNCTION__); GELOGE(FAILED, "SetListStr failed."); return FAILED); @@ -106,6 +109,8 @@ Status TaskGenerator::GetTaskInfo(Model &model, ComputeGraphPtr &graph, uint64_t for (const TaskDef &task_def_temp : task_def_list) { TaskDef *task_def = model_task_def.add_task(); if (task_def == nullptr) { + REPORT_INNER_ERROR("E19999", "Add task_def in ModelTaskDef fail, session_id:%lu, graph:%s, model:%s, when %s", + session_id, graph->GetName().c_str(), model.GetName().c_str(), __FUNCTION__); GELOGE(FAILED, "task_def is nullptr."); return FAILED; } @@ -126,30 +131,44 @@ Status TaskGenerator::AddModelTaskToModel(const ModelTaskDef &model_task_def, ui RunContext &run_context) { GE_CHK_BOOL_EXEC( AttrUtils::SetInt(model, MODEL_ATTR_TASK_GEN_BASE_ADDR, reinterpret_cast(run_context.dataMemBase)), + REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for model:%s when %s", + MODEL_ATTR_TASK_GEN_BASE_ADDR.c_str(), model.GetName().c_str(), __FUNCTION__); GELOGE(FAILED, "SetInt MODEL_ATTR_TASK_GEN_BASE_ADDR failed."); return FAILED); GE_CHK_BOOL_EXEC( AttrUtils::SetInt(model, MODEL_ATTR_TASK_GEN_WEIGHT_ADDR, reinterpret_cast(run_context.weightMemBase)), + REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for model:%s when %s", + MODEL_ATTR_TASK_GEN_WEIGHT_ADDR.c_str(), model.GetName().c_str(), __FUNCTION__); GELOGE(FAILED, "SetInt MODEL_ATTR_TASK_GEN_WEIGHT_ADDR failed."); return FAILED); GE_CHK_BOOL_EXEC(AttrUtils::SetInt(model, ATTR_MODEL_TASK_GEN_VAR_ADDR, reinterpret_cast(var_mem_base_)), + REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for model:%s when %s", + ATTR_MODEL_TASK_GEN_VAR_ADDR.c_str(), model.GetName().c_str(), __FUNCTION__); GELOGE(FAILED, "SetInt ATTR_MODEL_TASK_GEN_VAR_ADDR failed."); return FAILED); GE_CHK_BOOL_EXEC(AttrUtils::SetInt(model, ATTR_MODEL_VAR_SIZE, var_mem_size_), + REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for model:%s when %s", + ATTR_MODEL_VAR_SIZE.c_str(), model.GetName().c_str(), __FUNCTION__); GELOGE(FAILED, "SetInt ATTR_MODEL_VAR_SIZE failed."); return FAILED); GE_CHK_BOOL_EXEC(AttrUtils::SetInt(model, MODEL_ATTR_SESSION_ID, session_id), + REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for mode:%s when %s", + MODEL_ATTR_SESSION_ID.c_str(), model.GetName().c_str(), __FUNCTION__); GELOGE(FAILED, "SetInt MODEL_ATTR_SESSION_ID failed."); return FAILED); size_t task_size = model_task_def.ByteSizeLong(); ge::Buffer serial_buff(task_size); if (!model_task_def.SerializePartialToArray(serial_buff.GetData(), static_cast(task_size))) { + REPORT_INNER_ERROR("E19999", "model_task_def's serialize failed, model name = %s, task_size=%zu when %s", + model.GetName().c_str(), task_size, __FUNCTION__); GELOGE(FAILED, "model_task_def's serialize failed, model name = %s, task_size=%zu.", model.GetName().c_str(), task_size); return FAILED; } if (!AttrUtils::SetZeroCopyBytes(model, MODEL_ATTR_TASKS, std::move(serial_buff))) { + REPORT_INNER_ERROR("E19999", "Set model task to model failed, model name = %s, task_size=%zu.", + model.GetName().c_str(), task_size, __FUNCTION__); GELOGE(FAILED, "Set model task to model failed, model name = %s, task_size=%zu.", model.GetName().c_str(), task_size); return FAILED; @@ -167,7 +186,10 @@ Status TaskGenerator::UpdateOpIsVarAttr(const OpDescPtr &op_desc, uint64_t sessi for (int64_t input : input_offsets) { input_var.push_back(VarManager::Instance(session_id)->IsVarAddr(input)); } - GE_CHK_BOOL_EXEC(AttrUtils::SetListBool(op_desc, kIsInputVar, input_var), GELOGE(FAILED, "SetListBool failed."); + GE_CHK_BOOL_EXEC(AttrUtils::SetListBool(op_desc, kIsInputVar, input_var), + REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s) when %s", kIsInputVar, + op_desc->GetName().c_str(), op_desc->GetType().c_str(), __FUNCTION__); + GELOGE(FAILED, "SetListBool failed."); return FAILED); } @@ -177,7 +199,10 @@ Status TaskGenerator::UpdateOpIsVarAttr(const OpDescPtr &op_desc, uint64_t sessi for (int64_t output : output_offsets) { output_var.push_back(VarManager::Instance(session_id)->IsVarAddr(output)); } - GE_CHK_BOOL_EXEC(AttrUtils::SetListBool(op_desc, kIsOutputVar, output_var), GELOGE(FAILED, "SetListBool failed."); + GE_CHK_BOOL_EXEC(AttrUtils::SetListBool(op_desc, kIsOutputVar, output_var), + REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s) when %s", kIsOutputVar, + op_desc->GetName().c_str(), op_desc->GetType().c_str(), __FUNCTION__); + GELOGE(FAILED, "SetListBool failed."); return FAILED); } return SUCCESS; @@ -252,6 +277,7 @@ Status TaskGenerator::GenerateTask(RunContext &run_context, ComputeGraphPtr &gra GELOGD("Beign to generate task, graph name is %s.", graph->GetName().c_str()); std::shared_ptr ge_lib = GELib::GetInstance(); if ((ge_lib == nullptr) || !ge_lib->InitFlag()) { + REPORT_INNER_ERROR("E19999", "Check GELib instance not init before when %s", __FUNCTION__); GELOGE(GE_CLI_GE_NOT_INITIALIZED, "GenerateTask failed."); return GE_CLI_GE_NOT_INITIALIZED; } @@ -319,6 +345,8 @@ Status TaskGenerator::GenerateTask(RunContext &run_context, ComputeGraphPtr &gra } auto kernel_info_store = ops_kernel_manager.GetOpsKernelInfoStore(op_kernel_lib_name); if (kernel_info_store == nullptr) { + REPORT_INNER_ERROR("E19999", "Get ops kernel info store failed for op:%s(%s), op_kernel_name:%s when %s", + node->GetName().c_str(), node->GetType().c_str(), op_kernel_lib_name.c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "No ops kernel store or ops kernel builder found. node:%s(%s), op_kernel_lib_name=%s.", name.c_str(), @@ -344,6 +372,8 @@ Status TaskGenerator::GenerateTask(RunContext &run_context, ComputeGraphPtr &gra auto ret = OpsKernelBuilderManager::Instance().GenerateTask(*node, run_context, task_def_list); GE_TIMESTAMP_ADD(GenerateTask); if (ret != SUCCESS) { + REPORT_CALL_ERROR("E19999", "Call OpsKernelBuilderManager GenerateTask fail for op:%s(%s) when %s", + node->GetName().c_str(), node->GetType().c_str(), __FUNCTION__); GELOGE(ret, "Call %s to generate node[name:%s(%s), id:%ld, stream_id:%ld] task failed.", op_kernel_lib_name.c_str(), name.c_str(), type.c_str(), op_id, stream_id); return ret; @@ -353,6 +383,9 @@ Status TaskGenerator::GenerateTask(RunContext &run_context, ComputeGraphPtr &gra size_t task_list_size_after = task_def_list.size(); // If tasks is reduced if (task_list_size_after < task_list_size_before) { + REPORT_INNER_ERROR("E19999", "Call %s to generate node[name:%s(%s), id:%ld, stream_id:%ld] task " + "but task num from %zu to %zu, check invalid", op_kernel_lib_name.c_str(), name.c_str(), + type.c_str(), op_id, stream_id, task_list_size_before, task_list_size_after); GELOGE(FAILED, "Call %s to generate node[name:%s(%s), id:%ld, stream_id:%ld] task. but task num from %zu to %zu.", op_kernel_lib_name.c_str(), name.c_str(), type.c_str(), op_id, stream_id, task_list_size_before, task_list_size_after); @@ -417,6 +450,9 @@ Status TaskGenerator::GenerateTaskForFusionNode(FusionTaskInfo &fusion_task_info size_t task_list_size_before = task_def_list.size(); OpsKernelInfoStorePtr kernel_info_store = ops_kernel_manager.GetOpsKernelInfoStore(op_kernel_lib_name); if (kernel_info_store == nullptr) { + REPORT_INNER_ERROR("E19999", "Get ops kernel info store failed for op:%s(%s), op_kernel_name:%s when %s", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), + op_kernel_lib_name.c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Fusion: No ops kernel store or ops kernel builder found. fusion_node:%s(%s), op_kernel_lib_name=%s.", fusion_node_name.c_str(), fusion_node_type.c_str(), op_kernel_lib_name.c_str()); @@ -433,6 +469,9 @@ Status TaskGenerator::GenerateTaskForFusionNode(FusionTaskInfo &fusion_task_info int64_t op_id = op_desc->GetId(); int64_t stream_id = op_desc->GetStreamId(); if (stream_id < 0 || stream_id >= (int64_t)run_context.graphStreamList.size()) { + REPORT_INNER_ERROR("E19999", "Fusion: fusion_node[name:%s(%s), id:%ld] stream id is invalid, " + "stream list size=%zu, when %s", fusion_node_name.c_str(), fusion_node_type.c_str(), + op_id, run_context.graphStreamList.size(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Fusion: fusion_node[name:%s(%s), id:%ld] stream id is invalid, stream list size=%zu", fusion_node_name.c_str(), fusion_node_type.c_str(), op_id, run_context.graphStreamList.size()); return INTERNAL_ERROR; @@ -444,6 +483,9 @@ Status TaskGenerator::GenerateTaskForFusionNode(FusionTaskInfo &fusion_task_info op_kernel_lib_name.c_str(), fusion_node_name.c_str(), fusion_node_type.c_str(), op_id, stream_id); ret = OpsKernelBuilderManager::Instance().GenerateTask(*fusion_node, run_context, task_def_list); if (ret != SUCCESS) { + REPORT_CALL_ERROR("E19999", " Call %s to generate fusion_node:[fusion_node_name:%s(%s), " + "id:%ld, stream_id:%ld] task failed when %s", op_kernel_lib_name.c_str(), + fusion_node_name.c_str(), fusion_node_type.c_str(), op_id, stream_id, __FUNCTION__); GELOGE(ret, "Fusion: Call %s to generate fusion_node:[fusion_node_name:%s(%s), " "id:%ld, stream_id:%ld] task failed.", @@ -455,6 +497,10 @@ Status TaskGenerator::GenerateTaskForFusionNode(FusionTaskInfo &fusion_task_info size_t task_list_size_after = task_def_list.size(); // if tasks is reduced if (task_list_size_after < task_list_size_before) { + REPORT_INNER_ERROR("E19999", "InsertProfilingTask for fusion_node:[fusion_node_name:%s(%s), " + "id:%ld, stream_id:%ld] task, but task num from %zu to %zu, check invalid when %s", + op_kernel_lib_name.c_str(), fusion_node_name.c_str(), fusion_node_type.c_str(), + op_id, stream_id, task_list_size_before, task_list_size_after, __FUNCTION__); GELOGE(FAILED, "Fusion: Call %s to generate fusion_node:[fusion_node_name:%s(%s), " "id:%ld, stream_id:%ld] task. but task num from %zu to %zu.", @@ -489,6 +535,8 @@ Status TaskGenerator::GenerateTaskForFusionNode(FusionTaskInfo &fusion_task_info Status TaskGenerator::UpdateAnchorStatus(const NodePtr &node) { if (NodeUtils::SetAllAnchorStatus(node) != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "SetAllAnchorStatus fail for op:%s(%s) when %s", + node->GetName().c_str(), node->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "NodeUtils::SetAllAnchorStatus failed."); return INTERNAL_ERROR; } @@ -496,6 +544,8 @@ Status TaskGenerator::UpdateAnchorStatus(const NodePtr &node) { auto peer_anchor = anchor->GetPeerOutAnchor(); if (peer_anchor == nullptr) { if (AnchorUtils::SetStatus(anchor, ANCHOR_SUSPEND) != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Set in peer anchor status fail for op:%s(%s), anchor_index:%d, when %s", + node->GetName().c_str(), node->GetType().c_str(), anchor->GetIdx(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "AnchorUtils::SetStatus failed."); return INTERNAL_ERROR; } @@ -506,11 +556,15 @@ Status TaskGenerator::UpdateAnchorStatus(const NodePtr &node) { bool is_const = NodeUtils::GetConstOpType(peer_anchor->GetOwnerNode(), const_type); if (is_const && (const_type == CONSTANT)) { if (AnchorUtils::SetStatus(anchor, ANCHOR_CONST) != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Set in anchor CONST status fail for op:%s(%s), anchor_index:%d, when %s", + node->GetName().c_str(), node->GetType().c_str(), anchor->GetIdx(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "AnchorUtils::SetStatus failed."); return INTERNAL_ERROR; } } else { if (AnchorUtils::SetStatus(anchor, ANCHOR_DATA) != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Set in anchor DATA status fail for op:%s(%s), anchor_index:%d, when %s", + node->GetName().c_str(), node->GetType().c_str(), anchor->GetIdx(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "AnchorUtils::SetStatus failed."); return INTERNAL_ERROR; } @@ -523,12 +577,15 @@ Status TaskGenerator::UpdateAnchorStatus(const NodePtr &node) { Status TaskGenerator::MarkNodeAndSetIndex(ComputeGraphPtr &graph) { auto ge_lib = GELib::GetInstance(); if ((ge_lib == nullptr) || !ge_lib->InitFlag()) { + REPORT_INNER_ERROR("E19999", "Check GELib instance not init before when %s", __FUNCTION__); GELOGE(GE_CLI_GE_NOT_INITIALIZED, "GE is not initialized or is finalized."); return GE_CLI_GE_NOT_INITIALIZED; } const auto all_nodes = graph->GetNodes(graph->GetGraphUnknownFlag()); if (all_nodes.empty()) { + REPORT_INNER_ERROR("E19999", "Check param all_nodes empty in graph:%s when %s", + graph->GetName().c_str(), __FUNCTION__); GELOGE(GE_GRAPH_GRAPH_NODE_NULL, "Graph's node is empty"); return GE_GRAPH_GRAPH_NODE_NULL; } @@ -584,6 +641,9 @@ Status TaskGenerator::MarkFirstAndLastOps(const vector &ops, bool is_ for (auto &op_desc : continuous_ops) { string op_kernel_lib_name = op_desc->GetOpKernelLibName(); if (op_kernel_lib_name.empty()) { + REPORT_INNER_ERROR("E19999", "Get ops kernel info store failed for op:%s(%s), op_kernel_name:%s when %s", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), + op_kernel_lib_name.c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "node:%s(%s) get op kernel lib failed.", op_desc->GetName().c_str(), op_desc->GetType().c_str()); return INTERNAL_ERROR; @@ -599,9 +659,17 @@ Status TaskGenerator::MarkFirstAndLastOps(const vector &ops, bool is_ for (auto &it : first_and_last_ops) { auto &op_pair = it.second; - GE_CHK_BOOL_EXEC(ge::AttrUtils::SetBool(op_pair.first, kIsFirstNode, true), GELOGE(FAILED, "SetBool failed."); + GE_CHK_BOOL_EXEC(ge::AttrUtils::SetBool(op_pair.first, kIsFirstNode, true), + REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s) when %s", kIsFirstNode, + op_pair.first->GetName().c_str(), op_pair.first->GetType().c_str(), + __FUNCTION__); + GELOGE(FAILED, "SetBool failed."); return FAILED); - GE_CHK_BOOL_EXEC(ge::AttrUtils::SetBool(op_pair.second, kIsLastNode, true), GELOGE(FAILED, "SetBool failed."); + GE_CHK_BOOL_EXEC(ge::AttrUtils::SetBool(op_pair.second, kIsLastNode, true), + REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s) when %s", kIsLastNode, + op_pair.second->GetName().c_str(), op_pair.second->GetType().c_str(), + __FUNCTION__); + GELOGE(FAILED, "SetBool failed."); return FAILED); } } @@ -906,6 +974,8 @@ Status TaskGenerator::InsertProfilingArTaskBefore(const OpDescPtr &op_desc, std: for (size_t i = 0; i < all_reduce_nodes.size(); i++) { if (all_reduce_nodes[i] == node_index) { GE_IF_BOOL_EXEC(TypeUtils::CheckUint64MulOverflow(i, kProfilingArStep), + REPORT_INNER_ERROR("E19999", "Multiply result is out of range when calc profiling ar log id " + "for node:%s(%s)", op_desc->GetName().c_str(), op_desc->GetType().c_str()); GELOGE(FAILED, "Multiply result is out of range."); return FAILED); ar_log_id = i * kProfilingArStep + kProfilingArStartLogid; @@ -998,6 +1068,8 @@ Status TaskGenerator::InsertProfilingArTaskAfter(const OpDescPtr &op_desc, std:: for (size_t i = 0; i < all_reduce_nodes.size(); i++) { if (all_reduce_nodes[i] == node_index) { GE_IF_BOOL_EXEC(TypeUtils::CheckUint64MulOverflow(i, kProfilingArStep), + REPORT_INNER_ERROR("E19999", "Multiply result is out of range when calc profiling ar log id " + "for node:%s(%s)", op_desc->GetName().c_str(), op_desc->GetType().c_str()); GELOGE(FAILED, "Multiply result is out of range."); return FAILED); ar_log_id = i * kProfilingArStep + kProfilingArEndLogid; @@ -1107,6 +1179,7 @@ Status TaskGenerator::SetUnknownShapeStream(RunContext &run_context, rtStream_t run_context.stream = stream; rtError_t rt_ret = rtModelBindStream(run_context.model, stream, 0); if (rt_ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "Call rtModelBindStream fail, ret:0x%X when %s", rt_ret, __FUNCTION__); GELOGE(FAILED, "Call rt api failed, ret: 0x%X", rt_ret); GE_CHK_RT_RET(rtStreamDestroy(stream)); return FAILED; diff --git a/ge/graph/common/bcast.cc b/ge/graph/common/bcast.cc index 7948ff14..95a93897 100644 --- a/ge/graph/common/bcast.cc +++ b/ge/graph/common/bcast.cc @@ -73,6 +73,8 @@ Status BCast::SetShapeDifferentInfo(const kVecInt &x, const kVecInt &y) { y_bcast_i = x_i; grad_y_reduce_idx_.push_back(n - 1 - i); } else { + REPORT_INNER_ERROR("E19999", "SetShapeDifferentInfo failed. Two tensor shapes are not compatible " + "according to the broadcasting rule."); GELOGE(domi::PARAM_INVALID, "SetShapeDifferentInfo failed. Two tensor shapes are not compatible " "according to the broadcasting rule."); diff --git a/ge/graph/common/bcast.h b/ge/graph/common/bcast.h index 9df1c422..9b5b2538 100644 --- a/ge/graph/common/bcast.h +++ b/ge/graph/common/bcast.h @@ -111,11 +111,14 @@ class BCast { const std::function &func) { Status ret; if (func == nullptr) { + REPORT_INNER_ERROR("E19999", "Check param func nullptr when %s", __FUNCTION__); GELOGE(domi::PARAM_INVALID, "Param func is null"); return domi::PARAM_INVALID; } // Min input num is 2 if (input.size() < kMinDimNum) { + REPORT_INNER_ERROR("E19999", "Param input.size():%zu < %zu, check invalid when %s", + input.size(), kMinDimNum, __FUNCTION__); GELOGE(domi::PARAM_INVALID, "Input size is smaller than two."); return domi::PARAM_INVALID; } @@ -149,11 +152,14 @@ class BCast { Status BCastComputeCheck(const std::vector &input, std::vector &v_output, const std::function &func) { if (func == nullptr) { + REPORT_INNER_ERROR("E19999", "Check param func nullptr when %s", __FUNCTION__); GELOGE(PARAM_INVALID, "Param func is null"); return PARAM_INVALID; } // Min input num is 2 if (input.size() < kMinDimNum) { + REPORT_INNER_ERROR("E19999", "Param input.size():%zu < %zu, check invalid when %s", + input.size(), kMinDimNum, __FUNCTION__); GELOGE(PARAM_INVALID, "Input size is smaller than two."); return PARAM_INVALID; } @@ -179,6 +185,7 @@ class BCast { auto value = func((*(reinterpret_cast(x1_data) + x_index)), (*(reinterpret_cast(x2_data) + y_index)), data_type, ret); if (ret != SUCCESS) { + REPORT_INNER_ERROR("E19999", "BCastComputeCheck func execute failed, datatype is %d.", data_type); GELOGE(ret, "BCastComputeCheck func execute failed, datatype is %d.", data_type); return ret; } diff --git a/ge/graph/common/omg_util.cc b/ge/graph/common/omg_util.cc index 5c76d0a1..b0d64a41 100644 --- a/ge/graph/common/omg_util.cc +++ b/ge/graph/common/omg_util.cc @@ -36,6 +36,8 @@ Status GetOriginalType(const ge::NodePtr &node, string &type) { GE_CHECK_NOTNULL(node->GetOpDesc()); bool ret = ge::AttrUtils::GetStr(node->GetOpDesc(), ATTR_NAME_FRAMEWORK_ORIGINAL_TYPE, type); if (!ret) { + REPORT_INNER_ERROR("E19999", "Get Attr:%s fail for op:%s(%s) when %s", ATTR_NAME_FRAMEWORK_ORIGINAL_TYPE.c_str(), + node->GetName().c_str(), node->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Get FrameWorkOp original type [%s]", type.c_str()); return INTERNAL_ERROR; } @@ -55,6 +57,8 @@ Status SetStreamLabel(const ge::NodePtr &node, const std::string &label) { GE_CHECK_NOTNULL(tmp_desc); if (!AttrUtils::SetStr(tmp_desc, ge::ATTR_NAME_STREAM_LABEL, label)) { + REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s) when %s", ATTR_NAME_STREAM_LABEL.c_str(), + node->GetName().c_str(), node->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "Op: %s set ATTR_NAME_STREAM_LABEL failed", node->GetName().c_str()); return FAILED; } @@ -72,6 +76,8 @@ Status SetCycleEvent(const ge::NodePtr &node) { OpDescPtr tmp_desc = node->GetOpDesc(); GE_CHECK_NOTNULL(tmp_desc); if (!AttrUtils::SetBool(tmp_desc, ge::ATTR_NAME_STREAM_CYCLE_EVENT_FLAG, true)) { + REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s) when %s", ATTR_NAME_STREAM_CYCLE_EVENT_FLAG.c_str(), + node->GetName().c_str(), node->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "Op: %s set ATTR_NAME_STREAM_CYCLE_EVENT_FLAG failed", node->GetName().c_str()); return FAILED; } @@ -90,6 +96,8 @@ Status SetActiveLabelList(const ge::NodePtr &node, const std::vectorGetOpDesc(); GE_CHECK_NOTNULL(tmp_desc); if (!AttrUtils::SetListStr(tmp_desc, ge::ATTR_NAME_ACTIVE_LABEL_LIST, active_label_list)) { + REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s) when %s", ATTR_NAME_ACTIVE_LABEL_LIST.c_str(), + node->GetName().c_str(), node->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "Op: %s set ATTR_NAME_ACTIVE_LABEL_LIST failed", node->GetName().c_str()); return FAILED; } @@ -108,6 +116,8 @@ Status SetSwitchBranchNodeLabel(const ge::NodePtr &node, const std::string &bran OpDescPtr tmp_desc = node->GetOpDesc(); GE_CHECK_NOTNULL(tmp_desc); if (!AttrUtils::SetStr(tmp_desc, ge::ATTR_NAME_SWITCH_BRANCH_NODE_LABEL, branch_label)) { + REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s) when %s", ATTR_NAME_SWITCH_BRANCH_NODE_LABEL.c_str(), + node->GetName().c_str(), node->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "Op: %s set ATTR_NAME_SWITCH_BRANCH_NODE_LABEL failed", node->GetName().c_str()); return FAILED; } @@ -126,6 +136,8 @@ Status SetSwitchTrueBranchFlag(const ge::NodePtr &node, bool value) { OpDescPtr tmp_desc = node->GetOpDesc(); GE_CHECK_NOTNULL(tmp_desc); if (!AttrUtils::SetBool(tmp_desc, ge::ATTR_NAME_SWITCH_TRUE_BRANCH_FLAG, value)) { + REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s) when %s", ATTR_NAME_SWITCH_TRUE_BRANCH_FLAG.c_str(), + node->GetName().c_str(), node->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "Op: %s set ATTR_NAME_SWITCH_TRUE_BRANCH_FLAG failed", node->GetName().c_str()); return FAILED; } @@ -144,6 +156,8 @@ Status SetOriginalNodeName(const ge::NodePtr &node, const std::string &orig_name OpDescPtr tmp_desc = node->GetOpDesc(); GE_CHECK_NOTNULL(tmp_desc); if (!AttrUtils::SetStr(tmp_desc, ge::ATTR_NAME_ORIG_NODE_NAME, orig_name)) { + REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s) when %s", ATTR_NAME_ORIG_NODE_NAME.c_str(), + node->GetName().c_str(), node->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "Op: %s set ATTR_NAME_ORIG_NODE_NAME failed", node->GetName().c_str()); return FAILED; } @@ -161,6 +175,8 @@ Status SetCyclicDependenceFlag(const ge::NodePtr &node) { OpDescPtr tmp_desc = node->GetOpDesc(); GE_CHECK_NOTNULL(tmp_desc); if (!AttrUtils::SetBool(tmp_desc, ge::ATTR_NAME_CYCLIC_DEPENDENCE_FLAG, true)) { + REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s) when %s", ATTR_NAME_CYCLIC_DEPENDENCE_FLAG.c_str(), + node->GetName().c_str(), node->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "Op: %s set ATTR_NAME_CYCLIC_DEPENDENCE_FLAG failed", node->GetName().c_str()); return FAILED; } @@ -180,6 +196,8 @@ Status SetNextIteration(const ge::NodePtr &node, const std::string &next) { GE_CHECK_NOTNULL(tmp_desc); if (!AttrUtils::SetStr(tmp_desc, ge::ATTR_NAME_NEXT_ITERATION, next)) { + REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s) when %s", ATTR_NAME_NEXT_ITERATION.c_str(), + node->GetName().c_str(), node->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "Op: %s set ATTR_NAME_NEXT_ITERATION failed", node->GetName().c_str()); return FAILED; } diff --git a/ge/graph/execute/graph_execute.cc b/ge/graph/execute/graph_execute.cc index 1aee756c..d8d5c88d 100755 --- a/ge/graph/execute/graph_execute.cc +++ b/ge/graph/execute/graph_execute.cc @@ -40,6 +40,7 @@ GraphExecutor::~GraphExecutor() { rtError_t rt_ret; rt_ret = rtFreeHost(buffer_addr); if (rt_ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "Call rtFreeHost fail, ret:0x%X when %s", rt_ret, __FUNCTION__); GELOGE(RT_FAILED, "[GraphManager] subgraph free buffer failed, ret: 0x%X", rt_ret); } } @@ -51,14 +52,17 @@ GraphExecutor::~GraphExecutor() { Status GraphExecutor::SetCondition(std::mutex *mutex, std::condition_variable *cond, std::shared_ptr listener) { if (mutex == nullptr) { + REPORT_INNER_ERROR("E19999", "Check param mutex nullptr when %s", __FUNCTION__); GELOGE(GE_GRAPH_PARAM_NULLPTR, "[SetCondition] input param mutex is nullptr."); return GE_GRAPH_PARAM_NULLPTR; } if (cond == nullptr) { + REPORT_INNER_ERROR("E19999", "Check param cond nullptr when %s", __FUNCTION__); GELOGE(GE_GRAPH_PARAM_NULLPTR, "[SetCondition] input param cond is nullptr."); return GE_GRAPH_PARAM_NULLPTR; } if (listener == nullptr) { + REPORT_INNER_ERROR("E19999", "Check param listener nullptr when %s", __FUNCTION__); GELOGE(GE_GRAPH_PARAM_NULLPTR, "[SetCondition] input param listener is nullptr."); return GE_GRAPH_PARAM_NULLPTR; } @@ -75,6 +79,7 @@ Status GraphExecutor::SetCondition(std::mutex *mutex, std::condition_variable *c Status GraphExecutor::SetGraphContext(GraphContextPtr graph_context_ptr) { if (graph_context_ptr == nullptr) { + REPORT_INNER_ERROR("E19999", "Check param graph_context_ptr nullptr when %s", __FUNCTION__); GELOGE(GE_GRAPH_PARAM_NULLPTR, "[SetGraphContext] input param graph_context_ptr is nullptr"); return GE_GRAPH_PARAM_NULLPTR; } @@ -101,6 +106,7 @@ Status GraphExecutor::FreeInOutBuffer() { rtError_t rt_ret; rt_ret = rtFreeHost(*iter); if (rt_ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "Call rtFreeHost fail, ret:0x%X when %s", rt_ret, __FUNCTION__); GELOGE(RT_FAILED, "[GraphManager] subgraph free buffer failed, ret: 0x%X", rt_ret); (void)buffer_addr_.erase(buffer_addr_.begin(), iter); return GE_GRAPH_FREE_FAILED; @@ -146,6 +152,8 @@ Status GraphExecutor::MallocInOutBuffer(const std::vector &buffer_size void *tmp_buf = nullptr; rt_ret = rtMallocHost(&tmp_buf, buffer_size[i]); if (rt_ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "Call rtMallocHost fail, size:%lu, ret:0x%X when %s", + buffer_size[i], rt_ret, __FUNCTION__); GELOGE(RT_FAILED, "[GraphManager] subgraph malloc buffer failed, ret: 0x%X", rt_ret); return GE_GRAPH_MALLOC_FAILED; } @@ -191,6 +199,8 @@ Status GraphExecutor::PrepareInputData(const std::vector &input_tensor rtError_t rt_ret = rtMemcpy(addrVec[i], bufferSizeVec[i], in_tensor->GetData().data(), in_tensor->GetData().size(), RT_MEMCPY_HOST_TO_HOST); if (rt_ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "Call rtMemcpy fail, dst_size:%lu, src_size:%zu, ret:0x%X when %s", + bufferSizeVec[i], in_tensor->GetData().size(), rt_ret, __FUNCTION__); GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); return RT_FAILED; } @@ -250,6 +260,8 @@ Status GraphExecutor::SyncExecuteModel(uint32_t model_id, const std::vectorResetResult() != SUCCESS) { + REPORT_CALL_ERROR("E19999", "Call graph_run_listener_.ResetResult fail, model_id:%u, when %s", + model_id, __FUNCTION__); GELOGE(GE_GRAPH_EXECUTE_FAILED, "Reset result failed"); return GE_GRAPH_EXECUTE_FAILED; } @@ -273,6 +285,8 @@ Status GraphExecutor::SyncExecuteModel(uint32_t model_id, const std::vectorGetResultCode(); if (result_code != SUCCESS && result_code != END_OF_SEQUENCE) { + REPORT_CALL_ERROR("E19999", "Graph_run_listener_ run fail, result:%u, model_id:%u, when %s", + result_code, model_id, __FUNCTION__); GELOGE(GE_GRAPH_EXECUTE_FAILED, "[GraphExecutor] execute model failed, ret=%u, modelId=%u.", result_code, model_id); return GE_GRAPH_EXECUTE_FAILED; @@ -281,10 +295,14 @@ Status GraphExecutor::SyncExecuteModel(uint32_t model_id, const std::vector outBufTmp(new (std::nothrow) uint8_t[outputDataTmp.length]); if (outBufTmp == nullptr) { + REPORT_INNER_ERROR("E19999", "New output buffer fail, length:%lu, model:%u, when %s", + outputDataTmp.length, model_id, __FUNCTION__); GELOGE(FAILED, "Failed to allocate memory."); return FAILED; } @@ -292,6 +310,8 @@ Status GraphExecutor::SyncExecuteModel(uint32_t model_id, const std::vectorGetSubgraphInstanceNames(); if (graph_names.empty() || graph_names.size() > kMaxCaseBranch) { + REPORT_INNER_ERROR("E19999", "Node:%s(%s) subgraph size: %zu, check invalid when %s", case_desc->GetName().c_str(), + case_desc->GetType().c_str(), graph_names.size(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Node: %s has invalid subgraph, graph size: %zu.", case_desc->GetName().c_str(), graph_names.size()); return FAILED; @@ -67,6 +69,8 @@ Status CaseOpLabelMaker::Run(uint32_t &label_index) { parent_node_->GetName() + "/StreamActive_" + std::to_string(index); // rtStreamActive NodePtr stream_active = AddStreamActive(graph, stream_active_name); if (stream_active == nullptr) { + REPORT_CALL_ERROR("E19999", "Add StreamActive node in graph:%s fail when %s", + graph->GetName().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Subgraph: %s add stream active failed.", graph->GetName().c_str()); return FAILED; } @@ -75,6 +79,8 @@ Status CaseOpLabelMaker::Run(uint32_t &label_index) { std::string label_set_name = parent_node_->GetName() + "/LabelSet_" + std::to_string(index); // rtLabelSet NodePtr label = AddLabelSetEnter(graph, label_set_name, curr_label_index, stream_active); if (label == nullptr) { + REPORT_CALL_ERROR("E19999", "Add LabelSetEnter node in graph:%s fail when %s", + graph->GetName().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Subgraph: %s add label set failed.", graph->GetName().c_str()); return FAILED; } @@ -88,6 +94,8 @@ Status CaseOpLabelMaker::Run(uint32_t &label_index) { // middle node, add goto node to tail. std::string label_goto_name = parent_node_->GetName() + "/LabelGoto_" + std::to_string(index); // rtLabelGoto if (AddLabelGotoLeave(graph, label_goto_name, last_label_index) == nullptr) { + REPORT_CALL_ERROR("E19999", "Add LabelGotoLeave node in graph:%s fail when %s", + graph->GetName().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Subgraph: %s add label goto failed.", graph->GetName().c_str()); return FAILED; } @@ -95,6 +103,8 @@ Status CaseOpLabelMaker::Run(uint32_t &label_index) { // last node, add label node to tail. std::string last_label_name = parent_node_->GetName() + "/LabelSet_Last"; // rtLabelSet if (AddLabelSetLeave(graph, last_label_name, last_label_index) == nullptr) { + REPORT_CALL_ERROR("E19999", "Add LabelSetLeave node in graph:%s fail when %s", + graph->GetName().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Subgraph: %s add label set failed.", graph->GetName().c_str()); return FAILED; } @@ -110,12 +120,16 @@ Status CaseOpLabelMaker::Run(uint32_t &label_index) { const GeTensorDesc &pred_desc = case_desc->GetInputDesc(kCasePredIndex); NodePtr switch_node = AddLabelSwitchEnter(first_graph, label_switch_name, pred_desc, switch_labels); if (switch_node == nullptr) { + REPORT_CALL_ERROR("E19999", "Add LabelSwitchEnter node in graph:%s fail when %s", + first_graph->GetName().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Subgraph: %s add label switch failed.", first_graph->GetName().c_str()); return FAILED; } // Link control edge to then branch head. if (GraphUtils::AddEdge(switch_node->GetOutControlAnchor(), first_label->GetInControlAnchor()) != SUCCESS) { + REPORT_CALL_ERROR("E19999", "Add ctrl edge from %s to %s in graph:%s fail when %s", switch_node->GetName().c_str(), + first_label->GetName().c_str(), first_graph->GetName().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "LabelSwitchByIndex: Add ctrl edge to %s failed.", first_label->GetName().c_str()); return FAILED; } @@ -123,6 +137,8 @@ Status CaseOpLabelMaker::Run(uint32_t &label_index) { uint32_t parent_index = 0; // Case cond input is first. const std::string data_name = parent_node_->GetName() + "/SwitchIndexData"; if (AddLabelSwitchIndex(first_graph, data_name, pred_desc, switch_node, parent_index) == nullptr) { + REPORT_CALL_ERROR("E19999", "Add LabelSwitchIndex node in graph:%s fail when %s", + first_graph->GetName().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Subgraph: %s add switch input failed.", first_graph->GetName().c_str()); return FAILED; } diff --git a/ge/graph/label/if_label_maker.cc b/ge/graph/label/if_label_maker.cc index d07f7984..655381f0 100644 --- a/ge/graph/label/if_label_maker.cc +++ b/ge/graph/label/if_label_maker.cc @@ -43,6 +43,10 @@ Status IfOpLabelMaker::Run(uint32_t &label_index) { const std::string then_branch_name = if_desc->GetSubgraphInstanceName(kThenBranchIndex); const std::string else_branch_name = if_desc->GetSubgraphInstanceName(kElseBranchIndex); if (then_branch_name.empty() || else_branch_name.empty()) { + REPORT_INNER_ERROR("E19999", "Node:%s(%s), check subgraph invalid, " + "then branch graph: %s, else branch graph: %s, when %s", + if_desc->GetName().c_str(), if_desc->GetType().c_str(), + then_branch_name.c_str(), else_branch_name.c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Node: %s has invalid subgraph, then branch: %s, else branch: %s.", if_desc->GetName().c_str(), then_branch_name.c_str(), else_branch_name.c_str()); return FAILED; @@ -66,32 +70,44 @@ Status IfOpLabelMaker::Run(uint32_t &label_index) { NodePtr then_stream_active = AddStreamActive(then_sub_graph, then_active_name); if (then_stream_active == nullptr) { + REPORT_CALL_ERROR("E19999", "Add StreamActive node in graph:%s fail when %s", + then_sub_graph->GetName().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Subgraph: %s add stream active failed.", then_sub_graph->GetName().c_str()); return FAILED; } NodePtr then_enter_label = AddLabelSetEnter(then_sub_graph, then_label_name, then_enter_index, then_stream_active); if (then_enter_label == nullptr) { + REPORT_CALL_ERROR("E19999", "Add LabelSetEnter node in graph:%s fail when %s", + then_sub_graph->GetName().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Subgraph: %s add label set failed.", then_sub_graph->GetName().c_str()); return FAILED; } if (AddLabelGotoLeave(then_sub_graph, then_leave_name, else_leave_index) == nullptr) { + REPORT_CALL_ERROR("E19999", "Add LabelGotoLeave node in graph:%s fail when %s", + then_sub_graph->GetName().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Subgraph: %s add label goto failed.", then_sub_graph->GetName().c_str()); return FAILED; } NodePtr else_stream_active = AddStreamActive(else_sub_graph, else_active_name); if (else_stream_active == nullptr) { + REPORT_CALL_ERROR("E19999", "Add StreamActive node in graph:%s fail when %s", + else_stream_active->GetName().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Subgraph: %s add stream active failed.", else_sub_graph->GetName().c_str()); return FAILED; } if (AddLabelSetEnter(else_sub_graph, else_enter_name, else_enter_index, else_stream_active) == nullptr) { + REPORT_CALL_ERROR("E19999", "Add LabelSetEnter node in graph:%s fail when %s", + else_sub_graph->GetName().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Subgraph: %s add label set failed.", else_sub_graph->GetName().c_str()); return FAILED; } if (AddLabelSetLeave(else_sub_graph, else_leave_name, else_leave_index) == nullptr) { + REPORT_CALL_ERROR("E19999", "Add LabelSetLeave node in graph:%s fail when %s", + else_sub_graph->GetName().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Subgraph: %s add label set failed.", else_sub_graph->GetName().c_str()); return FAILED; } @@ -103,12 +119,16 @@ Status IfOpLabelMaker::Run(uint32_t &label_index) { const GeTensorDesc &pred_desc = if_desc->GetInputDesc(kIfPredIndex); NodePtr switch_node = AddLabelSwitchEnter(then_sub_graph, then_enter_name, pred_desc, switch_labels); if (switch_node == nullptr) { + REPORT_CALL_ERROR("E19999", "Add LabelSwitchEnter node in graph:%s fail when %s", + then_sub_graph->GetName().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Subgraph: %s add label switch failed.", then_sub_graph->GetName().c_str()); return FAILED; } // Link control edge to then branch head. if (GraphUtils::AddEdge(switch_node->GetOutControlAnchor(), then_enter_label->GetInControlAnchor()) != SUCCESS) { + REPORT_CALL_ERROR("E19999", "Add ctrl edge from %s to %s in graph:%s fail when %s", switch_node->GetName().c_str(), + then_enter_label->GetName().c_str(), then_sub_graph->GetName().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "LabelSwitchByIndex: Add ctrl edge to %s failed.", then_enter_label->GetName().c_str()); return FAILED; } @@ -116,6 +136,8 @@ Status IfOpLabelMaker::Run(uint32_t &label_index) { uint32_t parent_index = 0; // If cond input is first. const std::string data_name = parent_node_->GetName() + "/SwitchIndexData"; if (AddLabelSwitchIndex(then_sub_graph, data_name, pred_desc, switch_node, parent_index) == nullptr) { + REPORT_CALL_ERROR("E19999", "Add LabelSwitchIndex node in graph:%s fail when %s", + then_sub_graph->GetName().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Subgraph: %s add switch input failed.", then_sub_graph->GetName().c_str()); return FAILED; } diff --git a/ge/graph/label/label_maker.cc b/ge/graph/label/label_maker.cc index 0e1e571c..03d2e87e 100644 --- a/ge/graph/label/label_maker.cc +++ b/ge/graph/label/label_maker.cc @@ -56,6 +56,8 @@ void LabelMaker::LinkToGraphHead(const ComputeGraphPtr &graph, const NodePtr &no } if (GraphUtils::AddEdge(node->GetOutControlAnchor(), n->GetInControlAnchor()) != SUCCESS) { + REPORT_CALL_ERROR("E19999", "Add ctrl edge from %s to %s in graph:%s fail when %s", node->GetName().c_str(), + n->GetName().c_str(), graph->GetName().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Add ctrl edge from %s to %s failed.", node->GetName().c_str(), n->GetName().c_str()); } } @@ -78,6 +80,8 @@ void LabelMaker::LinkToGraphTail(const ComputeGraphPtr &graph, const NodePtr &no } if (GraphUtils::AddEdge(tail->GetOutControlAnchor(), node->GetInControlAnchor()) != SUCCESS) { + REPORT_CALL_ERROR("E19999", "Add ctrl edge from %s to %s in graph:%s fail when %s", tail->GetName().c_str(), + node->GetName().c_str(), graph->GetName().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Add ctrl edge from %s to %s failed.", tail->GetName().c_str(), node->GetName().c_str()); } return; @@ -96,6 +100,7 @@ NodePtr LabelMaker::AddStreamActive(const ComputeGraphPtr &graph, const std::str const auto &node_list = graph->GetDirectNode(); if (node_list.empty()) { + REPORT_INNER_ERROR("E19999", "Check param graph has no node when %s", graph->GetName().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "LabelSet: Graph %s node is empty.", graph->GetName().c_str()); return nullptr; } @@ -131,6 +136,7 @@ NodePtr LabelMaker::AddLabelSetEnter(const ComputeGraphPtr &graph, const std::st const auto &node_list = graph->GetDirectNode(); if (node_list.empty()) { + REPORT_INNER_ERROR("E19999", "Check param graph has no node when %s", graph->GetName().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "LabelSet: Graph %s node is empty.", graph->GetName().c_str()); return nullptr; } @@ -145,6 +151,8 @@ NodePtr LabelMaker::AddLabelSetEnter(const ComputeGraphPtr &graph, const std::st GE_CHECK_NOTNULL_EXEC(label_set, return nullptr); if (GraphUtils::AddEdge(label_set->GetOutControlAnchor(), stream_active->GetInControlAnchor()) != SUCCESS) { + REPORT_CALL_ERROR("E19999", "Add ctrl edge from %s to %s in graph:%s fail when %s", label_set->GetName().c_str(), + stream_active->GetName().c_str(), graph->GetName().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Add ctrl edge from %s to %s failed.", label_set->GetName().c_str(), stream_active->GetName().c_str()); return nullptr; @@ -193,6 +201,7 @@ NodePtr LabelMaker::AddLabelGotoEnter(const ComputeGraphPtr &graph, const std::s const auto &node_list = graph->GetDirectNode(); auto it = node_list.begin(); if (it == node_list.end()) { + REPORT_INNER_ERROR("E19999", "Check param graph has no node when %s", graph->GetName().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "LabelGoto: Graph %s node is empty.", graph->GetName().c_str()); return nullptr; } @@ -205,6 +214,8 @@ NodePtr LabelMaker::AddLabelGotoEnter(const ComputeGraphPtr &graph, const std::s (void)AttrUtils::SetInt(op_desc, ATTR_NAME_LABEL_SWITCH_INDEX, index); NodePtr label_goto = graph->AddNodeFront(op_desc); if (label_goto == nullptr) { + REPORT_CALL_ERROR("E19999", "Add node:%s(%s) to graph:%s fail when %s", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), graph->GetName().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "LabelGoto: Add to graph %s failed.", graph->GetName().c_str()); return nullptr; } @@ -253,6 +264,7 @@ NodePtr LabelMaker::AddLabelSwitchEnter(const ComputeGraphPtr &graph, const std: const auto &node_list = graph->GetDirectNode(); auto it = node_list.begin(); if (it == node_list.end()) { + REPORT_INNER_ERROR("E19999", "Check param graph has no node when %s", graph->GetName().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "LabelSwitchByIndex: Graph %s node is empty.", graph->GetName().c_str()); return nullptr; } @@ -263,17 +275,23 @@ NodePtr LabelMaker::AddLabelSwitchEnter(const ComputeGraphPtr &graph, const std: GELOGI("LabelSwitchByIndex: Create node %s.", op_desc->GetName().c_str()); if (op_desc->AddInputDesc(desc) != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Add input desc into node:%s(%s) in graph:%s fail when %s", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), graph->GetName().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "LabelSwitchByIndex: Add input desc failed."); return nullptr; } if (!AttrUtils::SetListInt(op_desc, ATTR_NAME_LABEL_SWITCH_LIST, labels)) { + REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s) when %s", ATTR_NAME_LABEL_SWITCH_LIST.c_str(), + op_desc->GetName().c_str(), op_desc->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "LabelSwitchByIndex: Add %s failed.", ATTR_NAME_LABEL_SWITCH_INDEX.c_str()); return nullptr; } NodePtr label_switch = graph->AddNodeFront(op_desc); if (label_switch == nullptr) { + REPORT_CALL_ERROR("E19999", "Add node:%s(%s) to graph:%s ahead fail when %s", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), graph->GetName().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "LabelSwitchByIndex: Add to graph %s failed.", graph->GetName().c_str()); return nullptr; } @@ -300,11 +318,15 @@ NodePtr LabelMaker::AddLabelSwitchLeave(const ComputeGraphPtr &graph, const std: GELOGI("LabelSwitchByIndex: Create node %s.", op_desc->GetName().c_str()); if (op_desc->AddInputDesc(desc) != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Add input desc into node:%s(%s) in graph:%s fail when %s", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), graph->GetName().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "LabelSwitchByIndex: Add input desc failed."); return nullptr; } if (!AttrUtils::SetListInt(op_desc, ATTR_NAME_LABEL_SWITCH_LIST, labels)) { + REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s) when %s", ATTR_NAME_LABEL_SWITCH_LIST.c_str(), + op_desc->GetName().c_str(), op_desc->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "LabelSwitchByIndex: Add %s failed.", ATTR_NAME_LABEL_SWITCH_INDEX.c_str()); return nullptr; } @@ -336,15 +358,21 @@ NodePtr LabelMaker::AddLabelSwitchIndex(const ComputeGraphPtr &graph, const std: GELOGI("Data: Create node %s.", op_desc->GetName().c_str()); if (op_desc->AddInputDesc(desc) != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Add input desc into node:%s(%s) in graph:%s fail when %s", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), graph->GetName().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "LabelSwitchByIndex: Add data input desc failed."); return nullptr; } if (op_desc->AddOutputDesc(desc) != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Add output desc into node:%s(%s) in graph:%s fail when %s", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), graph->GetName().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "LabelSwitchByIndex: Add data output desc failed."); return nullptr; } if (!AttrUtils::SetInt(op_desc, ATTR_NAME_PARENT_NODE_INDEX, parent_index)) { + REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s) when %s", ATTR_NAME_PARENT_NODE_INDEX.c_str(), + op_desc->GetName().c_str(), op_desc->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "LabelSwitchByIndex: Add %s failed.", ATTR_NAME_PARENT_NODE_INDEX.c_str()); return nullptr; } @@ -354,6 +382,8 @@ NodePtr LabelMaker::AddLabelSwitchIndex(const ComputeGraphPtr &graph, const std: // Link control edge to graph head. if (GraphUtils::AddEdge(op_data->GetOutDataAnchor(0), sw_node->GetInDataAnchor(0)) != SUCCESS) { + REPORT_CALL_ERROR("E19999", "Add ctrl edge from %s to %s in graph:%s fail when %s", op_data->GetName().c_str(), + sw_node->GetName().c_str(), graph->GetName().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "LabelSwitchByIndex: Add input edge to %s failed.", op_data->GetName().c_str()); return nullptr; } diff --git a/ge/graph/label/partitioned_call_label_maker.cc b/ge/graph/label/partitioned_call_label_maker.cc index 0be738f0..a556e832 100644 --- a/ge/graph/label/partitioned_call_label_maker.cc +++ b/ge/graph/label/partitioned_call_label_maker.cc @@ -39,12 +39,17 @@ Status PartitionedCallLabelMaker::Run(uint32_t &label_index) { std::string sub_graph_name = call_desc->GetSubgraphInstanceName(kSubGraphIndex); if (sub_graph_name.empty()) { + REPORT_INNER_ERROR("E19999", "Node:%s(%s) subgraph_index:%d name is empty, check invalid when %s", + call_desc->GetName().c_str(), call_desc->GetType().c_str(), kSubGraphIndex, __FUNCTION__); GELOGE(INTERNAL_ERROR, "Node: %s has no subgraph name.", sub_graph_name.c_str()); return FAILED; } ComputeGraphPtr sub_graph = parent_graph_->GetSubgraph(sub_graph_name); if (sub_graph == nullptr) { + REPORT_INNER_ERROR("E19999", "Node:%s(%s) subgraph_name:%s is not exist in parent_graph, check invalid when %s", + call_desc->GetName().c_str(), call_desc->GetType().c_str(), + sub_graph_name.c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Node: %s has no subgraph.", sub_graph_name.c_str()); return FAILED; } @@ -52,6 +57,8 @@ Status PartitionedCallLabelMaker::Run(uint32_t &label_index) { const std::string stream_active_name = parent_node_->GetName() + "/StreamActive"; // rtStreamActive NodePtr stream_active = AddStreamActive(sub_graph, stream_active_name); if (stream_active == nullptr) { + REPORT_CALL_ERROR("E19999", "Add StreamActive node in graph:%s fail when %s", + sub_graph->GetName().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Subgraph: %s add stream active node failed.", sub_graph->GetName().c_str()); return FAILED; } diff --git a/ge/graph/label/while_label_maker.cc b/ge/graph/label/while_label_maker.cc index 83aad7c9..eaa320ea 100644 --- a/ge/graph/label/while_label_maker.cc +++ b/ge/graph/label/while_label_maker.cc @@ -44,6 +44,9 @@ Status WhileOpLabelMaker::Run(uint32_t &label_index) { std::string cond_name = while_desc->GetSubgraphInstanceName(kCondBranchIndex); std::string body_name = while_desc->GetSubgraphInstanceName(kBodyBranchIndex); if (cond_name.empty() || body_name.empty()) { + REPORT_INNER_ERROR("E19999", "Node:%s(%s) cond subgraph index:%d or body subgraph index:%d name is empty, " + "check invalid when %s", while_desc->GetName().c_str(), while_desc->GetType().c_str(), + kCondBranchIndex, kBodyBranchIndex, __FUNCTION__); GELOGE(INTERNAL_ERROR, "Node: %s has invalid subgraph, cond branch: %s, body branch: %s.", while_desc->GetName().c_str(), cond_name.c_str(), body_name.c_str()); return FAILED; @@ -67,32 +70,44 @@ Status WhileOpLabelMaker::Run(uint32_t &label_index) { NodePtr cond_stream_active = AddStreamActive(cond_graph, cond_active_name); if (cond_stream_active == nullptr) { + REPORT_CALL_ERROR("E19999", "Add StreamActive node in graph:%s fail when %s", + cond_graph->GetName().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Subgraph: %s add stream active failed.", cond_graph->GetName().c_str()); return FAILED; } if (AddLabelSetEnter(cond_graph, cond_enter_name, cond_enter_index, cond_stream_active) == nullptr) { + REPORT_CALL_ERROR("E19999", "Add LabelSetEnter node in graph:%s fail when %s", + cond_graph->GetName().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Subgraph: %s add label set failed.", cond_graph->GetName().c_str()); return FAILED; } NodePtr body_stream_active = AddStreamActive(body_graph, body_active_name); if (body_stream_active == nullptr) { + REPORT_CALL_ERROR("E19999", "Add StreamActive node in graph:%s fail when %s", + body_graph->GetName().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Subgraph: %s add stream active failed.", body_graph->GetName().c_str()); return FAILED; } if (AddLabelSetEnter(body_graph, body_enter_name, body_enter_index, body_stream_active) == nullptr) { + REPORT_CALL_ERROR("E19999", "Add LabelSetEnter node in graph:%s fail when %s", + body_graph->GetName().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Subgraph: %s add label set failed.", body_graph->GetName().c_str()); return FAILED; } if (AddLabelGotoLeave(body_graph, goto_leave_name, cond_enter_index) == nullptr) { + REPORT_CALL_ERROR("E19999", "Add LabelGotoLeave node in graph:%s fail when %s", + body_graph->GetName().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Subgraph: %s add label goto failed.", body_graph->GetName().c_str()); return FAILED; } if (AddLabelSetLeave(body_graph, body_leave_name, body_leave_index) == nullptr) { + REPORT_CALL_ERROR("E19999", "Add LabelSetLeave node in graph:%s fail when %s", + body_graph->GetName().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Subgraph: %s add label set failed.", body_graph->GetName().c_str()); return FAILED; } @@ -109,6 +124,8 @@ Status WhileOpLabelMaker::Run(uint32_t &label_index) { const std::vector switch_labels = {body_leave_index, body_enter_index}; NodePtr switch_node = AddLabelSwitchLeave(cond_graph, cond_leave_name, pred_desc, switch_labels); if (switch_node == nullptr) { + REPORT_CALL_ERROR("E19999", "Add LabelSwitchLeave node in graph:%s fail when %s", + cond_graph->GetName().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Subgraph: %s add label switch failed.", cond_graph->GetName().c_str()); return FAILED; } @@ -124,6 +141,9 @@ Status WhileOpLabelMaker::Run(uint32_t &label_index) { InDataAnchorPtr in_anchor = all_in_data.at(kCondOutputIndex); GE_CHECK_NOTNULL(in_anchor); if (GraphUtils::AddEdge(in_anchor->GetPeerOutAnchor(), switch_node->GetInDataAnchor(kCondOutputIndex)) != SUCCESS) { + REPORT_CALL_ERROR("E19999", "Add ctrl edge from %s to %s in graph:%s fail when %s", + in_anchor->GetPeerOutAnchor()->GetOwnerNode()->GetName().c_str(), + switch_node->GetName().c_str(), cond_graph ->GetName().c_str(), __FUNCTION__); GELOGE(FAILED, "Node: %s Add pred data input failed.", switch_node->GetName().c_str()); return FAILED; } diff --git a/ge/graph/load/model_manager/task_info/end_graph_task_info.cc b/ge/graph/load/model_manager/task_info/end_graph_task_info.cc index c306c650..8b30e92e 100644 --- a/ge/graph/load/model_manager/task_info/end_graph_task_info.cc +++ b/ge/graph/load/model_manager/task_info/end_graph_task_info.cc @@ -27,6 +27,7 @@ namespace ge { Status EndGraphTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *davinci_model) { GELOGI("InitEndGraphTaskInfo Init Start."); if (davinci_model == nullptr) { + REPORT_INNER_ERROR("E19999", "Check param davinci_model nullptr when EndGraphTaskInfo %s", __FUNCTION__); GELOGE(PARAM_INVALID, "davinci_model is null!"); return PARAM_INVALID; } @@ -52,6 +53,7 @@ Status EndGraphTaskInfo::Distribute() { GELOGI("Start to call rtEndGraphEx"); rtError_t rt_ret = rtEndGraphEx(model_, stream_, kDumpFlag); if (rt_ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "Call rtEndGraphEx fail ret:0x%X, when EndGraphTaskInfo %s", rt_ret, __FUNCTION__); GELOGE(RT_FAILED, "Call rtEndGraphEx failed, ret: 0x%x", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } @@ -59,6 +61,7 @@ Status EndGraphTaskInfo::Distribute() { GELOGI("Start to call rtEndGraph"); rtError_t rt_ret = rtEndGraph(model_, stream_); if (rt_ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "Call rtEndGraph fail ret:0x%X, when EndGraphTaskInfo %s", rt_ret, __FUNCTION__); GELOGE(RT_FAILED, "Call rtEndGraph failed, ret: 0x%x", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } @@ -68,6 +71,7 @@ Status EndGraphTaskInfo::Distribute() { uint32_t stream_id = 0; rtError_t rt_ret = rtModelGetTaskId(davinci_model_->GetRtModelHandle(), &task_id, &stream_id); if (rt_ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "Call rtModelGetTaskId fail ret:0x%X, when EndGraphTaskInfo %s", rt_ret, __FUNCTION__); GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } diff --git a/ge/graph/load/model_manager/task_info/event_record_task_info.cc b/ge/graph/load/model_manager/task_info/event_record_task_info.cc index f736c386..4ffeee66 100755 --- a/ge/graph/load/model_manager/task_info/event_record_task_info.cc +++ b/ge/graph/load/model_manager/task_info/event_record_task_info.cc @@ -23,6 +23,7 @@ namespace ge { Status EventRecordTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *davinci_model) { GELOGI("EventRecordTaskInfo Init Start."); if (davinci_model == nullptr) { + REPORT_INNER_ERROR("E19999", "Check param davinci_model nullptr when EventRecordTaskInfo %s", __FUNCTION__); GELOGE(PARAM_INVALID, "davinci_model is null!"); return PARAM_INVALID; } @@ -34,6 +35,8 @@ Status EventRecordTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *da const auto &eventList = davinci_model->GetEventList(); if (task_def.event_id() >= eventList.size()) { + REPORT_INNER_ERROR("E19999", "Task event_id:%u > model event size:%zu, check invalid when EventRecordTaskInfo %s", + task_def.event_id(), eventList.size(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "event list size:%zu, cur:%u!", eventList.size(), task_def.event_id()); return INTERNAL_ERROR; } @@ -47,6 +50,7 @@ Status EventRecordTaskInfo::Distribute() { GELOGI("EventRecordTaskInfo Distribute Start."); rtError_t rt_ret = rtEventRecord(event_, stream_); if (rt_ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "Call rtEventRecord fail ret:0x%X, when EventRecordTaskInfo %s", rt_ret, __FUNCTION__); GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } diff --git a/ge/graph/load/model_manager/task_info/event_wait_task_info.cc b/ge/graph/load/model_manager/task_info/event_wait_task_info.cc index 34058502..b5795eff 100755 --- a/ge/graph/load/model_manager/task_info/event_wait_task_info.cc +++ b/ge/graph/load/model_manager/task_info/event_wait_task_info.cc @@ -23,6 +23,7 @@ namespace ge { Status EventWaitTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *davinci_model) { GELOGI("EventWaitTaskInfo Init Start."); if (davinci_model == nullptr) { + REPORT_INNER_ERROR("E19999", "Check param davinci_model nullptr when EventWaitTaskInfo %s", __FUNCTION__); GELOGE(PARAM_INVALID, "davinci_model is null!"); return PARAM_INVALID; } @@ -34,6 +35,8 @@ Status EventWaitTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *davi const auto &eventList = davinci_model->GetEventList(); if (task_def.event_id() >= eventList.size()) { + REPORT_INNER_ERROR("E19999", "Task event_id:%u > model event size:%zu, check invalid when EventWaitTaskInfo %s", + task_def.event_id(), eventList.size(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "event list size:%zu, cur:%u!", eventList.size(), task_def.event_id()); return INTERNAL_ERROR; } @@ -48,12 +51,16 @@ Status EventWaitTaskInfo::Distribute() { GELOGI("EventWaitTaskInfo Distribute Start."); rtError_t rt_ret = rtStreamWaitEvent(stream_, event_); if (rt_ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "Call rtStreamWaitEvent fail ret:0x%X, when EventWaitTaskInfo %s", + rt_ret, __FUNCTION__); GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } rt_ret = rtEventReset(event_, stream_); if (rt_ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "Call rtEventReset fail ret:0x%X, when EventWaitTaskInfo %s", + rt_ret, __FUNCTION__); GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } diff --git a/ge/graph/load/model_manager/task_info/fusion_start_task_info.cc b/ge/graph/load/model_manager/task_info/fusion_start_task_info.cc index 6feea9e4..1c2f95b7 100755 --- a/ge/graph/load/model_manager/task_info/fusion_start_task_info.cc +++ b/ge/graph/load/model_manager/task_info/fusion_start_task_info.cc @@ -23,6 +23,7 @@ namespace ge { Status FusionStartTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *davinci_model) { GELOGI("FusionStartTaskInfo Init Start."); if (davinci_model == nullptr) { + REPORT_INNER_ERROR("E19999", "Check param davinci_model nullptr when FusionStartTaskInfo %s", __FUNCTION__); GELOGE(PARAM_INVALID, "davinci_model is null!"); return PARAM_INVALID; } @@ -39,6 +40,8 @@ Status FusionStartTaskInfo::Distribute() { GELOGI("FusionStartTaskInfo Distribute Start."); rtError_t rt_ret = rtKernelFusionStart(stream_); if (rt_ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "Call rtKernelFusionStart fail ret:0x%X, when FusionStartTaskInfo %s", + rt_ret, __FUNCTION__); GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } diff --git a/ge/graph/load/model_manager/task_info/fusion_stop_task_info.cc b/ge/graph/load/model_manager/task_info/fusion_stop_task_info.cc index 22d1589c..6b0035b9 100755 --- a/ge/graph/load/model_manager/task_info/fusion_stop_task_info.cc +++ b/ge/graph/load/model_manager/task_info/fusion_stop_task_info.cc @@ -23,6 +23,7 @@ namespace ge { Status FusionStopTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *davinci_model) { GELOGI("FusionStopTaskInfo Init Start."); if (davinci_model == nullptr) { + REPORT_INNER_ERROR("E19999", "Check param davinci_model nullptr when FusionStopTaskInfo %s", __FUNCTION__); GELOGE(PARAM_INVALID, "davinci_model is null!"); return PARAM_INVALID; } @@ -39,6 +40,8 @@ Status FusionStopTaskInfo::Distribute() { GELOGI("FusionStopTaskInfo Distribute Start."); rtError_t rt_ret = rtKernelFusionEnd(stream_); if (rt_ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "Call rtKernelFusionEnd fail ret:0x%X, when FusionStopTaskInfo %s", + rt_ret, __FUNCTION__); GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } diff --git a/ge/graph/load/model_manager/task_info/hccl_task_info.cc b/ge/graph/load/model_manager/task_info/hccl_task_info.cc index 2d0ad560..9179ca71 100644 --- a/ge/graph/load/model_manager/task_info/hccl_task_info.cc +++ b/ge/graph/load/model_manager/task_info/hccl_task_info.cc @@ -30,6 +30,7 @@ HcclTaskInfo::~HcclTaskInfo() { if (private_def_ != nullptr) { rtError_t ret = rtFreeHost(private_def_); if (ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "Call rtFreeHost fail ret:0x%X, when HcclTaskInfo %s", ret, __FUNCTION__); GELOGE(RT_FAILED, "Call rtFree Fail, ret = 0x%X.", ret); } private_def_ = nullptr; @@ -41,6 +42,7 @@ HcclTaskInfo::~HcclTaskInfo() { Status HcclTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *davinci_model) { GELOGI("HcclTaskInfo Init Start."); if (davinci_model == nullptr) { + REPORT_INNER_ERROR("E19999", "Check param davinci_model nullptr when HcclTaskInfo %s", __FUNCTION__); GELOGE(PARAM_INVALID, "davinci_model is null!"); return PARAM_INVALID; } @@ -67,22 +69,30 @@ Status HcclTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *davinci_m // Only in Horovod scenario should get the inputName and GeShape ret = HcomOmeUtil::GetHorovodInputs(op_desc, kernel_hccl_infos_); if (ret != SUCCESS) { + REPORT_CALL_ERROR("E19999", "Call GetHorovodInputs fail for op:%s(%s), when HcclTaskInfo %s", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), __FUNCTION__); GELOGE(ret, "davinci_model: GetHorovodInputs fail! domi error: %u", ret); return ret; } Status dmrt = HcomOmeUtil::GetHcclDataType(op_desc, kernel_hccl_infos_); if (dmrt != SUCCESS) { + REPORT_CALL_ERROR("E19999", "Call GetHcclDataType fail for op:%s(%s), when HcclTaskInfo %s", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), __FUNCTION__); GELOGE(dmrt, "davinci_model: GetHcomDataType fail! domi error: %u", dmrt); return dmrt; } dmrt = HcomOmeUtil::GetHcclCount(op_desc, kernel_hccl_infos_); if (dmrt != SUCCESS) { + REPORT_CALL_ERROR("E19999", "Call GetHcclCount fail for op:%s(%s), when HcclTaskInfo %s", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), __FUNCTION__); GELOGE(dmrt, "davinci_model: GetHcomCount fail! domi error: %u", dmrt); return dmrt; } // Only HCOMBROADCAST and HVDCALLBACKBROADCAST need to get the rootId dmrt = HcomOmeUtil::GetAllRootId(op_desc, kernel_hccl_infos_); if (dmrt != SUCCESS) { + REPORT_CALL_ERROR("E19999", "Call GetAllRootId fail for op:%s(%s), when HcclTaskInfo %s", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), __FUNCTION__); GELOGE(dmrt, "davinci_model: Get rootId fail! domi error: %u", dmrt); return dmrt; } @@ -169,12 +179,16 @@ Status HcclTaskInfo::CreateStream(int64_t stream_num, DavinciModel *davinci_mode rtError_t rt_ret = rtStreamCreateWithFlags(&stream, davinci_model->Priority(), RT_STREAM_PERSISTENT | RT_STREAM_FORCE_COPY); if (rt_ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "Call rtStreamCreateWithFlags fail ret:0x%X, stream_idx:%ld, stream_num:%ld, " + "when HcclTaskInfo %s", rt_ret, i, stream_num, __FUNCTION__); GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } // Create slave stream, inactive by default, activated by hccl rt_ret = rtModelBindStream(davinci_model->GetRtModelHandle(), stream, RT_MODEL_WAIT_ACTIVE_STREAM); if (rt_ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "Call rtModelBindStream fail ret:0x%X, stream_idx:%ld, stream_num:%ld, " + "when HcclTaskInfo %s", rt_ret, i, stream_num, __FUNCTION__); GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); (void)rtStreamDestroy(stream); return RT_ERROR_TO_GE_STATUS(rt_ret); @@ -192,6 +206,7 @@ Status HcclTaskInfo::CreateStream(int64_t stream_num, DavinciModel *davinci_mode Status HcclTaskInfo::Distribute() { GELOGI("HcclTaskInfo Distribute Start. begin to call function LoadTask in hccl."); if (ops_kernel_store_ == nullptr) { + REPORT_INNER_ERROR("E19999", "Check param ops_kernel_store_ nullptr when HcclTaskInfo %s", __FUNCTION__); GELOGE(INTERNAL_ERROR, "ops kernel store is null."); return INTERNAL_ERROR; } @@ -201,6 +216,7 @@ Status HcclTaskInfo::Distribute() { TransToGETaskInfo(ge_task); auto result = ops_kernel_info_store->LoadTask(ge_task); if (result != HCCL_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Call ops_kernel_info_store LoadTask fail when HcclTaskInfo %s", __FUNCTION__); GELOGE(INTERNAL_ERROR, "davinci_model : load task fail, return ret: %u", result); return INTERNAL_ERROR; } @@ -316,6 +332,8 @@ void HcclTaskInfo::GetPrivateDefByTaskDef(const domi::TaskDef &task) { private_def_len_ = private_def_temp.size(); rtError_t ret = rtMallocHost(&private_def_, private_def_len_); if (ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "Call rtMallocHost fail ret:0x%X, size:%u, when HcclTaskInfo %s", + ret, private_def_len_, __FUNCTION__); GELOGE(RT_FAILED, "Call rtMallocHost Fail, ret = 0x%X.", ret); return; } @@ -323,6 +341,8 @@ void HcclTaskInfo::GetPrivateDefByTaskDef(const domi::TaskDef &task) { ret = rtMemcpy(private_def_, private_def_len_, task.private_def().c_str(), private_def_len_, RT_MEMCPY_HOST_TO_HOST); if (ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "Call rtMemcpy fail ret:0x%X, size:%u, when HcclTaskInfo %s", + ret, private_def_len_, __FUNCTION__); GELOGE(RT_FAILED, "Call rtMemcpy Fail, ret = 0x%X.", ret); return; } diff --git a/ge/graph/load/model_manager/task_info/kernel_ex_task_info.cc b/ge/graph/load/model_manager/task_info/kernel_ex_task_info.cc index 2317f961..4ab946aa 100644 --- a/ge/graph/load/model_manager/task_info/kernel_ex_task_info.cc +++ b/ge/graph/load/model_manager/task_info/kernel_ex_task_info.cc @@ -75,11 +75,15 @@ Status KernelExTaskInfo::InitTaskExtInfo(const std::string &ext_info, const OpDe } auto rt_ret = rtMalloc(&ext_info_addr_, ext_handle->GetExtInfoLen(), RT_MEMORY_HBM); GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, + REPORT_CALL_ERROR("E19999", "Call rtMalloc fail, size:%zu, ret:0x%X, when KernelExTaskInfo %s", + ext_info.size(), rt_ret, __FUNCTION__); GELOGE(RT_FAILED, "rtMalloc ext_info error: 0x%X, size=%zu", rt_ret, ext_info.size()); return RT_ERROR_TO_GE_STATUS(rt_ret);) rt_ret = rtMemcpy(ext_info_addr_, ext_handle->GetExtInfoLen(), ext_handle->GetExtInfo(), ext_handle->GetExtInfoLen(), RT_MEMCPY_HOST_TO_DEVICE); GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, + REPORT_CALL_ERROR("E19999", "Call rtMemcpy fail, size:%zu, ret:0x%X, when KernelExTaskInfo %s", + ext_handle->GetExtInfoLen(), rt_ret, __FUNCTION__); GELOGE(RT_FAILED, "rtMemcpy ext_info error: 0x%X, size=%zu", rt_ret, ext_info.size()); return RT_ERROR_TO_GE_STATUS(rt_ret);) return SUCCESS; @@ -101,6 +105,8 @@ Status KernelExTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *davin uint32_t op_index = kernel_ex_def.op_index(); OpDescPtr op_desc = davinci_model_->GetOpByIndex(op_index); if (op_desc == nullptr) { + REPORT_INNER_ERROR("E19999", "Can't get op_desc from davinci_model by index:%u when KernelExTaskInfo %s", + op_index, __FUNCTION__); GELOGE(INTERNAL_ERROR, "Init aicpu task info error, index is out of range!"); return INTERNAL_ERROR; } @@ -108,6 +114,9 @@ Status KernelExTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *davin // 2. Reconstruct kernelExDef.args to STR_FWK_OP_KERNEL STR_FWK_OP_KERNEL fwk_op_kernel = {0}; if (sizeof(STR_FWK_OP_KERNEL) < kernel_ex_def.args_size()) { + REPORT_INNER_ERROR("E19999", "Param kernel_ex_def.args_size():%u > sizeof(STR_FWK_OP_KERNEL):%zu, " + "check invalid when KernelExTaskInfo %s", kernel_ex_def.args_size(), sizeof(STR_FWK_OP_KERNEL), + __FUNCTION__); GELOGE(FAILED, "sizeof STR_FWK_OP_KERNEL is: %zu, but args_size is: %u", sizeof(STR_FWK_OP_KERNEL), kernel_ex_def.args_size()); return FAILED; @@ -115,6 +124,8 @@ Status KernelExTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *davin errno_t sec_ret = memcpy_s(&fwk_op_kernel, sizeof(STR_FWK_OP_KERNEL), kernel_ex_def.args().data(), kernel_ex_def.args_size()); if (sec_ret != EOK) { + REPORT_CALL_ERROR("E19999", "Call memcpy_s fail, size:%zu, ret:0x%X, when KernelExTaskInfo %s", + sizeof(STR_FWK_OP_KERNEL), sec_ret, __FUNCTION__); GELOGE(FAILED, "memcpy failed, ret: %d", sec_ret); return FAILED; } @@ -136,12 +147,17 @@ Status KernelExTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *davin uint64_t kernel_id = fwk_op_kernel.fwkKernelBase.fwk_kernel.kernelID; GE_IF_BOOL_EXEC(ModelManager::GetInstance()->CreateAicpuKernel(session_id, davinci_model->Id(), davinci_model->SubModelId(), kernel_id) != SUCCESS, + REPORT_CALL_ERROR("E19999", "CreateAicpuKernel fail, session_id:%lu, model_id:%u, kernel_id:%lu " + "when KernelExTaskInfo %s", + session_id, davinci_model->Id(), kernel_id, __FUNCTION__); GELOGE(FAILED, "CreateAicpuKernel error."); return FAILED;) // 2.3 Create session GE_CHECK_NOTNULL(ModelManager::GetInstance()); ret = ModelManager::GetInstance()->CreateAicpuSession(session_id); GE_IF_BOOL_EXEC(ret != SUCCESS, + REPORT_CALL_ERROR("E19999", "CreateAicpuSession fail, session_id:%lu when KernelExTaskInfo %s", + session_id, __FUNCTION__); GELOGE(ret, "CreateAicpuSession error. session id: %lu", session_id); return ret;) @@ -152,7 +168,10 @@ Status KernelExTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *davin static_cast(reinterpret_cast(input_output_addr)); void *workspace_base_addr = nullptr; rtError_t rt_ret = rtMalloc(&workspace_base_addr, kernel_ex_def.task_info_size(), RT_MEMORY_HBM); - GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, GELOGE(RT_FAILED, "rtMalloc error, ret: Ox%X", rt_ret); + GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, + REPORT_CALL_ERROR("E19999", "Call rtMalloc fail, size:%u, ret:0x%X, when KernelExTaskInfo %s", + kernel_ex_def.task_info_size(), rt_ret, __FUNCTION__); + GELOGE(RT_FAILED, "rtMalloc error, ret: Ox%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret);); rt_ret = rtMemcpy(workspace_base_addr, kernel_ex_def.task_info_size(), kernel_ex_def.task_info().data(), kernel_ex_def.task_info_size(), RT_MEMCPY_HOST_TO_DEVICE); @@ -163,12 +182,18 @@ Status KernelExTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *davin fwk_op_kernel.fwkKernelBase.fwk_kernel.extInfoAddr = reinterpret_cast(ext_info_addr_); rt_ret = rtMalloc(&kernel_buf_, kernel_buf_size_, RT_MEMORY_HBM); - GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, GELOGE(RT_FAILED, "rtMalloc error: 0x%X", rt_ret); + GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, + REPORT_CALL_ERROR("E19999", "Call rtMalloc fail ret:0x%X, size:%u, when KernelExTaskInfo %s", + rt_ret, kernel_buf_size_, __FUNCTION__); + GELOGE(RT_FAILED, "rtMalloc error: 0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret);) rt_ret = rtMemcpy(kernel_buf_, kernel_buf_size_, static_cast(&fwk_op_kernel), kernel_buf_size_, RT_MEMCPY_HOST_TO_DEVICE); - GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, GELOGE(RT_FAILED, "rtMemcpy error, ret: Ox%X", rt_ret); + GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, + REPORT_CALL_ERROR("E19999", "Call rtMemcpy fail ret:0x%X, size:%u, when KernelExTaskInfo %s", + rt_ret, kernel_buf_size_, __FUNCTION__); + GELOGE(RT_FAILED, "rtMemcpy error, ret: Ox%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret);) SetIoAddrs(op_desc); @@ -186,6 +211,8 @@ Status KernelExTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *davin const vector workspace_data_addrs = ModelUtils::GetWorkspaceDataAddrs(rts_param, op_desc); if (workspace_data_addrs.empty()) { + REPORT_CALL_ERROR("E19999", "workspace_data_addrs is empty in op:%s(%s), check invalid when KernelExTaskInfo %s", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "workspace_data_addrs is empty."); return FAILED; } @@ -200,11 +227,17 @@ Status KernelExTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *davin auto addrs_size = sizeof(uint64_t) * (io_addrs.size()); if (addrs_size > 0) { rtError_t rt_ret = rtMalloc(&input_output_addr_, addrs_size, RT_MEMORY_HBM); - GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, GELOGE(RT_FAILED, "rtMalloc error, ret: 0x%X", rt_ret); + GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, + REPORT_CALL_ERROR("E19999", "Call rtMalloc fail ret:0x%X, size:%u, when KernelExTaskInfo %s", + rt_ret, addrs_size, __FUNCTION__); + GELOGE(RT_FAILED, "rtMalloc error, ret: 0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret);) rt_ret = rtMemcpy(input_output_addr_, addrs_size, io_addrs.data(), addrs_size, RT_MEMCPY_HOST_TO_DEVICE); - GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, GELOGE(RT_FAILED, "rtMemcpy to input_output_addr_ error: 0x%X", rt_ret); + GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, + REPORT_CALL_ERROR("E19999", "Call rtMemcpy fail ret:0x%X, size:%u, when KernelExTaskInfo %s", + rt_ret, addrs_size, __FUNCTION__); + GELOGE(RT_FAILED, "rtMemcpy to input_output_addr_ error: 0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret);) InitDumpTask(input_output_addr_, op_desc); @@ -223,12 +256,18 @@ Status KernelExTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *davin // 4. Return result rtError_t rt_ret = rtMalloc(&kernel_buf_, sizeof(STR_FWK_OP_KERNEL), RT_MEMORY_HBM); - GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, GELOGE(RT_FAILED, "rtMalloc error: 0x%X", rt_ret); + GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, + REPORT_CALL_ERROR("E19999", "Call rtMalloc fail ret:0x%X, size:%zu, when KernelExTaskInfo %s", + rt_ret, sizeof(STR_FWK_OP_KERNEL), __FUNCTION__); + GELOGE(RT_FAILED, "rtMalloc error: 0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret);) rt_ret = rtMemcpy(kernel_buf_, sizeof(STR_FWK_OP_KERNEL), static_cast(&fwk_op_kernel), sizeof(STR_FWK_OP_KERNEL), RT_MEMCPY_HOST_TO_DEVICE); - GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, GELOGE(RT_FAILED, "rtMemcpy error, ret: Ox%X", rt_ret); + GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, + REPORT_CALL_ERROR("E19999", "Call rtMemcpy fail ret:0x%X, size:%zu, when KernelExTaskInfo %s", + rt_ret, sizeof(STR_FWK_OP_KERNEL), __FUNCTION__); + GELOGE(RT_FAILED, "rtMemcpy error, ret: Ox%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret);) davinci_model_->SetZeroCopyAddr(op_desc, io_addrs, io_addrs.data(), input_output_addr_, addrs_size, 0); @@ -250,6 +289,8 @@ Status KernelExTaskInfo::CalculateArgs(const domi::TaskDef &task_def, DavinciMod uint32_t op_index = kernel_ex_def.op_index(); OpDescPtr op_desc = davinci_model->GetOpByIndex(op_index); if (op_desc == nullptr) { + REPORT_INNER_ERROR("E19999", "Can't get op_desc from davinci_model by index:%u when KernelExTaskInfo %s", + op_index, __FUNCTION__); GELOGE(INTERNAL_ERROR, "Init aicpu task info error, index is out of range!"); return INTERNAL_ERROR; } @@ -267,6 +308,9 @@ Status KernelExTaskInfo::CalculateArgs(const domi::TaskDef &task_def, DavinciMod if (AttrUtils::GetStr(op_desc, ATTR_DYNAMIC_SHAPE_FIXED_ADDR, peer_input_name) && !peer_input_name.empty()) { uint32_t output_index = davinci_model->GetFixedAddrOutputIndex(peer_input_name); if (output_index > outputs_size) { + REPORT_INNER_ERROR("E19999", "The output size[%zu] and output index[%u] in op:%s(%s) are inconsistent, " + "check invalid when KernelExTaskInfo %s", outputs_size, output_index, + op_desc->GetName().c_str(), op_desc->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "The output size[%zu] and output index[%u] are inconsistent.", outputs_size, output_index); return FAILED; } @@ -293,6 +337,9 @@ void KernelExTaskInfo::SetIoAddrs(const OpDescPtr &op_desc) { if (AttrUtils::GetStr(op_desc, ATTR_DYNAMIC_SHAPE_FIXED_ADDR, peer_input_name)) { uint32_t output_index = davinci_model_->GetFixedAddrOutputIndex(peer_input_name); if (output_index > output_data_addrs.size()) { + REPORT_INNER_ERROR("E19999", "The output data addr size[%zu] and output index[%u] in op:%s(%s) are inconsistent" + ", check invalid when KernelExTaskInfo %s", output_data_addrs.size(), output_index, + op_desc->GetName().c_str(), op_desc->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "The output data addr size[%zu] and output index[%u] are inconsistent.", output_data_addrs.size(), output_index); return; @@ -323,17 +370,25 @@ Status KernelExTaskInfo::CopyTaskInfo(const domi::KernelExDef &kernel_def, const const vector workspace_data_sizes = ModelUtils::GetWorkspaceSize(op_desc); const vector workspace_data_addrs = ModelUtils::GetWorkspaceDataAddrs(rts_param, op_desc); if (workspace_data_addrs.empty() || workspace_data_sizes.empty()) { + REPORT_INNER_ERROR("E19999", "Node:%s(%s) workspace addr:%zu or size:%zu empty, check invalid " + "when KernelExTaskInfo %s", op_desc->GetName().c_str(), op_desc->GetType().c_str(), + workspace_data_addrs.size(), workspace_data_sizes.size(), __FUNCTION__); GELOGE(FAILED, "Node:%s invalid workspace, addrs is %zu, size is %zu.", op_desc->GetName().c_str(), workspace_data_addrs.size(), workspace_data_sizes.size()); return FAILED; } if (workspace_data_addrs[0] == nullptr) { + REPORT_INNER_ERROR("E19999", "Node:%s(%s) workspace addr is nullptr, check invalid when KernelExTaskInfo %s", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "Node:%s workspace addrs is null.", op_desc->GetName().c_str()); return FAILED; } if (workspace_data_sizes[0] < static_cast(kernel_def.task_info_size())) { + REPORT_INNER_ERROR("E19999", "Node:%s(%s) workspace size:%ld < task info size:%d, check invalid " + "when KernelExTaskInfo %s", op_desc->GetName().c_str(), op_desc->GetType().c_str(), + workspace_data_sizes[0], kernel_def.task_info_size(), __FUNCTION__); GELOGE(FAILED, "Node:%s workspace size is %ld, task info size is %d.", op_desc->GetName().c_str(), workspace_data_sizes[0], kernel_def.task_info_size()); return FAILED; @@ -342,6 +397,8 @@ Status KernelExTaskInfo::CopyTaskInfo(const domi::KernelExDef &kernel_def, const rtError_t rt_ret = rtMemcpy(workspace_data_addrs[0], kernel_def.task_info_size(), kernel_def.task_info().data(), kernel_def.task_info_size(), RT_MEMCPY_HOST_TO_DEVICE); if (rt_ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "Call rtMemcpy fail ret:0x%X, size:%d, when KernelExTaskInfo %s", + rt_ret, kernel_def.task_info_size(), __FUNCTION__); GELOGE(RT_FAILED, "rtMemcpy error: 0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } @@ -353,11 +410,13 @@ Status KernelExTaskInfo::Distribute() { GELOGI("KernelExTaskInfo Distribute Start."); rtError_t rt_ret = rtKernelLaunchEx(kernel_buf_, kernel_buf_size_, dump_flag_, stream_); if (rt_ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "Call rtKernelLaunchEx fail ret:0x%X when KernelExTaskInfo %s", rt_ret, __FUNCTION__); GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } if (davinci_model_ == nullptr) { + REPORT_INNER_ERROR("E19999", "Check param davinci_model nullptr when KernelExTaskInfo %s", __FUNCTION__); GELOGE(PARAM_INVALID, "davinci_model_ is null."); return PARAM_INVALID; } @@ -366,6 +425,7 @@ Status KernelExTaskInfo::Distribute() { uint32_t stream_id = 0; // for profiling rt_ret = rtModelGetTaskId(davinci_model_->GetRtModelHandle(), &task_id, &stream_id); if (rt_ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "Call rtModelGetTaskId fail ret:0x%X when KernelExTaskInfo %s", rt_ret, __FUNCTION__); GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } diff --git a/ge/graph/load/model_manager/task_info/kernel_task_info.cc b/ge/graph/load/model_manager/task_info/kernel_task_info.cc index 9e0250de..a6eaa6b7 100755 --- a/ge/graph/load/model_manager/task_info/kernel_task_info.cc +++ b/ge/graph/load/model_manager/task_info/kernel_task_info.cc @@ -93,8 +93,13 @@ Status KernelTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *davinci // new aicpu kernel(rtCpuKernelLaunch) no need to check function if (kernel_type_ == ccKernelType::CCE_AI_CORE) { rtError_t rt_ret = rtGetFunctionByName(const_cast(kernel_def.stub_func().c_str()), &stub_func_); - GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, GELOGE(RT_FAILED, "execute rtGetFunctionByName failed. stub_func: %s", - kernel_def.stub_func().c_str()); + GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, + REPORT_CALL_ERROR("E19999", "Call rtGetFunctionByName fail for op:%s(%s), " + "bin_file_key:%s, ret:0x%X, when KernelTaskInfo %s", + op_desc_->GetName().c_str(), op_desc_->GetType().c_str(), + kernel_def.stub_func().c_str(), rt_ret, __FUNCTION__); + GELOGE(RT_FAILED, "execute rtGetFunctionByName failed. stub_func: %s", + kernel_def.stub_func().c_str()); return RT_ERROR_TO_GE_STATUS(rt_ret);); } else if (kernel_type_ == ccKernelType::TE) { // get bin_file_key @@ -103,11 +108,18 @@ Status KernelTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *davinci const char *bin_file_key = davinci_model_->GetRegisterStub(op_desc_->GetName(), session_graph_model_id); rtError_t rt_ret = rtGetFunctionByName(bin_file_key, &stub_func_); GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, + REPORT_CALL_ERROR("E19999", "Call rtGetFunctionByName fail for op:%s(%s), " + "bin_file_key:%s, ret:0x%X, when KernelTaskInfo %s", + op_desc_->GetName().c_str(), op_desc_->GetType().c_str(), + bin_file_key, rt_ret, __FUNCTION__); GELOGE(RT_FAILED, "execute rtGetFunctionByName failed. bin_file_key: %s", bin_file_key); return RT_ERROR_TO_GE_STATUS(rt_ret);); } if (context.origin_op_index_size() > CC_FUSION_OP_MAX) { + REPORT_INNER_ERROR("E19999", "context.origin_op_index_size():%d is more than CC_FUSION_OP_MAX(%d), op:%s(%s) ," + "check invalid when KernelTaskInfo %s", context.origin_op_index_size(), CC_FUSION_OP_MAX, + op_desc_->GetName().c_str(), op_desc_->GetType().c_str(), __FUNCTION__); GELOGE(PARAM_INVALID, "context.origin_op_index_size() is more than CC_FUSION_OP_MAX(%d)", CC_FUSION_OP_MAX); return PARAM_INVALID; } @@ -120,6 +132,9 @@ Status KernelTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *davinci ctx_.opIndex = context.op_index(); uint16_t *args_offset_tmp = reinterpret_cast(const_cast(context.args_offset().data())); if (context.args_offset().size() / sizeof(uint16_t) < 1) { + REPORT_INNER_ERROR("E19999", "context.args_offset().size():%zu / sizeof(uint16_t) less than 1, op:%s(%s) ," + "check invalid when KernelTaskInfo %s", context.args_offset().size(), + op_desc_->GetName().c_str(), op_desc_->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "context.args_offset().size() / sizeof(uint16_t) less than 1"); return FAILED; } @@ -132,6 +147,8 @@ Status KernelTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *davinci ret = InitAicpuTask(context.op_index(), kernel_def); } else { if (kernel_def.args().empty() || args_size_ == 0) { + REPORT_INNER_ERROR("E19999", "kernel_def.args() is empty, op:%s(%s), check invalid when KernelTaskInfo %s", + op_desc_->GetName().c_str(), op_desc_->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "args is null."); return FAILED; } @@ -164,6 +181,8 @@ void KernelTaskInfo::UpdateSKTTaskId() { if (davinci_model_ != nullptr) { rtError_t rt_ret = rtModelGetTaskId(davinci_model_->GetRtModelHandle(), &task_id, &stream_id); if (rt_ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "Call rtModelGetTaskId fail for op:%s(%s), ret:0x%X, when KernelTaskInfo %s", + op_desc_->GetName().c_str(), op_desc_->GetType().c_str(), rt_ret, __FUNCTION__); GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); return; } @@ -182,6 +201,8 @@ void KernelTaskInfo::UpdateTaskId() { if (davinci_model_ != nullptr) { rtError_t rt_ret = rtModelGetTaskId(davinci_model_->GetRtModelHandle(), &task_id, &stream_id); if (rt_ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "Call rtModelGetTaskId fail for op:%s(%s), ret:0x%X, when KernelTaskInfo %s", + op_desc_->GetName().c_str(), op_desc_->GetType().c_str(), rt_ret, __FUNCTION__); GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); return; } @@ -237,6 +258,8 @@ Status KernelTaskInfo::SuperKernelLaunch() { static_cast(skt_info.last_sm_desc), skt_info.last_stream, skt_info.last_dump_flag); if (rt_ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "Call rtKernelLaunchWithFlag fail for op:%s(%s), ret:0x%X, when KernelTaskInfo %s", + op_desc_->GetName().c_str(), op_desc_->GetType().c_str(), rt_ret, __FUNCTION__); GELOGE(RT_FAILED, "SuperKernelLaunch: Call rt api failed, ret: 0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } @@ -249,6 +272,8 @@ Status KernelTaskInfo::SuperKernelLaunch() { // Init super kernel factory Status ge_ret = factory->Init(); if (ge_ret != SUCCESS) { + REPORT_CALL_ERROR("E19999", "Call SuperKernelFactory init fail for op:%s(%s), ret:0x%X, when KernelTaskInfo %s", + op_desc_->GetName().c_str(), op_desc_->GetType().c_str(), ge_ret, __FUNCTION__); GELOGE(ge_ret, "SuperKernelLaunch: SuperKernelFactory init failed"); return ge_ret; } @@ -256,6 +281,9 @@ Status KernelTaskInfo::SuperKernelLaunch() { std::unique_ptr superKernel = nullptr; ge_ret = factory->FuseKernels(skt_kernel_list, skt_arg_list, skt_info.last_block_dim, superKernel); if (ge_ret != SUCCESS) { + REPORT_CALL_ERROR("E19999", "Call SuperKernelFactory FuseKernels fail for op:%s(%s), ret:0x%X, " + "when KernelTaskInfo %s", + op_desc_->GetName().c_str(), op_desc_->GetType().c_str(), ge_ret, __FUNCTION__); GELOGE(ge_ret, "SuperKernelLaunch: fuse call failed"); return ge_ret; } @@ -263,6 +291,8 @@ Status KernelTaskInfo::SuperKernelLaunch() { skt_dump_flag_ = GetDumpFlag(); ge_ret = superKernel->Launch(skt_info.last_stream, skt_dump_flag_); if (ge_ret != SUCCESS) { + REPORT_CALL_ERROR("E19999", "Call SuperKernelFactory Launch fail for op:%s(%s), ret:0x%X, when KernelTaskInfo %s", + op_desc_->GetName().c_str(), op_desc_->GetType().c_str(), ge_ret, __FUNCTION__); GELOGE(ge_ret, "SuperKernelLaunch: launch failed"); return ge_ret; } @@ -301,11 +331,14 @@ Status KernelTaskInfo::SaveSuperKernelInfo() { bool KernelTaskInfo::IsMarkedLastNode() { if (davinci_model_ == nullptr) { + REPORT_INNER_ERROR("E19999", "Check param davinci_model nullptr when KernelTaskInfo %s", __FUNCTION__); GELOGE(PARAM_INVALID, "davinci_model is null!"); return false; } OpDescPtr op_desc = davinci_model_->GetOpByIndex(ctx_.opIndex); if (op_desc == nullptr) { + REPORT_INNER_ERROR("E19999", "Can't get op_desc from davinci_model by index:%u when KernelTaskInfo %s", + ctx_.opIndex, __FUNCTION__); GELOGE(INTERNAL_ERROR, "InitTVMTaskInfo error, index is out of range!"); return false; } @@ -316,11 +349,14 @@ bool KernelTaskInfo::IsMarkedLastNode() { bool KernelTaskInfo::IsMarkedFirstNode() { if (davinci_model_ == nullptr) { + REPORT_INNER_ERROR("E19999", "Check param davinci_model nullptr when KernelTaskInfo %s", __FUNCTION__); GELOGE(PARAM_INVALID, "davinci_model is null!"); return false; } OpDescPtr op_desc = davinci_model_->GetOpByIndex(ctx_.opIndex); if (op_desc == nullptr) { + REPORT_INNER_ERROR("E19999", "Can't get op_desc from davinci_model by index:%u when KernelTaskInfo %s", + ctx_.opIndex, __FUNCTION__); GELOGE(INTERNAL_ERROR, "InitTVMTaskInfo error, index is out of range!"); return false; } @@ -361,6 +397,8 @@ Status KernelTaskInfo::SuperKernelDistribute() { rtError_t rt_ret = rtKernelLaunchWithFlag(stub_func_, block_dim_, args_, args_size_, static_cast(sm_desc_), stream_, dump_flag_); if (rt_ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "Call rtKernelLaunchWithFlag fail for op:%s(%s), ret:0x%X, when KernelTaskInfo %s", + op_desc_->GetName().c_str(), op_desc_->GetType().c_str(), rt_ret, __FUNCTION__); GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); return rt_ret; } @@ -425,6 +463,9 @@ Status KernelTaskInfo::Distribute() { } } if (rt_ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "Call rtKernelLaunchWithFlag or rtCpuKernelLaunchWithFlag fail for op:%s(%s), " + "ret:0x%X, when KernelTaskInfo %s", + op_desc_->GetName().c_str(), op_desc_->GetType().c_str(), rt_ret, __FUNCTION__); GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } @@ -459,12 +500,16 @@ Status KernelTaskInfo::CopyNoncontinuousArgs(uint16_t offset) { // copy io addr errno_t sec_ret = memcpy_s(args_addr.get() + offset, addr_size, io_addrs_.data(), addr_size); if (sec_ret != EOK) { + REPORT_CALL_ERROR("E19999", "Call memcpy_s fail, size:%zu, ret:0x%X, when KernelTaskInfo %s", + addr_size, sec_ret, __FUNCTION__); GELOGE(FAILED, "memcpy failed, ret: %d", sec_ret); return FAILED; } // copy args to device rtError_t rt_ret = rtMemcpy(args_, args_size_, args_addr.get(), args_size_, RT_MEMCPY_HOST_TO_DEVICE); + REPORT_CALL_ERROR("E19999", "Call rtMemcpy fail for op:%s(%s), size:%u, ret:0x%X, when KernelTaskInfo %s", + op_desc_->GetName().c_str(), op_desc_->GetType().c_str(), args_size_, rt_ret, __FUNCTION__); if (rt_ret != RT_ERROR_NONE) { GELOGE(RT_FAILED, "Call rt api(rtMemcpy) failed, ret: 0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); @@ -514,6 +559,8 @@ Status KernelTaskInfo::Release() { ret = (sm_desc_ != nullptr) ? rtMemFreeManaged(sm_desc_) : RT_ERROR_NONE; if (ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "Call rtMemFreeManaged fail for op:%s(%s), ret:0x%X, when KernelTaskInfo %s", + op_desc_->GetName().c_str(), op_desc_->GetType().c_str(), ret, __FUNCTION__); GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", static_cast(ret)); return RT_ERROR_TO_GE_STATUS(ret); } @@ -544,12 +591,16 @@ Status KernelTaskInfo::UpdateL2Data(const domi::KernelDef &kernel_def) { rtError_t rt_ret = rtMemAllocManaged(&sm_desc_, sm_desc.size(), RT_MEMORY_SPM); if (rt_ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "Call rtMemAllocManaged fail for op:%s(%s), ret:0x%X, when KernelTaskInfo %s", + op_desc_->GetName().c_str(), op_desc_->GetType().c_str(), rt_ret, __FUNCTION__); GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } rt_ret = rtMemcpy(sm_desc_, sm_desc.size(), sm_desc.data(), sm_desc.size(), RT_MEMCPY_HOST_TO_DEVICE); if (rt_ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "Call rtMemcpy fail for op:%s(%s), size:%zu, ret:0x%X, when KernelTaskInfo %s", + op_desc_->GetName().c_str(), op_desc_->GetType().c_str(), sm_desc.size(), rt_ret, __FUNCTION__); GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } @@ -596,6 +647,8 @@ Status KernelTaskInfo::InitTVMTask(uint16_t offset, const domi::KernelDef &kerne args_addr = std::unique_ptr(new (std::nothrow) uint8_t[args_size_]); errno_t sec_ret = memcpy_s(args_addr.get(), args_size_, kernel_def.args().data(), args_size_); if (sec_ret != EOK) { + REPORT_CALL_ERROR("E19999", "Call memcpy_s fail, size:%u, ret:0x%X, when KernelTaskInfo %s", + args_size_, sec_ret, __FUNCTION__); GELOGE(FAILED, "memcpy failed, ret: %d", sec_ret); return FAILED; } @@ -638,6 +691,8 @@ Status KernelTaskInfo::InitTVMTask(uint16_t offset, const domi::KernelDef &kerne // malloc args memory rt_ret = rtMalloc(&args_, args_size_, RT_MEMORY_HBM); if (rt_ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "Call rtMalloc fail for op:%s(%s), size:%u, ret:0x%X, when KernelTaskInfo %s", + op_desc_->GetName().c_str(), op_desc_->GetType().c_str(), args_size_, rt_ret, __FUNCTION__); GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } @@ -645,11 +700,17 @@ Status KernelTaskInfo::InitTVMTask(uint16_t offset, const domi::KernelDef &kerne // copy orign args rt_ret = rtMemcpy(args_, args_size_, kernel_def.args().data(), args_size_, RT_MEMCPY_HOST_TO_DEVICE); if (rt_ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "Call rtMemcpy fail for op:%s(%s), size:%u, ret:0x%X, when KernelTaskInfo %s", + op_desc_->GetName().c_str(), op_desc_->GetType().c_str(), args_size_, rt_ret, __FUNCTION__); GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } if ((args_size_ <= offset) || (args_size_ - offset < kAddrLen * tensor_device_addrs.size())) { + REPORT_INNER_ERROR("E19999", "offset:%u >= kernelInfo.argsSize:%u or copy content:%zu beyond applied memory:%u, " + "check invalid in op:%s(%s), when KernelTaskInfo %s", + offset, args_size_, kAddrLen * tensor_device_addrs.size(), args_size_ - offset, + op_desc_->GetName().c_str(), op_desc_->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "offset >= kernelInfo.argsSize or copy content beyond applied memory."); return FAILED; } @@ -658,12 +719,17 @@ Status KernelTaskInfo::InitTVMTask(uint16_t offset, const domi::KernelDef &kerne rt_ret = rtMemcpy(static_cast(args_) + offset, args_size_ - offset, tensor_device_addrs.data(), kAddrLen * tensor_device_addrs.size(), RT_MEMCPY_HOST_TO_DEVICE); if (rt_ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "Call rtMemcpy fail for op:%s(%s), size:%u, ret:0x%X, when KernelTaskInfo %s", + op_desc_->GetName().c_str(), op_desc_->GetType().c_str(), + args_size_ - offset, rt_ret, __FUNCTION__); GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } sec_ret = memcpy_s(args_addr.get() + offset, args_size_ - offset, tensor_device_addrs.data(), kAddrLen * tensor_device_addrs.size()); if (sec_ret != EOK) { + REPORT_CALL_ERROR("E19999", "Call memcpy_s fail, size:%u, ret:0x%X, when KernelTaskInfo %s", + args_size_ - offset, sec_ret, __FUNCTION__); GELOGE(FAILED, "memcpy failed, ret: %d", sec_ret); return FAILED; } @@ -708,6 +774,8 @@ Status KernelTaskInfo::InitAICPUCustomTask(uint32_t op_index, const domi::Kernel GELOGI("Do InitAICPUCustomTask"); OpDescPtr op_desc = davinci_model_->GetOpByIndex(op_index); if (op_desc == nullptr) { + REPORT_INNER_ERROR("E19999", "Can't get op_desc from davinci_model by index:%u when KernelTaskInfo %s", + op_index, __FUNCTION__); GELOGE(INTERNAL_ERROR, "index is out of range, index: %u", op_index); return INTERNAL_ERROR; } @@ -718,11 +786,17 @@ Status KernelTaskInfo::InitAICPUCustomTask(uint32_t op_index, const domi::Kernel const uint32_t kCustomAicpuArgsLen = 5; ctx_.argsOffset = new (std::nothrow) uint16_t[kCustomAicpuArgsLen](); if (ctx_.argsOffset == nullptr) { + REPORT_INNER_ERROR("E19999", "New ctx_.argsOffset fail, size:%u, op:%s(%s), when KernelTaskInfo %s", + kCustomAicpuArgsLen, op_desc_->GetName().c_str(), op_desc_->GetType().c_str(), __FUNCTION__); GELOGE(PARAM_INVALID, "ctx_.argsOffset is null!"); return PARAM_INVALID; } if (context.args_offset().size() / sizeof(uint16_t) < kCustomAicpuArgsLen) { + REPORT_INNER_ERROR("E19999", "context.args_offset().size():%zu / sizeof(uint16_t) is less than " + "kCustomAicpuArgsLen:%u, op:%s(%s), check invalid when KernelTaskInfo %s", + context.args_offset().size(), kCustomAicpuArgsLen, + op_desc_->GetName().c_str(), op_desc_->GetType().c_str(), __FUNCTION__); GELOGE(PARAM_INVALID, "context.args_offset().size() / sizeof(uint16_t) is less than kCustomAicpuArgsLen"); return PARAM_INVALID; } @@ -743,24 +817,32 @@ Status KernelTaskInfo::InitAICPUCustomTask(uint32_t op_index, const domi::Kernel // attrHandle Buffer buffer; if (!AttrUtils::GetBytes(op_desc, ATTR_NAME_OPATTR, buffer)) { + REPORT_INNER_ERROR("E19999", "Get Attr:%s in op:%s(%s) fail when KernelTaskInfo %s", ATTR_NAME_OPATTR.c_str(), + op_desc->GetName().c_str(), op_desc->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "can't find opattr bytes!."); return FAILED; } uint32_t op_attr_size = buffer.GetSize(); if (op_attr_size == 0) { + REPORT_INNER_ERROR("E19999", "Attr:%s in op:%s(%s) size is 0, check invalid when KernelTaskInfo %s", + ATTR_NAME_OPATTR.c_str(), op_desc->GetName().c_str(), op_desc->GetType().c_str(), __FUNCTION__); GELOGE(PARAM_INVALID, "param op_attr_size is out of range"); return PARAM_INVALID; } rtError_t rt_ret = rtMalloc(&custom_info_.attr_handle, op_attr_size, RT_MEMORY_HBM); if (rt_ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "Call rtMalloc fail for op:%s(%s), size:%u, ret:0x%X, when KernelTaskInfo %s", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), op_attr_size, rt_ret, __FUNCTION__); GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } rt_ret = rtMemcpy(custom_info_.attr_handle, op_attr_size, buffer.GetData(), op_attr_size, RT_MEMCPY_HOST_TO_DEVICE); if (rt_ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "Call rtMemcpy fail for op:%s(%s), size:%u, ret:0x%X, when KernelTaskInfo %s", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), op_attr_size, rt_ret, __FUNCTION__); GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } @@ -770,6 +852,10 @@ Status KernelTaskInfo::InitAICPUCustomTask(uint32_t op_index, const domi::Kernel for (uint32_t i = 0; i < kCustomAicpuArgsLen; ++i) { if (kernel_def.args().size() < ((size_t)ctx_.argsOffset[i] + sizeof(uint64_t))) { + REPORT_INNER_ERROR("E19999", "ctx.argsOffset[%u]: %u + sizeof(uint64_t): %zu >= kernelDef.args().size():%zu, " + "op:%s(%s) check invalid when KernelTaskInfo %s", i, (uint32_t)ctx_.argsOffset[i], + sizeof(uint64_t), kernel_def.args().size(), + op_desc->GetName().c_str(), op_desc->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "ctx.argsOffset[%u]: %u + sizeof(uint64_t): %zu >= kernelDef.args().size():%zu", i, (uint32_t)ctx_.argsOffset[i], sizeof(uint64_t), kernel_def.args().size()); return FAILED; @@ -788,6 +874,8 @@ Status KernelTaskInfo::InitAICPUCustomTask(uint32_t op_index, const domi::Kernel rt_ret = rtMalloc(&args_, args_size_, RT_MEMORY_HBM); if (rt_ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "Call rtMalloc fail for op:%s(%s), size:%u, ret:0x%X, when KernelTaskInfo %s", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), args_size_, rt_ret, __FUNCTION__); GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } @@ -795,6 +883,9 @@ Status KernelTaskInfo::InitAICPUCustomTask(uint32_t op_index, const domi::Kernel rt_ret = rtMemcpy(args_, kernel_def.args_size(), kernel_def.args().data(), kernel_def.args_size(), RT_MEMCPY_HOST_TO_DEVICE); if (rt_ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "Call rtMemcpy fail for op:%s(%s), size:%u, ret:0x%X, when KernelTaskInfo %s", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), + kernel_def.args_size(), rt_ret, __FUNCTION__); GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } @@ -809,6 +900,7 @@ Status KernelTaskInfo::InitAICPUCustomTask(uint32_t op_index, const domi::Kernel Status KernelTaskInfo::InitCceTask(const domi::KernelDef &kernel_def) { GELOGI("Do InitCCETask"); if (davinci_model_ == nullptr) { + REPORT_INNER_ERROR("E19999", "Check param davinci_model nullptr when KernelTaskInfo %s", __FUNCTION__); GELOGE(PARAM_INVALID, "davinci_model is null!"); return PARAM_INVALID; } @@ -823,6 +915,8 @@ Status KernelTaskInfo::InitCceTask(const domi::KernelDef &kernel_def) { if (context.is_flowtable()) { if (flowtable.empty()) { + REPORT_INNER_ERROR("E19999", "kernel_def.flowtable is empty, op:%s(%s), check invalid when KernelTaskInfo %s", + op_desc_->GetName().c_str(), op_desc_->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "flowtable is null."); return FAILED; } @@ -857,6 +951,9 @@ Status KernelTaskInfo::InitCceTask(const domi::KernelDef &kernel_def) { // args rtError_t rt_ret = rtMalloc(&args_, kernel_def.args_size(), RT_MEMORY_HBM); if (rt_ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "Call rtMalloc fail for op:%s(%s), size:%u, ret:0x%X, when KernelTaskInfo %s", + op_desc_->GetName().c_str(), op_desc_->GetType().c_str(), + kernel_def.args_size(), rt_ret, __FUNCTION__); GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } @@ -865,6 +962,9 @@ Status KernelTaskInfo::InitCceTask(const domi::KernelDef &kernel_def) { rt_ret = rtMemcpy(args_, kernel_def.args_size(), kernel_def.args().data(), kernel_def.args_size(), RT_MEMCPY_HOST_TO_DEVICE); if (rt_ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "Call rtMemcpy fail for op:%s(%s), size:%u, ret:0x%X, when KernelTaskInfo %s", + op_desc_->GetName().c_str(), op_desc_->GetType().c_str(), + kernel_def.args_size(), rt_ret, __FUNCTION__); GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } @@ -873,12 +973,16 @@ Status KernelTaskInfo::InitCceTask(const domi::KernelDef &kernel_def) { if (!sm_desc.empty()) { rt_ret = rtMemAllocManaged(&sm_desc_, sm_desc.size(), RT_MEMORY_SPM); if (rt_ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "Call rtMemAllocManaged fail for op:%s(%s), ret:0x%X, when KernelTaskInfo %s", + op_desc_->GetName().c_str(), op_desc_->GetType().c_str(), rt_ret, __FUNCTION__); GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } rt_ret = rtMemcpy(sm_desc_, sm_desc.size(), sm_desc.data(), sm_desc.size(), RT_MEMCPY_HOST_TO_DEVICE); if (rt_ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "Call rtMemcpy fail for op:%s(%s), size:%zu, ret:0x%X, when KernelTaskInfo %s", + op_desc_->GetName().c_str(), op_desc_->GetType().c_str(), sm_desc.size(), rt_ret, __FUNCTION__); GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } @@ -893,6 +997,8 @@ Status KernelTaskInfo::InitAicpuTask(uint32_t op_index, const domi::KernelDef &k OpDescPtr op_desc = davinci_model_->GetOpByIndex(op_index); if (op_desc == nullptr) { + REPORT_INNER_ERROR("E19999", "Can't get op_desc from davinci_model by index:%u when KernelTaskInfo %s", + op_index, __FUNCTION__); GELOGE(INTERNAL_ERROR, "index is out of range, index: %u", op_index); return INTERNAL_ERROR; } @@ -910,6 +1016,8 @@ Status KernelTaskInfo::InitAicpuTask(uint32_t op_index, const domi::KernelDef &k GE_PRINT_DYNAMIC_MEMORY(new, "cce task physical memory.", sizeof(uint8_t) * args_size_) errno_t sec_ret = memcpy_s(args_addr.get(), args_size_, kernel_def.args().data(), args_size_); if (sec_ret != EOK) { + REPORT_CALL_ERROR("E19999", "Call memcpy_s fail, size:%u, ret:0x%X, when KernelTaskInfo %s", + args_size_, sec_ret, __FUNCTION__); GELOGE(FAILED, "memcpy failed, ret: %d", sec_ret); return FAILED; } @@ -944,6 +1052,8 @@ Status KernelTaskInfo::InitAicpuTask(uint32_t op_index, const domi::KernelDef &k auto addrs_size = sizeof(uint64_t) * io_addrs.size(); sec_ret = memcpy_s(reinterpret_cast(io_addr), addrs_size, io_addrs.data(), addrs_size); if (sec_ret != EOK) { + REPORT_CALL_ERROR("E19999", "Call memcpy_s fail, size:%u, ret:0x%X, when KernelTaskInfo %s", + addrs_size, sec_ret, __FUNCTION__); GELOGE(FAILED, "memcpy failed, ret: %d", sec_ret); return FAILED; } @@ -952,6 +1062,8 @@ Status KernelTaskInfo::InitAicpuTask(uint32_t op_index, const domi::KernelDef &k // malloc device memory for args rtError_t rt_ret = rtMalloc(static_cast(&args_), args_size_, RT_MEMORY_HBM); if (rt_ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "Call rtMalloc fail for op:%s(%s), size:%u, ret:0x%X, when KernelTaskInfo %s", + op_desc_->GetName().c_str(), op_desc_->GetType().c_str(), args_size_, rt_ret, __FUNCTION__); GELOGE(RT_FAILED, "Call rt api(rtMalloc) failed, ret: 0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } @@ -960,6 +1072,8 @@ Status KernelTaskInfo::InitAicpuTask(uint32_t op_index, const domi::KernelDef &k // copy args to device rt_ret = rtMemcpy(args_, args_size_, args_addr.get(), args_size_, RT_MEMCPY_HOST_TO_DEVICE); if (rt_ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "Call rtMemcpy fail for op:%s(%s), size:%u, ret:0x%X, when KernelTaskInfo %s", + op_desc_->GetName().c_str(), op_desc_->GetType().c_str(), args_size_, rt_ret, __FUNCTION__); GELOGE(RT_FAILED, "Call rt api(rtMemcpy) failed, ret: 0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } @@ -1032,12 +1146,18 @@ Status KernelTaskInfo::InitAicpuTaskExtInfo(const std::string &ext_info) { } auto rt_ret = rtMalloc(&aicpu_ext_info_addr_, ext_handle->GetExtInfoLen(), RT_MEMORY_HBM); if (rt_ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "Call rtMalloc fail for op:%s(%s), size:%zu, ret:0x%X, when KernelTaskInfo %s", + op_desc_->GetName().c_str(), op_desc_->GetType().c_str(), + ext_handle->GetExtInfoLen(), rt_ret, __FUNCTION__); GELOGE(RT_FAILED, "rtMalloc ext_info error: 0x%X, size=%zu", rt_ret, ext_info.size()); return RT_ERROR_TO_GE_STATUS(rt_ret); } rt_ret = rtMemcpy(aicpu_ext_info_addr_, ext_handle->GetExtInfoLen(), ext_handle->GetExtInfo(), ext_handle->GetExtInfoLen(), RT_MEMCPY_HOST_TO_DEVICE); if (rt_ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "Call rtMemcpy fail for op:%s(%s), size:%zu, ret:0x%X, when KernelTaskInfo %s", + op_desc_->GetName().c_str(), op_desc_->GetType().c_str(), + ext_handle->GetExtInfoLen(), rt_ret, __FUNCTION__); GELOGE(RT_FAILED, "rtMemcpy ext_info error: 0x%X, size=%zu", rt_ret, ext_info.size()); return RT_ERROR_TO_GE_STATUS(rt_ret); } @@ -1055,6 +1175,9 @@ Status KernelTaskInfo::StoreInputOutputTensor(const std::vector &input_d // inputDescs rtError_t rt_ret = rtMalloc(&custom_info_.input_descs, sizeof(opTensor_t) * input_size, RT_MEMORY_HBM); if (rt_ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "Call rtMalloc fail for op:%s(%s), size:%zu, ret:0x%X, when KernelTaskInfo %s", + op_desc_->GetName().c_str(), op_desc_->GetType().c_str(), + sizeof(opTensor_t) * input_size, rt_ret, __FUNCTION__); GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } @@ -1063,6 +1186,9 @@ Status KernelTaskInfo::StoreInputOutputTensor(const std::vector &input_d rt_ret = rtMemcpy(static_cast(custom_info_.input_descs) + i, sizeof(opTensor_t), const_cast(&input_descs[i]), sizeof(opTensor_t), RT_MEMCPY_HOST_TO_DEVICE); if (rt_ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "Call rtMemcpy fail for op:%s(%s), size:%zu, ret:0x%X, when KernelTaskInfo %s", + op_desc_->GetName().c_str(), op_desc_->GetType().c_str(), + sizeof(opTensor_t), rt_ret, __FUNCTION__); GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } @@ -1071,6 +1197,9 @@ Status KernelTaskInfo::StoreInputOutputTensor(const std::vector &input_d // inputAddrs rt_ret = rtMalloc(&custom_info_.input_addrs, sizeof(opTensor_t) * input_size, RT_MEMORY_HBM); if (rt_ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "Call rtMalloc fail for op:%s(%s), size:%zu, ret:0x%X, when KernelTaskInfo %s", + op_desc_->GetName().c_str(), op_desc_->GetType().c_str(), + sizeof(opTensor_t) * input_size, rt_ret, __FUNCTION__); GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } @@ -1079,6 +1208,9 @@ Status KernelTaskInfo::StoreInputOutputTensor(const std::vector &input_d rt_ret = rtMemcpy(custom_info_.input_addrs, kAddrLen * input_size, &input_data_addrs[0], kAddrLen * input_size, RT_MEMCPY_HOST_TO_DEVICE); if (rt_ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "Call rtMemcpy fail for op:%s(%s), size:%zu, ret:0x%X, when KernelTaskInfo %s", + op_desc_->GetName().c_str(), op_desc_->GetType().c_str(), + kAddrLen * input_size, rt_ret, __FUNCTION__); GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } @@ -1087,6 +1219,9 @@ Status KernelTaskInfo::StoreInputOutputTensor(const std::vector &input_d // outputDescs rt_ret = rtMalloc(&custom_info_.output_descs, sizeof(opTensor_t) * output_size, RT_MEMORY_HBM); if (rt_ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "Call rtMalloc fail for op:%s(%s), size:%zu, ret:0x%X, when KernelTaskInfo %s", + op_desc_->GetName().c_str(), op_desc_->GetType().c_str(), + sizeof(opTensor_t) * output_size, rt_ret, __FUNCTION__); GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } @@ -1094,6 +1229,9 @@ Status KernelTaskInfo::StoreInputOutputTensor(const std::vector &input_d rt_ret = rtMemcpy(static_cast(custom_info_.output_descs) + i, sizeof(opTensor_t), const_cast(&input_descs[i]), sizeof(opTensor_t), RT_MEMCPY_HOST_TO_DEVICE); if (rt_ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "Call rtMemcpy fail for op:%s(%s), size:%zu, ret:0x%X, when KernelTaskInfo %s", + op_desc_->GetName().c_str(), op_desc_->GetType().c_str(), + sizeof(opTensor_t), rt_ret, __FUNCTION__); GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } @@ -1102,6 +1240,9 @@ Status KernelTaskInfo::StoreInputOutputTensor(const std::vector &input_d // outputAddrs rt_ret = rtMalloc(&custom_info_.output_addrs, sizeof(opTensor_t) * output_size, RT_MEMORY_HBM); if (rt_ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "Call rtMalloc fail for op:%s(%s), size:%zu, ret:0x%X, when KernelTaskInfo %s", + op_desc_->GetName().c_str(), op_desc_->GetType().c_str(), + sizeof(opTensor_t) * output_size, rt_ret, __FUNCTION__); GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } @@ -1110,6 +1251,9 @@ Status KernelTaskInfo::StoreInputOutputTensor(const std::vector &input_d rt_ret = rtMemcpy(custom_info_.output_addrs, kAddrLen * output_size, &output_data_addrs[0], kAddrLen * output_size, RT_MEMCPY_HOST_TO_DEVICE); if (rt_ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "Call rtMemcpy fail for op:%s(%s), size:%zu, ret:0x%X, when KernelTaskInfo %s", + op_desc_->GetName().c_str(), op_desc_->GetType().c_str(), + kAddrLen * output_size, rt_ret, __FUNCTION__); GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } @@ -1126,11 +1270,17 @@ Status KernelTaskInfo::SetContext(const domi::KernelDef &kernel_def) { ctx_.isFlowtable = context.is_flowtable(); ctx_.argsCount = context.args_count(); if (ctx_.argsCount == 0) { + REPORT_INNER_ERROR("E19999", "kernel_def.context.args_count is 0, op:%s(%s), check invalid when KernelTaskInfo %s", + op_desc_->GetName().c_str(), op_desc_->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "check argsCount fail:%u.", ctx_.argsCount); return INTERNAL_ERROR; } if (context.args_offset().size() / sizeof(uint16_t) < ctx_.argsCount) { + REPORT_INNER_ERROR("E19999", "param [context.args_offset().size():%zu / sizeof(uint16_t)] " + "is less than [ctx_.argsCount:%u], op:%s(%s), check invalid when KernelTaskInfo %s", + context.args_offset().size(), ctx_.argsCount, + op_desc_->GetName().c_str(), op_desc_->GetType().c_str(), __FUNCTION__); GELOGE(PARAM_INVALID, "param [context.args_offset().size() / sizeof(uint16_t)] is less than [ctx_.argsCount]"); return PARAM_INVALID; } @@ -1138,6 +1288,8 @@ Status KernelTaskInfo::SetContext(const domi::KernelDef &kernel_def) { // ctx_.argsOffset stores the offset of the internal information of agrs_, equal to the ctx_.argsCount ctx_.argsOffset = new (std::nothrow) uint16_t[ctx_.argsCount](); if (ctx_.argsOffset == nullptr) { + REPORT_INNER_ERROR("E19999", "New ctx_.argsOffset fail, size:%u, op:%s(%s), when KernelTaskInfo %s", + ctx_.argsCount, op_desc_->GetName().c_str(), op_desc_->GetType().c_str(), __FUNCTION__); GELOGE(PARAM_INVALID, "(param [ctx_.argsOffset] must not be null."); return PARAM_INVALID; } @@ -1155,6 +1307,7 @@ void KernelTaskInfo::FreeRtMem(void **ptr) { } rtError_t ret = rtFree(*ptr); if (ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "Call rtFree fail, ret:0x%X, when KernelTaskInfo %s", ret, __FUNCTION__); GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", ret); } @@ -1202,6 +1355,9 @@ Status KernelTaskInfo::CceUpdateKernelArgs(const domi::KernelContext &context, u if (handle == nullptr) { error = mmDlerror(); GE_IF_BOOL_EXEC(error == nullptr, error = ""); + REPORT_INNER_ERROR("E19999", "Failed in dlopen:%s, dlerror:%s, op:%s(%s), when KernelTaskInfo %s", + canonicalPath.c_str(), error, + op_desc_->GetName().c_str(), op_desc_->GetType().c_str(), __FUNCTION__); GELOGE(GE_PLGMGR_SO_NOT_EXIST, "Failed in dlopen %s! ", error); return FAILED; } @@ -1210,6 +1366,9 @@ Status KernelTaskInfo::CceUpdateKernelArgs(const domi::KernelContext &context, u auto cceUpdateKernelArgs = (ccStatus_t(*)(ccOpContext &, uint64_t, uint64_t, uint64_t, void *, uint64_t, void *))mmDlsym(handle, const_cast(update_kernel_args.c_str())); if (cceUpdateKernelArgs == nullptr) { + REPORT_INNER_ERROR("E19999", "No symbol:%s in %s, op:%s(%s), check invalid when KernelTaskInfo %s", + update_kernel_args.c_str(), canonicalPath.c_str(), + op_desc_->GetName().c_str(), op_desc_->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "Failed to invoke function ccUpdateKernelArgs"); if (mmDlclose(handle) != 0) { error = mmDlerror(); @@ -1234,6 +1393,8 @@ Status KernelTaskInfo::CceUpdateKernelArgs(const domi::KernelContext &context, u return FAILED; } if (cc_ret != CC_STATUS_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Call cceUpdateKernelArgs fail, op:%s(%s), ret:0x%X, when KernelTaskInfo %s", + op_desc_->GetName().c_str(), op_desc_->GetType().c_str(), cc_ret, __FUNCTION__); GELOGE(CCE_FAILED, "Call cce api failed, ret: 0x%X", cc_ret); return CCE_FAILED; } @@ -1247,6 +1408,9 @@ Status KernelTaskInfo::SetFlowtable(std::string &flowtable, const domi::KernelDe if (context.is_flowtable()) { rtError_t rt_ret = rtMalloc(&flowtable_, flowtable.size(), RT_MEMORY_HBM); if (rt_ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "Call rtMalloc fail, op:%s(%s), size:%zu, ret:0x%X, when KernelTaskInfo %s", + op_desc_->GetName().c_str(), op_desc_->GetType().c_str(), + flowtable.size(), rt_ret, __FUNCTION__); GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } @@ -1254,6 +1418,9 @@ Status KernelTaskInfo::SetFlowtable(std::string &flowtable, const domi::KernelDe rt_ret = rtMemcpy(flowtable_, flowtable.size(), flowtable.data(), flowtable.size(), RT_MEMCPY_HOST_TO_DEVICE); if (rt_ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "Call rtMemcpy fail, op:%s(%s), size:%zu, ret:0x%X, when KernelTaskInfo %s", + op_desc_->GetName().c_str(), op_desc_->GetType().c_str(), + flowtable.size(), rt_ret, __FUNCTION__); GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } @@ -1263,6 +1430,12 @@ Status KernelTaskInfo::SetFlowtable(std::string &flowtable, const domi::KernelDe if (kernel_def.args().size() < ((reinterpret_cast(const_cast(context.args_offset().data())))[0] + sizeof(uint64_t))) { + REPORT_INNER_ERROR( + "E19999", "(context.args_offset().data()))[0]:%u + sizeof(uint64_t):%zu > " + "kernelDef.args().size():%zu, op:%s(%s), check invalid when %s", + (uint32_t)((reinterpret_cast(const_cast(context.args_offset().data())))[0]), + sizeof(uint64_t), kernel_def.args().size(), + op_desc_->GetName().c_str(), op_desc_->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "(context.args_offset().data()))[0]:%u + sizeof(uint64_t):%zu > kernelDef.args().size():%zu", (uint32_t)((reinterpret_cast(const_cast(context.args_offset().data())))[0]), sizeof(uint64_t), kernel_def.args().size()); diff --git a/ge/graph/load/model_manager/task_info/super_kernel/super_kernel.cc b/ge/graph/load/model_manager/task_info/super_kernel/super_kernel.cc index 65dca3b3..68a1d364 100644 --- a/ge/graph/load/model_manager/task_info/super_kernel/super_kernel.cc +++ b/ge/graph/load/model_manager/task_info/super_kernel/super_kernel.cc @@ -26,15 +26,24 @@ Status SuperKernel::Launch(rtStream_t stream, uint32_t dump_flag) { reinterpret_cast(static_cast(this->GetNavTableSize()))}; rtError_t rt_ret = rtMalloc(reinterpret_cast(&device_args_addr_), sizeof(args), RT_MEMORY_HBM); - GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, GELOGE(RT_FAILED, "rtMalloc failied. error: 0x%X", rt_ret); + GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, + REPORT_CALL_ERROR("E19999", "Call rtMalloc fail, size:%u, ret:0x%X when %s", + sizeof(args), rt_ret, __FUNCTION__); + GELOGE(RT_FAILED, "rtMalloc failied. error: 0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret);) rt_ret = rtMemcpy(reinterpret_cast(device_args_addr_), sizeof(args), reinterpret_cast(args), sizeof(args), RT_MEMCPY_HOST_TO_DEVICE); - GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, GELOGE(RT_FAILED, "rtMemcpy failied. error: 0x%X", rt_ret); + GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, + REPORT_CALL_ERROR("E19999", "Call rtMemcpy fail, size:%u, ret:0x%X when %s", + sizeof(args), rt_ret, __FUNCTION__); + GELOGE(RT_FAILED, "rtMemcpy failied. error: 0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret);) rt_ret = rtKernelLaunchWithFlag((void *const)func_stub_, block_dim_, device_args_addr_, sizeof(args), NULL, stream, dump_flag); - GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, GELOGE(RT_FAILED, "rtKernelLaunchWithFlag failied. error: 0x%X", rt_ret); + GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, + REPORT_CALL_ERROR("E19999", "Call rtKernelLaunchWithFlag fail, dump_flag:%u, ret:0x%X when %s", + dump_flag, rt_ret, __FUNCTION__); + GELOGE(RT_FAILED, "rtKernelLaunchWithFlag failied. error: 0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret);) return SUCCESS; } diff --git a/ge/graph/load/model_manager/task_info/super_kernel/super_kernel_factory.cc b/ge/graph/load/model_manager/task_info/super_kernel/super_kernel_factory.cc index 4e22cd7c..297ac43c 100644 --- a/ge/graph/load/model_manager/task_info/super_kernel/super_kernel_factory.cc +++ b/ge/graph/load/model_manager/task_info/super_kernel/super_kernel_factory.cc @@ -35,14 +35,16 @@ Status SuperKernelFactory::Init() { } rtError_t rt_ret; rt_ret = rtGetFunctionByName(this->sk_stub_name_.c_str(), &this->func_stub_); - GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, GELOGE(RT_FAILED, - "rtGetFunctionByName " - "failed. stub_func: %s, please export LD_LIBRARY_PATH for " - "libcce_aicore.so", - this->sk_stub_name_.c_str()); + GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, + REPORT_CALL_ERROR("E19999", "Call rtGetFunctionByName fail, stub_func:%s, ret:0x%X, when %s", + this->sk_stub_name_.c_str(), rt_ret, __FUNCTION__); + GELOGE(RT_FAILED, "rtGetFunctionByName failed. stub_func: %s, please export LD_LIBRARY_PATH for " + "libcce_aicore.so", this->sk_stub_name_.c_str()); return RT_ERROR_TO_GE_STATUS(rt_ret);) rt_ret = rtGetAddrByFun(this->func_stub_, &this->func_ptr_); - GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, GELOGE(RT_FAILED, "rtGetAddrByFun failed. error: 0x%X", rt_ret); + GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, + REPORT_CALL_ERROR("E19999", "Call rtGetAddrByFun fail, ret:0x%X, when %s", rt_ret, __FUNCTION__); + GELOGE(RT_FAILED, "rtGetAddrByFun failed. error: 0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret);) GELOGD( "SKT: fuseKernels super_kernel_template subFunc %p, device func " @@ -98,7 +100,9 @@ Status SuperKernelFactory::FuseKernels(const std::vector &stub_func_list for (unsigned i = 0; i < stub_func_list.size(); i++) { void *sub_device_func = nullptr; rt_ret = rtGetAddrByFun(stub_func_list[i], &sub_device_func); - GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, GELOGE(RT_FAILED, "rtGetAddrByFun failed. error: 0x%X", rt_ret); + GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, + REPORT_CALL_ERROR("E19999", "Call rtGetAddrByFun fail, ret:0x%X, when %s", rt_ret, __FUNCTION__); + GELOGE(RT_FAILED, "rtGetAddrByFun failed. error: 0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret);) GELOGD("SKT: fuseKernels subFunc %p, device func address %p", stub_func_list[i], sub_device_func); // store two uint64_t address @@ -109,11 +113,17 @@ Status SuperKernelFactory::FuseKernels(const std::vector &stub_func_list GELOGD("SKT: fuseKernels args base address %lu", nav_table[i * kFusedKernelSizeUnit + 1]); } rt_ret = rtMalloc(reinterpret_cast(&hbm_nav_table_addr), nav_table_size, RT_MEMORY_HBM); - GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, GELOGE(RT_FAILED, "rtMalloc failed. error: 0x%X", rt_ret); + GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, + REPORT_CALL_ERROR("E19999", "Call rtMalloc fail, size:%lu, ret:0x%X, when %s", + nav_table_size, rt_ret, __FUNCTION__); + GELOGE(RT_FAILED, "rtMalloc failed. error: 0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret);) rt_ret = rtMemcpy(reinterpret_cast(hbm_nav_table_addr), nav_table_size, reinterpret_cast(nav_table.get()), nav_table_size, RT_MEMCPY_HOST_TO_DEVICE); - GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, GELOGE(RT_FAILED, "rtMemcpy failed. error: 0x%X", rt_ret); + GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, + REPORT_CALL_ERROR("E19999", "Call rtMemcpy fail, size:%lu, ret:0x%X when %s", + nav_table_size, rt_ret, __FUNCTION__); + GELOGE(RT_FAILED, "rtMemcpy failed. error: 0x%X", rt_ret); GE_CHK_RT(rtFree(hbm_nav_table_addr)); return RT_ERROR_TO_GE_STATUS(rt_ret);) // Create the necessary metadata for the super kernel h = diff --git a/inc/framework/common/debug/log.h b/inc/framework/common/debug/log.h index 43fb3224..a60edce2 100644 --- a/inc/framework/common/debug/log.h +++ b/inc/framework/common/debug/log.h @@ -19,6 +19,7 @@ #include #include +#include #include "runtime/rt.h" #include "common/string_util.h" @@ -105,6 +106,7 @@ do { \ bool b = (expr); \ if (!b) { \ + REPORT_INNER_ERROR("E19999", __VA_ARGS__); \ GELOGE(_status, __VA_ARGS__); \ return _status; \ } \ @@ -193,6 +195,7 @@ { \ bool b = (expr); \ if (b) { \ + REPORT_INNER_ERROR("E19999", __VA_ARGS__); \ DOMI_LOGE(__VA_ARGS__); \ exec_expr; \ return _status; \ diff --git a/metadef b/metadef index ac0de021..99934058 160000 --- a/metadef +++ b/metadef @@ -1 +1 @@ -Subproject commit ac0de0213755e49360a9467eb5b13e13a752a35b +Subproject commit 99934058bfdceaae740acf2c8cadd316b51c00c7 diff --git a/parser b/parser index eff7e2ec..0d4703aa 160000 --- a/parser +++ b/parser @@ -1 +1 @@ -Subproject commit eff7e2ecc54ef7887581acd6ea66356de6872e3a +Subproject commit 0d4703aa893e90f23ba8a2dbd8903e028680213f