From 12b0168265a07a13831e3b2a2d952554bbb56654 Mon Sep 17 00:00:00 2001 From: wangxiaotian22 Date: Thu, 19 Nov 2020 10:32:35 +0800 Subject: [PATCH 01/18] fix atomic process bug in loop condition. atomic out node link to atomic input node(hccl), will build error --- ge/graph/passes/atomic_addr_clean_pass.cc | 71 +++++++++++++++++++++++++------ ge/graph/passes/atomic_addr_clean_pass.h | 3 ++ 2 files changed, 60 insertions(+), 14 deletions(-) diff --git a/ge/graph/passes/atomic_addr_clean_pass.cc b/ge/graph/passes/atomic_addr_clean_pass.cc index 690dee27..e1b0e4b5 100755 --- a/ge/graph/passes/atomic_addr_clean_pass.cc +++ b/ge/graph/passes/atomic_addr_clean_pass.cc @@ -248,10 +248,50 @@ bool AtomicAddrCleanPass::IsAtomicOp(const NodePtr &node) { if (op_desc == nullptr) { return false; } + + if (CheckAtomicFromOpsKernel(node)) { + return true; + } + + // 2.Check atomic attr in node + std::map> node_workspace_offset; + bool has_atomic_input = op_desc->HasAttr(ATOMIC_ATTR_INPUT_INDEX); + bool has_atomic_output = op_desc->HasAttr(ATOMIC_ATTR_OUTPUT_INDEX); + node_workspace_offset = op_desc->TryGetExtAttr(EXT_ATTR_ATOMIC_WORKSPACE_OFFSET, node_workspace_offset); + if (!has_atomic_input && !has_atomic_output && node_workspace_offset.empty()) { + return false; + } + + if (!has_atomic_input && has_atomic_output && node_workspace_offset.empty()) { + std::vector atomic_output_index; + (void) ge::AttrUtils::GetListInt(op_desc, ATOMIC_ATTR_OUTPUT_INDEX, atomic_output_index); + bool is_all_output_peer_also_atomic = true; + for (auto &output_index : atomic_output_index) { + if (!IsOutputIndexPeerInputAtomic(node, output_index)) { + is_all_output_peer_also_atomic = false; + break; + } + } + if (is_all_output_peer_also_atomic) { + GELOGI("all out peer node input atomic, skip this out atomic process, node name: %s", node->GetName().c_str()); + return false; + } + } + + graphStatus ret = op_desc->SetAttr(ATOMIC_ATTR_IS_ATOMIC_NODE, GeAttrValue::CreateFrom(true)); + if (ret != GRAPH_SUCCESS) { + GELOGW("set attr ATOMIC_ATTR_IS_ATOMIC_NODE fail."); + } + GELOGD("Recognized atomic op %s from attr.", op_desc->GetName().c_str()); + return true; +} + +// just hccl may mark atomic from ops kernel now, and hccl's atomic if for all input +bool AtomicAddrCleanPass::CheckAtomicFromOpsKernel(const NodePtr &node) { // 1.Check if isAtomic attrs exist for HCOM std::shared_ptr instance_ptr = GELib::GetInstance(); if ((instance_ptr == nullptr) || (!instance_ptr->InitFlag())) { - GELOGW("GELib not initialized"); + GELOGW("GELib not initialized, atomic from ops kernel judge false, node_name: %s", node->GetName().c_str()); return false; } @@ -259,38 +299,41 @@ bool AtomicAddrCleanPass::IsAtomicOp(const NodePtr &node) { vector op_info_vec = ops_kernel_manager.GetOpsKernelInfo(op_desc->GetType()); for (const auto &op_info : op_info_vec) { if (op_info.isAtomic) { - GELOGI("Recognized atomic op %s from DNN_HCCL engine.", op_desc->GetName().c_str()); // check peer input is DATA for (auto &in_data_anchor : node->GetAllInDataAnchors()) { if (in_data_anchor->GetPeerOutAnchor() != nullptr && in_data_anchor->GetPeerOutAnchor()->GetOwnerNode() != nullptr) { auto peer_in_node = in_data_anchor->GetPeerOutAnchor()->GetOwnerNode(); if (peer_in_node->GetType() == DATA) { - GELOGI("Recognized atomic op %s from DNN_HCCL engine and input is DATA.", op_desc->GetName().c_str()); + GELOGI("Recognized atomic op %s from %s engine and input is DATA.", op_desc->GetName().c_str(), op_info.engine.c_str()); return false; } } } + GELOGI("Recognized atomic op %s from %s engine.", op_desc->GetName().c_str(), op_info.engine.c_str()); hcom_node_vec_.push_back(node); return true; } } - // 2.Check atomic attr in node - std::map> node_workspace_offset; - bool has_atomic_input = op_desc->HasAttr(ATOMIC_ATTR_INPUT_INDEX); - bool has_atomic_output = op_desc->HasAttr(ATOMIC_ATTR_OUTPUT_INDEX); - node_workspace_offset = op_desc->TryGetExtAttr(EXT_ATTR_ATOMIC_WORKSPACE_OFFSET, node_workspace_offset); - if (!has_atomic_input && !has_atomic_output && node_workspace_offset.empty()) { +} + +bool AtomicAddrCleanPass::IsOutputIndexPeerInputAtomic(const ge::NodePtr &node, int64_t output_index) { + auto out_data_anchor = node->GetAllInDataAnchors().at(output_index); + if (out_data_anchor == nullptr) { return false; } - graphStatus ret = op_desc->SetAttr(ATOMIC_ATTR_IS_ATOMIC_NODE, GeAttrValue::CreateFrom(true)); - if (ret != GRAPH_SUCCESS) { - GELOGW("set attr ATOMIC_ATTR_IS_ATOMIC_NODE fail."); + for (auto input_anchor : out_data_anchor->GetPeerInDataAnchors()) { + auto output_node = input_anchor->GetOwnerNode(); + // just hccl may mark atomic from ops kernel now, and hccl's atomic if for all input + // hccl's attr ATOMIC_ATTR_INPUT_INDEX mark on CalcOpRunningParam, can't be get here + if (CheckAtomicFromOpsKernel(output_node)) { + return true; + } } - GELOGD("Recognized atomic op %s from FE engine.", op_desc->GetName().c_str()); - return true; + return false; } + /// /// @brief Clear Status, used for subgraph pass /// @return SUCCESS diff --git a/ge/graph/passes/atomic_addr_clean_pass.h b/ge/graph/passes/atomic_addr_clean_pass.h index ad60b7b5..9adeb611 100755 --- a/ge/graph/passes/atomic_addr_clean_pass.h +++ b/ge/graph/passes/atomic_addr_clean_pass.h @@ -84,6 +84,9 @@ class AtomicAddrCleanPass : public GraphPass { Status HandleDispersedAtomicNodes(ComputeGraphPtr &graph, const std::vector &atomic_node_vec, std::vector &common_atomic_nodes); + bool CheckAtomicFromOpsKernel(const NodePtr &node); + + bool IsOutputIndexPeerInputAtomic(const ge::NodePtr &node, int64_t output_index); vector hcom_node_vec_; bool is_loop_graph_ = false; From 079abb175b200f3bd8e9ffca42def6463061a6c3 Mon Sep 17 00:00:00 2001 From: wangxiaotian22 Date: Thu, 19 Nov 2020 10:54:19 +0800 Subject: [PATCH 02/18] fix --- ge/graph/passes/atomic_addr_clean_pass.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ge/graph/passes/atomic_addr_clean_pass.cc b/ge/graph/passes/atomic_addr_clean_pass.cc index e1b0e4b5..88d6b3fe 100755 --- a/ge/graph/passes/atomic_addr_clean_pass.cc +++ b/ge/graph/passes/atomic_addr_clean_pass.cc @@ -296,7 +296,7 @@ bool AtomicAddrCleanPass::CheckAtomicFromOpsKernel(const NodePtr &node) { } OpsKernelManager &ops_kernel_manager = instance_ptr->OpsKernelManagerObj(); - vector op_info_vec = ops_kernel_manager.GetOpsKernelInfo(op_desc->GetType()); + vector op_info_vec = ops_kernel_manager.GetOpsKernelInfo(node->GetType()); for (const auto &op_info : op_info_vec) { if (op_info.isAtomic) { // check peer input is DATA @@ -305,12 +305,12 @@ bool AtomicAddrCleanPass::CheckAtomicFromOpsKernel(const NodePtr &node) { in_data_anchor->GetPeerOutAnchor()->GetOwnerNode() != nullptr) { auto peer_in_node = in_data_anchor->GetPeerOutAnchor()->GetOwnerNode(); if (peer_in_node->GetType() == DATA) { - GELOGI("Recognized atomic op %s from %s engine and input is DATA.", op_desc->GetName().c_str(), op_info.engine.c_str()); + GELOGI("Recognized atomic op %s from %s engine and input is DATA.", node->GetName().c_str(), op_info.engine.c_str()); return false; } } } - GELOGI("Recognized atomic op %s from %s engine.", op_desc->GetName().c_str(), op_info.engine.c_str()); + GELOGI("Recognized atomic op %s from %s engine.", node->GetName().c_str(), op_info.engine.c_str()); hcom_node_vec_.push_back(node); return true; } @@ -318,7 +318,7 @@ bool AtomicAddrCleanPass::CheckAtomicFromOpsKernel(const NodePtr &node) { } bool AtomicAddrCleanPass::IsOutputIndexPeerInputAtomic(const ge::NodePtr &node, int64_t output_index) { - auto out_data_anchor = node->GetAllInDataAnchors().at(output_index); + auto out_data_anchor = node->GetAllOutDataAnchors().at(output_index); if (out_data_anchor == nullptr) { return false; } From c5abd20f91e636ee9ecc64271e82fda6d32a74d1 Mon Sep 17 00:00:00 2001 From: wangxiaotian22 Date: Thu, 19 Nov 2020 14:22:00 +0800 Subject: [PATCH 03/18] fix --- ge/graph/passes/atomic_addr_clean_pass.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/ge/graph/passes/atomic_addr_clean_pass.cc b/ge/graph/passes/atomic_addr_clean_pass.cc index 88d6b3fe..8932bef3 100755 --- a/ge/graph/passes/atomic_addr_clean_pass.cc +++ b/ge/graph/passes/atomic_addr_clean_pass.cc @@ -315,6 +315,7 @@ bool AtomicAddrCleanPass::CheckAtomicFromOpsKernel(const NodePtr &node) { return true; } } + return false; } bool AtomicAddrCleanPass::IsOutputIndexPeerInputAtomic(const ge::NodePtr &node, int64_t output_index) { From c0dc27f58073e35c5349e05cadcb2a65ad2db971 Mon Sep 17 00:00:00 2001 From: wangxiaotian22 Date: Thu, 19 Nov 2020 21:11:20 +0800 Subject: [PATCH 04/18] fix comment --- ge/graph/passes/atomic_addr_clean_pass.cc | 6 +++--- ge/graph/passes/atomic_addr_clean_pass.h | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ge/graph/passes/atomic_addr_clean_pass.cc b/ge/graph/passes/atomic_addr_clean_pass.cc index 8932bef3..18cac856 100755 --- a/ge/graph/passes/atomic_addr_clean_pass.cc +++ b/ge/graph/passes/atomic_addr_clean_pass.cc @@ -266,7 +266,7 @@ bool AtomicAddrCleanPass::IsAtomicOp(const NodePtr &node) { std::vector atomic_output_index; (void) ge::AttrUtils::GetListInt(op_desc, ATOMIC_ATTR_OUTPUT_INDEX, atomic_output_index); bool is_all_output_peer_also_atomic = true; - for (auto &output_index : atomic_output_index) { + for (const auto &output_index : atomic_output_index) { if (!IsOutputIndexPeerInputAtomic(node, output_index)) { is_all_output_peer_also_atomic = false; break; @@ -318,13 +318,13 @@ bool AtomicAddrCleanPass::CheckAtomicFromOpsKernel(const NodePtr &node) { return false; } -bool AtomicAddrCleanPass::IsOutputIndexPeerInputAtomic(const ge::NodePtr &node, int64_t output_index) { +bool AtomicAddrCleanPass::IsOutputIndexPeerInputAtomic(const NodePtr &node, int64_t output_index) { auto out_data_anchor = node->GetAllOutDataAnchors().at(output_index); if (out_data_anchor == nullptr) { return false; } - for (auto input_anchor : out_data_anchor->GetPeerInDataAnchors()) { + for (const auto input_anchor : out_data_anchor->GetPeerInDataAnchors()) { auto output_node = input_anchor->GetOwnerNode(); // just hccl may mark atomic from ops kernel now, and hccl's atomic if for all input // hccl's attr ATOMIC_ATTR_INPUT_INDEX mark on CalcOpRunningParam, can't be get here diff --git a/ge/graph/passes/atomic_addr_clean_pass.h b/ge/graph/passes/atomic_addr_clean_pass.h index 9adeb611..420ddd01 100755 --- a/ge/graph/passes/atomic_addr_clean_pass.h +++ b/ge/graph/passes/atomic_addr_clean_pass.h @@ -86,7 +86,7 @@ class AtomicAddrCleanPass : public GraphPass { bool CheckAtomicFromOpsKernel(const NodePtr &node); - bool IsOutputIndexPeerInputAtomic(const ge::NodePtr &node, int64_t output_index); + bool IsOutputIndexPeerInputAtomic(const NodePtr &node, int64_t output_index); vector hcom_node_vec_; bool is_loop_graph_ = false; From 62591e6cf91b056df020219cf24376814fe47901 Mon Sep 17 00:00:00 2001 From: wangxiaotian22 Date: Mon, 23 Nov 2020 14:55:03 +0800 Subject: [PATCH 05/18] atomic fix for unknownshape graph --- ge/graph/passes/atomic_addr_clean_pass.cc | 34 +++++++++++++++++-------------- ge/graph/passes/atomic_addr_clean_pass.h | 2 +- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/ge/graph/passes/atomic_addr_clean_pass.cc b/ge/graph/passes/atomic_addr_clean_pass.cc index 18cac856..b62f86c7 100755 --- a/ge/graph/passes/atomic_addr_clean_pass.cc +++ b/ge/graph/passes/atomic_addr_clean_pass.cc @@ -33,10 +33,12 @@ namespace ge { Status AtomicAddrCleanPass::Run(ComputeGraphPtr graph) { GE_CHECK_NOTNULL(graph); GELOGD("AtomicAddrCleanPass begin."); + bool is_unknown_graph = graph->GetGraphUnknownFlag(); + // 1.Recoginze atomic and loop mark vector atomic_node_vec; for (NodePtr &node : graph->GetDirectNode()) { - if (IsAtomicOp(node)) { + if (IsAtomicOp(node, is_unknown_graph)) { atomic_node_vec.push_back(node); } if (!is_loop_graph_ && node->GetType() == LOOPCOND) { @@ -50,7 +52,6 @@ Status AtomicAddrCleanPass::Run(ComputeGraphPtr graph) { return SUCCESS; } - bool is_unknown_graph = graph->GetGraphUnknownFlag(); if (is_unknown_graph) { GELOGD("Graph[%s] is unknown graph. It will call fe interface to compile op.", graph->GetName().c_str()); GE_CHK_STATUS_RET(CompileUnknownGraphOp(atomic_node_vec)); @@ -242,7 +243,7 @@ Status AtomicAddrCleanPass::LinkToAtomicNode(const NodePtr &atomic_node, NodePtr return SUCCESS; } -bool AtomicAddrCleanPass::IsAtomicOp(const NodePtr &node) { +bool AtomicAddrCleanPass::IsAtomicOp(const NodePtr &node, bool is_unknown_graph) { GE_IF_BOOL_EXEC(node == nullptr, GELOGE(FAILED, "node is null."); return false); OpDescPtr op_desc = node->GetOpDesc(); if (op_desc == nullptr) { @@ -262,19 +263,21 @@ bool AtomicAddrCleanPass::IsAtomicOp(const NodePtr &node) { return false; } - if (!has_atomic_input && has_atomic_output && node_workspace_offset.empty()) { - std::vector atomic_output_index; - (void) ge::AttrUtils::GetListInt(op_desc, ATOMIC_ATTR_OUTPUT_INDEX, atomic_output_index); - bool is_all_output_peer_also_atomic = true; - for (const auto &output_index : atomic_output_index) { - if (!IsOutputIndexPeerInputAtomic(node, output_index)) { - is_all_output_peer_also_atomic = false; - break; + if (!is_unknown_graph) { + if (!has_atomic_input && has_atomic_output && node_workspace_offset.empty()) { + std::vector atomic_output_index; + (void) ge::AttrUtils::GetListInt(op_desc, ATOMIC_ATTR_OUTPUT_INDEX, atomic_output_index); + bool is_all_output_peer_also_atomic = true; + for (const auto &output_index : atomic_output_index) { + if (!IsOutputIndexPeerInputAtomic(node, output_index)) { + is_all_output_peer_also_atomic = false; + break; + } + } + if (is_all_output_peer_also_atomic) { + GELOGI("all out peer node input atomic, skip this out atomic process, node name: %s", node->GetName().c_str()); + return false; } - } - if (is_all_output_peer_also_atomic) { - GELOGI("all out peer node input atomic, skip this out atomic process, node name: %s", node->GetName().c_str()); - return false; } } @@ -342,6 +345,7 @@ bool AtomicAddrCleanPass::IsOutputIndexPeerInputAtomic(const NodePtr &node, int6 Status AtomicAddrCleanPass::ClearStatus() { hcom_node_vec_.clear(); return SUCCESS; + } Status AtomicAddrCleanPass::CompileUnknownGraphOp(const vector &atomic_node_vec) { diff --git a/ge/graph/passes/atomic_addr_clean_pass.h b/ge/graph/passes/atomic_addr_clean_pass.h index 420ddd01..64bc604b 100755 --- a/ge/graph/passes/atomic_addr_clean_pass.h +++ b/ge/graph/passes/atomic_addr_clean_pass.h @@ -72,7 +72,7 @@ class AtomicAddrCleanPass : public GraphPass { * @param node * @return */ - bool IsAtomicOp(const NodePtr &node); + bool IsAtomicOp(const NodePtr &node, bool is_unknown_graph); /** * Handle atomic node in unknown graph From c84c73ada5f84c3513fa436025163117487e7440 Mon Sep 17 00:00:00 2001 From: baker Date: Wed, 25 Nov 2020 10:16:41 +0800 Subject: [PATCH 06/18] save and get compile failed msg --- ge/graph/manager/graph_manager.cc | 8 +++++--- ge/graph/manager/graph_manager.h | 3 ++- ge/graph/optimize/graph_optimize.cc | 7 ++++++- ge/graph/optimize/graph_optimize.h | 3 ++- ge/ir_build/ge_ir_build.cc | 7 +++++++ 5 files changed, 22 insertions(+), 6 deletions(-) diff --git a/ge/graph/manager/graph_manager.cc b/ge/graph/manager/graph_manager.cc index 654bea16..86df5d5b 100755 --- a/ge/graph/manager/graph_manager.cc +++ b/ge/graph/manager/graph_manager.cc @@ -552,7 +552,7 @@ Status GraphManager::OptimizeSubGraphWithMultiThreads(ComputeGraphPtr compute_gr (void) AttrUtils::SetStr(subgraph->GetSubGraph(), ATTR_NAME_OP_COMPILE_STRATEGY, op_compile_strategy); } std::future f = executor.commit(GraphManager::ProcessSubGraphWithMultiThreads, this, - compute_graph->GetGraphID(), subgraph, session_id, GetThreadLocalContext()); + compute_graph->GetGraphID(), subgraph, compute_graph, session_id, GetThreadLocalContext()); if (!f.valid()) { GELOGE(FAILED, "Future is invalid"); return FAILED; @@ -567,7 +567,7 @@ Status GraphManager::OptimizeSubGraphWithMultiThreads(ComputeGraphPtr compute_gr (void) AttrUtils::SetStr(subgraph->GetSubGraph(), ATTR_NAME_OP_COMPILE_STRATEGY, op_compile_strategy); } std::future f = executor.commit(GraphManager::ProcessSubGraphWithMultiThreads, this, - compute_graph->GetGraphID(), subgraph, session_id, + compute_graph->GetGraphID(), subgraph, compute_graph, session_id, GetThreadLocalContext()); if (!f.valid()) { GELOGE(FAILED, "Future is invalid"); @@ -2510,7 +2510,8 @@ Status GraphManager::CheckAndReleaseMemory(const GeModelPtr &ge_model, const Gra } Status GraphManager::ProcessSubGraphWithMultiThreads(GraphManager *graph_manager, GraphId root_graph_id, - const SubGraphInfoPtr &sub_graph_info_ptr, uint64_t session_id, + const SubGraphInfoPtr &sub_graph_info_ptr, + const ComputeGraphPtr &compute_graph, uint64_t session_id, const GEThreadLocalContext &ge_context) { if (sub_graph_info_ptr != nullptr && graph_manager != nullptr) { GetContext().SetSessionId(session_id); @@ -2526,6 +2527,7 @@ Status GraphManager::ProcessSubGraphWithMultiThreads(GraphManager *graph_manager GE_CHECK_NOTNULL(compute_graph_tmp); compute_graph_tmp->SetSessionID(session_id); Status ret = graph_manager->GetCompilerStages(root_graph_id).optimizer.OptimizeSubGraph(compute_graph_tmp, + compute_graph, engine_name); if (ret != SUCCESS) { GELOGE(ret, "SubGraph optimize Failed %s", engine_name.c_str()); diff --git a/ge/graph/manager/graph_manager.h b/ge/graph/manager/graph_manager.h index b0dde0c0..83aebeb6 100644 --- a/ge/graph/manager/graph_manager.h +++ b/ge/graph/manager/graph_manager.h @@ -218,7 +218,8 @@ class GraphManager { std::shared_ptr GetModelListener() const { return graph_run_listener_; } static Status ProcessSubGraphWithMultiThreads(GraphManager *graph_manager, GraphId root_graph_id, - const SubGraphInfoPtr &sub_graph_info_ptr, uint64_t session_id, + const SubGraphInfoPtr &sub_graph_info_ptr, + const ComputeGraphPtr &compute_graph, uint64_t session_id, const GEThreadLocalContext &ge_context); Status ParseInputsDims(const std::vector &input_tensor); Status DistinguishGetNextAndData(ComputeGraphPtr &graph, vector &data_nodes, diff --git a/ge/graph/optimize/graph_optimize.cc b/ge/graph/optimize/graph_optimize.cc index 931d529b..c6ba0b06 100644 --- a/ge/graph/optimize/graph_optimize.cc +++ b/ge/graph/optimize/graph_optimize.cc @@ -77,7 +77,8 @@ void AddNodeInputProperty(ComputeGraphPtr &compute_graph) { } } -Status GraphOptimize::OptimizeSubGraph(ComputeGraphPtr &compute_graph, const std::string &engine_name) { +Status GraphOptimize::OptimizeSubGraph(ComputeGraphPtr &compute_graph, const ComputeGraphPtr &parent_graph, + const std::string &engine_name) { if (compute_graph == nullptr) { GELOGE(GE_GRAPH_OPTIMIZE_COMPUTE_GRAPH_NULL, "[OptimizeSubGraph]: compute_graph is nullptr."); return GE_GRAPH_OPTIMIZE_COMPUTE_GRAPH_NULL; @@ -106,6 +107,10 @@ Status GraphOptimize::OptimizeSubGraph(ComputeGraphPtr &compute_graph, const std for (auto iter = graph_optimizer.begin(); iter != graph_optimizer.end(); ++iter) { Status ret = (*iter)->OptimizeFusedGraphAfterGraphSlice(*(compute_graph)); if (ret != SUCCESS) { + auto root_graph = ge::GraphUtils::FindRootGraph(parent_graph); + if (root_graph != nullptr) { + ErrorManager.GetInstance().SaveMstuneCompileFailedMsg(root_graph->GetName()); + } GELOGE(ret, "[OptimizeSubGraph][OptimizeFusedGraphAfterGraphSlice]: graph optimize failed, ret:%d", ret); return ret; } diff --git a/ge/graph/optimize/graph_optimize.h b/ge/graph/optimize/graph_optimize.h index 78d580b7..969b4720 100755 --- a/ge/graph/optimize/graph_optimize.h +++ b/ge/graph/optimize/graph_optimize.h @@ -42,7 +42,8 @@ class GraphOptimize { ~GraphOptimize() = default; // subgraph optimize - Status OptimizeSubGraph(ComputeGraphPtr &compute_graph, const std::string &engine_name); + Status OptimizeSubGraph(ComputeGraphPtr &compute_graph, const ComputeGraphPtr &parent_graph, + const std::string &engine_name); // original graph optimize Status OptimizeOriginalGraph(ComputeGraphPtr &compute_graph); diff --git a/ge/ir_build/ge_ir_build.cc b/ge/ir_build/ge_ir_build.cc index 242b38a0..6fc3b7b7 100644 --- a/ge/ir_build/ge_ir_build.cc +++ b/ge/ir_build/ge_ir_build.cc @@ -164,6 +164,13 @@ graphStatus aclgrphBuildInitializeImpl(std::map &globa } } GELOGW("gelib has been initialized!"); + + std::string path_base = ge::GELib::GetPath(); + int ret = ErrorManager::GetInstance().Init(path_base); + if (ret != 0) { + DOMI_LOGE("ErrorManager init fail !"); + return GRAPH_FAILED; + } return GRAPH_SUCCESS; } From 0f6a645f5e06429cb728567f81df967d624e11d7 Mon Sep 17 00:00:00 2001 From: baker Date: Wed, 25 Nov 2020 11:14:36 +0800 Subject: [PATCH 07/18] fix not flag --- inc/external/ge/ge_api_types.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/inc/external/ge/ge_api_types.h b/inc/external/ge/ge_api_types.h index c439a8bf..cb2fbdef 100644 --- a/inc/external/ge/ge_api_types.h +++ b/inc/external/ge/ge_api_types.h @@ -364,8 +364,8 @@ static const char *const OPTYPELIST_FOR_IMPLMODE = ge::OPTYPELIST_FOR_IMPLMODE.c static const char *const DEBUG_DIR = ge::DEBUG_DIR; static const char *const OP_COMPILER_CACHE_DIR = ge::OP_COMPILER_CACHE_DIR; static const char *const OP_COMPILER_CACHE_MODE = ge::OP_COMPILER_CACHE_MODE; -static const char *const MDL_BANK_PATH_FLAG = ge::MDL_BANK_PATH_FLAG.c_str(); -static const char *const OP_BANK_PATH_FLAG = ge::OP_BANK_PATH_FLAG.c_str(); +static const char *const MDL_BANK_PATH = ge::MDL_BANK_PATH_FLAG.c_str(); +static const char *const OP_BANK_PATH = ge::OP_BANK_PATH_FLAG.c_str(); static const char *const OP_DEBUG_LEVEL = ge::OP_DEBUG_LEVEL.c_str(); // for interface: aclgrphBuildModel @@ -387,8 +387,8 @@ const std::set ir_builder_suppported_options = {INPUT_FORMAT, DEBUG_DIR, OP_COMPILER_CACHE_DIR, OP_COMPILER_CACHE_MODE, - MDL_BANK_PATH_FLAG, - OP_BANK_PATH_FLAG}; + MDL_BANK_PATH, + OP_BANK_PATH}; // for interface: aclgrphParse const std::set ir_parser_suppported_options = {INPUT_FORMAT, From eb058ac96931b14b63a7c27babddf58cb00df560 Mon Sep 17 00:00:00 2001 From: chenyemeng Date: Thu, 26 Nov 2020 10:24:16 +0800 Subject: [PATCH 08/18] rm empty_tensor inputs for merge --- ge/graph/passes/merge_pass.cc | 27 +++++++++++++++++++++++++++ ge/graph/passes/merge_pass.h | 1 + 2 files changed, 28 insertions(+) diff --git a/ge/graph/passes/merge_pass.cc b/ge/graph/passes/merge_pass.cc index d2340037..c6bae2a2 100644 --- a/ge/graph/passes/merge_pass.cc +++ b/ge/graph/passes/merge_pass.cc @@ -34,6 +34,11 @@ using domi::SUCCESS; namespace ge { const int kValueIndexOutputIndex = 1; +bool IsEmptyTensor(const GeShape &shpae) { + const auto &dims = shape.GetDims(); + return std::any_of(dims.begin(), dims.end(), [](int64_t dim) { return dim == 0; }); +} + Status MergePass::Run(NodePtr &node) { GELOGD("MergePass running"); if (node == nullptr) { @@ -53,6 +58,11 @@ Status MergePass::Run(NodePtr &node) { return PARAM_INVALID; } + if (OptimizeEmptyTensorInput(node) != SUCCESS) { + GELOGE(FAILED, "[%s] remove empty_tensor inputs failed.", node->GetName().c_str()); + return FAILED; + } + auto in_data_nodes = node->GetInDataNodes(); switch (in_data_nodes.size()) { case 0: { @@ -202,4 +212,21 @@ bool MergePass::IsMergeInputNeedOptimized(NodePtr &node) const { } return true; } + +Status MergePass::OptimizeEmptyTensorInput(const NodePtr &node) const { + for (const auto &in_data_anchor : node->GetAllInDataAnchors()) { + const auto &peer_data_anchor = in_data_anchor->GetPeerOutAnchor(); + if (peer_data_anchor == nullptr) { + continue; + } + const auto &op_desc = peer_data_anchor->GetOwnerNode()->GetOpDesc(); + if (op_desc == nullptr) { + continue; + } + if (IsEmptyTensor(op_desc->GetOutputDesc(peer_data_anchor->GetIdx()).GetShape())) { + return GraphUtils::RemoveEdge(peer_data_anchor, in_data_anchor) == GRAPH_SUCCESS ? SUCCESS : FAILED; + } + } + return SUCCESS; +} } // namespace ge diff --git a/ge/graph/passes/merge_pass.h b/ge/graph/passes/merge_pass.h index 2cdb5022..c297a86e 100755 --- a/ge/graph/passes/merge_pass.h +++ b/ge/graph/passes/merge_pass.h @@ -29,6 +29,7 @@ class MergePass : public BaseNodePass { Status ChangeIndexToConstant(NodePtr &node, int &value_index); Status CreateConstByValue(NodePtr &node, int value_index, OpDescPtr &op_desc); bool IsMergeInputNeedOptimized(NodePtr &node) const; + static Status OptimizeEmptyTensorInput(const NodePtr &node) const; }; } // namespace ge #endif // GE_GRAPH_PASSES_MERGE_PASS_H_ From 17d1e9dac5b581bbcc9a0648a27297a20808231e Mon Sep 17 00:00:00 2001 From: chenyemeng Date: Thu, 26 Nov 2020 10:46:48 +0800 Subject: [PATCH 09/18] rm empty_tensor inputs for merge --- ge/graph/passes/merge_pass.cc | 2 +- ge/graph/passes/merge_pass.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ge/graph/passes/merge_pass.cc b/ge/graph/passes/merge_pass.cc index c6bae2a2..a65c32fd 100644 --- a/ge/graph/passes/merge_pass.cc +++ b/ge/graph/passes/merge_pass.cc @@ -213,7 +213,7 @@ bool MergePass::IsMergeInputNeedOptimized(NodePtr &node) const { return true; } -Status MergePass::OptimizeEmptyTensorInput(const NodePtr &node) const { +Status MergePass::OptimizeEmptyTensorInput(const NodePtr &node) { for (const auto &in_data_anchor : node->GetAllInDataAnchors()) { const auto &peer_data_anchor = in_data_anchor->GetPeerOutAnchor(); if (peer_data_anchor == nullptr) { diff --git a/ge/graph/passes/merge_pass.h b/ge/graph/passes/merge_pass.h index c297a86e..464f2172 100755 --- a/ge/graph/passes/merge_pass.h +++ b/ge/graph/passes/merge_pass.h @@ -29,7 +29,7 @@ class MergePass : public BaseNodePass { Status ChangeIndexToConstant(NodePtr &node, int &value_index); Status CreateConstByValue(NodePtr &node, int value_index, OpDescPtr &op_desc); bool IsMergeInputNeedOptimized(NodePtr &node) const; - static Status OptimizeEmptyTensorInput(const NodePtr &node) const; + static Status OptimizeEmptyTensorInput(const NodePtr &node); }; } // namespace ge #endif // GE_GRAPH_PASSES_MERGE_PASS_H_ From a910c80fbf6e3efd98ddf96fd7d231fef4678492 Mon Sep 17 00:00:00 2001 From: chenyemeng Date: Thu, 26 Nov 2020 11:00:18 +0800 Subject: [PATCH 10/18] rm empty_tensor inputs for merge --- ge/graph/passes/merge_pass.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ge/graph/passes/merge_pass.cc b/ge/graph/passes/merge_pass.cc index a65c32fd..fb485e2b 100644 --- a/ge/graph/passes/merge_pass.cc +++ b/ge/graph/passes/merge_pass.cc @@ -34,7 +34,7 @@ using domi::SUCCESS; namespace ge { const int kValueIndexOutputIndex = 1; -bool IsEmptyTensor(const GeShape &shpae) { +bool IsEmptyTensor(const GeShape &shape) { const auto &dims = shape.GetDims(); return std::any_of(dims.begin(), dims.end(), [](int64_t dim) { return dim == 0; }); } From ec4ada340ac5f271c12ab5f52b889ebd40ea938d Mon Sep 17 00:00:00 2001 From: chenyemeng Date: Thu, 26 Nov 2020 15:22:53 +0800 Subject: [PATCH 11/18] rm empty_tensor inputs for merge --- ge/graph/passes/merge_pass.cc | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/ge/graph/passes/merge_pass.cc b/ge/graph/passes/merge_pass.cc index fb485e2b..a50893a9 100644 --- a/ge/graph/passes/merge_pass.cc +++ b/ge/graph/passes/merge_pass.cc @@ -219,12 +219,18 @@ Status MergePass::OptimizeEmptyTensorInput(const NodePtr &node) { if (peer_data_anchor == nullptr) { continue; } - const auto &op_desc = peer_data_anchor->GetOwnerNode()->GetOpDesc(); - if (op_desc == nullptr) { + if ((peer_data_anchor->GetOwnerNode() == nullptr) || + (peer_data_anchor->GetOwnerNode()->GetOpDesc() == nullptr)) { continue; } + const auto &op_desc = peer_data_anchor->GetOwnerNode()->GetOpDesc(); if (IsEmptyTensor(op_desc->GetOutputDesc(peer_data_anchor->GetIdx()).GetShape())) { - return GraphUtils::RemoveEdge(peer_data_anchor, in_data_anchor) == GRAPH_SUCCESS ? SUCCESS : FAILED; + if (GraphUtils::RemoveEdge(peer_data_anchor, in_data_anchor) != GRAPH_SUCCESS) { + GELOGE(FAILED, "Remove data edge %s:%d->%s:%d failed.", + op_desc->GetName().c_str(), peer_data_anchor->GetIdx(), + node->GetName().c_str(), in_data_anchor->GetIdx()); + return FAILED; + } } } return SUCCESS; From 680e63f88c1d1bdcd689079c0ae4e931483deb6d Mon Sep 17 00:00:00 2001 From: chenyemeng Date: Thu, 26 Nov 2020 16:05:13 +0800 Subject: [PATCH 12/18] rm empty_tensor inputs for merge --- ge/graph/passes/merge_pass.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ge/graph/passes/merge_pass.cc b/ge/graph/passes/merge_pass.cc index a50893a9..0b367614 100644 --- a/ge/graph/passes/merge_pass.cc +++ b/ge/graph/passes/merge_pass.cc @@ -231,6 +231,9 @@ Status MergePass::OptimizeEmptyTensorInput(const NodePtr &node) { node->GetName().c_str(), in_data_anchor->GetIdx()); return FAILED; } + GELOGD("Remove data edge %s:%d->%s:%d", + op_desc->GetName().c_str(), peer_data_anchor->GetIdx(), + node->GetName().c_str(), in_data_anchor->GetIdx()); } } return SUCCESS; From c81248f5b59befc470110cbb9ef27c5be9015c65 Mon Sep 17 00:00:00 2001 From: yanghaoran Date: Thu, 26 Nov 2020 16:35:09 +0800 Subject: [PATCH 13/18] fix support for mindspore tests compilation --- CMakeLists.txt | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2a30c479..7b745974 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -21,6 +21,13 @@ set(ASCEND_ATC_DIR ${ASCEND_DIR}/atc/lib64) set(ASCEND_ACL_DIR ${ASCEND_DIR}/acllib/lib64) set(STATIC_ACL_LIB ${ASCEND_ACL_DIR}) +set(ASCEND_MS_RUNTIME_PATH ${ASCEND_RUNTIME_DIR} ${ASCEND_ACL_DIR} ${ASCEND_ATC_DIR}) +set(ASCEND_MS_DRIVER_PATH ${ASCEND_DRIVER_DIR} ${ASCEND_DRIVER_COMMON_DIR}) +set(ATLAS_RUNTIME_DIR ${ASCEND_DIR}/ascend-toolkit/latest/fwkacllib/lib64) +set(ATLAS_ACL_DIR ${ASCEND_DIR}/ascend-toolkit/latest/acllib/lib64) +set(ATLAS_ATC_DIR ${ASCEND_DIR}/ascend-toolkit/latest/atc/lib64) +set(ATLAS_MS_RUNTIME_PATH ${ATLAS_RUNTIME_DIR} ${ATLAS_ACL_DIR} ${ATLAS_ATC_DIR}) + option(ENABLE_OPEN_SRC "Enable graphengine compile in opensource." FALSE) if (ENABLE_OPEN_SRC) @@ -129,14 +136,6 @@ if (ENABLE_OPEN_SRC) #add_subdirectory(metadef/graph) #add_subdirectory(metadef/register) elseif (ENABLE_D OR ENABLE_ACL) - - set(ASCEND_MS_RUNTIME_PATH ${ASCEND_RUNTIME_DIR} ${ASCEND_ACL_DIR} ${ASCEND_ATC_DIR}) - set(ASCEND_MS_DRIVER_PATH ${ASCEND_DRIVER_DIR} ${ASCEND_DRIVER_COMMON_DIR}) - set(ATLAS_RUNTIME_DIR ${ASCEND_DIR}/ascend-toolkit/latest/fwkacllib/lib64) - set(ATLAS_ACL_DIR ${ASCEND_DIR}/ascend-toolkit/latest/acllib/lib64) - set(ATLAS_ATC_DIR ${ASCEND_DIR}/ascend-toolkit/latest/atc/lib64) - set(ATLAS_MS_RUNTIME_PATH ${ATLAS_RUNTIME_DIR} ${ATLAS_ACL_DIR} ${ATLAS_ATC_DIR}) - # compiling with MindSpore include(cmake/external_libs/protobuf_static.cmake) include(cmake/external_libs/protoc.cmake) @@ -158,11 +157,18 @@ elseif (ENABLE_D OR ENABLE_ACL) set(METADEF_DIR ${CMAKE_CURRENT_LIST_DIR}/metadef) add_subdirectory(metadef) -elseif(ENABLE_MS_TESTCASE) +elseif(ENABLE_MS_TESTCASES) include(cmake/external_libs/protobuf_static.cmake) + include(cmake/external_libs/protoc.cmake) include(cmake/external_libs/securec.cmake) + include(cmake/FindModule.cmake) include(cmake/intf_pub_linux.cmake) + # common libraries + find_module(slog libslog.so ${ASCEND_MS_DRIVER_PATH}) + find_module(error_manager liberror_manager.so ${ASCEND_MS_RUNTIME_PATH} ${ATLAS_MS_RUNTIME_PATH}) + find_module(static_mmpa libmmpa.a ${ASCEND_MS_RUNTIME_PATH} ${ATLAS_MS_RUNTIME_PATH}) + set(METADEF_DIR ${CMAKE_CURRENT_LIST_DIR}/metadef) add_subdirectory(metadef) else() From 36c4b68924d122b5af2bf58e2fcb1bae08368cf9 Mon Sep 17 00:00:00 2001 From: zhengyuanhua Date: Fri, 27 Nov 2020 09:57:45 +0800 Subject: [PATCH 14/18] external interface modify: string change to ascendstring --- ge/client/ge_api.cc | 121 +++++++++++++++++++++++++++++- ge/graph/manager/graph_manager.cc | 27 +++++++ ge/graph/manager/graph_manager.h | 6 ++ ge/graph/passes/multi_batch_clone_pass.cc | 15 +++- ge/ir_build/ge_ir_build.cc | 53 ++++++++++++- ge/session/inner_session.cc | 19 +++++ ge/session/inner_session.h | 4 + ge/session/session_manager.cc | 20 +++++ ge/session/session_manager.h | 3 + inc/external/ge/ge_api.h | 34 +++++++++ inc/external/ge/ge_api_error_codes.h | 18 +++++ inc/external/ge/ge_api_types.h | 42 ++++++++++- inc/external/ge/ge_ir_build.h | 14 +++- metadef | 2 +- parser | 2 +- 15 files changed, 371 insertions(+), 9 deletions(-) diff --git a/ge/client/ge_api.cc b/ge/client/ge_api.cc index ef8d5622..9ecc3016 100644 --- a/ge/client/ge_api.cc +++ b/ge/client/ge_api.cc @@ -76,7 +76,7 @@ Status CheckOptionsValid(const std::map &options) { } // Initialize GE, prepare for execution, call GELib::Initialize -Status GEInitialize(const std::map &options) { +Status GEInitializeImpl(const std::map &options) { GELOGT(TRACE_INIT, "GEInitialize start"); // 0.check init status if (g_ge_initialized) { @@ -127,6 +127,26 @@ Status GEInitialize(const std::map &options) { return ret; } +// Initialize GE, prepare for execution, call GELib::Initialize +Status GEInitialize(const std::map &options) { + return GEInitializeImpl(options); +} + +Status GEInitialize(const std::map &options) { + std::map str_options; + for (auto & option : options) { + if (option.first.GetString() == nullptr || option.second.GetString() == nullptr) { + GELOGE(FAILED, "GEInitialize options is nullptr."); + return FAILED; + } + std::string key = option.first.GetString(); + std::string val = option.second.GetString(); + str_options[key] = val; + } + return GEInitializeImpl(str_options); +} + + // GE finalize, releasing all resources Status GEFinalize() { GELOGT(TRACE_INIT, "GEFinalize start"); @@ -202,6 +222,46 @@ Session::Session(const std::map &options) { GELOGT(TRACE_STOP, "Session Constructor finished"); } +Session::Session(const std::map &options) { + GELOGT(TRACE_INIT, "Session Constructor start"); + // check init status + sessionId_ = 0; + if (!g_ge_initialized) { + GELOGE(GE_CLI_GE_NOT_INITIALIZED, "GE is not initialized."); + return; + } + // call Initialize + std::shared_ptr instance_ptr = ge::GELib::GetInstance(); + if (instance_ptr == nullptr || !instance_ptr->InitFlag()) { + GELOGE(GE_CLI_GE_NOT_INITIALIZED, "Session Constructor failed"); + return; + } + + GELOGT(TRACE_RUNNING, "Creating session"); + std::map str_options; + for (auto &option : options) { + if (option.first.GetString() == nullptr || option.second.GetString() == nullptr) { + GELOGE(FAILED, "Session options is nullptr."); + return; + } + std::string key = option.first.GetString(); + std::string val = option.second.GetString(); + str_options[key] = val; + } + uint64_t session_id = 0; + Status ret = instance_ptr->SessionManagerObj().CreateSession(str_options, session_id); + GELOGT(TRACE_RUNNING, "Session id is %lu", session_id); + + // check return status, return, update session id if success + if (ret == SUCCESS) { + sessionId_ = session_id; + } else { + GELOGE(ret, "Session constructor failed, session Id not initialized"); + return; + } + GELOGT(TRACE_STOP, "Session Constructor finished"); +} + // session destructor Session::~Session() { GELOGT(TRACE_INIT, "Session Destructor start"); @@ -260,6 +320,34 @@ Status Session::AddGraph(uint32_t graph_id, const Graph &graph, const std::map &options) { + GELOGT(TRACE_INIT, "Start to add graph in Session. graph_id: %u, session_id: %lu.", graph_id, sessionId_); + std::shared_ptr instance_ptr = ge::GELib::GetInstance(); + if (instance_ptr == nullptr || !instance_ptr->InitFlag()) { + GELOGE(GE_CLI_GE_NOT_INITIALIZED, "AddGraph failed in Session."); + return FAILED; + } + GELOGD("Adding graph to session"); + std::map str_options; + for (auto &option : options) { + if (option.first.GetString() == nullptr || option.second.GetString() == nullptr) { + GELOGE(FAILED, "AddGraph options is nullptr."); + return FAILED; + } + std::string key = option.first.GetString(); + std::string val = option.second.GetString(); + str_options[key] = val; + } + Status ret = instance_ptr->SessionManagerObj().AddGraph(sessionId_, graph_id, graph, str_options); + if (ret != SUCCESS) { + GELOGE(ret, "AddGraph failed in Session."); + return FAILED; + } + GELOGD("AddGraph finished in Session."); + return ret; +} + Status Session::AddGraphWithCopy(uint32_t graph_id, const Graph &graph) { std::map options; return AddGraphWithCopy(graph_id, graph, options); @@ -387,6 +475,14 @@ Status Session::RegisterCallBackFunc(const std::string &key, const pCallBackFunc return ge::GELib::GetInstance()->SessionManagerObj().RegisterCallBackFunc(sessionId_, key, callback); } +Status Session::RegisterCallBackFunc(const char *key, const session::pCallBackFunc &callback) { + std::string str_key; + if (key != nullptr) { + str_key = key; + } + return ge::GELib::GetInstance()->SessionManagerObj().RegisterCallBackFunc(sessionId_, str_key, callback); +} + Status Session::BuildGraph(uint32_t graph_id, const std::vector &inputs) { std::shared_ptr instance_ptr = ge::GELib::GetInstance(); if (instance_ptr == nullptr || !instance_ptr->InitFlag()) { @@ -436,6 +532,29 @@ Status Session::GetVariables(const std::vector &var_names, std::vec return SUCCESS; } +Status Session::GetVariables(const std::vector &var_names, std::vector &var_values) { + auto instance_ptr = ge::GELib::GetInstance(); + if (instance_ptr == nullptr || !instance_ptr->InitFlag()) { + GELOGE(GE_CLI_GE_NOT_INITIALIZED, "SessionConstructor failed"); + return FAILED; + } + GELOGT(TRACE_RUNNING, "Get Variables"); + std::vector str_var_names; + for (auto &var_name : var_names) { + if (var_name.GetString() == nullptr) { + GELOGE(FAILED, "GetVariables name is nullptr."); + return FAILED; + } + str_var_names.emplace_back(var_name.GetString()); + } + Status ret = ge::GELib::GetInstance()->SessionManagerObj().GetVariables(sessionId_, str_var_names, var_values); + if (ret != SUCCESS) { + GELOGE(ret, "SessionManager RunGraphAsync failed"); + return FAILED; + } + return SUCCESS; +} + bool Session::IsGraphNeedRebuild(uint32_t graph_id) { return ge::GELib::GetInstance()->SessionManagerObj().IsGraphNeedRebuild(sessionId_, graph_id); } diff --git a/ge/graph/manager/graph_manager.cc b/ge/graph/manager/graph_manager.cc index 5e6f8577..36663fec 100755 --- a/ge/graph/manager/graph_manager.cc +++ b/ge/graph/manager/graph_manager.cc @@ -1865,12 +1865,30 @@ Status GraphManager::RegisterCallBackFunc( return SUCCESS; } +Status GraphManager::RegisterCallBackFunc( + const std::string &key, + const std::function &)> &callback) { + std::lock_guard lock(member_mutex_); + GELOGI("[GraphManager] RegisterCallBackFunc, key=%s.", key.c_str()); + callback_map_[key] = callback; + return SUCCESS; +} + Status GraphManager::PushSummaryData2ME(const GraphId &graph_id, const std::map &summary_data) { std::lock_guard lock(member_mutex_); GELOGI("[GraphManager] PushSummaryData2ME, dataSize=%zu.", summary_data.size()); auto itr = me_callback_map_.find(kSummary); if (itr == me_callback_map_.end()) { + auto iter = callback_map_.find(kSummary); + if (iter != callback_map_.end()) { + std::map tmp_summary_data; + for (auto &data : summary_data) { + AscendString tmp(data.first.c_str()); + tmp_summary_data[tmp] = data.second; + } + return iter->second(graph_id, tmp_summary_data); + } GELOGE(FAILED, "[GraphManager] PushSummaryData2ME failed, not found summary callback."); return FAILED; } @@ -1882,6 +1900,15 @@ Status GraphManager::PushSaveData2ME(const GraphId &graph_id, const std::map tmp_save_data; + for (auto &data : save_data) { + AscendString tmp(data.first.c_str()); + tmp_save_data[tmp] = data.second; + } + return iter->second(graph_id, tmp_save_data); + } GELOGE(FAILED, "[GraphManager] PushSaveData2ME failed, not found checkpoint callback."); return FAILED; } diff --git a/ge/graph/manager/graph_manager.h b/ge/graph/manager/graph_manager.h index 897ebf92..b0dde0c0 100644 --- a/ge/graph/manager/graph_manager.h +++ b/ge/graph/manager/graph_manager.h @@ -163,6 +163,10 @@ class GraphManager { const std::string &key, const std::function &)> &callback); + Status RegisterCallBackFunc( + const std::string &key, + const std::function &)> &callback); + const bool GetTrainFlag() const { return options_.train_graph_flag; } bool IsGraphNeedRebuild(uint32_t graph_id); @@ -390,6 +394,8 @@ class GraphManager { // summary and checkpoint callback function list for ME, key is summary or checkpoint std::map &)>> me_callback_map_; + std::map &)>> callback_map_; + bool init_flag_; GraphManagerOptions options_; diff --git a/ge/graph/passes/multi_batch_clone_pass.cc b/ge/graph/passes/multi_batch_clone_pass.cc index 732844e5..87d9749a 100755 --- a/ge/graph/passes/multi_batch_clone_pass.cc +++ b/ge/graph/passes/multi_batch_clone_pass.cc @@ -610,11 +610,17 @@ Status MultiBatchClonePass::CreateSubgraphs(const ComputeGraphPtr &graph, const /// Status MultiBatchClonePass::PostProcSubgraph(const ComputeGraphPtr &graph) { auto func_desc = case_node_->GetOpDesc(); + domi::ParseSubgraphFuncV2 parse_func_v2 = nullptr; auto post_func = domi::OpRegistry::Instance()->GetParseSubgraphPostFunc(func_desc->GetType()); if (post_func == nullptr) { GELOGW("The subgraph post func for node %s type %s is null.", case_node_->GetName().c_str(), case_node_->GetType().c_str()); - return FAILED; + if (domi::OpRegistry::Instance()->GetParseSubgraphPostFunc(func_desc->GetType(), parse_func_v2) != SUCCESS || + parse_func_v2 == nullptr) { + GELOGW("The subgraph new post func v2 for node %s type %s is null", case_node_->GetName().c_str(), + case_node_->GetType().c_str()); + return FAILED; + } } for (const auto &name : func_desc->GetSubgraphInstanceNames()) { @@ -629,7 +635,12 @@ Status MultiBatchClonePass::PostProcSubgraph(const ComputeGraphPtr &graph) { "Subgraph: %s get subgraph name failed.", subgraph->GetName().c_str()); auto graph = GraphUtils::CreateGraphFromComputeGraph(subgraph); - auto ret = post_func(subgraph_name, graph); + Status ret = FAILED; + if (post_func != nullptr) { + ret = post_func(subgraph_name, graph); + } else if (parse_func_v2 != nullptr) { + ret = parse_func_v2(subgraph_name.c_str(), graph); + } if (ret != SUCCESS) { GELOGE(FAILED, "Failed to post-process subgraph %s on node %s type %s", graph.GetName().c_str(), case_node_->GetName().c_str(), case_node_->GetType().c_str()); diff --git a/ge/ir_build/ge_ir_build.cc b/ge/ir_build/ge_ir_build.cc index 53f758a4..6aa63762 100644 --- a/ge/ir_build/ge_ir_build.cc +++ b/ge/ir_build/ge_ir_build.cc @@ -141,7 +141,7 @@ static void LoadOpsProto() { (void)manager->Initialize(option_tmp); } -graphStatus aclgrphBuildInitialize(std::map global_options) { +graphStatus aclgrphBuildInitializeImpl(std::map &global_options) { GELOGD("Enter aclgrphInitialize start!"); // check global options if (CheckGlobalOptions(global_options) != GRAPH_SUCCESS) { @@ -167,6 +167,24 @@ graphStatus aclgrphBuildInitialize(std::map global_opt return GRAPH_SUCCESS; } +graphStatus aclgrphBuildInitialize(std::map global_options) { + return aclgrphBuildInitializeImpl(global_options); +} + +graphStatus aclgrphBuildInitialize(std::map &global_options) { + std::map tmp_global_options; + for (auto &option : global_options) { + if (option.first.GetString() == nullptr || option.second.GetString() == nullptr) { + GELOGE(GRAPH_FAILED, "AclgrphBuildInitialize option is nullptr."); + return GRAPH_FAILED; + } + std::string key = option.first.GetString(); + std::string val = option.second.GetString(); + tmp_global_options[key] = val; + } + return aclgrphBuildInitializeImpl(tmp_global_options); +} + void aclgrphBuildFinalize() { if (ge::GELib::GetInstance() != nullptr && ge::GELib::GetInstance()->InitFlag()) { (void)ge::GELib::GetInstance()->Finalize(); @@ -453,6 +471,24 @@ graphStatus aclgrphBuildModel(const ge::Graph &graph, const std::map &build_options, + ModelBufferData &model) { + GELOGD("Enter aclmdlBuildModel process!"); + std::map tmp_build_options; + for (auto &option : build_options) { + if (option.first.GetString() == nullptr || option.second.GetString() == nullptr) { + GELOGE(GRAPH_FAILED, "AclgrphBuildInitialize option is nullptr."); + return GRAPH_FAILED; + } + std::string key = option.first.GetString(); + std::string val = option.second.GetString(); + tmp_build_options[key] = val; + } + + Impl builder; + return builder.BuildModel(graph, tmp_build_options, model); +} + graphStatus aclgrphSaveModel(const string &output_file, const ModelBufferData &model) { GELOGD("Enter aclmdlSaveModel process!"); if (model.data.get() == nullptr || model.length == 0) { @@ -463,6 +499,21 @@ graphStatus aclgrphSaveModel(const string &output_file, const ModelBufferData &m static_cast(model.length)); } +graphStatus aclgrphSaveModel(const char *output_file, const ModelBufferData &model) { + GELOGD("Enter aclmdlSaveModel process!"); + if (model.data.get() == nullptr || model.length == 0) { + GELOGE(GRAPH_PARAM_INVALID, "Input model is illegal"); + return GRAPH_PARAM_INVALID; + } + if (output_file == nullptr) { + GELOGE(GRAPH_PARAM_INVALID, "Output file is nullptr."); + return GRAPH_PARAM_INVALID; + } + std::string str_output_file = output_file; + return FileSaver::SaveToFile((str_output_file + ".om"), reinterpret_cast(model.data.get()), + static_cast(model.length)); +} + graphStatus aclgrphGetIRVersion(int *major_version, int *minor_version, int *patch_version) { GELOGD("Enter aclgrphGetIRVersion process!"); GE_CHECK_NOTNULL(major_version); diff --git a/ge/session/inner_session.cc b/ge/session/inner_session.cc index ec85d9ac..c4f8a53b 100755 --- a/ge/session/inner_session.cc +++ b/ge/session/inner_session.cc @@ -254,6 +254,25 @@ Status InnerSession::RegisterCallBackFunc( return SUCCESS; } +Status InnerSession::RegisterCallBackFunc( + const std::string &key, + const std::function &)> &callback) { + std::lock_guard lock(resource_mutex_); + if (!init_flag_) { + GELOGE(GE_SESS_INIT_FAILED, "[InnerSession:%lu] initialize failed.", session_id_); + return GE_SESS_INIT_FAILED; + } + UpdateThreadContext(std::map{}); + Status ret = graph_manager_.RegisterCallBackFunc(key, callback); + if (ret != SUCCESS) { + GELOGE(ret, "[InnerSession:%lu] register %s callback function failed.", session_id_, key.c_str()); + return ret; + } + + GELOGI("[InnerSession:%lu] register %s callback function success.", session_id_, key.c_str()); + return SUCCESS; +} + Status InnerSession::BuildGraph(uint32_t graph_id, const std::vector &inputs) { UpdateThreadContext(graph_id); GELOGI("[InnerSession:%lu] build graph on session, graph_id=%u.", session_id_, graph_id); diff --git a/ge/session/inner_session.h b/ge/session/inner_session.h index db7a2c92..5cab43d8 100644 --- a/ge/session/inner_session.h +++ b/ge/session/inner_session.h @@ -62,6 +62,10 @@ class InnerSession { const std::string &key, const std::function &)> &callback); + Status RegisterCallBackFunc( + const std::string &key, + const std::function &)> &callback); + const GraphManager &getGraphManagerObj() const; bool IsGraphNeedRebuild(uint32_t graph_id); diff --git a/ge/session/session_manager.cc b/ge/session/session_manager.cc index 69a62923..5d5a299a 100755 --- a/ge/session/session_manager.cc +++ b/ge/session/session_manager.cc @@ -276,6 +276,26 @@ Status SessionManager::RegisterCallBackFunc( return innerSession->RegisterCallBackFunc(key, callback); } +Status SessionManager::RegisterCallBackFunc( + SessionId session_id, const std::string &key, + const std::function &)> &callback) { + if (!init_flag_) { + GELOGE(GE_SESSION_MANAGER_NOT_INIT, "Session manager is not initialized."); + return GE_SESSION_MANAGER_NOT_INIT; + } + SessionPtr innerSession = nullptr; + { + std::lock_guard lock(mutex_); + std::map::iterator it = session_manager_map_.find(session_id); + if (it == session_manager_map_.end()) { + return GE_SESSION_NOT_EXIST; + } else { + innerSession = it->second; + } + } + return innerSession->RegisterCallBackFunc(key, callback); +} + Status SessionManager::BuildGraph(SessionId session_id, uint32_t graph_id, const std::vector &inputs) { if (!init_flag_) { GELOGE(GE_SESSION_MANAGER_NOT_INIT, "Session manager is not initialized."); diff --git a/ge/session/session_manager.h b/ge/session/session_manager.h index ac901c3a..da23219c 100644 --- a/ge/session/session_manager.h +++ b/ge/session/session_manager.h @@ -158,6 +158,9 @@ class SessionManager { Status RegisterCallBackFunc( SessionId session_id, const std::string &key, const std::function &)> &callback); + Status RegisterCallBackFunc( + SessionId session_id, const std::string &key, + const std::function &)> &callback); bool IsGraphNeedRebuild(SessionId session_id, uint32_t graph_id); diff --git a/inc/external/ge/ge_api.h b/inc/external/ge/ge_api.h index 8fd4b944..9c26ebf8 100644 --- a/inc/external/ge/ge_api.h +++ b/inc/external/ge/ge_api.h @@ -29,16 +29,26 @@ namespace ge { typedef uint32_t (*pCallBackFunc)(uint32_t graph_id, const std::map ¶ms_list); +namespace session { +typedef uint32_t (*pCallBackFunc)(uint32_t graph_id, const std::map ¶ms_list); +} + // Initialize GE +ATTRIBUTED_DEPRECATED(Status GEInitialize(const std::map &)) Status GEInitialize(const std::map &options); +Status GEInitialize(const std::map &options); + // Finalize GE, release all resources Status GEFinalize(); class GE_FUNC_DEV_VISIBILITY GE_FUNC_HOST_VISIBILITY Session { public: + ATTRIBUTED_DEPRECATED(Session(const std::map &)) explicit Session(const std::map &options); + explicit Session(const std::map &options); + ~Session(); /// @@ -57,10 +67,21 @@ class GE_FUNC_DEV_VISIBILITY GE_FUNC_HOST_VISIBILITY Session { /// @param [in] options graph options /// @return Status result of function /// + ATTRIBUTED_DEPRECATED(Status AddGraph(uint32_t, const Graph &, const std::map &)) Status AddGraph(uint32_t graphId, const Graph &graph, const std::map &options); /// /// @ingroup client + /// @brief add a graph with a specific graphId and graphOptions + /// @param [in] graphId graph id + /// @param [in] graph the graph + /// @param [in] options graph options + /// @return Status result of function + /// + Status AddGraph(uint32_t graphId, const Graph &graph, const std::map &options); + + /// + /// @ingroup client /// @brief add a copy graph with a specific graphId /// @param [in] graphId graph id /// @param [in] graph the graph @@ -124,10 +145,20 @@ class GE_FUNC_DEV_VISIBILITY GE_FUNC_HOST_VISIBILITY Session { /// @param [out] var_values: variable values /// @return Status result of function /// + ATTRIBUTED_DEPRECATED(Status GetVariables(const std::vector &, std::vector &)) Status GetVariables(const std::vector &var_names, std::vector &var_values); /// /// @ingroup ge_graph + /// @brief get variables in the session with specific session id + /// @param [in] var_names: variable names + /// @param [out] var_values: variable values + /// @return Status result of function + /// + Status GetVariables(const std::vector &var_names, std::vector &var_values); + + /// + /// @ingroup ge_graph /// @brief register callback func with specific summary or checkpoint by users /// @param [in] key: func key /// @param [in] callback: callback specific summary or checkpoint. @@ -135,8 +166,11 @@ class GE_FUNC_DEV_VISIBILITY GE_FUNC_HOST_VISIBILITY Session { /// Please ensure that the implementation of the function is trusted. /// @return Status result of function /// + ATTRIBUTED_DEPRECATED(Status RegisterCallBackFunc(const char *, const session::pCallBackFunc &)) Status RegisterCallBackFunc(const std::string &key, const pCallBackFunc &callback); + Status RegisterCallBackFunc(const char *key, const session::pCallBackFunc &callback); + bool IsGraphNeedRebuild(uint32_t graphId); private: diff --git a/inc/external/ge/ge_api_error_codes.h b/inc/external/ge/ge_api_error_codes.h index 1a25b86d..0ea2d064 100644 --- a/inc/external/ge/ge_api_error_codes.h +++ b/inc/external/ge/ge_api_error_codes.h @@ -22,6 +22,12 @@ #include "ge_error_codes.h" namespace ge { +#ifdef __GNUC__ +#define ATTRIBUTED_DEPRECATED(replacement) __attribute__((deprecated("Please use " #replacement " instead."))) +#else +#define ATTRIBUTED_DEPRECATED(replacement) __declspec(deprecated("Please use " #replacement " instead.")) +#endif + class StatusFactory { public: static StatusFactory *Instance() { @@ -37,6 +43,17 @@ class StatusFactory { err_desc_[err] = desc; } + void RegisterErrorNo(uint32_t err, const char *desc) { + if (desc == nullptr) { + return; + } + std::string error_desc = desc; + if (err_desc_.find(err) != err_desc_.end()) { + return; + } + err_desc_[err] = error_desc; + } + std::string GetErrDesc(uint32_t err) { auto iter_find = err_desc_.find(err); if (iter_find == err_desc_.end()) { @@ -56,6 +73,7 @@ class StatusFactory { class ErrorNoRegisterar { public: ErrorNoRegisterar(uint32_t err, const std::string &desc) { StatusFactory::Instance()->RegisterErrorNo(err, desc); } + ErrorNoRegisterar(uint32_t err, const char *desc) { StatusFactory::Instance()->RegisterErrorNo(err, desc); } ~ErrorNoRegisterar() {} }; diff --git a/inc/external/ge/ge_api_types.h b/inc/external/ge/ge_api_types.h index 28de4307..c439a8bf 100644 --- a/inc/external/ge/ge_api_types.h +++ b/inc/external/ge/ge_api_types.h @@ -65,7 +65,47 @@ const char *const OPTION_EXEC_ENABLE_TAILING_OPTIMIZATION = "ge.exec.isTailingOp // Option key: memory init const char *const GRAPH_MEMORY_MAX_SIZE = "ge.graphMemoryMaxSize"; const char *const VARIABLE_MEMORY_MAX_SIZE = "ge.variableMemoryMaxSize"; - +namespace configure_option { +const char *const STREAM_NUM = "ge.streamNum"; +const char *const HEAD_STREAM = "ge.headStream"; +const char *const PERF_LEVEL = "ge.perfLevel"; +const char *const ENCRYPT_MODE = "ge.encryptMode"; +const char *const EK_FILE = "ge.ekFile"; +const char *const CERT_FILE = "ge.certFile"; +const char *const HW_KEY_FILE = "ge.hwKeyFile"; +const char *const PRIVATE_KEY_FILE = "ge.privateKeyFile"; +const char *const FRAMEWORK_TYPE = "ge.frameworkType"; +const char *const CALIBRATION_CONF_FILE = "ge.calibrationConfFile"; +const char *const INSERT_OP_FILE = "ge.insertOpFile"; +const char *const OUTPUT_NODE_NAME = "ge.outputNodeName"; +const char *const COMPRESS_FLAG = "ge.compressFlag"; +const char *const PRECISION_MODE = "ge.exec.precision_mode"; +const char *const SINGLE_OP_FLAG = "ge.exec.single_op"; +const char *const TRAIN_FLAG = "ge.trainFlag"; +const char *const RUN_FLAG = "ge.runFlag"; +const char *const LOCAL_FMKOP_FLAG = "ge.enabledLocalFmkop"; +const char *const TBE_PLUGIN_PATH_FLAG = "ge.TBE_plugin_path"; +const char *const DDK_VERSION_FLAG = "ge.DDK_version"; +const char *const GE_FE_FLAG = "ge.feFlag"; +const char *const STREAM_MAX_PARALLEL_NUM = "ge.streamMaxParallelNum"; +const char *const OUTPUT_DATATYPE = "ge.outputDatatype"; +const char *const OP_SELECT_IMPL_MODE = "ge.opSelectImplmode"; +const char *const OPTYPELIST_FOR_IMPLMODE = "ge.optypelistForImplmode"; +const char *const HCOM_PARALLEL = "ge.hcomParallel"; +const char *const AUTO_TUNE_MODE = "ge.autoTuneMode"; +const char *const SOC_VERSION = "ge.socVersion"; +const char *const CORE_TYPE = "ge.engineType"; +const char *const AICORE_NUM = "ge.aicoreNum"; +const char *const L1_FUSION = "ge.l1Fusion"; +const char *const BUFFER_OPTIMIZE = "ge.bufferOptimize"; +const char *const ENABLE_SMALL_CHANNEL = "ge.enableSmallChannel"; +const char *const ENABLE_COMPRESS_WEIGHT = "ge.enableCompressWeight"; +const char *const FUSION_SWITCH_FILE = "ge.fusionSwitchFile"; +const char *const SAVE_ORIGINAL_MODEL = "ge.saveOriginalModel"; +const char *const ORIGINAL_MODEL_FILE = "ge.originalModelFile"; +const char *const INPUT_FP16_NODES = "ge.INPUT_NODES_SET_FP16"; +const char *const OP_DEBUG_LEVEL = "ge.opDebugLevel"; +} // namespace configure_option // Configure stream num by Session constructor options param, // its value should be int32_t type, default value is "1" const std::string STREAM_NUM = "ge.streamNum"; diff --git a/inc/external/ge/ge_ir_build.h b/inc/external/ge/ge_ir_build.h index 7ac54781..778ec21d 100644 --- a/inc/external/ge/ge_ir_build.h +++ b/inc/external/ge/ge_ir_build.h @@ -44,8 +44,11 @@ struct ModelBufferData { * @retval GRAPH_SUCCESS The function is successfully executed. * @retval OtherValues Failure */ +ATTRIBUTED_DEPRECATED(graphStatus aclgrphBuildInitialize(std::map &)) graphStatus aclgrphBuildInitialize(std::map global_options); +graphStatus aclgrphBuildInitialize(std::map &global_options); + /** * @ingroup AscendCL * @brief build model.Notice the model is stored in buffer @@ -63,9 +66,14 @@ void aclgrphBuildFinalize(); * @retval GRAPH_SUCCESS The function is successfully executed. * @retval OtherValues Failure */ +ATTRIBUTED_DEPRECATED(graphStatus aclgrphBuildModel(const ge::Graph &, const std::map &, + ModelBufferData &)) graphStatus aclgrphBuildModel(const ge::Graph &graph, const std::map &build_options, ModelBufferData &model); +graphStatus aclgrphBuildModel(const ge::Graph &graph, const std::map &build_options, + ModelBufferData &model); + /** * @ingroup AscendCL * @brief save model buffer to file @@ -75,8 +83,11 @@ graphStatus aclgrphBuildModel(const ge::Graph &graph, const std::map Date: Fri, 27 Nov 2020 10:59:30 +0800 Subject: [PATCH 15/18] fix static protobuf install path error --- cmake/external_libs/protobuf_static.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/external_libs/protobuf_static.cmake b/cmake/external_libs/protobuf_static.cmake index 23630d44..1780090f 100755 --- a/cmake/external_libs/protobuf_static.cmake +++ b/cmake/external_libs/protobuf_static.cmake @@ -42,7 +42,7 @@ include(GNUInstallDirs) add_library(ascend_protobuf_static_lib STATIC IMPORTED) set_target_properties(ascend_protobuf_static_lib PROPERTIES - IMPORTED_LOCATION ${PROTOBUF_STATIC_PKG_DIR}/lib64/libascend_protobuf.a + IMPORTED_LOCATION ${PROTOBUF_STATIC_PKG_DIR}/${CMAKE_INSTALL_LIBDIR}/libascend_protobuf.a ) add_library(ascend_protobuf_static INTERFACE) From 8c1d64e1375446b19677df1a53ca5aa8be67e9c9 Mon Sep 17 00:00:00 2001 From: "gengchao4@huawei.com" Date: Fri, 27 Nov 2020 11:23:33 +0800 Subject: [PATCH 16/18] bugfix for graph_partition of optune --- ge/graph/partition/graph_partition.cc | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/ge/graph/partition/graph_partition.cc b/ge/graph/partition/graph_partition.cc index 9e053020..6a1fbb34 100755 --- a/ge/graph/partition/graph_partition.cc +++ b/ge/graph/partition/graph_partition.cc @@ -382,11 +382,18 @@ graphStatus ge::GraphPartitioner::AddPlaceHolderEndInSrcDstGraph(const AnchorPtr GELOGW("SetInt anchorIndex failed");) GE_IF_BOOL_EXEC(!pld_op_desc->SetExtAttr("parentNode", src_node), GELOGW("SetPldExtAttr parentNode failed");) - - OpDescPtr src_node_op_desc = src_node->GetOpDesc(); - GE_CHECK_NOTNULL(src_node_op_desc); GE_IF_BOOL_EXEC(!AttrUtils::SetStr(pld_op_desc, ATTR_NAME_PLD_FRONT_NODE_ENGINE_NAME, - src_node_op_desc->GetOpEngineName()), GELOGW("SetStr frontNodeEngineName failed");) + src_node_opdesc->GetOpEngineName()), GELOGW("SetStr frontNodeEngineName failed");) + std::string l2_info_attr; + if (AttrUtils::GetStr(src_node_opdesc, "_task_L2FusionInfo", l2_info_attr)) { + GE_IF_BOOL_EXEC(!AttrUtils::SetStr(pld_op_desc, "_task_L2FusionInfo", l2_info_attr), + GELOGW("SetStr l2_info_attr failed");) + } + int64_t anchor_index_for_lxfusion; + if (AttrUtils::GetInt(src_node_opdesc, "_data_anchor_index_for_lxfusion", anchor_index_for_lxfusion)) { + GE_IF_BOOL_EXEC(!AttrUtils::SetInt(pld_op_desc, "_data_anchor_index_for_lxfusion", anchor_index_for_lxfusion), + GELOGW("SetInt anchor_index_for_lxfusion failed");) + } // do not care over flow graph_info_.num_of_pld_end_++; // replace output_desc of pld with input node's output desc From 15aa9afb8bf1ba45bda189f6e4d5c91d0f1a8fec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E7=AC=91=E5=A4=A9?= Date: Fri, 27 Nov 2020 16:49:08 +0800 Subject: [PATCH 17/18] =?UTF-8?q?=E5=9B=9E=E9=80=80=20'Pull=20Request=20!4?= =?UTF-8?q?34=20:=20atomic=20loop=20condition=20bug=20fix'?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ge/graph/passes/atomic_addr_clean_pass.cc | 84 +++++++------------------------ ge/graph/passes/atomic_addr_clean_pass.h | 5 +- 2 files changed, 19 insertions(+), 70 deletions(-) diff --git a/ge/graph/passes/atomic_addr_clean_pass.cc b/ge/graph/passes/atomic_addr_clean_pass.cc index 4cbc42d1..60742eb1 100755 --- a/ge/graph/passes/atomic_addr_clean_pass.cc +++ b/ge/graph/passes/atomic_addr_clean_pass.cc @@ -33,12 +33,10 @@ namespace ge { Status AtomicAddrCleanPass::Run(ComputeGraphPtr graph) { GE_CHECK_NOTNULL(graph); GELOGD("AtomicAddrCleanPass begin."); - bool is_unknown_graph = graph->GetGraphUnknownFlag(); - // 1.Recoginze atomic and loop mark vector atomic_node_vec; for (NodePtr &node : graph->GetDirectNode()) { - if (IsAtomicOp(node, is_unknown_graph)) { + if (IsAtomicOp(node)) { atomic_node_vec.push_back(node); } if (!is_loop_graph_ && node->GetType() == LOOPCOND) { @@ -52,6 +50,7 @@ Status AtomicAddrCleanPass::Run(ComputeGraphPtr graph) { return SUCCESS; } + bool is_unknown_graph = graph->GetGraphUnknownFlag(); if (is_unknown_graph) { GELOGD("Graph[%s] is unknown graph. It will call fe interface to compile op.", graph->GetName().c_str()); GE_CHK_STATUS_RET(CompileUnknownGraphOp(atomic_node_vec)); @@ -243,101 +242,55 @@ Status AtomicAddrCleanPass::LinkToAtomicNode(const NodePtr &atomic_node, NodePtr return SUCCESS; } -bool AtomicAddrCleanPass::IsAtomicOp(const NodePtr &node, bool is_unknown_graph) { +bool AtomicAddrCleanPass::IsAtomicOp(const NodePtr &node) { GE_IF_BOOL_EXEC(node == nullptr, GELOGE(FAILED, "node is null."); return false); OpDescPtr op_desc = node->GetOpDesc(); if (op_desc == nullptr) { return false; } - - if (CheckAtomicFromOpsKernel(node)) { - return true; - } - - // 2.Check atomic attr in node - std::map> node_workspace_offset; - bool has_atomic_input = op_desc->HasAttr(ATOMIC_ATTR_INPUT_INDEX); - bool has_atomic_output = op_desc->HasAttr(ATOMIC_ATTR_OUTPUT_INDEX); - node_workspace_offset = op_desc->TryGetExtAttr(EXT_ATTR_ATOMIC_WORKSPACE_OFFSET, node_workspace_offset); - if (!has_atomic_input && !has_atomic_output && node_workspace_offset.empty()) { - return false; - } - - if (!is_unknown_graph) { - if (!has_atomic_input && has_atomic_output && node_workspace_offset.empty()) { - std::vector atomic_output_index; - (void) ge::AttrUtils::GetListInt(op_desc, ATOMIC_ATTR_OUTPUT_INDEX, atomic_output_index); - bool is_all_output_peer_also_atomic = true; - for (const auto &output_index : atomic_output_index) { - if (!IsOutputIndexPeerInputAtomic(node, output_index)) { - is_all_output_peer_also_atomic = false; - break; - } - } - if (is_all_output_peer_also_atomic) { - GELOGI("all out peer node input atomic, skip this out atomic process, node name: %s", node->GetName().c_str()); - return false; - } - } - } - - graphStatus ret = op_desc->SetAttr(ATOMIC_ATTR_IS_ATOMIC_NODE, GeAttrValue::CreateFrom(true)); - if (ret != GRAPH_SUCCESS) { - GELOGW("set attr ATOMIC_ATTR_IS_ATOMIC_NODE fail."); - } - GELOGD("Recognized atomic op %s from attr.", op_desc->GetName().c_str()); - return true; -} - -// just hccl may mark atomic from ops kernel now, and hccl's atomic if for all input -bool AtomicAddrCleanPass::CheckAtomicFromOpsKernel(const NodePtr &node) { // 1.Check if isAtomic attrs exist for HCOM std::shared_ptr instance_ptr = GELib::GetInstance(); if ((instance_ptr == nullptr) || (!instance_ptr->InitFlag())) { - GELOGW("GELib not initialized, atomic from ops kernel judge false, node_name: %s", node->GetName().c_str()); + GELOGW("GELib not initialized"); return false; } OpsKernelManager &ops_kernel_manager = instance_ptr->OpsKernelManagerObj(); - vector op_info_vec = ops_kernel_manager.GetOpsKernelInfo(node->GetType()); + vector op_info_vec = ops_kernel_manager.GetOpsKernelInfo(op_desc->GetType()); for (const auto &op_info : op_info_vec) { if (op_info.isAtomic) { + GELOGI("Recognized atomic op %s from DNN_HCCL engine.", op_desc->GetName().c_str()); // check peer input is DATA for (auto &in_data_anchor : node->GetAllInDataAnchors()) { if (in_data_anchor->GetPeerOutAnchor() != nullptr && in_data_anchor->GetPeerOutAnchor()->GetOwnerNode() != nullptr) { auto peer_in_node = in_data_anchor->GetPeerOutAnchor()->GetOwnerNode(); if (peer_in_node->GetType() == DATA) { - GELOGI("Recognized atomic op %s from %s engine and input is DATA.", node->GetName().c_str(), op_info.engine.c_str()); + GELOGI("Recognized atomic op %s from DNN_HCCL engine and input is DATA.", op_desc->GetName().c_str()); return false; } } } - GELOGI("Recognized atomic op %s from %s engine.", node->GetName().c_str(), op_info.engine.c_str()); hcom_node_vec_.push_back(node); return true; } } - return false; -} - -bool AtomicAddrCleanPass::IsOutputIndexPeerInputAtomic(const NodePtr &node, int64_t output_index) { - auto out_data_anchor = node->GetAllOutDataAnchors().at(output_index); - if (out_data_anchor == nullptr) { + // 2.Check atomic attr in node + std::map> node_workspace_offset; + bool has_atomic_input = op_desc->HasAttr(ATOMIC_ATTR_INPUT_INDEX); + bool has_atomic_output = op_desc->HasAttr(ATOMIC_ATTR_OUTPUT_INDEX); + node_workspace_offset = op_desc->TryGetExtAttr(EXT_ATTR_ATOMIC_WORKSPACE_OFFSET, node_workspace_offset); + if (!has_atomic_input && !has_atomic_output && node_workspace_offset.empty()) { return false; } - for (const auto input_anchor : out_data_anchor->GetPeerInDataAnchors()) { - auto output_node = input_anchor->GetOwnerNode(); - // just hccl may mark atomic from ops kernel now, and hccl's atomic if for all input - // hccl's attr ATOMIC_ATTR_INPUT_INDEX mark on CalcOpRunningParam, can't be get here - if (CheckAtomicFromOpsKernel(output_node)) { - return true; - } + graphStatus ret = op_desc->SetAttr(ATOMIC_ATTR_IS_ATOMIC_NODE, GeAttrValue::CreateFrom(true)); + if (ret != GRAPH_SUCCESS) { + GELOGW("set attr ATOMIC_ATTR_IS_ATOMIC_NODE fail."); } - return false; + GELOGD("Recognized atomic op %s from FE engine.", op_desc->GetName().c_str()); + return true; } - /// /// @brief Clear Status, used for subgraph pass /// @return SUCCESS @@ -345,7 +298,6 @@ bool AtomicAddrCleanPass::IsOutputIndexPeerInputAtomic(const NodePtr &node, int6 Status AtomicAddrCleanPass::ClearStatus() { hcom_node_vec_.clear(); return SUCCESS; - } Status AtomicAddrCleanPass::CompileUnknownGraphOp(const vector &atomic_node_vec) { diff --git a/ge/graph/passes/atomic_addr_clean_pass.h b/ge/graph/passes/atomic_addr_clean_pass.h index 64bc604b..ad60b7b5 100755 --- a/ge/graph/passes/atomic_addr_clean_pass.h +++ b/ge/graph/passes/atomic_addr_clean_pass.h @@ -72,7 +72,7 @@ class AtomicAddrCleanPass : public GraphPass { * @param node * @return */ - bool IsAtomicOp(const NodePtr &node, bool is_unknown_graph); + bool IsAtomicOp(const NodePtr &node); /** * Handle atomic node in unknown graph @@ -84,9 +84,6 @@ class AtomicAddrCleanPass : public GraphPass { Status HandleDispersedAtomicNodes(ComputeGraphPtr &graph, const std::vector &atomic_node_vec, std::vector &common_atomic_nodes); - bool CheckAtomicFromOpsKernel(const NodePtr &node); - - bool IsOutputIndexPeerInputAtomic(const NodePtr &node, int64_t output_index); vector hcom_node_vec_; bool is_loop_graph_ = false; From 7c94c444b4b7e182f591e1475b15e598ac75bcc4 Mon Sep 17 00:00:00 2001 From: baker Date: Fri, 27 Nov 2020 17:11:02 +0800 Subject: [PATCH 18/18] save and get compile failed msg --- ge/graph/optimize/graph_optimize.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ge/graph/optimize/graph_optimize.cc b/ge/graph/optimize/graph_optimize.cc index c6ba0b06..5ea462a5 100644 --- a/ge/graph/optimize/graph_optimize.cc +++ b/ge/graph/optimize/graph_optimize.cc @@ -109,7 +109,7 @@ Status GraphOptimize::OptimizeSubGraph(ComputeGraphPtr &compute_graph, const Com if (ret != SUCCESS) { auto root_graph = ge::GraphUtils::FindRootGraph(parent_graph); if (root_graph != nullptr) { - ErrorManager.GetInstance().SaveMstuneCompileFailedMsg(root_graph->GetName()); + ErrorManager::GetInstance().SaveMstuneCompileFailedMsg(root_graph->GetName()); } GELOGE(ret, "[OptimizeSubGraph][OptimizeFusedGraphAfterGraphSlice]: graph optimize failed, ret:%d", ret); return ret;