common log optimize

build.sh

@@ -173,7 +173,7 @@ build_graphengine()
     TARGET="ge_runner fwk_atc.bin ${TARGET}"
   elif [ "x${PLATFORM}" = "xinference" ]
   then
-    TARGET="ge_compiler atc_atc.bin opensrc_ascendcl ${TARGET}"
+    TARGET="ge_compiler atc_atc.bin ge_executor_shared ${TARGET}"
   elif [ "X$ENABLE_GE_UT" = "Xon" ]
   then
     TARGET="ut_libgraph ut_libge_multiparts_utest ut_libge_others_utest ut_libge_kernel_utest ut_libge_distinct_load_utest"
@@ -183,7 +183,7 @@ build_graphengine()
   elif [ "x${PLATFORM}" = "xall" ]
   then
     # build all the target
-    TARGET="ge_runner ge_compiler fwk_atc.bin atc_atc.bin opensrc_ascendcl ${TARGET}"
+    TARGET="ge_runner ge_compiler fwk_atc.bin atc_atc.bin ge_executor_shared ${TARGET}"
   fi
 
   make ${VERBOSE} ${TARGET} -j${THREAD_NUM} && make install
@@ -250,6 +250,7 @@ generate_package()
   NNENGINE_PATH="plugin/nnengine/ge_config"
   OPSKERNEL_PATH="plugin/opskernel"
 
+  ACL_LIB=("libge_common.so" "libgraph.so" "libregister.so" "liberror_manager.so" "libge_executor.so")
   ATC_LIB=("libc_sec.so" "libge_common.so" "libge_compiler.so" "libgraph.so" "libregister.so" "liberror_manager.so")
   FWK_LIB=("libge_common.so" "libge_runner.so" "libgraph.so" "libregister.so" "liberror_manager.so")
   PLUGIN_OPSKERNEL=("libge_local_engine.so" "libge_local_opskernel_builder.so" "libhost_cpu_engine.so" "libhost_cpu_opskernel_builder.so" "optimizer_priority.pbtxt")
@@ -303,6 +304,11 @@ generate_package()
     find ${OUTPUT_PATH}/${GRAPHENGINE_LIB_PATH} -maxdepth 1 -name "$lib" -exec cp -f {} ${OUTPUT_PATH}/${FWK_PATH} \;
   done
 
+  for lib in "${ACL_LIB[@]}";
+  do
+    find ${OUTPUT_PATH}/${GRAPHENGINE_LIB_PATH} -maxdepth 1 -name "$lib" -exec cp -f {} ${OUTPUT_PATH}/${ACL_PATH} \;
+  done
+
   for lib in "${ATC_LIB[@]}";
   do
     find ${OUTPUT_PATH}/${GRAPHENGINE_LIB_PATH} -maxdepth 1 -name "$lib" -exec cp -f {} ${OUTPUT_PATH}/${ATC_PATH} \;
@@ -310,7 +316,6 @@ generate_package()
 
   find ./lib/atclib -name atc.bin -exec cp {} "${OUTPUT_PATH}/${ATC_BIN_PATH}" \;
   find ./lib/fwkacl -name atc.bin -exec cp {} "${OUTPUT_PATH}/${FWK_BIN_PATH}" \;
-  find ${OUTPUT_PATH}/${GRAPHENGINE_LIB_PATH} -maxdepth 1 -name "libascendcl.so" -exec cp -f {} ${OUTPUT_PATH}/${ACL_PATH} \;
 
   cp -r ${OUTPUT_PATH}/../metadef/inc/external/* ${ATC_INCLUDE_PATH}
   cp -r ${OUTPUT_PATH}/../parser/inc/external/* ${ATC_INCLUDE_PATH}

ge/CMakeLists.txt

@@ -25,7 +25,7 @@ set(PROTO_HEADER_LIST
     "${METADEF_DIR}/proto/insert_op.proto"
     "${METADEF_DIR}/proto/ge_ir.proto"
     "${METADEF_DIR}/proto/fwk_adapter.proto"
-    "${METADEF_DIR}/proto/op_mapping_info.proto"
+    "${METADEF_DIR}/proto/op_mapping.proto"
 )
 
 protobuf_generate(ge PROTO_SRCS PROTO_HDRS ${PROTO_LIST})
@@ -108,6 +108,7 @@ set(TRAIN_SRC_LIST
     "common/helper/model_cache_helper.cc"
     "common/profiling/profiling_manager.cc"
     "common/dump/dump_manager.cc"
+    "common/dump/exception_dumper.cc"
     "common/dump/dump_properties.cc"
     "common/dump/opdebug_register.cc"
     "common/dump/dump_op.cc"
@@ -339,7 +340,6 @@ set(TRAIN_SRC_LIST
     "init/gelib.cc"
     "model/ge_model.cc"
     "model/ge_root_model.cc"
-    "omm/csa_interact.cc"
     "opskernel_manager/ops_kernel_manager.cc"
     "opskernel_manager/ops_kernel_builder_manager.cc"
     "session/inner_session.cc"
@@ -401,7 +401,7 @@ set(TRAIN_SRC_LIST
     "ir_build/attr_options/utils.cc"
     "ir_build/attr_options/keep_dtype_option.cc"
     "ir_build/attr_options/weight_compress_option.cc"
-    "ir_build/atc_ir_common.cc"
+    "ir_build/option_utils.cc"
     "graph/build/memory/memory_assigner.cc"
     "graph/build/memory/graph_mem_assigner.cc"
     "graph/build/memory/binary_block_mem_assigner.cc"
@@ -414,7 +414,6 @@ set(TRAIN_SRC_LIST
 
 set(INFER_SRC_LIST
     "graph/manager/trans_var_data_utils.cc"
-    "omm/csa_interact.cc"
     "common/fp16_t.cc"
     "common/formats/utils/formats_trans_utils.cc"
     "common/formats/format_transfers/datatype_transfer.cc"
@@ -437,6 +436,7 @@ set(INFER_SRC_LIST
     "common/formats/formats.cc"
     "common/profiling/profiling_manager.cc"
     "common/dump/dump_properties.cc"
+    "common/dump/exception_dumper.cc"
     "common/dump/dump_manager.cc"
     "common/dump/dump_op.cc"
     "common/dump/opdebug_register.cc"
@@ -661,7 +661,7 @@ set(INFER_SRC_LIST
     "ir_build/attr_options/utils.cc"
     "ir_build/attr_options/keep_dtype_option.cc"
     "ir_build/attr_options/weight_compress_option.cc"
-    "ir_build/atc_ir_common.cc"
+    "ir_build/option_utils.cc"
     "graph/preprocess/insert_op/ge_aipp_op.cc"
     "graph/preprocess/insert_op/util_insert_aipp_op.cc"
     "hybrid/node_executor/aicpu/aicpu_ext_info.cc"

ge/client/ge_api.cc

@@ -100,7 +100,7 @@ Status GEInitializeImpl(const std::map<string, string> &options) {
     GELOGW("GEInitialize is called more than once");
     return SUCCESS;
   }
-  ErrorManager::GetInstance().SetStage(ErrorMessage::kInitialize, ErrorMessage::kOpsProtoInit);
+  ErrorManager::GetInstance().SetStage(error_message::kInitialize, error_message::kOpsProtoInit);
   // Load OpsProto lib plugin
   std::string opsproto_path;
   GetOpsProtoPath(opsproto_path);
@@ -119,7 +119,7 @@ Status GEInitializeImpl(const std::map<string, string> &options) {
     return FAILED;
   }
 
-  ErrorManager::GetInstance().SetStage(ErrorMessage::kInitialize, ErrorMessage::kOther);
+  ErrorManager::GetInstance().SetStage(error_message::kInitialize, error_message::kOther);
   // check options is valid
   GE_TIMESTAMP_START(CheckOptionsValid);
   if (CheckOptionsValid(options) != SUCCESS) {
@@ -127,13 +127,13 @@ Status GEInitializeImpl(const std::map<string, string> &options) {
   }
   GE_TIMESTAMP_END(CheckOptionsValid, "GEInitialize::CheckOptionsValid");
 
-  ErrorManager::GetInstance().SetStage(ErrorMessage::kInitialize, ErrorMessage::kOpsProtoInit);
+  ErrorManager::GetInstance().SetStage(error_message::kInitialize, error_message::kOpsProtoInit);
   GE_TIMESTAMP_START(InitPreparation);
   TBEPluginManager::Instance().InitPreparation(options);
   GE_TIMESTAMP_END(InitPreparation, "GEInitialize::InitPreparation");
   // call Initialize
   GELOGT(TRACE_RUNNING, "Initializing environment");
-  ErrorManager::GetInstance().SetStage(ErrorMessage::kInitialize, ErrorMessage::kOther);
+  ErrorManager::GetInstance().SetStage(error_message::kInitialize, error_message::kOther);
   GE_TIMESTAMP_START(GELibInitialize);
   ret = ge::GELib::Initialize(options);
   GE_TIMESTAMP_END(GELibInitialize, "GEInitialize::GELibInitialize");
@@ -154,7 +154,7 @@ Status GEInitializeImpl(const std::map<string, string> &options) {
 
 // Initialize GE, prepare for execution, call GELib::Initialize
 Status GEInitialize(const std::map<string, string> &options) {
-  ErrorManager::GetInstance().SetStage(ErrorMessage::kInitialize, ErrorMessage::kOther);
+  ErrorManager::GetInstance().SetStage(error_message::kInitialize, error_message::kOther);
   if (DlogReportInitialize() != SUCCESS) {
     GELOGW("Dlog report device log initialize failed.");
   }
@@ -162,7 +162,7 @@ Status GEInitialize(const std::map<string, string> &options) {
 }
 
 Status GEInitialize(const std::map<AscendString, AscendString> &options) {
-  ErrorManager::GetInstance().SetStage(ErrorMessage::kInitialize, ErrorMessage::kOther);
+  ErrorManager::GetInstance().SetStage(error_message::kInitialize, error_message::kOther);
   std::map<std::string, std::string> str_options;
   for (auto &option : options) {
     if (option.first.GetString() == nullptr || option.second.GetString() == nullptr) {
@@ -191,7 +191,7 @@ Status GEFinalize() {
     return SUCCESS;
   }
 
-  ErrorManager::GetInstance().SetStage(ErrorMessage::kFinalize, ErrorMessage::kFinalize);
+  ErrorManager::GetInstance().SetStage(error_message::kFinalize, error_message::kFinalize);
   ErrorManager::GetInstance().GenWorkStreamIdDefault();
   GELOGT(TRACE_INIT, "GEFinalize start");
 
@@ -243,7 +243,7 @@ std::string GEGetWarningMsg() {
 
 // Initialize session，which calls innerSession
 Session::Session(const std::map<string, string> &options) {
-  ErrorManager::GetInstance().SetStage(ErrorMessage::kInitialize, ErrorMessage::kOther);
+  ErrorManager::GetInstance().SetStage(error_message::kInitialize, error_message::kOther);
   GELOGT(TRACE_INIT, "Session Constructor start");
 
   ErrorManager::GetInstance().GenWorkStreamIdDefault();
@@ -259,7 +259,7 @@ Session::Session(const std::map<string, string> &options) {
   // call Initialize
   std::shared_ptr<GELib> instance_ptr = ge::GELib::GetInstance();
   if (instance_ptr == nullptr || !instance_ptr->InitFlag()) {
-    GELOGE(GE_CLI_GE_NOT_INITIALIZED, 
+    GELOGE(GE_CLI_GE_NOT_INITIALIZED,
            "[Construct][Session]Failed, GELib instance is nullptr or it is not InitFlag");
     return;
   }
@@ -280,7 +280,7 @@ Session::Session(const std::map<string, string> &options) {
 }
 
 Session::Session(const std::map<AscendString, AscendString> &options) {
-  ErrorManager::GetInstance().SetStage(ErrorMessage::kInitialize, ErrorMessage::kOther);
+  ErrorManager::GetInstance().SetStage(error_message::kInitialize, error_message::kOther);
   GELOGT(TRACE_INIT, "Session Constructor start");
 
   ErrorManager::GetInstance().GenWorkStreamIdDefault();
@@ -331,7 +331,7 @@ Session::Session(const std::map<AscendString, AscendString> &options) {
 
 // session destructor
 Session::~Session() {
-  ErrorManager::GetInstance().SetStage(ErrorMessage::kFinalize, ErrorMessage::kFinalize);
+  ErrorManager::GetInstance().SetStage(error_message::kFinalize, error_message::kFinalize);
   GELOGT(TRACE_INIT, "Session Destructor start");
   // 0.check init status
   if (!g_ge_initialized) {
@@ -371,7 +371,7 @@ Session::~Session() {
 
 // Add Graph
 Status Session::AddGraph(uint32_t graph_id, const Graph &graph) {
-  ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kOther);
+  ErrorManager::GetInstance().SetStage(error_message::kModelCompile, error_message::kOther);
   std::map<std::string, std::string> options;
   ErrorManager::GetInstance().GenWorkStreamIdBySessionGraph(sessionId_, graph_id);
   return AddGraph(graph_id, graph, options);
@@ -379,7 +379,7 @@ Status Session::AddGraph(uint32_t graph_id, const Graph &graph) {
 
 // Add Graph
 Status Session::AddGraph(uint32_t graph_id, const Graph &graph, const std::map<std::string, std::string> &options) {
-  ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kOther);
+  ErrorManager::GetInstance().SetStage(error_message::kModelCompile, error_message::kOther);
   GELOGT(TRACE_INIT, "Start to add graph in Session. graph_id: %u, session_id: %lu.", graph_id, sessionId_);
   ErrorManager::GetInstance().GenWorkStreamIdBySessionGraph(sessionId_, graph_id);
   std::shared_ptr<GELib> instance_ptr = ge::GELib::GetInstance();
@@ -393,7 +393,7 @@ Status Session::AddGraph(uint32_t graph_id, const Graph &graph, const std::map<s
   GELOGD("Adding graph to session");
   Status ret = instance_ptr->SessionManagerObj().AddGraph(sessionId_, graph_id, graph, options);
   if (ret != SUCCESS) {
-    GELOGE(ret, 
+    GELOGE(ret,
            "[Add][Graph]Failed, error code:%u, session_id:%lu, graph_id:%u.",
 	   ret, sessionId_, graph_id);
     return FAILED;
@@ -405,7 +405,7 @@ Status Session::AddGraph(uint32_t graph_id, const Graph &graph, const std::map<s
 //Add Graph
 Status Session::AddGraph(uint32_t graph_id, const Graph &graph,
                          const std::map<AscendString, AscendString> &options) {
-  ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kOther);
+  ErrorManager::GetInstance().SetStage(error_message::kModelCompile, error_message::kOther);
   GELOGT(TRACE_INIT, "Start to add graph in Session. graph_id: %u, session_id: %lu.", graph_id, sessionId_);
   ErrorManager::GetInstance().GenWorkStreamIdBySessionGraph(sessionId_, graph_id);
   std::shared_ptr<GELib> instance_ptr = ge::GELib::GetInstance();
@@ -441,7 +441,7 @@ Status Session::AddGraph(uint32_t graph_id, const Graph &graph,
 }
 
 Status Session::AddGraphWithCopy(uint32_t graph_id, const Graph &graph) {
-  ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kOther);
+  ErrorManager::GetInstance().SetStage(error_message::kModelCompile, error_message::kOther);
   ErrorManager::GetInstance().GenWorkStreamIdBySessionGraph(sessionId_, graph_id);
   std::map<AscendString, AscendString> options;
   return AddGraphWithCopy(graph_id, graph, options);
@@ -450,7 +450,7 @@ Status Session::AddGraphWithCopy(uint32_t graph_id, const Graph &graph) {
 // Add Graph With Copy
 Status Session::AddGraphWithCopy(uint32_t graph_id, const Graph &graph,
                                  const std::map<AscendString, AscendString> &options) {
-  ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kOther);
+  ErrorManager::GetInstance().SetStage(error_message::kModelCompile, error_message::kOther);
   GELOGT(TRACE_INIT, "Start to add graph in Session. graph_id: %u, session_id: %lu.", graph_id, sessionId_);
   ErrorManager::GetInstance().GenWorkStreamIdBySessionGraph(sessionId_, graph_id);
   std::shared_ptr<GELib> instance_ptr = ge::GELib::GetInstance();
@@ -479,7 +479,7 @@ Status Session::AddGraphWithCopy(uint32_t graph_id, const Graph &graph,
 
 // Remove Graph
 Status Session::RemoveGraph(uint32_t graph_id) {
-  ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kOther);
+  ErrorManager::GetInstance().SetStage(error_message::kModelCompile, error_message::kOther);
   GELOGT(TRACE_INIT, "Session RemoveGraph start");
 
   ErrorManager::GetInstance().GenWorkStreamIdBySessionGraph(sessionId_, graph_id);
@@ -560,7 +560,7 @@ void PrintOutputResult(std::vector<Tensor> &outputs) {
 
 // Run Graph
 Status Session::RunGraph(uint32_t graph_id, const std::vector<Tensor> &inputs, std::vector<Tensor> &outputs) {
-  ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kOther);
+  ErrorManager::GetInstance().SetStage(error_message::kModelCompile, error_message::kOther);
   GELOGT(TRACE_INIT, "Session RunGraph start");
 
   ErrorManager::GetInstance().GenWorkStreamIdBySessionGraph(sessionId_, graph_id);
@@ -615,7 +615,7 @@ Status Session::RegisterCallBackFunc(const char *key, const session::pCallBackFu
 
 // Build Graph
 Status Session::BuildGraph(uint32_t graph_id, const std::vector<InputTensorInfo> &inputs) {
-  ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kOther);
+  ErrorManager::GetInstance().SetStage(error_message::kModelCompile, error_message::kOther);
   ErrorManager::GetInstance().GenWorkStreamIdBySessionGraph(sessionId_, graph_id);
   std::shared_ptr<GELib> instance_ptr = ge::GELib::GetInstance();
   if (instance_ptr == nullptr || !instance_ptr->InitFlag()) {
@@ -643,7 +643,7 @@ Status Session::BuildGraph(uint32_t graph_id, const std::vector<InputTensorInfo>
 // Run Graph Asynchronously
 Status Session::RunGraphAsync(uint32_t graph_id, const std::vector<InputTensorInfo> &inputs,
                               RunAsyncCallback callback) {
-  ErrorManager::GetInstance().SetStage(ErrorMessage::kModelExecute, ErrorMessage::kModelExecute);
+  ErrorManager::GetInstance().SetStage(error_message::kModelExecute, error_message::kModelExecute);
   ErrorManager::GetInstance().GenWorkStreamIdBySessionGraph(sessionId_, graph_id);
   std::shared_ptr<GELib> instance_ptr = ge::GELib::GetInstance();
   if (instance_ptr == nullptr || !instance_ptr->InitFlag()) {
@@ -672,11 +672,11 @@ Status Session::RunGraphAsync(uint32_t graph_id, const std::vector<InputTensorIn
 
 // Get Variables
 Status Session::GetVariables(const std::vector<std::string> &var_names, std::vector<Tensor> &var_values) {
-  ErrorManager::GetInstance().SetStage(ErrorMessage::kModelExecute, ErrorMessage::kModelExecute);
+  ErrorManager::GetInstance().SetStage(error_message::kModelExecute, error_message::kModelExecute);
   ErrorManager::GetInstance().GenWorkStreamIdDefault();
   auto instance_ptr = ge::GELib::GetInstance();
   if (instance_ptr == nullptr || !instance_ptr->InitFlag()) {
-    GELOGE(GE_CLI_GE_NOT_INITIALIZED, 
+    GELOGE(GE_CLI_GE_NOT_INITIALIZED,
            "[Get][Variables]Failed, the GELib instance is nullptr or is not InitFlag.");
     REPORT_INNER_ERROR("E19999",
                        "GetVariables failed, the GELib instance is nullptr or is not InitFlag.");
@@ -693,7 +693,7 @@ Status Session::GetVariables(const std::vector<std::string> &var_names, std::vec
 
 // Get Variables
 Status Session::GetVariables(const std::vector<AscendString> &var_names, std::vector<Tensor> &var_values) {
-  ErrorManager::GetInstance().SetStage(ErrorMessage::kModelExecute, ErrorMessage::kModelExecute);
+  ErrorManager::GetInstance().SetStage(error_message::kModelExecute, error_message::kModelExecute);
   ErrorManager::GetInstance().GenWorkStreamIdDefault();
   auto instance_ptr = ge::GELib::GetInstance();
   if (instance_ptr == nullptr || !instance_ptr->InitFlag()) {

ge/client/proto/ge_ir.proto

@@ -31,6 +31,8 @@ enum DataType
     DT_STRING_REF = 24;        // string_ref type
     DT_DUAL      = 25;              /**< dual output type */
     DT_VARIANT = 26;           // variant type
+    DT_BF16 = 27;              // bf16 type
+    DT_INT4 = 28;              // int4 type
 }
 
 message AttrDef

ge/common/auth/file_saver.cc

@@ -48,8 +48,8 @@ Status FileSaver::OpenFile(int32_t &fd, const std::string &file_path) {
   fd = mmOpen2(real_path, M_RDWR | M_CREAT | O_TRUNC, mode);
   if (fd == EN_INVALID_PARAM || fd == EN_ERROR) {
     // -1: Failed to open file; - 2: Illegal parameter
-    GELOGE(FAILED, "[Open][File]Failed. mmpa_errno = %d, %s", fd, strerror(errno));
-    REPORT_INNER_ERROR("E19999", "Open file failed, mmpa_errno = %d, error:%s.",
+    GELOGE(FAILED, "[Open][File]Failed. errno:%d, errmsg:%s", fd, strerror(errno));
+    REPORT_INNER_ERROR("E19999", "Open file failed, errno:%d, errmsg:%s.",
                        fd, strerror(errno));
     return FAILED;
   }
@@ -67,9 +67,9 @@ Status FileSaver::WriteData(const void *data, uint32_t size, int32_t fd) {
     while (size > size_1g) {
       write_count = mmWrite(fd, reinterpret_cast<void *>(seek), size_1g);
       if (write_count == EN_INVALID_PARAM || write_count == EN_ERROR) {
-        GELOGE(FAILED, "[Write][Data]Failed, mmpa_errorno = %ld, error:%s",
+        GELOGE(FAILED, "[Write][Data]Failed, errno:%ld, errmsg:%s",
                write_count, strerror(errno));
-        REPORT_INNER_ERROR("E19999", "Write data failed, mmpa_errorno = %ld, error:%s.",
+        REPORT_INNER_ERROR("E19999", "Write data failed, errno:%ld, errmsg:%s.",
                            write_count, strerror(errno));
         return FAILED;
       }

ge/common/debug/memory_dumper.cc

@@ -59,17 +59,17 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status MemoryDumper::DumpToFile
   int32_t mmpa_ret = mmWrite(fd, data, len);
   // mmWrite return -1:Failed to write data to file；return -2:Invalid parameter
   if (mmpa_ret == EN_ERROR || mmpa_ret == EN_INVALID_PARAM) {
-    GELOGE(FAILED, "[Write][Data]Failed, errno = %d, error:%s", mmpa_ret, strerror(errno));
-    REPORT_INNER_ERROR("E19999", "Write data failed, errno = %d, error:%s.",
+    GELOGE(FAILED, "[Write][Data]Failed, errno:%d, errmsg:%s", mmpa_ret, strerror(errno));
+    REPORT_INNER_ERROR("E19999", "Write data failed, errno:%d, errmsg:%s.",
                        mmpa_ret, strerror(errno));
     ret = FAILED;
   }
 
   // Close the file
   if (mmClose(fd) != EN_OK) {  // mmClose return 0: success
-    GELOGE(FAILED, "[Close][File]Failed, error_code:%u, filename:%s.", ret, filename);
-    REPORT_INNER_ERROR("E19999", "Close file failed, error_code:%u, filename:%s.",
-                       ret, filename);
+    GELOGE(FAILED, "[Close][File]Failed, error_code:%u, filename:%s errmsg:%s.", ret, filename, strerror(errno));
+    REPORT_INNER_ERROR("E19999", "Close file failed, error_code:%u, filename:%s errmsg:%s.",
+                       ret, filename, strerror(errno));
     ret = FAILED;
   }
 
@@ -111,8 +111,8 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status MemoryDumper::Dump(void
   int32_t mmpa_ret = mmWrite(fd_, data, len);
   // mmWrite return -1:failed to write data to file；return -2:invalid parameter
   if (mmpa_ret == EN_ERROR || mmpa_ret == EN_INVALID_PARAM) {
-    GELOGE(FAILED, "[Write][Data]Failed, errno = %d, error:%s", mmpa_ret, strerror(errno));
-    REPORT_INNER_ERROR("E19999", "Write data to file failed, errno = %d, error:%s.",
+    GELOGE(FAILED, "[Write][Data]Failed, errno:%d, errmsg:%s", mmpa_ret, strerror(errno));
+    REPORT_INNER_ERROR("E19999", "Write data to file failed, errno:%d, errmsg:%s.",
                        mmpa_ret, strerror(errno));
     return FAILED;
   }
@@ -128,7 +128,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status MemoryDumper::Dump(void
 void MemoryDumper::Close() noexcept {
   // Close file
   if (fd_ != kInvalidFd && mmClose(fd_) != EN_OK) {
-    GELOGW("Close file failed.");
+    GELOGW("Close file failed, errmsg:%s.", strerror(errno));
   }
   fd_ = kInvalidFd;
 }
@@ -151,7 +151,7 @@ int MemoryDumper::OpenFile(const char *filename) {
     GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(prefix_path.length() >= MMPA_MAX_PATH,
         return kInvalidFd, "Prefix path is too long!");
     GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(mmRealPath(prefix_path.c_str(), tmp_path, MMPA_MAX_PATH) != EN_OK, return kInvalidFd,
-                                   "Dir %s does not exit.", prefix_path.c_str());
+                                   "Dir %s does not exit, errmsg:%s.", prefix_path.c_str(), strerror(errno));
     real_path = std::string(tmp_path) + last_path;)
   GE_IF_BOOL_EXEC(
     path_split_pos == -1 || path_split_pos == 0,
@@ -164,9 +164,9 @@ int MemoryDumper::OpenFile(const char *filename) {
   // Using the O_EXCL, if the file already exists,return failed to avoid privilege escalation vulnerability.
   mmMode_t mode = M_IRUSR | M_IWUSR;
 
-  int32_t fd = mmOpen2(real_path.c_str(), M_RDWR | M_CREAT | O_TRUNC, mode);
+  int32_t fd = mmOpen2(real_path.c_str(), M_RDWR | M_CREAT | M_APPEND, mode);
   if (fd == EN_ERROR || fd == EN_INVALID_PARAM) {
-    GELOGE(kInvalidFd, "[Open][File]Failed. errno = %d, error:%s, filename:%s.",
+    GELOGE(kInvalidFd, "[Open][File]Failed. errno:%d, errmsg:%s, filename:%s.",
            fd, strerror(errno), filename);
     return kInvalidFd;
   }

ge/common/dump/dump_op.cc

@@ -26,7 +26,7 @@
 #include "graph/op_desc.h"
 #include "graph/utils/tensor_utils.h"
 #include "proto/ge_ir.pb.h"
-#include "proto/op_mapping_info.pb.h"
+#include "proto/op_mapping.pb.h"
 #include "runtime/mem.h"
 #include "aicpu/common/aicpu_task_struct.h"
 
@@ -64,7 +64,7 @@ void DumpOp::SetDynamicModelInfo(const string &dynamic_model_name, const string
 }
 
 static void SetOpMappingLoopAddr(uintptr_t step_id, uintptr_t loop_per_iter, uintptr_t loop_cond,
-                                 aicpu::dump::OpMappingInfo &op_mapping_info) {
+                                 toolkit::aicpu::dump::OpMappingInfo &op_mapping_info) {
   if (step_id != 0) {
     GELOGI("step_id exists.");
     op_mapping_info.set_step_id_addr(static_cast<uint64_t>(step_id));
@@ -87,11 +87,11 @@ static void SetOpMappingLoopAddr(uintptr_t step_id, uintptr_t loop_per_iter, uin
   }
 }
 
-Status DumpOp::DumpOutput(aicpu::dump::Task &task) {
+Status DumpOp::DumpOutput(toolkit::aicpu::dump::Task &task) {
   GELOGI("Start dump output in Launch dump op");
   const auto &output_descs = op_desc_->GetAllOutputsDesc();
   for (size_t i = 0; i < output_descs.size(); ++i) {
-    aicpu::dump::Output output;
+    toolkit::aicpu::dump::Output output;
     output.set_data_type(static_cast<int32_t>(DataTypeUtil::GetIrDataType(output_descs.at(i).GetDataType())));
     output.set_format(static_cast<int32_t>(output_descs.at(i).GetFormat()));
     for (auto dim : output_descs.at(i).GetShape().GetDims()) {
@@ -116,11 +116,11 @@ Status DumpOp::DumpOutput(aicpu::dump::Task &task) {
   return SUCCESS;
 }
 
-Status DumpOp::DumpInput(aicpu::dump::Task &task) {
+Status DumpOp::DumpInput(toolkit::aicpu::dump::Task &task) {
   GELOGI("Start dump input in Launch dump op");
   const auto &input_descs = op_desc_->GetAllInputsDesc();
   for (size_t i = 0; i < input_descs.size(); ++i) {
-    aicpu::dump::Input input;
+    toolkit::aicpu::dump::Input input;
     input.set_data_type(static_cast<int32_t>(DataTypeUtil::GetIrDataType(input_descs.at(i).GetDataType())));
     input.set_format(static_cast<int32_t>(input_descs.at(i).GetFormat()));
 
@@ -155,7 +155,7 @@ void DumpOp::SetDumpInfo(const DumpProperties &dump_properties, const OpDescPtr
   stream_ = stream;
 }
 
-Status DumpOp::ExecutorDumpOp(aicpu::dump::OpMappingInfo &op_mapping_info) {
+Status DumpOp::ExecutorDumpOp(toolkit::aicpu::dump::OpMappingInfo &op_mapping_info) {
   std::string proto_msg;
   size_t proto_size = op_mapping_info.ByteSizeLong();
   bool ret = op_mapping_info.SerializeToString(&proto_msg);
@@ -216,7 +216,11 @@ Status DumpOp::ExecutorDumpOp(aicpu::dump::OpMappingInfo &op_mapping_info) {
   return SUCCESS;
 }
 
-Status DumpOp::SetDumpModelName(aicpu::dump::OpMappingInfo &op_mapping_info) {
+Status DumpOp::SetDumpModelName(toolkit::aicpu::dump::OpMappingInfo &op_mapping_info) {
+  if (dynamic_model_name_.empty() && dynamic_om_name_.empty()) {
+    GELOGI("Single op dump, no need set model name");
+    return SUCCESS;
+  }
   std::set<std::string> model_list = dump_properties_.GetAllDumpModel();
   bool not_find_by_omname = model_list.find(dynamic_om_name_) == model_list.end();
   bool not_find_by_modelname = model_list.find(dynamic_model_name_) == model_list.end();
@@ -232,7 +236,7 @@ Status DumpOp::SetDumpModelName(aicpu::dump::OpMappingInfo &op_mapping_info) {
     }
   }
   if (!dump_model_name.empty() && dump_properties_.IsDumpOpen()) {
-    GELOGD("Dump model name is %s", dump_model_name.c_str());
+    GELOGI("Dump model name is %s", dump_model_name.c_str());
     op_mapping_info.set_model_name(dump_model_name);
   }
   return SUCCESS;
@@ -252,7 +256,7 @@ Status DumpOp::LaunchDumpOp() {
     REPORT_INNER_ERROR("E19999","Check device_id %d failed", device_id);
     return ACL_ERROR_GE_INTERNAL_ERROR;
   }
-  aicpu::dump::OpMappingInfo op_mapping_info;
+  toolkit::aicpu::dump::OpMappingInfo op_mapping_info;
   auto dump_path = dump_properties_.GetDumpPath() + std::to_string(device_id) + "/";
   op_mapping_info.set_dump_path(dump_path);
   op_mapping_info.set_flag(kAicpuLoadFlag);
@@ -271,7 +275,7 @@ Status DumpOp::LaunchDumpOp() {
   if (rt_ret != RT_ERROR_NONE) {
     GELOGW("call rtGetTaskIdAndStreamID failed, ret = 0x%X", rt_ret);
   }
-  aicpu::dump::Task task;
+  toolkit::aicpu::dump::Task task;
   task.set_task_id(task_id);
   task.set_stream_id(stream_id);
   task.mutable_op()->set_op_name(op_desc_->GetName());

ge/common/dump/dump_op.h

@@ -21,7 +21,7 @@
 
 #include "common/ge_inner_error_codes.h"
 #include "common/properties_manager.h"
-#include "proto/op_mapping_info.pb.h"
+#include "proto/op_mapping.pb.h"
 #include "runtime/stream.h"
 
 namespace ge {
@@ -37,10 +37,10 @@ class DumpOp {
   void SetDynamicModelInfo(const string &dynamic_model_name, const string &dynamic_om_name, uint32_t dynamic_model_id);
 
  private:
-  Status ExecutorDumpOp(aicpu::dump::OpMappingInfo &op_mapping_info);
-  Status DumpOutput(aicpu::dump::Task &task);
-  Status DumpInput(aicpu::dump::Task &task);
-  Status SetDumpModelName(aicpu::dump::OpMappingInfo &op_mapping_info);
+  Status ExecutorDumpOp(toolkit::aicpu::dump::OpMappingInfo &op_mapping_info);
+  Status DumpOutput(toolkit::aicpu::dump::Task &task);
+  Status DumpInput(toolkit::aicpu::dump::Task &task);
+  Status SetDumpModelName(toolkit::aicpu::dump::OpMappingInfo &op_mapping_info);
 
   DumpProperties dump_properties_;
   OpDescPtr op_desc_;

ge/common/dump/exception_dumper.cc

@@ -0,0 +1,241 @@
+/**
+ * Copyright 2019-2021 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "common/dump/exception_dumper.h"
+
+#include "common/ge/datatype_util.h"
+#include "common/debug/memory_dumper.h"
+#include "framework/common/debug/log.h"
+#include "graph/manager/util/debug.h"
+#include "graph/utils/tensor_utils.h"
+#include "graph/load/model_manager/model_utils.h"
+#include "proto/dump_task.pb.h"
+
+namespace {
+static uint64_t GetNowTime() {
+  uint64_t ret = 0;
+  mmTimeval tv;
+  if (mmGetTimeOfDay(&tv, nullptr) == 0) {
+    ret = tv.tv_sec * 1000000ULL + tv.tv_usec;
+  }
+
+  return ret;
+}
+
+static void ReplaceStringElem(std::string &str) {
+  for_each(str.begin(), str.end(), [](char &ch) {
+    if ((ch == ' ') || (ch == '.') || (ch == '/') || (ch == '\\')) {
+      ch = '_';
+    }
+  });
+}
+
+static void SetDumpData(const ge::OpDescInfo &op_desc_info, toolkit::dump::DumpData &dump_data) {
+  dump_data.set_version("2.0");
+  dump_data.set_dump_time(GetNowTime());
+  dump_data.set_op_name(op_desc_info.op_name);
+  for (size_t i = 0; i < op_desc_info.input_format.size(); ++i) {
+    toolkit::dump::OpInput input;
+    input.set_data_type(toolkit::dump::OutputDataType(
+        ge::DataTypeUtil::GetIrDataType(op_desc_info.input_data_type[i])));
+    input.set_format(toolkit::dump::OutputFormat(op_desc_info.input_format[i]));
+    for (auto dim : op_desc_info.input_shape[i]) {
+      input.mutable_shape()->add_dim(dim);
+    }
+    input.set_size(op_desc_info.input_size[i]);
+    GELOGI("[Set][DumpData] The input size int exception is %ld", op_desc_info.input_size[i]);
+    dump_data.mutable_input()->Add(std::move(input));
+  }
+
+  for (size_t j = 0; j < op_desc_info.output_format.size(); ++j) {
+    toolkit::dump::OpOutput output;
+    output.set_data_type(toolkit::dump::OutputDataType(
+        ge::DataTypeUtil::GetIrDataType(op_desc_info.output_data_type[j])));
+    output.set_format(toolkit::dump::OutputFormat(op_desc_info.output_format[j]));
+    for (auto dim : op_desc_info.output_shape[j]) {
+      output.mutable_shape()->add_dim(dim);
+    }
+    output.set_size(op_desc_info.output_size[j]);
+    GELOGI("[Set][DumpData] The output size int exception is %ld", op_desc_info.output_size[j]);
+    dump_data.mutable_output()->Add(std::move(output));
+  }
+}
+}  // namespace
+
+namespace ge {
+ExceptionDumper::~ExceptionDumper() {}
+
+void ExceptionDumper::SaveDumpOpInfo(const OpDescPtr &op, uint32_t task_id, uint32_t stream_id,
+                                     vector<void *> &input_addrs, vector<void *> &output_addrs) {
+  OpDescInfo op_desc_info;
+  SaveOpDescInfo(op, task_id, stream_id, op_desc_info);
+  op_desc_info.input_addrs = input_addrs;
+  op_desc_info.output_addrs = output_addrs;
+  op_desc_info_.emplace_back(std::move(op_desc_info));
+}
+
+void ExceptionDumper::SaveDumpOpInfo(const RuntimeParam &model_param, const OpDescPtr &op,
+                                     uint32_t task_id, uint32_t stream_id) {
+  OpDescInfo op_desc_info;
+  SaveOpDescInfo(op, task_id, stream_id, op_desc_info);
+  op_desc_info.input_addrs = ModelUtils::GetInputDataAddrs(model_param, op);
+  op_desc_info.output_addrs = ModelUtils::GetOutputDataAddrs(model_param, op);
+  op_desc_info_.emplace_back(std::move(op_desc_info));
+}
+
+void ExceptionDumper::SaveOpDescInfo(const OpDescPtr &op, uint32_t task_id, uint32_t stream_id,
+                                     OpDescInfo &op_desc_info) {
+  if (op == nullptr) {
+    GELOGW("[Save][OpExceptionInfo] op desc ptr is null.");
+    return;
+  }
+  GELOGD("[Save][OpExceptionInfo] Start to save dump op [%s] info of task_id: %u, stream_id: %u",
+         op->GetName().c_str(), task_id, stream_id);
+  op_desc_info.op_name = op->GetName();
+  op_desc_info.op_type = op->GetType();
+  op_desc_info.task_id = task_id;
+  op_desc_info.stream_id = stream_id;
+  for (size_t i = 0; i < op->GetAllInputsSize(); ++i) {
+    GeTensorDescPtr input_tensor_desc = op->MutableInputDesc(i);
+    if (input_tensor_desc == nullptr) {
+      continue;
+    }
+    op_desc_info.input_format.emplace_back(input_tensor_desc->GetFormat());
+    op_desc_info.input_shape.emplace_back(input_tensor_desc->GetShape().GetDims());
+    op_desc_info.input_data_type.emplace_back(input_tensor_desc->GetDataType());
+    int64_t input_size = 0;
+
+    if (TensorUtils::GetTensorSizeInBytes(*input_tensor_desc, input_size) != SUCCESS) {
+      GELOGW("[Save][OpExceptionInfo] Op [%s] get input size failed.", op->GetName().c_str());
+      return;
+    }
+    GELOGD("[Save][OpExceptionInfo] Save dump op info, the input size is %ld", input_size);
+    op_desc_info.input_size.emplace_back(input_size);
+  }
+  for (size_t j = 0; j < op->GetOutputsSize(); ++j) {
+    GeTensorDescPtr output_tensor_desc = op->MutableOutputDesc(j);
+    if (output_tensor_desc == nullptr) {
+      continue;
+    }
+    op_desc_info.output_format.emplace_back(output_tensor_desc->GetFormat());
+    op_desc_info.output_shape.emplace_back(output_tensor_desc->GetShape().GetDims());
+    op_desc_info.output_data_type.emplace_back(output_tensor_desc->GetDataType());
+    int64_t output_size = 0;
+    if (TensorUtils::GetTensorSizeInBytes(*output_tensor_desc, output_size) != SUCCESS) {
+      GELOGW("[Save][OpExceptionInfo] Op [%s] get output size failed.", op->GetName().c_str());
+      return;
+    }
+    GELOGD("[Save][OpExceptionInfo] Save dump op info, the output size is %ld.", output_size);
+    op_desc_info.output_size.emplace_back(output_size);
+  }
+}
+
+Status ExceptionDumper::DumpExceptionInfo(const std::vector<rtExceptionInfo> &exception_infos) const {
+  GELOGI("[Dump][Exception] Start to dump exception info");
+  for (const rtExceptionInfo &iter : exception_infos) {
+    OpDescInfo op_desc_info;
+    if (GetOpDescInfo(iter.streamid, iter.taskid, op_desc_info)) {
+      toolkit::dump::DumpData dump_data;
+      SetDumpData(op_desc_info, dump_data);
+      uint64_t now_time = GetNowTime();
+      std::string op_name = op_desc_info.op_name;
+      std::string op_type = op_desc_info.op_type;
+      ReplaceStringElem(op_name);
+      ReplaceStringElem(op_type);
+      string dump_file_path =
+        "./" + op_type + "." + op_name + "." + std::to_string(op_desc_info.task_id) + "." + std::to_string(now_time);
+      GELOGI("[Dump][Exception] The exception dump file path is %s", dump_file_path.c_str());
+
+      uint64_t proto_size = dump_data.ByteSizeLong();
+      std::unique_ptr<char[]> proto_msg(new (std::nothrow) char[proto_size]);
+      bool ret = dump_data.SerializeToArray(proto_msg.get(), proto_size);
+      if (!ret || proto_size == 0) {
+        REPORT_INNER_ERROR("E19999", "Serialize proto to string fail");
+        GELOGE(PARAM_INVALID, "[Dump][Exception] Dump data proto serialize failed");
+        return PARAM_INVALID;
+      }
+
+      GE_CHK_STATUS_RET(MemoryDumper::DumpToFile(dump_file_path.c_str(), &proto_size, sizeof(uint64_t)),
+                        "Failed to dump proto size");
+      GE_CHK_STATUS_RET(MemoryDumper::DumpToFile(dump_file_path.c_str(), proto_msg.get(), proto_size),
+                        "Failed to dump proto msg");
+      if (DumpExceptionInput(op_desc_info, dump_file_path) != SUCCESS) {
+        GELOGE(PARAM_INVALID, "[Dump][Exception] Dump exception input failed");
+        return PARAM_INVALID;
+      }
+
+      if (DumpExceptionOutput(op_desc_info, dump_file_path) != SUCCESS) {
+        GELOGE(PARAM_INVALID, "[Dump][Exception] Dump exception output failed");
+        return PARAM_INVALID;
+      }
+      GELOGI("[Dump][Exception] Dump exception info SUCCESS");
+    } else {
+      GELOGE(PARAM_INVALID, "[Dump][Exception] Get op desc info failed,task id:%u,stream id:%u",
+             iter.taskid, iter.streamid);
+      return PARAM_INVALID;
+    }
+  }
+  return SUCCESS;
+}
+
+bool ExceptionDumper::GetOpDescInfo(uint32_t stream_id, uint32_t task_id, OpDescInfo &op_desc_info) const {
+  GELOGI("[Get][OpDescInfo] There are %zu op need to dump.", op_desc_info_.size());
+  for (size_t index = 0; index < op_desc_info_.size(); ++index) {
+    OpDescInfo dump_op_info = op_desc_info_.at(index);
+    if (dump_op_info.task_id == task_id && dump_op_info.stream_id == stream_id) {
+      GELOGI("[Get][OpDescInfo] Find exception op [%s] of task_id: %u, stream_id: %u.",
+             dump_op_info.op_name.c_str(), task_id, stream_id);
+      op_desc_info = dump_op_info;
+      return true;
+    }
+  }
+  return false;
+}
+
+Status ExceptionDumper::DumpExceptionInput(const OpDescInfo &op_desc_info, const string &dump_file) const {
+  GELOGI("[Dump][ExceptionInput] Start to dump exception input");
+  for (size_t i = 0; i < op_desc_info.input_addrs.size(); i++) {
+    if (Debug::DumpDevMem(dump_file.data(), op_desc_info.input_addrs.at(i), op_desc_info.input_size.at(i)) != SUCCESS) {
+      GELOGE(PARAM_INVALID, "[Dump][ExceptionInput] Dump the %zu input data of op [%s] failed",
+             i, op_desc_info.op_name.c_str());
+      return PARAM_INVALID;
+    }
+  }
+  return SUCCESS;
+}
+
+Status ExceptionDumper::DumpExceptionOutput(const OpDescInfo &op_desc_info, const string &dump_file) const {
+  GELOGI("[Dump][ExceptionOutput] Start to dump exception output");
+  for (size_t i = 0; i < op_desc_info.output_addrs.size(); i++) {
+    if (Debug::DumpDevMem(dump_file.data(), op_desc_info.output_addrs.at(i), op_desc_info.output_size.at(i)) !=
+        SUCCESS) {
+      GELOGE(PARAM_INVALID, "[Dump][ExceptionInput] Dump the %zu input data of op [%s] failed",
+             i, op_desc_info.op_name.c_str());
+      return PARAM_INVALID;
+    }
+  }
+  return SUCCESS;
+}
+
+OpDescInfo *ExceptionDumper::MutableOpDescInfo(uint32_t task_id, uint32_t stream_id) {
+  for (OpDescInfo &op_desc_info : op_desc_info_) {
+    if (op_desc_info.task_id == task_id && op_desc_info.stream_id == stream_id) {
+      return &op_desc_info;
+    }
+  }
+  return nullptr;
+}
+}  // namespace ge

ge/common/dump/exception_dumper.h

@@ -0,0 +1,48 @@
+/**
+ * Copyright 2019-2021 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef GE_COMMON_DUMP_EXCEPTION_DUMPER_H_
+#define GE_COMMON_DUMP_EXCEPTION_DUMPER_H_
+
+#include <vector>
+
+#include "graph/op_desc.h"
+#include "framework/common/ge_types.h"
+#include "graph/load/model_manager/task_info/task_info.h"
+
+namespace ge {
+class ExceptionDumper {
+ public:
+  ExceptionDumper() = default;
+  ~ExceptionDumper();
+
+  void SaveDumpOpInfo(const OpDescPtr &op, uint32_t task_id, uint32_t stream_id,
+                      std::vector<void *> &input_addrs, std::vector<void *> &output_addrs);
+  void SaveDumpOpInfo(const RuntimeParam &model_param, const OpDescPtr &op, uint32_t task_id, uint32_t stream_id);
+  Status DumpExceptionInfo(const std::vector<rtExceptionInfo> &exception_infos) const;
+  bool GetOpDescInfo(uint32_t stream_id, uint32_t task_id, OpDescInfo &op_desc_info) const;
+  OpDescInfo *MutableOpDescInfo(uint32_t task_id, uint32_t stream_id);
+
+ private:
+  void SaveOpDescInfo(const OpDescPtr &op, uint32_t task_id, uint32_t stream_id, OpDescInfo &op_desc_info);
+  Status DumpExceptionInput(const OpDescInfo &op_desc_info, const std::string &dump_file) const;
+  Status DumpExceptionOutput(const OpDescInfo &op_desc_info, const std::string &dump_file) const;
+
+  std::vector<OpDescInfo> op_desc_info_;
+};
+}  // namespace ge
+
+#endif // GE_COMMON_DUMP_EXCEPTION_DUMPER_H_

ge/common/formats/format_transfers/format_transfer_nc1hwc0_nhwc.cc

@@ -66,7 +66,7 @@ Status CheckArgsForNc1hwc0ToNhwc(const TransArgs &args) {
   if (c0 <= 0) {
     GELOGE(ACL_ERROR_GE_DATATYPE_INVALID, "[Get][Cube]Failed, the data type %s is invalid",
            TypeUtils::DataTypeToSerialString(args.src_data_type).c_str());
-    REPORT_CALL_ERROR("E19999", "Failed to get cube size， the data type %s is invalid",
+    REPORT_CALL_ERROR("E19999", "Failed to get cube size, the data type %s is invalid",
                       TypeUtils::DataTypeToSerialString(args.src_data_type).c_str());
     return ACL_ERROR_GE_DATATYPE_INVALID;
   }
@@ -175,7 +175,8 @@ Status FormatTransferNc1hwc0Nhwc::TransFormat(const TransArgs &args, TransResult
                       ShapeToString(args.dst_shape).c_str(), ShapeToString(args.src_shape).c_str());
     return ACL_ERROR_GE_SHAPE_INVALID;
   }
-  GELOGD("Begin to trans format from NC1HWC0 to NCHW, src shape %s, data type %s, dst shape %s, memory size %ld",
+  GELOGD("[Trans][Format]Begin to trans format from NC1HWC0 to NCHW, "
+         "src shape %s, data type %s, dst shape %s, memory size %ld",
          ShapeToString(args.src_shape).c_str(), TypeUtils::DataTypeToSerialString(args.src_data_type).c_str(),
          ShapeToString(args.dst_shape).c_str(), total_size);
 

ge/common/formats/format_transfers/format_transfer_nchw_fz_c04.cc

@@ -189,9 +189,9 @@ Status TransFormatFromNchwToFzC04(const TransArgs &args, TransResult &result) {
     ret = memcpy_s(p_d + k * stride, protectSize, p_s + k * block, block);
     if (ret != EOK) {
       GELOGE(ACL_ERROR_GE_MEMORY_OPERATE_FAILED, "[Set][Memcpy]Failed, block %zu, stride %zu, "
-             "protect_size %ld， error_code %d", block, stride, protectSize, ret);
+             "protect_size %ld, error_code %d", block, stride, protectSize, ret);
       REPORT_CALL_ERROR("E19999", "[Set][Memcpy]Failed, block %zu, stride %zu, "
-                        "protect_size %ld， error_code %d", block, stride, protectSize, ret);
+                        "protect_size %ld, error_code %d", block, stride, protectSize, ret);
       return ACL_ERROR_GE_MEMORY_OPERATE_FAILED;
     }
     protectSize = protectSize - block;
@@ -304,8 +304,8 @@ Status PaddingNC(const TransArgs &args, TransArgs &args_tmp, std::shared_ptr<uin
                      p_s + (i * c * h * w + j * h * w) * size, block);
       if (ret != EOK) {
         GELOGE(ACL_ERROR_GE_MEMORY_OPERATE_FAILED, "[Set][Memcpy]Failed, block %zu, "
-               "protect_size %ld， error_code %d", block, protectSize, ret);
-        REPORT_CALL_ERROR("E19999", "[Set][Memcpy]Failed, block %zu, protect_size %ld，"
+               "protect_size %ld, error_code %d", block, protectSize, ret);
+        REPORT_CALL_ERROR("E19999", "[Set][Memcpy]Failed, block %zu, protect_size %ld, "
                           "error_code %d", block, protectSize, ret);
         return ACL_ERROR_GE_MEMORY_OPERATE_FAILED;
       }

ge/common/ge/op_tiling_manager.cc

@@ -53,7 +53,7 @@ std::string OpTilingManager::GetPath() {
       ErrorManager::GetInstance().ATCReportErrMessage(
           "E19024", {"env", "value", "situation"}, {"ASCEND_OPP_PATH", opp_path_env, "loading the tiling lib"});
       GELOGE(PARAM_INVALID, "[Load][TilingLib]Failed, as env 'ASCEND_OPP_PATH'[%s] "
-             "is invalid path.", opp_path_env);
+             "is invalid path. errmsg:%s", opp_path_env, strerror(errno));
       return std::string();
     }
     opp_path = resolved_path;
@@ -76,7 +76,7 @@ void OpTilingManager::LoadSo() {
   if (handle_bi == nullptr) {
     const char *error = mmDlerror();
     GE_IF_BOOL_EXEC(error == nullptr, error = "");
-    GELOGW("Failed to dlopen %s!", error);
+    GELOGW("Failed to dlopen %s! errmsg:%s", built_in_tiling_lib.c_str(), error);
   } else {
     handles_[built_in_name] = handle_bi;
   }
@@ -85,7 +85,7 @@ void OpTilingManager::LoadSo() {
   if (handle_ct == nullptr) {
     const char *error = mmDlerror();
     GE_IF_BOOL_EXEC(error == nullptr, error = "");
-    GELOGW("Failed to dlopen %s!", error);
+    GELOGW("Failed to dlopen %s! errmsg:%s", custom_tiling_lib.c_str(), error);
   } else {
     handles_[custom_name] = handle_ct;
   }

ge/common/ge/plugin_manager.cc

@@ -39,7 +39,7 @@ void PluginManager::ClearHandles_() noexcept {
     if (mmDlclose(handle.second) != 0) {
       const char *error = mmDlerror();
       GE_IF_BOOL_EXEC(error == nullptr, error = "");
-      GELOGW("Failed to close handle of %s: %s", handle.first.c_str(), error);
+      GELOGW("Failed to close handle of %s, errmsg:%s", handle.first.c_str(), error);
     }
   }
   handles_.clear();
@@ -50,7 +50,9 @@ PluginManager::~PluginManager() { ClearHandles_(); }
 string PluginManager::GetPath() {
   mmDlInfo dl_info;
   if (mmDladdr(reinterpret_cast<void *>(&PluginManager::GetPath), &dl_info) != EN_OK) {
-    GELOGW("Failed to read the shared library file path!");
+    const char *error = mmDlerror();
+    GE_IF_BOOL_EXEC(error == nullptr, error = "");
+    GELOGW("Failed to read the shared library file path! errmsg:%s", error);
     return string();
   } else {
     GE_IF_BOOL_EXEC(dl_info.dli_fname == nullptr, return string());
@@ -61,7 +63,7 @@ string PluginManager::GetPath() {
       return string();
     }
     if (mmRealPath(so_path.c_str(), path, MMPA_MAX_PATH) != EN_OK) {
-      GELOGW("Failed to get realpath of %s", so_path.c_str());
+      GELOGW("Failed to get realpath of %s, errmsg:%s", so_path.c_str(), strerror(errno));
       return string();
     }
 
@@ -137,18 +139,24 @@ Status PluginManager::LoadSo(const string &path, const vector<string> &func_chec
     for (const auto &func_name : func_check_list) {
       auto real_fn = (void (*)())mmDlsym(handle, const_cast<char *>(func_name.c_str()));
       if (real_fn == nullptr) {
+        const char *error = mmDlerror();
+        GE_IF_BOOL_EXEC(error == nullptr, error = "");
         ErrorManager::GetInstance().ATCReportErrMessage("E19012", {"function", "reason"},
             {"mmDlsym", FmtToStr(func_name) + " is skipped since function" +
             FmtToStr(func_name) + " is not existed!"});
         GELOGE(ACL_ERROR_GE_PLGMGR_PATH_INVALID,
-               "[Check][So]%s is skipped since function %s is not existed!",
-               func_name.c_str(), func_name.c_str());
+               "[Check][So]%s is skipped since function %s is not existed! errmsg:%s",
+               func_name.c_str(), func_name.c_str(), error);
         is_valid = false;
         break;
       }
     }
     if (!is_valid) {
-      GE_LOGE_IF(mmDlclose(handle), "[DLClose][Handle]Failed.");
+      if (mmDlclose(handle) != 0) {
+        const char *error = mmDlerror();
+        GE_IF_BOOL_EXEC(error == nullptr, error = "");
+        GELOGE(FAILED, "[DLClose][Handle]Failed. errmsg:%s", error);
+      }
       continue;
     }
 
@@ -212,21 +220,21 @@ Status PluginManager::Load(const string &path, const vector<string> &func_check_
   GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(path.length() >= MMPA_MAX_PATH, GELOGW("File path is too long!");
                                  return FAILED, "File path is too long!");
   if (mmRealPath(path.c_str(), canonical_path, MMPA_MAX_PATH) != EN_OK) {
-    GELOGW("Failed to get realpath of %s", path.c_str());
+    GELOGW("Failed to get realpath of %s, errmsg:%s", path.c_str(), strerror(errno));
     return SUCCESS;
   }
 
   INT32 is_dir = mmIsDir(canonical_path);
   // Lib plugin path not exist
   if (is_dir != EN_OK) {
-      GELOGW("Invalid path for load: %s", path.c_str());
+      GELOGW("Invalid path for load: %s, errmsg:%s", path.c_str(), strerror(errno));
       return SUCCESS;
   }
 
   mmDirent **entries = nullptr;
   auto ret = mmScandir(canonical_path, &entries, nullptr, nullptr);
   if (ret < EN_OK) {
-      GELOGW("scan dir failed. path = %s, ret = %d", canonical_path, ret);
+      GELOGW("scan dir failed. path = %s, ret = %d, errmsg = %s", canonical_path, ret, strerror(errno));
       return FAILED;
   }
   for (int i = 0; i < ret; ++i) {
@@ -283,13 +291,20 @@ Status PluginManager::Load(const string &path, const vector<string> &func_check_
     for (const auto &func_name : func_check_list) {
       auto real_fn = (void (*)())mmDlsym(handle, const_cast<char *>(func_name.c_str()));
       if (real_fn == nullptr) {
-        GELOGW("The %s is skipped since function %s is not existed!", file_name.c_str(), func_name.c_str());
+        const char *error = mmDlerror();
+        GE_IF_BOOL_EXEC(error == nullptr, error = "");
+        GELOGW("The %s is skipped since function %s is not existed! errmsg:%s",
+               file_name.c_str(), func_name.c_str(), error);
         is_valid = false;
         break;
       }
     }
     if (!is_valid) {
-      GE_LOGE_IF(mmDlclose(handle), "Failed to dlclose.");
+      if (mmDlclose(handle) != 0) {
+        const char *error = mmDlerror();
+        GE_IF_BOOL_EXEC(error == nullptr, error = "");
+        GELOGE(FAILED, "[DLClose][Handle]Failed. errmsg:%s", error);
+      }
       continue;
     }
 

ge/common/ge/plugin_manager.h

@@ -59,7 +59,11 @@ class PluginManager {
     for (const auto &handle : handles_) {
       auto real_fn = (R(*)(Types...))mmDlsym(handle.second, const_cast<char *>(func_name.c_str()));
       if (real_fn == nullptr) {
-        GELOGW("Failed to get function %s in %s!", func_name.c_str(), handle.first.c_str());
+        const char *error = mmDlerror();
+        if (error == nullptr) {
+          error = "";
+        }
+        GELOGW("Failed to get function %s in %s! errmsg:%s", func_name.c_str(), handle.first.c_str(), error);
         return GE_PLGMGR_FUNC_NOT_EXIST;
       } else {
         funcs[handle.first] = real_fn;
@@ -74,7 +78,11 @@ class PluginManager {
       // If the funcName is existed, signature of realFn can be casted to any type
       auto real_fn = (void (*)(Types...))mmDlsym(handle.second, const_cast<char *>(func_name.c_str()));
       if (real_fn == nullptr) {
-        GELOGW("Failed to invoke function %s in %s!", func_name.c_str(), handle.first.c_str());
+        const char *error = mmDlerror();
+        if (error == nullptr) {
+          error = "";
+        }
+        GELOGW("Failed to invoke function %s in %s! errmsg:%s", func_name.c_str(), handle.first.c_str(), error);
         return GE_PLGMGR_INVOKE_FAILED;
       } else {
         real_fn(args...);
@@ -89,7 +97,11 @@ class PluginManager {
       // If the funcName is existed, signature of realFn can be casted to any type
       auto real_fn = (void (*)(T))mmDlsym(handle.second, const_cast<char *>(func_name.c_str()));
       if (real_fn == nullptr) {
-        GELOGW("Failed to invoke function %s in %s!", func_name.c_str(), handle.first.c_str());
+        const char *error = mmDlerror();
+        if (error == nullptr) {
+          error = "";
+        }
+        GELOGW("Failed to invoke function %s in %s! errmsg:%s", func_name.c_str(), handle.first.c_str(), error);
         return GE_PLGMGR_INVOKE_FAILED;
       }
       typename std::remove_reference<T>::type arg_temp;
@@ -114,7 +126,11 @@ class PluginManager {
       // If the funcName is existed, signature of realFn can be casted to any type
       auto real_fn = (T2(*)(T1))mmDlsym(handle.second, const_cast<char *>(func_name.c_str()));
       if (real_fn == nullptr) {
-        GELOGW("Failed to invoke function %s in %s!", func_name.c_str(), handle.first.c_str());
+        const char *error = mmDlerror();
+        if (error == nullptr) {
+          error = "";
+        }
+        GELOGW("Failed to invoke function %s in %s! errmsg:%s", func_name.c_str(), handle.first.c_str(), error);
         return GE_PLGMGR_INVOKE_FAILED;
       } else {
         T2 res = real_fn(arg);
@@ -132,7 +148,11 @@ class PluginManager {
       // If the funcName is existed, signature of realFn can be casted to any type
       auto real_fn = (T(*)())mmDlsym(handle.second, const_cast<char *>(func_name.c_str()));
       if (real_fn == nullptr) {
-        GELOGW("Failed to invoke function %s in %s!", func_name.c_str(), handle.first.c_str());
+        const char *error = mmDlerror();
+        if (error == nullptr) {
+          error = "";
+        }
+        GELOGW("Failed to invoke function %s in %s! errmsg:%s", func_name.c_str(), handle.first.c_str(), error);
         return GE_PLGMGR_INVOKE_FAILED;
       } else {
         T res = real_fn();

ge/common/ge/tbe_plugin_manager.cc

@@ -69,7 +69,9 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status TBEPluginManager::Finali
 string TBEPluginManager::GetPath() {
   mmDlInfo dl_info;
   if (mmDladdr(reinterpret_cast<void *>(&TBEPluginManager::GetPath), &dl_info) != EN_OK) {
-    GELOGW("Failed to read so path!");
+    const char *error = mmDlerror();
+    GE_IF_BOOL_EXEC(error == nullptr, error = "");
+    GELOGW("Failed to read so path! errmsg:%s", error);
     return string();
   } else {
     string so_path = dl_info.dli_fname;
@@ -79,7 +81,7 @@ string TBEPluginManager::GetPath() {
       return string();
     }
     if (mmRealPath(so_path.c_str(), path, MMPA_MAX_PATH) != EN_OK) {
-      GELOGW("Failed to get realpath of %s", so_path.c_str());
+      GELOGW("Failed to get realpath of %s, errmsg:%s", so_path.c_str(), strerror(errno));
       return string();
     }
 
@@ -113,14 +115,14 @@ void TBEPluginManager::FindParserSo(const string &path, vector<string> &file_lis
   INT32 is_dir = mmIsDir(real_path.c_str());
   // Lib plugin path not exist
   if (is_dir != EN_OK) {
-      GELOGW("%s is not a dir.", real_path.c_str());
+      GELOGW("%s is not a dir. errmsg:%s", real_path.c_str(), strerror(errno));
       return;
   }
 
   mmDirent **entries = nullptr;
   auto ret = mmScandir(real_path.c_str(), &entries, nullptr, nullptr);
   if (ret < EN_OK) {
-      GELOGW("scan dir failed. path = %s, ret = %d", real_path.c_str(), ret);
+      GELOGW("scan dir failed. path = %s, ret = %d, errmsg = %s", real_path.c_str(), ret, strerror(errno));
       return;
   }
   for (int i = 0; i < ret; ++i) {

ge/common/helper/model_cache_helper.cc

@@ -441,11 +441,11 @@ Status ModelCacheHelper::SaveJsonToFile(const string &file_name, const Json &jso
   const int FILE_AUTHORITY = 0600;
   int fd = mmOpen2(path.c_str(), M_WRONLY | M_CREAT | O_TRUNC, FILE_AUTHORITY);
   if (fd < 0) {
-    GELOGW("Fail to open the file: %s.", path.c_str());
+    GELOGW("Fail to open the file:%s. errmsg:%s", path.c_str(), strerror(errno));
     return INTERNAL_ERROR;
   }
   if (mmClose(fd) != 0) {
-    GELOGW("Fail to close the file: %s.", path.c_str());
+    GELOGW("Fail to close the file:%s. errmsg:%s", path.c_str(), strerror(errno));
     return INTERNAL_ERROR;
   }
 

ge/common/helper/model_helper.cc

@@ -697,6 +697,7 @@ Status ModelHelper::GenerateGeRootModel(OmFileLoadHelper &om_load_helper) {
       is_first_model = false;
       root_model_->SetRootGraph(GraphUtils::GetComputeGraph(cur_model->GetGraph()));
       root_model_->SetModelId(cur_model->GetModelId());
+      root_model_->SetModelName(cur_model->GetName());
       model_ = cur_model;
       continue;
     }

ge/common/model_saver.cc

@@ -61,7 +61,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ModelSaver::SaveJsonToFi
   int32_t fd = mmOpen2(real_path, M_RDWR | M_CREAT | O_TRUNC, mode);
   if (fd == EN_ERROR || fd == EN_INVALID_PARAM) {
     ErrorManager::GetInstance().ATCReportErrMessage("E19001", {"file", "errmsg"}, {file_path, strerror(errno)});
-    GELOGE(FAILED, "[Open][File]Failed, file %s, error %s", file_path, strerror(errno));
+    GELOGE(FAILED, "[Open][File]Failed, file %s, errmsg:%s", file_path, strerror(errno));
     return FAILED;
   }
   const char *model_char = model_str.c_str();
@@ -72,13 +72,13 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ModelSaver::SaveJsonToFi
     ErrorManager::GetInstance().ATCReportErrMessage(
         "E19004", {"file", "errmsg"}, {file_path, strerror(errno)});
     // Need to both print the error info of mmWrite and mmClose, so return ret after mmClose
-    GELOGE(FAILED, "[Write][Data]Write to file failed. errno = %ld, %s", mmpa_ret, strerror(errno));
+    GELOGE(FAILED, "[Write][Data]To file %s failed. errno %ld, errmsg %s", ,file_path, mmpa_ret, strerror(errno));
     ret = FAILED;
   }
   // Close file
   if (mmClose(fd) != EN_OK) {
-    GELOGE(FAILED, "[Close][File]Failed, file %s", file_path);
-    REPORT_CALL_ERROR("E19999", "Close file %s failed", file_path);
+    REPORT_CALL_ERROR("E19999", "Close file %s failed, errmsg %s", file_path, strerror(errno));
+    GELOGE(FAILED, "Close file failed, file %s, errmsg %s", file_path, strerror(errno));
     ret = FAILED;
   }
   return ret;

ge/common/profiling/profiling_manager.cc

@@ -383,6 +383,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::ReportDa
     reporter_data.deviceId = device_id;
     ret = memcpy_s(reporter_data.tag, MSPROF_ENGINE_MAX_TAG_LEN + 1, tag_name.c_str(), tag_name.size());
     GE_IF_BOOL_EXEC(ret != EOK, GELOGE(ret, "Report data tag [%s] memcpy error!", tag_name.c_str()); return;);
+    std::lock_guard<std::mutex> lock(mutex_report_);
     for (size_t i = 0; i < index; ++i) {
       reporter_data.data = (unsigned char *)data.c_str() + report_max_len * i;
       reporter_data.dataLen = report_max_len;
@@ -403,7 +404,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::ReportDa
     reporter_data.dataLen = data.size();
     ret = memcpy_s(reporter_data.tag, MSPROF_ENGINE_MAX_TAG_LEN + 1, tag_name.c_str(), tag_name.size());
     GE_IF_BOOL_EXEC(ret != EOK, GELOGE(ret, "Report data tag [%s] memcpy error!", tag_name.c_str()); return;);
-
+    std::lock_guard<std::mutex> lock(mutex_report_);
     cb_ret = CallMsprofReport(reporter_data);
     GE_IF_BOOL_EXEC(cb_ret != 0, GELOGE(cb_ret, "Reporter data [%s] failed, ret:%d", tag_name.c_str(), cb_ret);
                     return;);

ge/common/profiling/profiling_manager.h

@@ -118,6 +118,7 @@ class FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ProfilingManager {
   map<uint32_t, DeviceSubsInfo> subs_dev_module_; // key: device_id, value: profiling on module
   uint32_t subscribe_count_;
   std::mutex mutex_;
+  std::mutex mutex_report_;
   MsprofCallback prof_cb_;
   std::string fp_point_;
   std::string bp_point_;

ge/common/proto/ge_ir.proto

@@ -31,6 +31,8 @@ enum DataType
     DT_STRING_REF = 24;        // string_ref type
     DT_DUAL      = 25;              /**< dual output type */
     DT_VARIANT = 26;           // variant type
+    DT_BF16 = 27;              // bf16 type
+    DT_INT4 = 28;              // int4 type
 }
 
 message AttrDef

ge/common/proto/op_mapping_info.proto → ge/common/proto/op_mapping.proto

@@ -1,5 +1,5 @@
 syntax = "proto3";
-package aicpu.dump;
+package toolkit.aicpu.dump;
 
 message Shape {
     repeated uint64 dim = 1;

ge/common/util.cc

@@ -122,7 +122,7 @@ long GetFileLength(const std::string &input_file) {
   GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(
     mmGetFileSize(input_file.c_str(), &file_length) != EN_OK,
     ErrorManager::GetInstance().ATCReportErrMessage("E19001", {"file", "errmsg"}, {input_file, strerror(errno)});
-    return kFileSizeOutLimitedOrOpenFailed, "Open file[%s] failed. %s", input_file.c_str(), strerror(errno));
+    return kFileSizeOutLimitedOrOpenFailed, "Open file[%s] failed. errmsg:%s", input_file.c_str(), strerror(errno));
 
   GE_CHK_BOOL_TRUE_EXEC_WITH_LOG((file_length == 0),
                                  ErrorManager::GetInstance().ATCReportErrMessage("E19015", {"filepath"}, {input_file});
@@ -226,7 +226,8 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY int CreateDirectory(const std::
         if (ret != 0) {
           if (errno != EEXIST) {
             ErrorManager::GetInstance().ATCReportErrMessage("E19006", {"path"}, {directory_path});
-            GELOGW("Can not create directory %s. Make sure the directory exists and writable.", directory_path.c_str());
+            GELOGW("Can not create directory %s. Make sure the directory exists and writable. errmsg:%s",
+                   directory_path.c_str(), strerror(errno));
             return ret;
           }
         }
@@ -237,7 +238,8 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY int CreateDirectory(const std::
   if (ret != 0) {
     if (errno != EEXIST) {
       ErrorManager::GetInstance().ATCReportErrMessage("E19006", {"path"}, {directory_path});
-      GELOGW("Can not create directory %s. Make sure the directory exists and writable.", directory_path.c_str());
+      GELOGW("Can not create directory %s. Make sure the directory exists and writable. errmsg:%s",
+             directory_path.c_str(), strerror(errno));
       return ret;
     }
   }
@@ -310,7 +312,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY bool ReadProtoFromMem(const cha
 FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY uint64_t GetCurrentTimestamp() {
   mmTimeval tv{};
   int ret = mmGetTimeOfDay(&tv, nullptr);
-  GE_LOGE_IF(ret != EN_OK, "Func gettimeofday may failed: ret=%d", ret);
+  GE_LOGE_IF(ret != EN_OK, "Func gettimeofday may failed, ret:%d, errmsg:%s", ret, strerror(errno));
   auto total_use_time = tv.tv_usec + tv.tv_sec * 1000000;  // 1000000: seconds to microseconds
   return static_cast<uint64_t>(total_use_time);
 }
@@ -318,7 +320,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY uint64_t GetCurrentTimestamp()
 FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY uint32_t GetCurrentSecondTimestap() {
   mmTimeval tv{};
   int ret = mmGetTimeOfDay(&tv, nullptr);
-  GE_LOGE_IF(ret != EN_OK, "Func gettimeofday may failed: ret=%d", ret);
+  GE_LOGE_IF(ret != EN_OK, "Func gettimeofday may failed, ret:%d, errmsg:%s", ret, strerror(errno));
   auto total_use_time = tv.tv_sec;  // seconds
   return static_cast<uint32_t>(total_use_time);
 }
@@ -568,7 +570,8 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status CheckPath(const char *pa
 
   INT32 is_dir = mmIsDir(path);
   if (is_dir != EN_OK) {
-    GELOGE(PATH_INVALID, "Open directory %s failed, maybe it is not exit or not a dir", path);
+    GELOGE(PATH_INVALID, "Open directory %s failed, maybe it is not exit or not a dir. errmsg:%s",
+           path, strerror(errno));
     return PATH_INVALID;
   }
 

ge/engine_manager/dnnengine_manager.cc

@@ -435,7 +435,7 @@ Status DNNEngineManager::ReadJsonFile(const std::string &file_path, JsonHandle h
   const char *file = file_path.data();
   if ((mmAccess2(file, M_F_OK)) != EN_OK) {
     if (engines_map_.size() != 0) {
-      GELOGE(FAILED, "The json file %s is not exist, %s", file_path.c_str(), strerror(errno));
+      GELOGE(FAILED, "The json file %s is not exist, errmsg:%s", file_path.c_str(), strerror(errno));
       return FAILED;
     } else {
       GELOGW("The json file %s is not needed.", file_path.c_str());

ge/executor/CMakeLists.txt

@@ -3,7 +3,7 @@ set(PROTO_LIST
     "${METADEF_DIR}/proto/ge_ir.proto"
     "${METADEF_DIR}/proto/insert_op.proto"
     "${METADEF_DIR}/proto/task.proto"
-    "${METADEF_DIR}/proto/op_mapping_info.proto"
+    "${METADEF_DIR}/proto/op_mapping.proto"
     "${METADEF_DIR}/proto/dump_task.proto"
 )
 
@@ -16,13 +16,13 @@ set(SRC_LIST
     "../common/ge/plugin_manager.cc"
     "../common/ge/op_tiling_manager.cc"
     "../common/dump/dump_properties.cc"
+    "../common/dump/exception_dumper.cc"
     "../common/dump/dump_manager.cc"
     "../common/dump/dump_op.cc"
     "../common/dump/opdebug_register.cc"
     "../common/profiling/ge_profiling.cc"
     "../graph/load/graph_loader.cc"
     "../graph/execute/graph_execute.cc"
-    "../omm/csa_interact.cc"
     "../graph/manager/graph_manager_utils.cc"
     "../graph/manager/graph_var_manager.cc"
     "../graph/manager/graph_mem_allocator.cc"
@@ -249,12 +249,11 @@ target_include_directories(ge_executor_shared PRIVATE
 
 target_link_options(ge_executor_shared PRIVATE
     -Wl,-Bsymbolic
+    -Wl,--exclude-libs,ALL
 )
 
 target_link_libraries(ge_executor_shared PRIVATE
     $<BUILD_INTERFACE:intf_pub>
-    msprofiler
-    static_mmpa
     -Wl,--no-as-needed
     ge_common
     runtime

ge/executor/module.mk

@@ -11,7 +11,6 @@ local_ge_executor_src_files :=  \
     ../common/profiling/ge_profiling.cc \
     ../graph/load/graph_loader.cc \
     ../graph/execute/graph_execute.cc \
-    ../omm/csa_interact.cc \
     ../graph/manager/graph_manager_utils.cc \
     ../graph/manager/graph_var_manager.cc \
     ../graph/manager/rdma_pool_allocator.cc \

ge/executor/proto/dump_task.proto

@@ -1,5 +1,5 @@
 syntax = "proto3";
-package toolkit.dumpdata;
+package toolkit.dump;
 
 enum OutputDataType {
     DT_UNDEFINED = 0;

ge/executor/proto/ge_ir.proto

@@ -31,6 +31,8 @@ enum DataType
     DT_STRING_REF = 24;        // string_ref type
     DT_DUAL      = 25;              /**< dual output type */
     DT_VARIANT = 26;           // variant type
+    DT_BF16 = 27;              // bf16 type
+    DT_INT4 = 28;              // int4 type
 }
 
 message AttrDef

ge/executor/proto/op_mapping_info.proto → ge/executor/proto/op_mapping.proto

@@ -1,5 +1,5 @@
 syntax = "proto3";
-package aicpu.dump;
+package toolkit.aicpu.dump;
 
 message Shape {
     repeated uint64 dim = 1;

ge/ge_inference.mk

@@ -4,7 +4,6 @@ COMMON_LOCAL_SRC_FILES := \
     proto/fusion_model.proto \
     proto/optimizer_priority.proto \
     graph/manager/trans_var_data_utils.cc \
-    omm/csa_interact.cc \
     common/fp16_t.cc \
     common/formats/utils/formats_trans_utils.cc \
     common/formats/format_transfers/datatype_transfer.cc \
@@ -73,7 +72,7 @@ BUILER_SRC_FILES := \
     ir_build/attr_options/utils.cc \
     ir_build/attr_options/keep_dtype_option.cc \
     ir_build/attr_options/weight_compress_option.cc \
-    ir_build/atc_ir_common.cc \
+    ir_build/option_utils.cc \
 
 ANALYZER_SRC_FILES:= \
     analyzer/analyzer.cc \

ge/ge_local_engine/engine/host_cpu_engine.cc

@@ -82,7 +82,9 @@ Status GetDataNumber(const GeTensorDesc &out_desc, uint64_t &data_num) {
 void HostCpuEngine::CloseSo() {
   for (auto handle : lib_handles_) {
     if (mmDlclose(handle) != 0) {
-      GELOGW("failed to close handle, message: %s", mmDlerror());
+      const char *error = mmDlerror();
+      error = (error == nullptr) ? "" : error;
+      GELOGW("failed to close handle, message: %s", error);
     }
   }
   lib_handles_.clear();
@@ -284,7 +286,7 @@ Status HostCpuEngine::ListSoFiles(const std::string &base_dir, std::vector<std::
   mmDirent **entries = nullptr;
   auto ret = mmScandir(real_path.c_str(), &entries, RegularFileFilterFn, nullptr);
   if (ret < 0) {
-    GELOGW("scan dir failed. path = %s, ret = %d", real_path.c_str(), ret);
+    GELOGW("scan dir failed. path = %s, ret = %d, errmsg = %s", real_path.c_str(), ret, strerror(errno));
     return INTERNAL_ERROR;
   }
 
@@ -324,7 +326,9 @@ Status HostCpuEngine::LoadLib(const std::string &lib_path) {
   GELOGI("To invoke dlopen on lib: %s", lib_path.c_str());
   auto handle = mmDlopen(lib_path.c_str(), MMPA_RTLD_NOW | MMPA_RTLD_GLOBAL);
   if (handle == nullptr) {
-    GELOGE(INTERNAL_ERROR, "Failed to invoke dlopen. path = %s, error = %s", lib_path.c_str(), mmDlerror());
+    const char *error = mmDlerror();
+    error = (error == nullptr) ? "" : error;
+    GELOGE(INTERNAL_ERROR, "Failed to invoke dlopen. path = %s, error = %s", lib_path.c_str(), error);
     return INTERNAL_ERROR;
   }
 

ge/ge_runner.mk

@@ -256,7 +256,6 @@ LIBGE_LOCAL_SRC_FILES := \
     init/gelib.cc \
     model/ge_model.cc \
     model/ge_root_model.cc \
-    omm/csa_interact.cc \
     opskernel_manager/ops_kernel_manager.cc \
     opskernel_manager/ops_kernel_builder_manager.cc \
     session/inner_session.cc \
@@ -316,7 +315,7 @@ LIBGE_LOCAL_SRC_FILES := \
     ir_build/attr_options/utils.cc \
     ir_build/attr_options/keep_dtype_option.cc \
     ir_build/attr_options/weight_compress_option.cc \
-    ir_build/atc_ir_common.cc \
+    ir_build/option_utils.cc \
 
 LIBCLIENT_LOCAL_SRC_FILES := \
     proto/ge_api.proto \

ge/generator/ge_generator.cc

@@ -377,7 +377,7 @@ Status GeGenerator::Initialize(const map<string, string> &options, OmgContext &o
     return MEMALLOC_FAILED;
   }
 
-  ErrorManager::GetInstance().SetStage(ErrorMessage::kInitialize, ErrorMessage::kOpsProtoInit);
+  ErrorManager::GetInstance().SetStage(error_message::kInitialize, error_message::kOpsProtoInit);
   string opsproto_path;
   GetOpsProtoPath(opsproto_path);
   GELOGI("Get opsproto path is %s", opsproto_path.c_str());
@@ -426,7 +426,7 @@ Status GeGenerator::Initialize(const map<string, string> &options, OmgContext &o
 }
 
 Status GeGenerator::Finalize() {
-  ErrorManager::GetInstance().SetStage(ErrorMessage::kFinalize, ErrorMessage::kFinalize);
+  ErrorManager::GetInstance().SetStage(error_message::kFinalize, error_message::kFinalize);
   GE_CHECK_NOTNULL_EXEC(impl_, return PARAM_INVALID);
   Status ret = impl_->graph_manager_.Finalize();
   if (ret != SUCCESS) {
@@ -438,14 +438,14 @@ Status GeGenerator::Finalize() {
 
 Status GeGenerator::GenerateOfflineModel(const Graph &graph, const string &file_name_prefix,
                                          const vector<GeTensor> &inputs) {
-  ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kOther);
+  ErrorManager::GetInstance().SetStage(error_message::kModelCompile, error_message::kOther);
   GELOGI("Start to generate offline model.");
   ModelBufferData model;
   return GenerateModel(graph, file_name_prefix, inputs, model, true);
 }
 
 Status GeGenerator::GenerateOnlineModel(const Graph &graph, const vector<GeTensor> &inputs, ModelBufferData &model) {
-  ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kOther);
+  ErrorManager::GetInstance().SetStage(error_message::kModelCompile, error_message::kOther);
   return GenerateModel(graph, "online", inputs, model, false);
 }
 
@@ -783,9 +783,7 @@ Status GeGenerator::BuildSingleOp(OpDescPtr &op_desc, const vector<GeTensor> &in
   GELOGD("Inputs size is %zu, outputs size is %zu.", inputs.size(), outputs.size());
   GE_CHECK_NOTNULL_EXEC(impl_, return PARAM_INVALID);
   impl_->is_offline_ = is_offline;
-  if (!is_offline) {
-    (void)AttrUtils::SetBool(op_desc, ATTR_SINGLE_OP_SCENE, true);
-  }
+  (void)AttrUtils::SetBool(op_desc, ATTR_SINGLE_OP_SCENE, true);
 
   if (CheckForSingleOp(op_desc, inputs, outputs) != SUCCESS) {
     GELOGE(PARAM_INVALID, "input param is invalid when build single op!");
@@ -824,7 +822,7 @@ Status GeGenerator::BuildSingleOp(OpDescPtr &op_desc, const vector<GeTensor> &in
     auto node = comp_graph->FindNode(op_desc->GetName());
     Status ret = CheckEngineTypeSupport(node, engine_type);
     if (ret != SUCCESS) {
-      GELOGE(ret, "[Check][EngineType]value:%d for node:%s not support", engine_type, node->GetName().c_str());
+      GELOGE(ret, "[Check][EngineType]not support node:%s with engine of %d.", node->GetName().c_str(), engine_type);
       return ret;
     }
   }
@@ -850,6 +848,11 @@ Status GeGenerator::BuildSingleOp(OpDescPtr &op_desc, const vector<GeTensor> &in
 
   bool all_shape = false;
   (void)AttrUtils::GetBool(op_desc, kAicpuAllshape, all_shape);
+  GELOGD("Node: %s, all_shape is %d, compile_flag is %d.", op_desc->GetName().c_str(), all_shape, compile_flag);
+  (void)AttrUtils::SetInt(ge_model, ATTR_NAME_BUILD_MODE, fuzz_compile_flag);
+  if (all_shape) {
+    (void)AttrUtils::SetBool(ge_model, kAicpuAllshape, all_shape);
+  }
   if (all_shape && CheckNoAicore(root_graph)) {
     GELOGD("Get aicpu all_shape kernel!");
     vector<GeTensor> inputs_dynamic;
@@ -859,8 +862,6 @@ Status GeGenerator::BuildSingleOp(OpDescPtr &op_desc, const vector<GeTensor> &in
     GE_CHK_STATUS_RET_NOLOG(
       impl_->SaveParams(ge_model, op_desc_tmp->GetType(), op_attrs, inputs_dynamic, outputs_dynamic));
   } else if (fuzz_compile_flag) {
-    GELOGD("Get fuzz build result of %s.", op_desc->GetName().c_str());
-    (void)AttrUtils::SetInt(ge_model, ATTR_NAME_BUILD_MODE, fuzz_compile_flag);
     GeAttrValue::LIST_NAMED_ATTRS fuzz_build_attrs;
     if (GetFuzzBuildAttrs(op_desc, ge_root_model, fuzz_build_attrs) != SUCCESS) {
       GELOGE(FAILED, "[Get][FuzzRet]Failed to get fuzz build result of %s.", op_desc->GetName().c_str());
@@ -892,7 +893,7 @@ Status GeGenerator::BuildSingleOp(OpDescPtr &op_desc, const vector<GeTensor> &in
 Status GeGenerator::BuildSingleOpModel(OpDescPtr &op_desc, const vector<GeTensor> &inputs,
                                        const vector<GeTensor> &outputs, const string &model_file_name,
                                        int32_t compile_flag) {
-  ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kOther);
+  ErrorManager::GetInstance().SetStage(error_message::kModelCompile, error_message::kOther);
   GELOGI("Start to build single op offline model, input size: %zu, output size: %zu", inputs.size(), outputs.size());
   ModelBufferData model_buff;
   OpEngineType engine_type = ENGINE_SYS;
@@ -916,7 +917,7 @@ Status GeGenerator::BuildSingleOpModel(OpDescPtr &op_desc, const vector<GeTensor
 Status GeGenerator::BuildSingleOpModel(OpDescPtr &op_desc, const vector<GeTensor> &inputs,
                                        const vector<GeTensor> &outputs, OpEngineType engine_type,
                                        ModelBufferData &model_buff) {
-  ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kOther);
+  ErrorManager::GetInstance().SetStage(error_message::kModelCompile, error_message::kOther);
   GELOGI("Start to build single op online, input size: %zu, output size: %zu", inputs.size(), outputs.size());
   Status status = BuildSingleOp(op_desc, inputs, outputs, kFileNameSuffix, engine_type, model_buff, false);
   GELOGI("Finish build single online model, status: %u", status);
@@ -926,7 +927,7 @@ Status GeGenerator::BuildSingleOpModel(OpDescPtr &op_desc, const vector<GeTensor
 Status GeGenerator::BuildSingleOpModel(OpDescPtr &op_desc, const vector<GeTensor> &inputs,
                                        const vector<GeTensor> &outputs, OpEngineType engine_type, int32_t compile_flag,
                                        ModelBufferData &model_buff) {
-  ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kOther);
+  ErrorManager::GetInstance().SetStage(error_message::kModelCompile, error_message::kOther);
   GELOGI("Start to build single op online, input size: %zu, output size: %zu", inputs.size(), outputs.size());
   Status status = BuildSingleOp(op_desc, inputs, outputs, kFileNameSuffix, engine_type, model_buff, false,
                                 compile_flag);
@@ -1072,7 +1073,7 @@ Status GeGenerator::Impl::BuildModel(const Graph &graph, const vector<GeTensor>
     ret = graph_manager_.BuildGraph(graph_id, inputs, ge_root_model, session_id);
   }
 
-  ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kOther);
+  ErrorManager::GetInstance().SetStage(error_message::kModelCompile, error_message::kOther);
   if (ret != SUCCESS) {
     GELOGE(GE_GENERATOR_GRAPH_MANAGER_BUILD_GRAPH_FAILED, "GraphManager build graph fail, graph id: %u", graph_id);
     ret = GE_GENERATOR_GRAPH_MANAGER_BUILD_GRAPH_FAILED;

ge/graph/build/graph_builder.cc

@@ -233,7 +233,7 @@ Status GraphBuilder::BuildForKnownShapeGraph(ComputeGraphPtr &comp_graph,
     return SUCCESS;
   }
 
-  ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kPreBuild);
+  ErrorManager::GetInstance().SetStage(error_message::kModelCompile, error_message::kPreBuild);
   GELOGI("Begin to build known shape graph[%s].", comp_graph->GetName().c_str());
   Status ret = SecondPartition(comp_graph);
   GE_CHK_STATUS_RET(ret, "Graph[%s] second partition Failed.", comp_graph->GetName().c_str());
@@ -264,7 +264,7 @@ Status GraphBuilder::BuildForKnownShapeGraph(ComputeGraphPtr &comp_graph,
   GE_TIMESTAMP_END(BuildModelForGetTask, "GraphBuilder::BuildModelForGetTask");
   GE_DUMP(comp_graph, "AfterBuildModel");
 
-  ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kTaskGenerate);
+  ErrorManager::GetInstance().SetStage(error_message::kModelCompile, error_message::kTaskGenerate);
   GE_TIMESTAMP_START(GetTaskInfo);
   ret = GetTaskInfo(builder, model_ptr, comp_graph, subgraph_map, session_id);
   GE_TIMESTAMP_END(GetTaskInfo, "GraphBuilder::GetTaskInfo");
@@ -274,7 +274,7 @@ Status GraphBuilder::BuildForKnownShapeGraph(ComputeGraphPtr &comp_graph,
     return ret;
   }
 
-  ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kOther);
+  ErrorManager::GetInstance().SetStage(error_message::kModelCompile, error_message::kOther);
   ge_model_ptr = MakeShared<ge::GeModel>();
   if (ge_model_ptr == nullptr) {
     return MEMALLOC_FAILED;
@@ -336,7 +336,7 @@ Status GraphBuilder::SetConstantInputOffset(ComputeGraphPtr &comp_graph) {
 
 Status GraphBuilder::BuildForUnknownShapeGraph(ComputeGraphPtr &comp_graph, GeModelPtr &ge_model_ptr,
                                                uint64_t session_id) {
-  ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kPreBuild);
+  ErrorManager::GetInstance().SetStage(error_message::kModelCompile, error_message::kPreBuild);
   GELOGI("Begin to build unknown shape graph[%s].", comp_graph->GetName().c_str());
   Graph2SubGraphInfoList subgraph_map;
   ge::ModelBuilder builder(session_id, comp_graph, subgraph_map, stream_max_parallel_num_, hcom_parallel_, build_mode_);
@@ -369,11 +369,11 @@ Status GraphBuilder::BuildForUnknownShapeGraph(ComputeGraphPtr &comp_graph, GeMo
   GE_CHK_STATUS_RET(builder.BuildModelForGetDynShapeTask(*model_ptr),
                     "Graph[%s] builder BuildModelForGetDynShapeTask() return fail.", comp_graph->GetName().c_str());
   GE_TIMESTAMP_END(BuildModelForGetDynShapeTask, "GraphBuilder::BuildModelForGetDynShapeTask");
-  ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kTaskGenerate);
+  ErrorManager::GetInstance().SetStage(error_message::kModelCompile, error_message::kTaskGenerate);
   GE_TIMESTAMP_START(GetTaskInfo);
   Status ret = GetTaskInfo(builder, model_ptr, comp_graph, subgraph_map, session_id);
   GE_TIMESTAMP_END(GetTaskInfo, "GraphBuilder::GetTaskInfo");
-  ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kOther);
+  ErrorManager::GetInstance().SetStage(error_message::kModelCompile, error_message::kOther);
 
   GraphUtils::DumpGEGraph(comp_graph, "AfterGetTask");
   GraphUtils::DumpGEGraphToOnnx(*comp_graph, "AfterGetTask");
@@ -395,24 +395,6 @@ Status GraphBuilder::BuildForHostCpuGraph(ComputeGraphPtr &comp_graph, GeModelPt
   return BuildForUnknownShapeGraph(comp_graph, ge_model_ptr, session_id);
 }
 
-static Status InsertMemcpyNode(const ComputeGraphPtr &graph, const OutDataAnchorPtr &out_anchor,
-                               const std::vector<InDataAnchorPtr> &in_anchors, const std::string &name) {
-  GE_CHECK_NOTNULL(out_anchor);
-  NodePtr in_node = out_anchor->GetOwnerNode();
-  GE_CHECK_NOTNULL(in_node);
-  OpDescBuilder op_desc_builder(name, MEMCPYASYNC);
-  OpDescPtr op_desc = op_desc_builder.AddInput("x", in_node->GetOpDesc()->GetOutputDesc(0))
-                                     .AddOutput("y", in_node->GetOpDesc()->GetOutputDesc(0))
-                                     .Build();
-  (void)AttrUtils::SetBool(op_desc, ATTR_NO_NEED_CONSTANT_FOLDING, false);
-  if (GraphUtils::InsertNodeAfter(out_anchor, in_anchors, graph->AddNode(op_desc)) != GRAPH_SUCCESS) {
-    REPORT_CALL_ERROR("E19999", "Insert IDENTITY node %s after %s failed", name.c_str(), in_node->GetName().c_str());
-    GELOGE(FAILED, "Insert IDENTITY node %s after %s failed.", name.c_str(), in_node->GetName().c_str());
-    return FAILED;
-  }
-  return SUCCESS;
-}
-
 Status GraphBuilder::MarkFpBpProfilingTaskAttr(ComputeGraphPtr &com_graph) {
   bool original_unknown_shape_flag = com_graph->GetGraphUnknownFlag();
   com_graph->SetGraphUnknownFlag(false);

ge/graph/build/memory/graph_mem_assigner.cc

@@ -427,6 +427,86 @@ bool IsContinuousInputConflict(const ge::NodePtr &node, const OpDescPtr &peer_op
   return false;
 }
 
+/// op1 -> node -> op2
+/// return true when node is ref from input, and op1 or op2 is reuse input from output
+bool GraphMemoryAssigner::IsRefFromInputOpCascade(const NodePtr &node) {
+  bool ref_from_input = false;
+  int32_t reuse_in_index = -1;
+  for (const auto &out_anchor : node->GetAllOutDataAnchors()) {
+    ref_from_input = GraphUtils::IsRefFromInput(out_anchor, reuse_in_index);
+    if (ref_from_input) {
+      GELOGD("IsRefFromInputOpCascade: cur node:%s:%d is ref", node->GetName().c_str(), reuse_in_index);
+      break;
+    }
+  }
+
+  for (const auto &in_anchor : node->GetAllInDataAnchors()) {
+    const auto &peer_out_anchor = in_anchor->GetPeerOutAnchor();
+    GE_IF_BOOL_EXEC(peer_out_anchor == nullptr, continue);
+    if (ref_from_input && GraphUtils::IsRefFromInput(peer_out_anchor, reuse_in_index)) {
+      GELOGD("IsRefFromInputOpCascade: in node[%s] is ref, reuse index is:%d",
+              peer_out_anchor->GetOwnerNode()->GetName().c_str(), reuse_in_index);
+      return true;
+    }
+  }
+
+  for (const auto &out_anchor : node->GetAllOutDataAnchors()) {
+    const auto &peer_in_anchors = out_anchor->GetPeerInDataAnchors();
+    for (const auto &peer_in_anchor : peer_in_anchors) {
+      auto peer_in_node = peer_in_anchor->GetOwnerNode();
+      GE_IF_BOOL_EXEC(peer_in_node == nullptr, continue);
+      for (const auto &peer_in_node_out_anchor : peer_in_node->GetAllOutDataAnchors()) {
+        if (ref_from_input && GraphUtils::IsRefFromInput(peer_in_node_out_anchor, reuse_in_index)) {
+          GELOGD("IsRefFromInputOpCascade: out node[%s] is ref, reuse index is:%d",
+                 peer_in_node_out_anchor->GetOwnerNode()->GetName().c_str(), reuse_in_index);
+          return true;
+        }
+      }
+    }
+  }
+  return false;
+}
+
+/// node:in0(in0 reuse out0) -> peer_node:out0
+/// update peer_node's 0th output offset with node's 0th output offset
+Status GraphMemoryAssigner::UpdateRefOpOffsetReverse(const NodePtr &node) {
+  map<int32_t, int32_t> out2ins;
+  GE_CHK_STATUS_RET(TryGetNodeRefIndexes(node, out2ins), "[Get][RefIndexes]fail for node:%s",
+                    node->GetName().c_str());
+  auto op_desc = node->GetOpDesc();
+  GE_CHECK_NOTNULL(op_desc);
+  vector<int64_t> output_list = op_desc->GetOutputOffset();
+  for (const auto &out2in : out2ins) {
+    auto reuse_in_anchor = node->GetInDataAnchor(out2in.second);
+    GE_CHECK_NOTNULL(reuse_in_anchor);
+    auto peer_out_anchor = reuse_in_anchor->GetPeerOutAnchor();
+    GE_CHECK_NOTNULL(peer_out_anchor);
+    auto peer_node = peer_out_anchor->GetOwnerNode();
+    GE_CHECK_NOTNULL(peer_node);
+    auto peer_op_desc = peer_node->GetOpDesc();
+    GE_CHECK_NOTNULL(peer_op_desc);
+    vector<int64_t> peer_output_list = peer_op_desc->GetOutputOffset();
+    if ((peer_out_anchor->GetIdx() >= static_cast<int>(peer_output_list.size()))
+        || (out2in.first >= static_cast<int32_t>(output_list.size()))) {
+      GELOGW("out of range, peer_out_anchor:%d, peer_output_list size:%zu, out2in:%d, output_list size:%zu",
+             peer_out_anchor->GetIdx(),
+             peer_output_list.size(),
+             out2in.first,
+             output_list.size());
+      continue;
+    }
+    peer_output_list.at(peer_out_anchor->GetIdx()) = output_list.at(out2in.first);
+    peer_op_desc->SetOutputOffset(peer_output_list);
+    GELOGD("UpdateRefOpOffsetReverse: Node[%s] output[%d] is set from node[%s] output index[%d] offset[%ld]",
+           peer_node->GetName().c_str(),
+           peer_out_anchor->GetIdx(),
+           node->GetName().c_str(),
+           out2in.first,
+           output_list.at(out2in.first));
+  }
+  return SUCCESS;
+}
+
 Status GraphMemoryAssigner::ReAssignContinuousMemory(bool is_loop_graph) {
   Status ret;
   // Stored nodes which need assign continuous input memory in `reverse topo order`
@@ -446,12 +526,16 @@ Status GraphMemoryAssigner::ReAssignContinuousMemory(bool is_loop_graph) {
     }
     // Assign continuous input memory
     bool continuous_input = ((continuous_type & kTypeInput) != 0) || ((continuous_type & kTypeInputNoPadding) != 0);
-    if (continuous_input) {
+    if (IsRefFromInputOpCascade(node)) {
+      nodes_stack.push_back(node);
+      GELOGD("Ref: Push node:%s to stack", node->GetName().c_str());
+    } else if (continuous_input) {
       if (AssignContinuousInputMemoryWithAtomicProcessDirectly(node, node_2_continuous_type)) {
         GE_CHK_STATUS_RET(AssignContinuousInputMemoryWithAtomicProcess(node, continuous_type),
                           "[Assign][Memory:Continuous:Input]fail for node:%s", node->GetName().c_str())
       } else {
         nodes_stack.push_back(node);
+        GELOGD("Continuous: Push node:%s to stack", node->GetName().c_str());
       }
     }
     // Assign continuous output memory
@@ -478,8 +562,13 @@ Status GraphMemoryAssigner::ReAssignContinuousMemory(bool is_loop_graph) {
       GELOGE(FAILED, "[Get][ContinuousType] find fail for node:%s", node->GetName().c_str());
       return FAILED;
     }
-    GE_CHK_STATUS_RET(AssignContinuousInputMemoryWithAtomicProcess(node, iter->second, true),
-                      "[Assign][Memory:Continuous:Input]fail for node:%s.", node->GetName().c_str())
+    if (((iter->second & kTypeInput) != 0) || ((iter->second & kTypeInputNoPadding) != 0)) {
+      GE_CHK_STATUS_RET(AssignContinuousInputMemoryWithAtomicProcess(node, iter->second, true),
+                        "[Assign][Memory:Continuous:Input]fail for node:%s.", node->GetName().c_str())
+    } else {
+      GE_CHK_STATUS_RET(UpdateRefOpOffsetReverse(node),
+                        "[Update][Memory:Reference:Output]fail for node:%s", node->GetName().c_str())
+    }
   }
   for (auto pair : memory_offset_) {
     GELOGD("[Reassign][Memory:Continuous]At last, memory type = %ld, mem offset = %zu", pair.first,
@@ -560,7 +649,7 @@ Status GraphMemoryAssigner::AssignContinuousInputMemory(const ge::NodePtr &node,
     bool is_allocated_first_input = is_continuous_input_allocated && (in_data_anchor->GetIdx() == 0);
     if (is_allocated_first_input) {
       std::map<int32_t, int32_t> out2ins;
-      GE_CHK_STATUS_RET(GetAllRef(node, out2ins), "[Get][AllRef]fail for node: %s", node->GetName().c_str());
+      GE_CHK_STATUS_RET(TryGetNodeRefIndexes(node, out2ins), "[Get][RefIndexes]fail for node: %s", node->GetName().c_str());
       // output is beginning offset, set offset for input; only support this case now
       if ((out2ins.size() == 1) && (out2ins.begin()->second == 0) && (reverse_refresh)) {
         auto peer_output_offset = output_list.at(peer_out_data_anchor->GetIdx());
@@ -1250,10 +1339,47 @@ Status GraphMemoryAssigner::CheckOffset() {
         return FAILED;
       }
     }
+    // check reuse input and output
+    GE_CHK_STATUS_RET(CheckRefNodeOffset(node), "[Check][Offset]fail for node: %s", node->GetName().c_str());
   }
+
   return SUCCESS;
 }
 
+ge::Status GraphMemoryAssigner::CheckRefNodeOffset(const NodePtr &node) {
+  GE_CHECK_NOTNULL(node);
+  std::map<int32_t, int32_t> out2ins;
+  GE_CHK_STATUS_RET(TryGetNodeRefIndexes(node, out2ins), "[Get][RefIndexes]fail for node: %s", node->GetName().c_str());
+  auto opdesc = node->GetOpDesc();
+  GE_CHECK_NOTNULL(opdesc);
+  auto output_list = opdesc->GetOutputOffset();
+  auto input_list = opdesc->GetInputOffset();
+  for (const auto &out2in : out2ins) {
+    auto out_i = out2in.first;
+    if (static_cast<size_t>(out_i) >= output_list.size()) {
+      std::string error = "Node" + FmtToStr(opdesc->GetName()) + "output offset size" +
+                          FmtToStr(output_list.size()) + "should bigger than ref out index" + FmtToStr(out_i);
+      GE_ERRORLOG_AND_ERRORMSG(ge::FAILED, error.c_str());
+      return ge::FAILED;
+    }
+    auto in_i = out2in.second;
+    if (static_cast<size_t>(in_i) >= input_list.size()) {
+      std::string error = "Node" + FmtToStr(opdesc->GetName()) + "input offset size" +
+                          FmtToStr(input_list.size()) + "should bigger than ref input index" + FmtToStr(in_i);
+      GE_ERRORLOG_AND_ERRORMSG(ge::FAILED, error.c_str());
+      return ge::FAILED;
+    }
+    if (output_list[out_i] != input_list[in_i]) {
+      std::string error = "Node" + FmtToStr(opdesc->GetName()) + "input offset " + FmtToStr(input_list[in_i]) +
+                          "should equal to output offset" + FmtToStr(output_list[out_i]) + "with ref in" +
+                          FmtToStr(in_i) + "to output" + FmtToStr(out_i);
+      GE_ERRORLOG_AND_ERRORMSG(ge::FAILED, error.c_str());
+      return ge::FAILED;
+    }
+  }
+  return ge::SUCCESS;
+}
+
 ge::Status GraphMemoryAssigner::SetInputOffset() {
   if (memory_offset_.empty()) {
     REPORT_INNER_ERROR("E19999", "InnerData memory_offset_ empty, not expected, graph_id:%u, graph_name:%s",
@@ -1330,6 +1456,8 @@ ge::Status GraphMemoryAssigner::UpdateOpInputOffset(const NodePtr &node, vector<
   origin_input_list = tmp_op_desc->GetInputOffset();
   int64_t valid_input_index = 0;
   bool has_mem_type_attr = ge::AttrUtils::GetListInt(tmp_op_desc, ATTR_NAME_INPUT_MEM_TYPE_LIST, memory_type);
+  std::map<int32_t, int32_t> out2ins;
+  GE_CHK_STATUS_RET(TryGetNodeRefIndexes(node, out2ins), "[Get][RefIndexes]fail for node: %s", node->GetName().c_str());
   for (const auto &anchor : node->GetAllInDataAnchors()) {
     vector<int64_t> output_list;
     auto peer_out_anchor = anchor->GetPeerOutAnchor();
@@ -1344,23 +1472,30 @@ ge::Status GraphMemoryAssigner::UpdateOpInputOffset(const NodePtr &node, vector<
     output_list = last_peer_out_op_desc->GetOutputOffset();
     auto out_index = static_cast<unsigned long>(peer_out_anchor->GetIdx());
     if (output_list.size() > static_cast<size_t>(out_index)) {
+      bool is_l1_type = false;
       int64_t input_offset = output_list.at(out_index);
       if (has_mem_type_attr && !origin_input_list.empty()) {
         auto input_size = tmp_op_desc->GetInputsSize();
         auto ori_input_offset_list_size = origin_input_list.size();
         auto mem_type_size = memory_type.size();
         if ((input_size != mem_type_size) || (input_size != ori_input_offset_list_size)) {
-            std::string error = "fusion: node" + FmtToStr(tmp_op_desc->GetName()) +
+            std::string error = "Node" + FmtToStr(tmp_op_desc->GetName()) +
                 + " input_size" + FmtToStr(input_size) + " diff from memory_type_size" +
                 FmtToStr(mem_type_size) + " from ori_input_offset_list_size" +
                 FmtToStr(ori_input_offset_list_size);
             GE_ERRORLOG_AND_ERRORMSG(ge::FAILED, error.c_str());
           return ge::FAILED;
         }
-        // not hbm keep orignal inputoffest
-        // hbm inputoffset = original inputoffset + outputoffset
-        input_offset = (memory_type[valid_input_index] == RT_MEMORY_L1 ? origin_input_list[valid_input_index]
-                       : origin_input_list[valid_input_index] + output_list.at(out_index));
+        GELOGD("Node[%s] input[%d] has origin offset[%ld]", tmp_op_desc->GetName().c_str(), anchor->GetIdx(),
+               origin_input_list[valid_input_index]);
+        // L1 keep original input_offset
+        is_l1_type = (memory_type[valid_input_index] == RT_MEMORY_L1);
+        if (is_l1_type) {
+          input_offset = origin_input_list[valid_input_index];
+        } else {
+          // hbm input_offset = original input_offset + output_offset
+          input_offset = origin_input_list[valid_input_index] + output_list.at(out_index);
+        }
       }
       const auto &in_node = GetKnownInputNode(peer_out_anchor->GetOwnerNode());
       if (in_node->GetType() == CONSTANT) {
@@ -1368,12 +1503,13 @@ ge::Status GraphMemoryAssigner::UpdateOpInputOffset(const NodePtr &node, vector<
         GE_CHK_STATUS(TensorUtils::GetDataOffset(tensor_desc, input_offset));
       }
 
-      GELOGD("%s node[%s] input[%ld] is set from node[%s] out index[%lu] offset[%ld]",
-             has_mem_type_attr ? "Fusion" : "",
-             tmp_op_desc->GetName().c_str(),
-             valid_input_index,
-             peer_out_anchor->GetOwnerNode()->GetOpDesc()->GetName().c_str(),
-             out_index,
+      if (!is_l1_type) {
+        // update ref output_offset when input change
+        GE_CHK_STATUS_RET(UpdateRefOpOutputOffset(node, out2ins, anchor->GetIdx(), input_offset),
+                          "[Update][RefOffset]fail for node: %s", node->GetName().c_str());
+      }
+      GELOGD("Node[%s] input[%d] is set from node[%s] out index[%lu] offset[%ld]", tmp_op_desc->GetName().c_str(),
+             anchor->GetIdx(), peer_out_anchor->GetOwnerNode()->GetOpDesc()->GetName().c_str(), out_index,
              input_offset);
       input_list.emplace_back(input_offset);
       valid_input_index++;
@@ -1382,6 +1518,30 @@ ge::Status GraphMemoryAssigner::UpdateOpInputOffset(const NodePtr &node, vector<
   return ge::SUCCESS;
 }
 
+ge::Status GraphMemoryAssigner::UpdateRefOpOutputOffset(const NodePtr &node, const std::map<int32_t, int32_t> &out2ins,
+                                                        const int ref_in, const int64_t input_offset) const {
+  auto opdesc = node->GetOpDesc();
+  GE_CHECK_NOTNULL(opdesc);
+  for (const auto &out2in : out2ins) {
+    auto out_i = out2in.first;
+    auto in_i = out2in.second;
+    if (in_i == ref_in) {
+      auto origin_output_list = opdesc->GetOutputOffset();
+      if (static_cast<size_t>(out_i) >= origin_output_list.size()) {
+        std::string error = "Node" + FmtToStr(opdesc->GetName()) + "output offset size" +
+                            FmtToStr(origin_output_list.size()) + "should bigger than ref out index" + FmtToStr(out_i);
+        GE_ERRORLOG_AND_ERRORMSG(ge::FAILED, error.c_str());
+        return ge::FAILED;
+      }
+      origin_output_list[out_i] = input_offset;
+      opdesc->SetOutputOffset(origin_output_list);
+      GELOGI("Node[%s] output[%d] is updated from reuse input index[%d] to offset[%ld]", opdesc->GetName().c_str(),
+             out_i, ref_in, input_offset);
+    }
+  }
+  return ge::SUCCESS;
+}
+
 ge::Status GraphMemoryAssigner::UpdateOpInputOffset(const NodePtr &node) const {
   GE_CHECK_NOTNULL(node->GetOpDesc());
   vector<int64_t> input_list;
@@ -1626,9 +1786,19 @@ void GraphMemoryAssigner::PrintMemoryOffset() {
   }
 }
 
-ge::Status GraphMemoryAssigner::GetAllRef(const NodePtr &node, map<int32_t, int32_t> &out2ins) {
+ge::Status GraphMemoryAssigner::TryGetNodeRefIndexes(const NodePtr &node, map<int32_t, int32_t> &out2ins) const{
+  // data and netoutput no need check because only data's output or netoutput's input is used
+  if (node->GetType() == DATA || node->GetType() == NETOUTPUT) {
+    return ge::SUCCESS;
+  }
   for (const auto &out_data_anchor : node->GetAllOutDataAnchors()) {
     int32_t reuse_in_index = -1;
+    // nopadding means output[0] reuse input[0], but as history reason,
+    // other output index also return true for mem assign in block_mem_assigner
+    if (GraphUtils::IsNoPaddingRefFromInput(out_data_anchor, reuse_in_index)) {
+      out2ins.emplace(out_data_anchor->GetIdx(), reuse_in_index);
+      return ge::SUCCESS;
+    }
     bool reuse_input_flag = GraphUtils::IsRefFromInput(out_data_anchor, reuse_in_index);
     if (reuse_input_flag) {
       if (node->GetInDataAnchor(reuse_in_index) != nullptr) {

ge/graph/build/memory/graph_mem_assigner.h

@@ -110,8 +110,11 @@ class GraphMemoryAssigner {
   ge::Status SetInputOffset();
 
   ge::Status UpdateOpInputOffset(const NodePtr &node) const;
+  ge::Status UpdateRefOpOutputOffset(const NodePtr &node, const std::map<int32_t, int32_t> &out2ins, const int ref_in,
+                                     const int64_t input_offset) const;
 
   ge::Status CheckOffset();
+  ge::Status CheckRefNodeOffset(const NodePtr &node);
 
   ge::Status AssignReferenceMemory();
 
@@ -125,7 +128,7 @@ class GraphMemoryAssigner {
 
   ge::Status ReAssignAtomicMemory(bool is_loop_graph);
 
-  ge::Status GetAllRef(const NodePtr &node, std::map<int32_t, int32_t> &out2ins);
+  ge::Status TryGetNodeRefIndexes(const NodePtr &node, std::map<int32_t, int32_t> &out2ins) const;
 
   bool AssignContinuousInputMemoryWithAtomicProcessDirectly(const NodePtr &input_continuous_node,
                                                             std::map<NodePtr, uint32_t> &node_2_continuous_type);
@@ -190,6 +193,10 @@ class GraphMemoryAssigner {
 
   Status AssignBufferPoolMemory();
 
+  bool IsRefFromInputOpCascade(const NodePtr &node);
+
+  Status UpdateRefOpOffsetReverse(const NodePtr &node);
+
   MemoryOffsetMap memory_offset_;
   ge::ComputeGraphPtr compute_graph_;
   HybridMemAssignerPtr mem_assigner_;

ge/graph/build/model_builder.cc

@@ -243,7 +243,7 @@ Status ModelBuilder::SetInputOutputDesc() {
     }
     // if user set input node format ND, the expected node for data and netoutput format is ND in
     // final graph.
-    if ((GetLocalOmgContext().format == domi::DOMI_TENSOR_ND) && (!node_op_desc->HasAttr("_is_single_op")) &&
+    if ((compute_graph_->GetParentGraph() == nullptr) && (GetLocalOmgContext().format == domi::DOMI_TENSOR_ND) && (!node_op_desc->HasAttr("_is_single_op")) &&
         ((node_op_desc->GetType() == DATA_TYPE) || (node_op_desc->GetType() == NETOUTPUT))) {
       auto inputDescsPtr = node_op_desc->GetAllInputsDescPtr();
       auto outputDescsPtr = node_op_desc->GetAllOutputsDescPtr();
@@ -794,7 +794,7 @@ Status ModelBuilder::PreBuildModel() {
 Status ModelBuilder::BuildModelForGetTask(ge::Model &model) {
   GE_CHK_STATUS_RET(AdjustInputTensorFlag(), "AdjustInputTensorFlag failed!");
 
-  ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kStreamAlloc);
+  ErrorManager::GetInstance().SetStage(error_message::kModelCompile, error_message::kStreamAlloc);
   // Assign logical streams.
   StreamAllocator stream_allocator(compute_graph_, subgraphs_);
   GE_TIMESTAMP_START(AssignLogicalStreams);
@@ -802,7 +802,7 @@ Status ModelBuilder::BuildModelForGetTask(ge::Model &model) {
                     "Assign logical streams failed.");
   GE_TIMESTAMP_END(AssignLogicalStreams, "GraphBuilder::AssignLogicalStreams");
 
-  ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kMemoryAlloc);
+  ErrorManager::GetInstance().SetStage(error_message::kModelCompile, error_message::kMemoryAlloc);
   // Assign functional op labels.
   auto root_graph = GraphUtils::FindRootGraph(compute_graph_);
   (void)AttrUtils::GetInt(*root_graph, ATTR_MODEL_LABEL_NUM, label_num_);
@@ -813,7 +813,7 @@ Status ModelBuilder::BuildModelForGetTask(ge::Model &model) {
                     "Assign Memory Failed!");
   GE_TIMESTAMP_END(AssignMemory, "GraphBuilder::AssignMemory");
 
-  ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kOther);
+  ErrorManager::GetInstance().SetStage(error_message::kModelCompile, error_message::kOther);
   GE_TIMESTAMP_START(SetInputOutputOffset);
   SetInputOutputOffsetPass input_output_offset;
   GE_CHK_STATUS_RET(input_output_offset.Run(compute_graph_), "Set input output offset failed.");
@@ -824,14 +824,14 @@ Status ModelBuilder::BuildModelForGetTask(ge::Model &model) {
   GE_CHK_STATUS_RET(CompileSingleOp(), "ATC builder CompileSingleOp() return fail.");
   GE_TIMESTAMP_EVENT_END(CompileSingleOp, "GraphBuilder::CompileSingleOp");
 
-  ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kStreamAlloc);
+  ErrorManager::GetInstance().SetStage(error_message::kModelCompile, error_message::kStreamAlloc);
   // Refresh real streams and insert event nodes.
   GE_TIMESTAMP_START(RefreshRealStream);
   GE_CHK_STATUS_RET(stream_allocator.RefreshRealStream(stream_num_, event_num_), "RefreshRealStream failed.");
   huge_streams_ = stream_allocator.GetHugeStreams();
   GE_TIMESTAMP_END(RefreshRealStream, "GraphBuilder::RefreshRealStream");
 
-  ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kOther);
+  ErrorManager::GetInstance().SetStage(error_message::kModelCompile, error_message::kOther);
   GE_TIMESTAMP_START(MergeWeights);
   GE_CHK_STATUS_RET(MergeWeights(), "MergeWeights Failed!");
   GE_TIMESTAMP_END(MergeWeights, "GraphBuilder::MergeWeights");

ge/graph/execute/graph_execute.cc

@@ -20,9 +20,11 @@
 #include <string>
 
 #include "graph/load/model_manager/model_manager.h"
-#include "omm/csa_interact.h"
+#include "graph/load/model_manager/davinci_model.h"
 
 namespace ge {
+using Uint32Pair = pair<uint32_t, uint32_t>;
+const uint32_t kInvalidModelId = UINT32_MAX;
 GraphExecutor::GraphExecutor()
     : init_flag_(false),
       train_graph_flag_(false),
@@ -380,7 +382,8 @@ Status GraphExecutor::ExecuteGraph(GraphId graph_id, const GeRootModelPtr &ge_ro
 }
 
 Status GraphExecutor::ExecuteGraphAsync(GraphId graph_id, const GeRootModelPtr &ge_root_model,
-                                        const std::vector<InputTensorInfo> &input_tensor) {
+                                        const std::vector<InputTensorInfo> &input_tensor,
+                                        const RunAsyncCallback& callback) {
   GELOGI("[GraphExecutor] Start to async execute graph, graph_id=%u", graph_id);
   if (graph_id != last_graph_id_) {
     auto ret = FreeExecuteMemory();
@@ -390,7 +393,7 @@ Status GraphExecutor::ExecuteGraphAsync(GraphId graph_id, const GeRootModelPtr &
   }
   last_graph_id_ = graph_id;
   GE_CHECK_NOTNULL_EXEC(ge_root_model, return FAILED);
-  Status ret = AsyncExecuteModel(ge_root_model->GetModelId(), input_tensor);
+  Status ret = AsyncExecuteModel(ge_root_model, input_tensor, callback);
   if (ret != SUCCESS) {
     GELOGE(GE_GRAPH_SYNC_MODEL_FAILED, "[GraphExecutor] AsyncExecuteModel Error!");
     return GE_GRAPH_SYNC_MODEL_FAILED;
@@ -400,11 +403,81 @@ Status GraphExecutor::ExecuteGraphAsync(GraphId graph_id, const GeRootModelPtr &
   return SUCCESS;
 }
 
-Status GraphExecutor::AsyncExecuteModel(uint32_t model_id, const std::vector<InputTensorInfo> &inputs) {
+bool CompareByLoad(const Uint32Pair &lhs, const Uint32Pair &rhs) {
+  return lhs.second < rhs.second;
+}
+
+uint32_t GraphExecutor::GetExecuteModelId(const GeRootModelPtr &ge_root_model) {
+  std::vector<uint32_t> model_ids = ge_root_model->GetAllModelId();
+  if (model_ids.empty()) {
+    return kInvalidModelId;
+  }
+  if (model_ids.size() == 1) {
+    return ge_root_model->GetModelId();
+  }
+  std::vector<Uint32Pair> model_id_to_loads;
+  auto model_manager = ModelManager::GetInstance();
+  GE_CHECK_NOTNULL(model_manager);
+  for (auto model_id : model_ids) {
+    auto davinci_model = model_manager->GetModel(model_id);
+    auto hybrid_model = model_manager->GetHybridModel(model_id);
+    if (hybrid_model == nullptr) {
+      GE_CHECK_NOTNULL(davinci_model);
+    }
+    uint32_t input_load = hybrid_model != nullptr ? hybrid_model->GetDataInputerSize() :
+                                                    davinci_model->GetDataInputerSize();
+    uint32_t running_load = hybrid_model != nullptr ? static_cast<uint32_t>(hybrid_model->GetRunningFlag()) :
+                                                      static_cast<uint32_t>(davinci_model->GetRunningFlag());
+    uint32_t load = input_load + running_load;
+    if (load == 0) {
+      return model_id;
+    }
+    model_id_to_loads.emplace_back(model_id, load);
+  }
+  sort(model_id_to_loads.begin(), model_id_to_loads.end(), CompareByLoad);
+  if (model_id_to_loads.empty()) {
+    return kInvalidModelId;
+  }
+  return model_id_to_loads.begin()->first;
+}
+
+Status GraphExecutor::SetCallback(uint32_t model_id, const GeRootModelPtr &ge_root_model,
+                                  const RunAsyncCallback &callback) {
+  auto model_manager = ge::ModelManager::GetInstance();
+  GE_CHECK_NOTNULL(model_manager);
+  if (model_manager->IsNeedHybridLoad(*ge_root_model)) {
+    auto model = model_manager->GetHybridModel(model_id);
+    GE_CHECK_NOTNULL(model);
+    if (model->SetRunAsyncListenerCallback(callback) != SUCCESS) {
+      GELOGE(FAILED, "SetRunAsyncListenerCallback failed.");
+      return FAILED;
+    }
+  } else {
+    auto model = model_manager->GetModel(model_id);
+    GE_CHECK_NOTNULL(model);
+    if (model->SetRunAsyncListenerCallback(callback) != SUCCESS) {
+      GELOGE(FAILED, "SetRunAsyncListenerCallback failed.");
+      return FAILED;
+    }
+  }
+  return SUCCESS;
+}
+
+Status GraphExecutor::AsyncExecuteModel(const GeRootModelPtr &ge_root_model, const std::vector<InputTensorInfo> &inputs,
+                                        const RunAsyncCallback &callback) {
+  uint32_t model_id = GetExecuteModelId(ge_root_model);
+  if (model_id == kInvalidModelId) {
+    GELOGE(INTERNAL_ERROR, "No valid model id.");
+    return INTERNAL_ERROR;
+  }
   try {
     auto model_manager = ge::ModelManager::GetInstance();
     GE_CHECK_NOTNULL(model_manager);
     GELOGI("RunAsync begin.model_id %u", model_id);
+    if (SetCallback(model_id, ge_root_model, callback) != SUCCESS) {
+      GELOGE(FAILED, "RunAsync: SetCallBack for model fail");
+      return FAILED;
+    }
 
     Status ret = model_manager->DataInputTensor(model_id, inputs);
     if (ret != SUCCESS) {
@@ -416,12 +489,10 @@ Status GraphExecutor::AsyncExecuteModel(uint32_t model_id, const std::vector<Inp
   } catch (std::bad_alloc &) {
     REPORT_INNER_ERROR("E19999", "Bad memory allocation exception occur failed");
     GELOGE(MEMALLOC_FAILED, "RunAsync failed, bad memory allocation occur !");
-    CsaInteract::GetInstance().WriteErrorCode(FAILED, ERROR_MODULE_FMK, JOBSUBSTATE_GRAPH_EXEC);
     return MEMALLOC_FAILED;
   } catch (...) {
     REPORT_INNER_ERROR("E19999", "Some exceptions occur failed");
     GELOGE(FAILED, "RunAsync failed, some exceptions occur !");
-    CsaInteract::GetInstance().WriteErrorCode(FAILED, ERROR_MODULE_FMK, JOBSUBSTATE_GRAPH_EXEC);
     return FAILED;
   }
 
@@ -435,18 +506,15 @@ Status GraphExecutor::DataInput(const InputData &input_data, OutputData &output_
     Status ret = model_manager->DataInput(input_data, output_data);
     if (ret != SUCCESS) {
       GELOGE(ret, "DataInput: DataInput failed.");
-      CsaInteract::GetInstance().WriteErrorCode(ret, ERROR_MODULE_FMK, JOBSUBSTATE_GRAPH_EXEC);
       return ret;
     }
   } catch (std::bad_alloc &) {
     REPORT_INNER_ERROR("E19999", "Bad memory allocation exception occur failed");
     GELOGE(MEMALLOC_FAILED, "DataInput failed, bad memory allocation occur !");
-    CsaInteract::GetInstance().WriteErrorCode(FAILED, ERROR_MODULE_FMK, JOBSUBSTATE_GRAPH_EXEC);
     return MEMALLOC_FAILED;
   } catch (...) {
     REPORT_INNER_ERROR("E19999", "Some exceptions occur failed");
     GELOGE(FAILED, "DataInput failed, some exceptions occur !");
-    CsaInteract::GetInstance().WriteErrorCode(FAILED, ERROR_MODULE_FMK, JOBSUBSTATE_GRAPH_EXEC);
     return FAILED;
   }
 
@@ -461,18 +529,15 @@ Status GraphExecutor::GetInputOutputDescInfo(const uint32_t model_id, vector<Inp
     Status ret = model_manager->GetInputOutputDescInfo(model_id, input_desc, output_desc);
     if (ret != SUCCESS) {
       GELOGE(ret, "GetInputOutputDescInfo  failed.");
-      CsaInteract::GetInstance().WriteErrorCode(ret, ERROR_MODULE_FMK, JOBSUBSTATE_GRAPH_EXEC);
       return ret;
     }
   } catch (std::bad_alloc &) {
     REPORT_INNER_ERROR("E19999", "Bad memory allocation exception occur failed");
     GELOGE(MEMALLOC_FAILED, "GetInputOutputDescInfo failed, bad memory allocation occur !");
-    CsaInteract::GetInstance().WriteErrorCode(FAILED, ERROR_MODULE_FMK, JOBSUBSTATE_GRAPH_EXEC);
     return MEMALLOC_FAILED;
   } catch (...) {
     REPORT_INNER_ERROR("E19999", "Some exceptions occur failed");
     GELOGE(FAILED, "GetInputOutputDescInfo failed, some exceptions occur !");
-    CsaInteract::GetInstance().WriteErrorCode(FAILED, ERROR_MODULE_FMK, JOBSUBSTATE_GRAPH_EXEC);
     return FAILED;
   }
 
@@ -490,18 +555,15 @@ Status GraphExecutor::GetInputOutputDescInfo(const uint32_t model_id, vector<Inp
                                                        new_model_desc);
     if (ret != SUCCESS) {
       GELOGE(ret, "GetInputOutputDescInfo  failed.");
-      CsaInteract::GetInstance().WriteErrorCode(ret, ERROR_MODULE_FMK, JOBSUBSTATE_GRAPH_EXEC);
       return ret;
     }
   } catch (std::bad_alloc &) {
     REPORT_INNER_ERROR("E19999", "Bad memory allocation exception occur failed");
     GELOGE(MEMALLOC_FAILED, "GetInputOutputDescInfo failed, bad memory allocation occur !");
-    CsaInteract::GetInstance().WriteErrorCode(FAILED, ERROR_MODULE_FMK, JOBSUBSTATE_GRAPH_EXEC);
     return MEMALLOC_FAILED;
   } catch (...) {
     REPORT_INNER_ERROR("E19999", "Some exceptions occur failed");
     GELOGE(FAILED, "GetInputOutputDescInfo failed, some exceptions occur !");
-    CsaInteract::GetInstance().WriteErrorCode(FAILED, ERROR_MODULE_FMK, JOBSUBSTATE_GRAPH_EXEC);
     return FAILED;
   }
 

ge/graph/execute/graph_execute.h

@@ -50,7 +50,7 @@ class GraphExecutor {
                       std::vector<GeTensor> &output_tensor);
 
   ge::Status ExecuteGraphAsync(GraphId graph_id, const GeRootModelPtr &ge_root_model,
-                               const std::vector<InputTensorInfo> &input_tensor);
+                               const std::vector<InputTensorInfo> &input_tensor, const RunAsyncCallback &callback);
 
   Status SetCondition(std::mutex *mutex, std::condition_variable *cond, std::shared_ptr<GraphModelListener> listener);
 
@@ -116,6 +116,8 @@ class GraphExecutor {
 
   static Status GetOpDescInfo(uint32_t device_id, uint32_t stream_id, uint32_t task_id, OpDescInfo &op_desc_info);
 
+  uint32_t GetExecuteModelId(const GeRootModelPtr &ge_root_model);
+
  private:
   Status PrepareInputData(const std::vector<GeTensor> &input_tensor, InputData &graph_input_data,
                           OutputData &graph_output_data, std::vector<InputOutputDescInfo> &output_desc);
@@ -123,7 +125,8 @@ class GraphExecutor {
   Status SyncExecuteModel(uint32_t model_id, const std::vector<GeTensor> &input_tensor,
                           std::vector<GeTensor> &output_tensor);
 
-  Status AsyncExecuteModel(uint32_t model_id, const std::vector<InputTensorInfo> &input_tensor);
+  Status AsyncExecuteModel(const GeRootModelPtr &ge_root_model, const std::vector<InputTensorInfo> &input_tensor,
+                           const RunAsyncCallback &callback);
 
   void InitModelIdInfo(std::vector<uint32_t> &out_model_id_info, std::vector<SubGraphInfoPtr> &sub_graph_vec,
                        uint32_t output_size);
@@ -132,6 +135,9 @@ class GraphExecutor {
 
   Status MallocInOutBuffer(const std::vector<uint64_t> &buffer_size, std::vector<void *> &data_addr);
 
+  static Status SetCallback(uint32_t model_id, const GeRootModelPtr &ge_root_model,
+                            const RunAsyncCallback &callback);
+
   bool init_flag_;
 
   bool train_graph_flag_;

ge/graph/load/graph_loader.cc

@@ -24,7 +24,6 @@
 #include "graph/ge_context.h"
 #include "graph/load/model_manager/model_manager.h"
 #include "graph/manager/graph_var_manager.h"
-#include "omm/csa_interact.h"
 
 namespace ge {
 Status GraphLoader::UnloadModel(uint32_t model_id) {
@@ -40,7 +39,6 @@ Status GraphLoader::UnloadModel(uint32_t model_id) {
   ret = model_manager->Unload(model_id);
   if (ret != SUCCESS) {
     GELOGE(ret, "UnloadModel: Unload failed. model id:%u", model_id);
-    CsaInteract::GetInstance().WriteErrorCode(ret, ERROR_MODULE_FMK, JOBSUBSTATE_GRAPH_UNLOAD);
     return ret;
   }
   GELOGI("UnLoad model success, model id:%u.", model_id);
@@ -55,7 +53,6 @@ Status GraphLoader::LoadModelOnline(uint32_t &model_id, const std::shared_ptr<ge
     REPORT_CALL_ERROR("E19999", "Call rtSetDevice failed, device_id:%u, ret:0x%X",
                       GetContext().DeviceId(), rt_ret);
     GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret);
-    CsaInteract::GetInstance().WriteErrorCode(rt_ret, ERROR_MODULE_RUNTIME, JOBSUBSTATE_GRAPH_LOAD);
     return RT_FAILED;
   }
   if (ge_root_model_ptr == nullptr) {
@@ -63,15 +60,12 @@ Status GraphLoader::LoadModelOnline(uint32_t &model_id, const std::shared_ptr<ge
     GELOGE(GE_GRAPH_PARAM_NULLPTR, "[LoadGraph] GE load graph model_ptr is nullptr.");
     return GE_GRAPH_PARAM_NULLPTR;
   }
-  model_id = ge_root_model_ptr->GetModelId();
 
   auto model_manager = ModelManager::GetInstance();
   GE_CHECK_NOTNULL(model_manager);
   Status ret = model_manager->LoadModelOnline(model_id, ge_root_model_ptr, listener);
   if (ret != SUCCESS) {
     GELOGE(ret, "LoadModel: Load failed. ret = %u", ret);
-    CsaInteract::GetInstance().WriteErrorCode(ret, ERROR_MODULE_FMK, JOBSUBSTATE_GRAPH_LOAD);
-
     rt_ret = rtDeviceReset(GetContext().DeviceId());
     if (rt_ret != RT_ERROR_NONE) {
       REPORT_CALL_ERROR("E19999", "Call rtDeviceReset failed, device_id:%u, ret:0x%X",
@@ -95,7 +89,6 @@ Status GraphLoader::LoadModelOnline(uint32_t &model_id, const std::shared_ptr<ge
     }
 
     GELOGE(ret, "LoadModel: Start failed.");
-    CsaInteract::GetInstance().WriteErrorCode(ret, ERROR_MODULE_FMK, JOBSUBSTATE_GRAPH_EXEC);
     return ret;
   }
   rt_ret = rtDeviceReset(GetContext().DeviceId());
@@ -248,7 +241,6 @@ Status GraphLoader::GetMemoryInfo(int64_t &free) {
     REPORT_CALL_ERROR("E19999", "Call rtSetDevice failed, device_id:%u, ret:0x%X",
                       GetContext().DeviceId(), rt_ret);
     GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret);
-    CsaInteract::GetInstance().WriteErrorCode(rt_ret, ERROR_MODULE_RUNTIME, JOBSUBSTATE_GRAPH_LOAD);
     return RT_FAILED;
   }
   size_t total_mem = 0;

ge/graph/load/model_manager/data_dumper.cc

@@ -35,7 +35,7 @@
 #include "graph/utils/tensor_utils.h"
 #include "proto/dump_task.pb.h"
 #include "proto/ge_ir.pb.h"
-#include "proto/op_mapping_info.pb.h"
+#include "proto/op_mapping.pb.h"
 #include "runtime/base.h"
 #include "runtime/mem.h"
 
@@ -72,24 +72,6 @@ static bool ParseNameIndex(const std::string &node_name_index, std::string &node
 static bool IsTensorDescWithSkipDumpAddrType(bool has_mem_type_attr, vector<int64_t> v_memory_type, size_t i) {
   return has_mem_type_attr && (v_memory_type[i] == RT_MEMORY_L1);
 }
-
-static uint64_t GetNowTime() {
-  uint64_t ret = 0;
-  mmTimeval tv;
-  if (mmGetTimeOfDay(&tv, nullptr) == 0) {
-    ret = tv.tv_sec * 1000000ULL + tv.tv_usec;
-  }
-
-  return ret;
-}
-
-static void ReplaceStringElem(std::string &str) {
-  for_each(str.begin(), str.end(), [](char &ch) {
-    if ((ch == ' ') || (ch == '.') || (ch == '/') || (ch == '\\')) {
-      ch = '_';
-    }
-  });
-}
 }  // namespace
 
 static int32_t GetIrDataType(ge::DataType data_type) {
@@ -194,66 +176,6 @@ void DataDumper::SaveOpDebugId(uint32_t task_id, uint32_t stream_id, void *op_de
   is_op_debug_ = is_op_debug;
 }
 
-void DataDumper::SaveDumpOpInfo(const RuntimeParam &model_param, const OpDescPtr &op, uint32_t task_id,
-                                uint32_t stream_id) {
-  GELOGD("Start SaveDumpOpInfo of task_id: %u, stream_id: %u", task_id, stream_id);
-  OpDescInfo op_desc_info;
-  op_desc_info.op_name = op->GetName();
-  op_desc_info.op_type = op->GetType();
-  op_desc_info.task_id = task_id;
-  op_desc_info.stream_id = stream_id;
-  for (size_t i = 0; i < op->GetAllInputsSize(); ++i) {
-    GeTensorDescPtr input_tensor_desc = op->MutableInputDesc(i);
-    if (input_tensor_desc == nullptr) {
-      continue;
-    }
-    op_desc_info.input_format.emplace_back(input_tensor_desc->GetFormat());
-    op_desc_info.input_shape.emplace_back(input_tensor_desc->GetShape().GetDims());
-    op_desc_info.input_data_type.emplace_back(input_tensor_desc->GetDataType());
-    int64_t input_size = 0;
-
-    if (TensorUtils::GetTensorSizeInBytes(*input_tensor_desc, input_size) != SUCCESS) {
-      GELOGW("Get input size failed");
-      return;
-    }
-    GELOGD("Save dump op info, the input size is %ld", input_size);
-    op_desc_info.input_size.emplace_back(input_size);
-  }
-  for (size_t j = 0; j < op->GetOutputsSize(); ++j) {
-    GeTensorDescPtr output_tensor_desc = op->MutableOutputDesc(j);
-    if (output_tensor_desc == nullptr) {
-      continue;
-    }
-    op_desc_info.output_format.emplace_back(output_tensor_desc->GetFormat());
-    op_desc_info.output_shape.emplace_back(output_tensor_desc->GetShape().GetDims());
-    op_desc_info.output_data_type.emplace_back(output_tensor_desc->GetDataType());
-    int64_t output_size = 0;
-    if (TensorUtils::GetTensorSizeInBytes(*output_tensor_desc, output_size) != SUCCESS) {
-      GELOGW("Get input size failed");
-      return;
-    }
-    GELOGD("Save dump op info, the output size is %ld", output_size);
-    op_desc_info.output_size.emplace_back(output_size);
-  }
-  op_desc_info.input_addrs = ModelUtils::GetInputDataAddrs(model_param, op);
-  op_desc_info.output_addrs = ModelUtils::GetOutputDataAddrs(model_param, op);
-
-  op_desc_info_.emplace_back(op_desc_info);
-}
-
-bool DataDumper::GetOpDescInfo(uint32_t stream_id, uint32_t task_id, OpDescInfo &op_desc_info) const {
-  GELOGI("There are %zu op need to dump.", op_desc_info_.size());
-  for (size_t index = 0; index < op_desc_info_.size(); ++index) {
-    OpDescInfo dump_op_info = op_desc_info_.at(index);
-    if (dump_op_info.task_id == task_id && dump_op_info.stream_id == stream_id) {
-      GELOGI("find exception op of task_id: %u, stream_id: %u.", task_id, stream_id);
-      op_desc_info = dump_op_info;
-      return true;
-    }
-  }
-  return false;
-}
-
 void DataDumper::SaveDumpTask(uint32_t task_id, uint32_t stream_id, const std::shared_ptr<OpDesc> &op_desc,
                               uintptr_t args) {
   if (op_desc == nullptr) {
@@ -295,7 +217,7 @@ void DataDumper::SaveDumpTask(uint32_t task_id, uint32_t stream_id, const std::s
 }
 
 static void SetOpMappingLoopAddr(uintptr_t step_id, uintptr_t loop_per_iter, uintptr_t loop_cond,
-                                 aicpu::dump::OpMappingInfo &op_mapping_info) {
+                                 toolkit::aicpu::dump::OpMappingInfo &op_mapping_info) {
   if (step_id != 0) {
     GELOGI("step_id exists.");
     op_mapping_info.set_step_id_addr(static_cast<uint64_t>(step_id));
@@ -312,7 +234,8 @@ static void SetOpMappingLoopAddr(uintptr_t step_id, uintptr_t loop_per_iter, uin
   }
 }
 
-Status DataDumper::GenerateOutput(aicpu::dump::Output &output, const OpDesc::Vistor<GeTensorDesc> &tensor_descs,
+Status DataDumper::GenerateOutput(toolkit::aicpu::dump::Output &output,
+                                  const OpDesc::Vistor<GeTensorDesc> &tensor_descs,
                                   const uintptr_t &addr, size_t index) {
   output.set_data_type(static_cast<int32_t>(GetIrDataType(tensor_descs.at(index).GetDataType())));
   output.set_format(static_cast<int32_t>(tensor_descs.at(index).GetFormat()));
@@ -343,7 +266,8 @@ Status DataDumper::GenerateOutput(aicpu::dump::Output &output, const OpDesc::Vis
   return SUCCESS;
 }
 
-Status DataDumper::DumpRefOutput(const DataDumper::InnerDumpInfo &inner_dump_info, aicpu::dump::Output &output,
+Status DataDumper::DumpRefOutput(const DataDumper::InnerDumpInfo &inner_dump_info,
+                                 toolkit::aicpu::dump::Output &output,
                                  size_t i, const std::string &node_name_index) {
   std::string dump_op_name;
   std::string input_or_output;
@@ -384,17 +308,9 @@ Status DataDumper::DumpRefOutput(const DataDumper::InnerDumpInfo &inner_dump_inf
   return SUCCESS;
 }
 
-Status DataDumper::DumpOutputWithTask(const InnerDumpInfo &inner_dump_info, aicpu::dump::Task &task) {
+Status DataDumper::DumpOutputWithTask(const InnerDumpInfo &inner_dump_info, toolkit::aicpu::dump::Task &task) {
   const auto &output_descs = inner_dump_info.op->GetAllOutputsDesc();
   const std::vector<void *> output_addrs = ModelUtils::GetOutputDataAddrs(*runtime_param_, inner_dump_info.op);
-  if (output_descs.size() != output_addrs.size()) {
-    REPORT_INNER_ERROR("E19999", "output_desc size:%zu != output addr size:%zu in op:%s(%s)",
-                       output_descs.size(), output_addrs.size(),
-                       inner_dump_info.op->GetName().c_str(), inner_dump_info.op->GetType().c_str());
-    GELOGE(PARAM_INVALID, "Invalid output desc addrs size %zu, op %s has %zu output desc.", output_addrs.size(),
-           inner_dump_info.op->GetName().c_str(), output_descs.size());
-    return PARAM_INVALID;
-  }
   std::vector<int64_t> v_memory_type;
   bool has_mem_type_attr = ge::AttrUtils::GetListInt(inner_dump_info.op, ATTR_NAME_OUTPUT_MEM_TYPE_LIST, v_memory_type);
   GE_RT_PARAM_INVALID_WITH_LOG_IF_TRUE(has_mem_type_attr && (v_memory_type.size() != output_descs.size()),
@@ -402,10 +318,33 @@ Status DataDumper::DumpOutputWithTask(const InnerDumpInfo &inner_dump_info, aicp
                                        inner_dump_info.op->GetName().c_str(), output_descs.size(),
                                        v_memory_type.size());
 
+  size_t no_need_dump_output_num = 0;
   for (size_t i = 0; i < output_descs.size(); ++i) {
-    aicpu::dump::Output output;
+    toolkit::aicpu::dump::Output output;
     std::string node_name_index;
     const auto &output_desc = output_descs.at(i);
+    int32_t calc_type = 0;
+    bool has_calc_type = ge::AttrUtils::GetInt(output_desc, ATTR_NAME_MEMORY_SIZE_CALC_TYPE, calc_type);
+    if (has_calc_type && (calc_type == static_cast<int32_t>(ge::MemorySizeCalcType::ALWAYS_EMPTY))) {
+      GELOGD("Node[%s] output[index:%zu] [name:%s] is an optional output, don't need to dump this output.",
+             inner_dump_info.op->GetName().c_str(), i, output_desc.GetName().c_str());
+      ++no_need_dump_output_num;
+      continue;
+    }
+
+    if (output_descs.size() - no_need_dump_output_num < output_addrs.size()) {
+      REPORT_INNER_ERROR("E19999", "The number of output does not match in op:%s(%s). The size[%zu] of output which is "
+                         "no need to dump should not greater than the size[%zu] of output descs minus the size[%zu] of "
+                         "output which is need to dump.", inner_dump_info.op->GetName().c_str(),
+                         inner_dump_info.op->GetType().c_str(), no_need_dump_output_num, output_descs.size(),
+                         output_addrs.size());
+      GELOGE(PARAM_INVALID, "The number of output does not match in op:%s(%s). The size[%zu] of output which is no need"
+             " to dump should not greater than the size[%zu] of output descs minus the size[%zu] of output which is "
+             "need to dump.", inner_dump_info.op->GetName().c_str(), inner_dump_info.op->GetType().c_str(),
+             no_need_dump_output_num, output_descs.size(), output_addrs.size());
+      return PARAM_INVALID;
+    }
+
     // check dump output tensor desc is redirected by attr ATTR_DATA_DUMP_REF
     if (AttrUtils::GetStr(&output_desc, ATTR_DATA_DUMP_REF, node_name_index)) {
       GE_CHK_STATUS_RET(DumpRefOutput(inner_dump_info, output, i, node_name_index), "DumpRefOutput failed");
@@ -433,14 +372,14 @@ Status DataDumper::DumpOutputWithTask(const InnerDumpInfo &inner_dump_info, aicp
   return SUCCESS;
 }
 
-Status DataDumper::DumpOutput(const InnerDumpInfo &inner_dump_info, aicpu::dump::Task &task) {
+Status DataDumper::DumpOutput(const InnerDumpInfo &inner_dump_info, toolkit::aicpu::dump::Task &task) {
   GELOGI("Start dump output");
   if (inner_dump_info.is_task) {
     // tbe or aicpu op, these ops are with task
     return DumpOutputWithTask(inner_dump_info, task);
   }
   // else data, const or variable op
-  aicpu::dump::Output output;
+  toolkit::aicpu::dump::Output output;
   auto output_tensor = inner_dump_info.op->GetOutputDescPtr(inner_dump_info.output_anchor_index);
   const std::vector<void *> output_addrs = ModelUtils::GetOutputDataAddrs(*runtime_param_, inner_dump_info.op);
   if (output_tensor == nullptr) {
@@ -485,7 +424,7 @@ Status DataDumper::DumpOutput(const InnerDumpInfo &inner_dump_info, aicpu::dump:
   return SUCCESS;
 }
 
-Status DataDumper::GenerateInput(aicpu::dump::Input &input, const OpDesc::Vistor<GeTensorDesc> &tensor_descs,
+Status DataDumper::GenerateInput(toolkit::aicpu::dump::Input &input, const OpDesc::Vistor<GeTensorDesc> &tensor_descs,
                                  const uintptr_t &addr, size_t index) {
   input.set_data_type(static_cast<int32_t>(GetIrDataType(tensor_descs.at(index).GetDataType())));
   input.set_format(static_cast<int32_t>(tensor_descs.at(index).GetFormat()));
@@ -510,8 +449,8 @@ Status DataDumper::GenerateInput(aicpu::dump::Input &input, const OpDesc::Vistor
   return SUCCESS;
 }
 
-Status DataDumper::DumpRefInput(const DataDumper::InnerDumpInfo &inner_dump_info, aicpu::dump::Input &input, size_t i,
-                                const std::string &node_name_index) {
+Status DataDumper::DumpRefInput(const DataDumper::InnerDumpInfo &inner_dump_info, toolkit::aicpu::dump::Input &input,
+                                size_t i, const std::string &node_name_index) {
   std::string dump_op_name;
   std::string input_or_output;
   size_t index;
@@ -551,7 +490,7 @@ Status DataDumper::DumpRefInput(const DataDumper::InnerDumpInfo &inner_dump_info
   return SUCCESS;
 }
 
-Status DataDumper::DumpInput(const InnerDumpInfo &inner_dump_info, aicpu::dump::Task &task) {
+Status DataDumper::DumpInput(const InnerDumpInfo &inner_dump_info, toolkit::aicpu::dump::Task &task) {
   GELOGI("Start dump input");
   const auto &input_descs = inner_dump_info.op->GetAllInputsDesc();
   const std::vector<void *> input_addrs = ModelUtils::GetInputDataAddrs(*runtime_param_, inner_dump_info.op);
@@ -570,7 +509,7 @@ Status DataDumper::DumpInput(const InnerDumpInfo &inner_dump_info, aicpu::dump::
                                        inner_dump_info.op->GetName().c_str(), input_descs.size(), v_memory_type.size());
 
   for (size_t i = 0; i < input_descs.size(); ++i) {
-    aicpu::dump::Input input;
+    toolkit::aicpu::dump::Input input;
     std::string node_name_index;
     // check dump input tensor desc is redirected by attr ATTR_DATA_DUMP_REF
     if (AttrUtils::GetStr(&input_descs.at(i), ATTR_DATA_DUMP_REF, node_name_index)) {
@@ -601,15 +540,15 @@ Status DataDumper::DumpInput(const InnerDumpInfo &inner_dump_info, aicpu::dump::
   return SUCCESS;
 }
 
-void DataDumper::GenerateOpBuffer(const int64_t &size, aicpu::dump::Task &task) {
-  aicpu::dump::OpBuffer op_buffer;
-  op_buffer.set_buffer_type(aicpu::dump::BufferType::L1);
+void DataDumper::GenerateOpBuffer(const int64_t &size, toolkit::aicpu::dump::Task &task) {
+  toolkit::aicpu::dump::OpBuffer op_buffer;
+  op_buffer.set_buffer_type(toolkit::aicpu::dump::BufferType::L1);
   op_buffer.set_address(reinterpret_cast<uintptr_t>(l1_fusion_addr_));
   op_buffer.set_size(size);
   task.mutable_buffer()->Add(std::move(op_buffer));
 }
 
-Status DataDumper::ExecuteLoadDumpInfo(aicpu::dump::OpMappingInfo &op_mapping_info) {
+Status DataDumper::ExecuteLoadDumpInfo(toolkit::aicpu::dump::OpMappingInfo &op_mapping_info) {
   std::string proto_str;
   size_t proto_size = op_mapping_info.ByteSizeLong();
   bool ret = op_mapping_info.SerializeToString(&proto_str);
@@ -653,7 +592,7 @@ Status DataDumper::ExecuteLoadDumpInfo(aicpu::dump::OpMappingInfo &op_mapping_in
   return SUCCESS;
 }
 
-Status DataDumper::ExecuteUnLoadDumpInfo(aicpu::dump::OpMappingInfo &op_mapping_info) {
+Status DataDumper::ExecuteUnLoadDumpInfo(toolkit::aicpu::dump::OpMappingInfo &op_mapping_info) {
   std::string proto_str;
   size_t proto_size = op_mapping_info.ByteSizeLong();
   bool ret = op_mapping_info.SerializeToString(&proto_str);
@@ -704,7 +643,7 @@ Status DataDumper::LoadDumpInfo() {
     GELOGD("op_list_ is empty");
   }
 
-  aicpu::dump::OpMappingInfo op_mapping_info;
+  toolkit::aicpu::dump::OpMappingInfo op_mapping_info;
 
   auto dump_path = dump_properties_.GetDumpPath() + std::to_string(device_id_) + "/";
   op_mapping_info.set_dump_path(dump_path);
@@ -733,11 +672,11 @@ Status DataDumper::LoadDumpInfo() {
   return SUCCESS;
 }
 
-Status DataDumper::BuildTaskInfo(aicpu::dump::OpMappingInfo &op_mapping_info) {
+Status DataDumper::BuildTaskInfo(toolkit::aicpu::dump::OpMappingInfo &op_mapping_info) {
   for (const auto &op_iter : op_list_) {
     auto op_desc = op_iter.op;
     GELOGD("Op %s in model begin to add task in op_mapping_info", op_desc->GetName().c_str());
-    aicpu::dump::Task task;
+    toolkit::aicpu::dump::Task task;
     task.set_end_graph(false);
     task.set_task_id(op_iter.task_id);
     task.set_stream_id(op_iter.stream_id);
@@ -785,10 +724,10 @@ Status DataDumper::BuildTaskInfo(aicpu::dump::OpMappingInfo &op_mapping_info) {
 }
 
 void DataDumper::SetEndGraphIdToAicpu(uint32_t task_id, uint32_t stream_id,
-                                      aicpu::dump::OpMappingInfo &op_mapping_info) {
+                                      toolkit::aicpu::dump::OpMappingInfo &op_mapping_info) {
   if (dump_properties_.GetDumpMode() == kDumpOutput || dump_properties_.GetDumpMode() == kDumpInput ||
       dump_properties_.GetDumpMode() == kDumpAll) {
-    aicpu::dump::Task task;
+    toolkit::aicpu::dump::Task task;
     task.set_end_graph(true);
     task.set_task_id(end_graph_task_id_);
     task.set_stream_id(end_graph_stream_id_);
@@ -797,7 +736,7 @@ void DataDumper::SetEndGraphIdToAicpu(uint32_t task_id, uint32_t stream_id,
     op_mapping_info.mutable_task()->Add(std::move(task));
 
     is_end_graph_ = true;
-    if (op_mapping_info.model_name_param_case() == aicpu::dump::OpMappingInfo::kModelName) {
+    if (op_mapping_info.model_name_param_case() == toolkit::aicpu::dump::OpMappingInfo::kModelName) {
       GELOGI("Add end_graph_info to aicpu, model_name is %s, task_id is %u, stream_id is %u",
           op_mapping_info.model_name().c_str(), end_graph_task_id_, end_graph_stream_id_);
       return;
@@ -807,10 +746,10 @@ void DataDumper::SetEndGraphIdToAicpu(uint32_t task_id, uint32_t stream_id,
 }
 
 void DataDumper::SetOpDebugIdToAicpu(uint32_t task_id, uint32_t stream_id, void *op_debug_addr,
-                                     aicpu::dump::OpMappingInfo &op_mapping_info) {
+                                     toolkit::aicpu::dump::OpMappingInfo &op_mapping_info) {
   if (is_op_debug_) {
     GELOGI("add op_debug_info to aicpu, task_id is %u, stream_id is %u", task_id, stream_id);
-    aicpu::dump::Task task;
+    toolkit::aicpu::dump::Task task;
     task.set_end_graph(false);
     task.set_task_id(task_id);
     task.set_stream_id(stream_id);
@@ -818,7 +757,7 @@ void DataDumper::SetOpDebugIdToAicpu(uint32_t task_id, uint32_t stream_id, void
     task.mutable_op()->set_op_type(OP_TYPE_OP_DEBUG);
 
     // set output
-    aicpu::dump::Output output;
+    toolkit::aicpu::dump::Output output;
     output.set_data_type(DT_UINT8);
     output.set_format(FORMAT_ND);
 
@@ -844,12 +783,12 @@ Status DataDumper::UnloadDumpInfo() {
   }
 
   GELOGI("UnloadDumpInfo start.");
-  aicpu::dump::OpMappingInfo op_mapping_info;
+  toolkit::aicpu::dump::OpMappingInfo op_mapping_info;
   op_mapping_info.set_model_id(model_id_);
   op_mapping_info.set_flag(kAicpuUnloadFlag);
 
   for (const auto &op_iter : op_list_) {
-    aicpu::dump::Task task;
+    toolkit::aicpu::dump::Task task;
     task.set_task_id(op_iter.task_id);
     task.set_stream_id(op_iter.stream_id);
     op_mapping_info.mutable_task()->Add(std::move(task));
@@ -904,98 +843,4 @@ void DataDumper::PrintCheckLog(string &dump_list_key) {
     }
   }
 }
-
-Status DataDumper::DumpExceptionInput(const OpDescInfo &op_desc_info, const string &dump_file) {
-  GELOGI("Start to dump exception input");
-  for (size_t i = 0; i < op_desc_info.input_addrs.size(); i++) {
-    if (Debug::DumpDevMem(dump_file.data(), op_desc_info.input_addrs.at(i), op_desc_info.input_size.at(i)) != SUCCESS) {
-      GELOGE(PARAM_INVALID, "Dump the %zu input data failed", i);
-      return PARAM_INVALID;
-    }
-  }
-  return SUCCESS;
-}
-
-Status DataDumper::DumpExceptionOutput(const OpDescInfo &op_desc_info, const string &dump_file) {
-  GELOGI("Start to dump exception output");
-  for (size_t i = 0; i < op_desc_info.output_addrs.size(); i++) {
-    if (Debug::DumpDevMem(dump_file.data(), op_desc_info.output_addrs.at(i), op_desc_info.output_size.at(i)) !=
-        SUCCESS) {
-      GELOGE(PARAM_INVALID, "Dump the %zu input data failed", i);
-      return PARAM_INVALID;
-    }
-  }
-  return SUCCESS;
-}
-
-Status DataDumper::DumpExceptionInfo(const std::vector<rtExceptionInfo> exception_infos) {
-  GELOGI("Start to dump exception info");
-  for (const rtExceptionInfo &iter : exception_infos) {
-    OpDescInfo op_desc_info;
-    if (GetOpDescInfo(iter.streamid, iter.taskid, op_desc_info)) {
-      toolkit::dumpdata::DumpData dump_data;
-      dump_data.set_version("2.0");
-      dump_data.set_dump_time(GetNowTime());
-      dump_data.set_op_name(op_desc_info.op_name);
-      for (size_t i = 0; i < op_desc_info.input_format.size(); ++i) {
-        toolkit::dumpdata::OpInput input;
-        input.set_data_type(toolkit::dumpdata::OutputDataType(GetIrDataType(op_desc_info.input_data_type[i])));
-        input.set_format(toolkit::dumpdata::OutputFormat(op_desc_info.input_format[i]));
-        for (auto dim : op_desc_info.input_shape[i]) {
-          input.mutable_shape()->add_dim(dim);
-        }
-        input.set_size(op_desc_info.input_size[i]);
-        GELOGI("The input size int exception is %ld", op_desc_info.input_size[i]);
-        dump_data.mutable_input()->Add(std::move(input));
-      }
-      for (size_t j = 0; j < op_desc_info.output_format.size(); ++j) {
-        toolkit::dumpdata::OpOutput output;
-        output.set_data_type(toolkit::dumpdata::OutputDataType(GetIrDataType(op_desc_info.output_data_type[j])));
-        output.set_format(toolkit::dumpdata::OutputFormat(op_desc_info.output_format[j]));
-        for (auto dim : op_desc_info.output_shape[j]) {
-          output.mutable_shape()->add_dim(dim);
-        }
-        output.set_size(op_desc_info.output_size[j]);
-        GELOGI("The output size int exception is %ld", op_desc_info.output_size[j]);
-        dump_data.mutable_output()->Add(std::move(output));
-      }
-      uint64_t now_time = GetNowTime();
-      std::string op_name = op_desc_info.op_name;
-      std::string op_type = op_desc_info.op_type;
-      ReplaceStringElem(op_name);
-      ReplaceStringElem(op_type);
-      string dump_file_path =
-          "./" + op_type + "." + op_name + "." + std::to_string(op_desc_info.task_id) + "." + std::to_string(now_time);
-      GELOGI("The exception dump file path is %s", dump_file_path.c_str());
-
-      uint64_t proto_size = dump_data.ByteSizeLong();
-      std::unique_ptr<char[]> proto_msg(new (std::nothrow) char[proto_size]);
-      bool ret = dump_data.SerializeToArray(proto_msg.get(), proto_size);
-      if (!ret || proto_size == 0) {
-        REPORT_INNER_ERROR("E19999", "Serialize proto to string fail");
-        GELOGE(PARAM_INVALID, "Dump data proto serialize failed");
-        return PARAM_INVALID;
-      }
-
-      GE_CHK_STATUS_RET(MemoryDumper::DumpToFile(dump_file_path.c_str(), &proto_size, sizeof(uint64_t)),
-                        "Failed to dump proto size");
-      GE_CHK_STATUS_RET(MemoryDumper::DumpToFile(dump_file_path.c_str(), proto_msg.get(), proto_size),
-                        "Failed to dump proto msg");
-      if (DumpExceptionInput(op_desc_info, dump_file_path) != SUCCESS) {
-        GELOGE(PARAM_INVALID, "Dump exception input failed");
-        return PARAM_INVALID;
-      }
-
-      if (DumpExceptionOutput(op_desc_info, dump_file_path) != SUCCESS) {
-        GELOGE(PARAM_INVALID, "Dump exception output failed");
-        return PARAM_INVALID;
-      }
-      GELOGI("Dump exception info SUCCESS");
-    } else {
-      GELOGE(PARAM_INVALID, "Get op desc info failed,task id:%u,stream id:%u", iter.taskid, iter.streamid);
-      return PARAM_INVALID;
-    }
-  }
-  return SUCCESS;
-}
 }  // namespace ge

ge/graph/load/model_manager/data_dumper.h

@@ -27,7 +27,7 @@
 #include "graph/node.h"
 #include "graph/compute_graph.h"
 #include "proto/ge_ir.pb.h"
-#include "proto/op_mapping_info.pb.h"
+#include "proto/op_mapping.pb.h"
 #include "runtime/mem.h"
 #include "task_info/task_info.h"
 #include "framework/common/ge_types.h"
@@ -70,8 +70,6 @@ class DataDumper {
 
   void SaveDumpInput(const std::shared_ptr<Node> &node);
 
-  void SaveDumpOpInfo(const RuntimeParam &model_param, const OpDescPtr &op, uint32_t task_id, uint32_t stream_id);
-
   // args is device memory stored first output addr
   void SaveDumpTask(uint32_t task_id, uint32_t stream_id, const std::shared_ptr<OpDesc> &op_desc, uintptr_t args);
   void SaveEndGraphId(uint32_t task_id, uint32_t stream_id);
@@ -87,14 +85,8 @@ class DataDumper {
 
   void SetDumpProperties(const DumpProperties &dump_properties) { dump_properties_ = dump_properties; }
   const DumpProperties &GetDumpProperties() const { return dump_properties_; }
-  bool GetOpDescInfo(uint32_t stream_id, uint32_t task_id, OpDescInfo &op_desc_info) const;
   const std::vector<OpDescInfo> &GetAllOpDescInfo() const { return op_desc_info_; }
 
-  // Dump exception info
-  Status DumpExceptionInput(const OpDescInfo &op_desc_info, const string &dump_file);
-  Status DumpExceptionOutput(const OpDescInfo &op_desc_info, const string &dump_file);
-  Status DumpExceptionInfo(const std::vector<rtExceptionInfo> exception_infos);
-
  private:
   void ReleaseDevMem(void **ptr) noexcept;
 
@@ -136,24 +128,25 @@ class DataDumper {
   DumpProperties dump_properties_;
 
   // Build task info of op mapping info
-  Status BuildTaskInfo(aicpu::dump::OpMappingInfo &op_mapping_info);
-  Status DumpOutput(const InnerDumpInfo &inner_dump_info, aicpu::dump::Task &task);
-  Status DumpRefOutput(const DataDumper::InnerDumpInfo &inner_dump_info, aicpu::dump::Output &output, size_t i,
-                       const std::string &node_name_index);
-  Status DumpOutputWithTask(const InnerDumpInfo &inner_dump_info, aicpu::dump::Task &task);
-  Status DumpInput(const InnerDumpInfo &inner_dump_info, aicpu::dump::Task &task);
-  Status DumpRefInput(const DataDumper::InnerDumpInfo &inner_dump_info, aicpu::dump::Input &input, size_t i,
-                      const std::string &node_name_index);
-  Status ExecuteLoadDumpInfo(aicpu::dump::OpMappingInfo &op_mapping_info);
-  void SetEndGraphIdToAicpu(uint32_t task_id, uint32_t stream_id, aicpu::dump::OpMappingInfo &op_mapping_info);
+  Status BuildTaskInfo(toolkit::aicpu::dump::OpMappingInfo &op_mapping_info);
+  Status DumpOutput(const InnerDumpInfo &inner_dump_info, toolkit::aicpu::dump::Task &task);
+  Status DumpRefOutput(const DataDumper::InnerDumpInfo &inner_dump_info, toolkit::aicpu::dump::Output &output,
+                       size_t i, const std::string &node_name_index);
+  Status DumpOutputWithTask(const InnerDumpInfo &inner_dump_info, toolkit::aicpu::dump::Task &task);
+  Status DumpInput(const InnerDumpInfo &inner_dump_info, toolkit::aicpu::dump::Task &task);
+  Status DumpRefInput(const DataDumper::InnerDumpInfo &inner_dump_info, toolkit::aicpu::dump::Input &input,
+                      size_t i, const std::string &node_name_index);
+  Status ExecuteLoadDumpInfo(toolkit::aicpu::dump::OpMappingInfo &op_mapping_info);
+  void SetEndGraphIdToAicpu(uint32_t task_id, uint32_t stream_id,
+                            toolkit::aicpu::dump::OpMappingInfo &op_mapping_info);
   void SetOpDebugIdToAicpu(uint32_t task_id, uint32_t stream_id, void *op_debug_addr,
-                           aicpu::dump::OpMappingInfo &op_mapping_info);
-  Status ExecuteUnLoadDumpInfo(aicpu::dump::OpMappingInfo &op_mapping_info);
-  Status GenerateInput(aicpu::dump::Input &input, const OpDesc::Vistor<GeTensorDesc> &tensor_descs,
+                           toolkit::aicpu::dump::OpMappingInfo &op_mapping_info);
+  Status ExecuteUnLoadDumpInfo(toolkit::aicpu::dump::OpMappingInfo &op_mapping_info);
+  Status GenerateInput(toolkit::aicpu::dump::Input &input, const OpDesc::Vistor<GeTensorDesc> &tensor_descs,
                        const uintptr_t &addr, size_t index);
-  Status GenerateOutput(aicpu::dump::Output &output, const OpDesc::Vistor<GeTensorDesc> &tensor_descs,
+  Status GenerateOutput(toolkit::aicpu::dump::Output &output, const OpDesc::Vistor<GeTensorDesc> &tensor_descs,
                         const uintptr_t &addr, size_t index);
-  void GenerateOpBuffer(const int64_t &size, aicpu::dump::Task &task);
+  void GenerateOpBuffer(const int64_t &size, toolkit::aicpu::dump::Task &task);
 };
 struct DataDumper::InnerDumpInfo {
   uint32_t task_id;

ge/graph/load/model_manager/data_inputer.h

@@ -134,6 +134,8 @@ class DataInputer {
   ///
   void Stop() { queue_.Stop(); }
 
+  uint32_t Size() { return queue_.Size(); }
+
  private:
   ///
   /// @ingroup domi_ome

ge/graph/load/model_manager/davinci_model.cc

@@ -50,7 +50,6 @@
 #include "graph/utils/type_utils.h"
 #include "init/gelib.h"
 #include "mmpa/mmpa_api.h"
-#include "omm/csa_interact.h"
 #include "runtime/base.h"
 #include "runtime/dev.h"
 #include "runtime/event.h"
@@ -2656,9 +2655,9 @@ Status DavinciModel::ReturnResult(uint32_t data_id, const bool rslt_flg, const b
     GE_CHECK_NOTNULL(model_manager);
     auto exception_infos = model_manager->GetExceptionInfos();
     if (exception_infos.size() > 0) {
-      GE_CHK_STATUS_RET(data_dumper_.DumpExceptionInfo(exception_infos), "Dump exception info failed");
+      GE_CHK_STATUS_RET(DumpExceptionInfo(exception_infos), "[Dump][Exception] Dump exception info failed.");
     } else {
-      GELOGI("Exception info is null");
+      GELOGI("[Dump][Exception] Exception info is null.");
     }
     GE_CHK_STATUS(listener_->OnComputeDone(model_id_, data_id, INTERNAL_ERROR, outputs), "OnComputeDone failed.");
     return INTERNAL_ERROR;
@@ -2718,7 +2717,6 @@ Status DavinciModel::ReturnNoOutput(uint32_t data_id) {
 
 void *DavinciModel::Run(DavinciModel *model) {
   GE_CHK_BOOL_EXEC(model != nullptr,
-                   CsaInteract::GetInstance().WriteErrorCode(FAILED, ERROR_MODULE_FMK, JOBSUBSTATE_GRAPH_EXEC);
                    return nullptr, "model_pointer is null!")
   bool seq_end_flag = false;
   uint32_t model_id = model->Id();
@@ -2735,17 +2733,20 @@ void *DavinciModel::Run(DavinciModel *model) {
   // DeviceReset before thread run finished!
   GE_MAKE_GUARD(not_used_var, [&] { GE_CHK_RT(rtDeviceReset(device_id)); });
 
-  ErrorManager::GetInstance().SetStage(ErrorMessage::kModelExecute, ErrorMessage::kModelExecute);
+  ErrorManager::GetInstance().SetStage(error_message::kModelExecute, error_message::kModelExecute);
   while (model->RunFlag()) {
+    // Model hasn't truly started runing before received data
+    model->SetRunningFlag(false);
     bool rslt_flg = true;
     if (model->GetDataInputer() == nullptr) {
       GELOGW("Data inputer is nullptr.");
-      CsaInteract::GetInstance().StoreInternalErrorCode(FAILED, ERROR_MODULE_FMK, JOBSUBSTATE_GRAPH_EXEC);
       break;
     }
 
     std::shared_ptr<InputDataWrapper> data_wrapper;
     Status ret = model->GetDataInputer()->Pop(data_wrapper);
+    // Model run indeedly start after received data.
+    model->SetRunningFlag(true);
     if (data_wrapper == nullptr || ret != SUCCESS) {
       GELOGI("data_wrapper is null!");
       continue;
@@ -2759,7 +2760,6 @@ void *DavinciModel::Run(DavinciModel *model) {
     ret = model->SyncVarData();
     GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(
         ret != SUCCESS, (void)model->ReturnResult(current_data.index, false, false, data_wrapper->GetOutput());
-        CsaInteract::GetInstance().StoreInternalErrorCode(ret, ERROR_MODULE_FMK, JOBSUBSTATE_GRAPH_EXEC);
         continue, "Copy input data to model failed.");  // [No need to check value]
     GE_IF_BOOL_EXEC(model->is_first_execute_, GE_TIMESTAMP_EVENT_END(Model_SyncVarData, "Model Run SyncVarData"));
 
@@ -2769,7 +2769,6 @@ void *DavinciModel::Run(DavinciModel *model) {
     ret = model->CopyInputData(current_data, false);
     GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(
         ret != SUCCESS, (void)model->ReturnResult(current_data.index, false, false, data_wrapper->GetOutput());
-        CsaInteract::GetInstance().StoreInternalErrorCode(ret, ERROR_MODULE_FMK, JOBSUBSTATE_GRAPH_EXEC);
         continue, "Copy input data to model failed.");  // [No need to check value]
     if (model->is_online_infer_dynamic_ && !model->is_getnext_sink_dynamic_) {
       model->cur_dynamic_dims_.clear();
@@ -2790,7 +2789,6 @@ void *DavinciModel::Run(DavinciModel *model) {
     rt_ret = rtModelExecute(model->rt_model_handle_, model->rt_model_stream_, 0);
     GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, rslt_flg = false;
                     (void)model->ReturnResult(current_data.index, false, false, data_wrapper->GetOutput());
-                    CsaInteract::GetInstance().WriteErrorCode(rt_ret, ERROR_MODULE_RUNTIME, JOBSUBSTATE_GRAPH_EXEC);
                     continue);
     GELOGI("rtModelExecute end");
     GE_IF_BOOL_EXEC(model->is_first_execute_, GE_TIMESTAMP_EVENT_END(rtModelExecute, "GraphExcute::rtModelExecute"));
@@ -2808,7 +2806,6 @@ void *DavinciModel::Run(DavinciModel *model) {
         rt_ret != RT_ERROR_NONE, rslt_flg = false; GELOGI("seq_end_flg: %d", seq_end_flag);
         (void)model->ReturnResult(current_data.index, false, seq_end_flag,
                                   data_wrapper->GetOutput());  // [No need to check value]
-        CsaInteract::GetInstance().StoreInternalErrorCode(rt_ret, ERROR_MODULE_RUNTIME, JOBSUBSTATE_GRAPH_EXEC);
         continue);
     }
 
@@ -2832,10 +2829,11 @@ void *DavinciModel::Run(DavinciModel *model) {
 
     model->iterator_count_++;
     model->is_first_execute_ = false;
-    GELOGI("run iterator count is %lu", model->iterator_count_);
+    // model run finished
+    model->SetRunningFlag(false);
+    GELOGI("run iterator count is %lu, model_id:%u", model->iterator_count_, model->model_id_);
   }
 
-  CsaInteract::GetInstance().WriteInternalErrorCode();
   GELOGI("Model run end, model id:%u", model->model_id_);
   return nullptr;
 }
@@ -2888,9 +2886,9 @@ Status DavinciModel::ModelRunStart() {
   int64_t maxDumpOpNum = std::strtol(opt.c_str(), nullptr, kDecimal);
   maxDumpOpNum_ = maxDumpOpNum;
 
-  error_context_ = ErrorManager::GetInstance().GetErrorContext();
+  error_context_ = ErrorManager::GetInstance().GetErrorManagerContext();
   CREATE_STD_THREAD(thread_id_, DavinciModel::Run, this);
-  GELOGI("model tread create success, model id:%u.", model_id_);
+  GELOGI("model thread create success, model id:%u.", model_id_);
   return SUCCESS;
 }
 
@@ -4340,4 +4338,43 @@ Status DavinciModel::InitL1DataDumperArgs() {
   return SUCCESS;
 }
 
+Status DavinciModel::SetRunAsyncListenerCallback(const RunAsyncCallback &callback) {
+  auto listener = dynamic_cast<RunAsyncListener *>(listener_.get());
+  GE_CHECK_NOTNULL(listener);
+  listener->SetCallback(callback);
+  return SUCCESS;
+}
+
+void DavinciModel::UpdateOpIOAddrs(uint32_t task_id, uint32_t stream_id, const std::vector<void *> &io_addrs) {
+  if (fixed_mem_base_ == reinterpret_cast<uintptr_t>(mem_base_)) {
+    GELOGD("[Update][OpIOAddrs] No need to update op input output addr.");
+    return;
+  }
+
+  OpDescInfo *op_desc_info = exception_dumper_.MutableOpDescInfo(task_id, stream_id);
+  if (op_desc_info == nullptr) {
+    GELOGW("[Update][OpIOAddrs] Find op desc failed, task_id: %u, stream_id: %u.", task_id, stream_id);
+    return;
+  }
+  size_t input_size = op_desc_info->input_addrs.size();
+  size_t output_size = op_desc_info->output_addrs.size();
+  if (input_size + output_size != io_addrs.size()) {
+    GELOGW("[Update][OpIOAddrs] Op[%s] input size[%zu] and output size[%zu] is not equal to io addr size[%zu]",
+           op_desc_info->op_name.c_str(), input_size, output_size, io_addrs.size());
+    return;
+  }
+
+  vector<void *> input_addrs;
+  vector<void *> output_addrs;
+  for (size_t i = 0; i < io_addrs.size(); i++) {
+    if (i < input_size) {
+      input_addrs.emplace_back(GetRunAddress(io_addrs[i]));
+    } else {
+      output_addrs.emplace_back(GetRunAddress(io_addrs[i]));
+    }
+  }
+  op_desc_info->input_addrs = input_addrs;
+  op_desc_info->output_addrs = output_addrs;
+  GELOGD("[Update][OpIOAddrs] Op [%s] update input output addr success.", op_desc_info->op_name.c_str());
+}
 }  // namespace ge

ge/graph/load/model_manager/davinci_model.h

@@ -29,6 +29,7 @@
 #include "common/helper/om_file_helper.h"
 #include "common/opskernel/ge_task_info.h"
 #include "common/properties_manager.h"
+#include "common/dump/exception_dumper.h"
 #include "common/dump/opdebug_register.h"
 #include "common/types.h"
 #include "framework/common/util.h"
@@ -221,6 +222,11 @@ class DavinciModel {
   ///
   DataInputer *const GetDataInputer() const { return data_inputer_; }
 
+  uint32_t GetDataInputerSize() {
+    GE_CHECK_NOTNULL(data_inputer_);
+    return data_inputer_->Size();
+  }
+
   // get Stream number
   uint32_t StreamNum() const { return runtime_param_.stream_num; }
 
@@ -418,7 +424,7 @@ class DavinciModel {
   ///
   uint64_t GetSessionId() const { return session_id_; }
 
-  const struct ErrorMessage::Context &GetErrorContext() const { return error_context_; }
+  const struct error_message::Context &GetErrorContext() const { return error_context_; }
 
   ///
   /// @ingroup ge
@@ -471,13 +477,17 @@ class DavinciModel {
   Status ReportProfilingData();
 
   void SaveDumpOpInfo(const RuntimeParam &model_param, const OpDescPtr &op, uint32_t task_id, uint32_t stream_id) {
-    data_dumper_.SaveDumpOpInfo(model_param, op, task_id, stream_id);
+    exception_dumper_.SaveDumpOpInfo(model_param, op, task_id, stream_id);
   }
 
   void SaveDumpTask(uint32_t task_id, uint32_t stream_id, const shared_ptr<OpDesc> &op_desc, uintptr_t args) {
     data_dumper_.SaveDumpTask(task_id, stream_id, op_desc, args);
   }
 
+  Status DumpExceptionInfo(const std::vector<rtExceptionInfo> &exception_infos) const {
+    return exception_dumper_.DumpExceptionInfo(exception_infos);
+  }
+
   void SetKnownShapeGlobalStep(void *global_step) {
     known_shape_global_step_ = global_step;
   }
@@ -557,8 +567,13 @@ class DavinciModel {
   const DumpProperties &GetDumpProperties() const { return data_dumper_.GetDumpProperties(); }
 
   bool GetOpDescInfo(uint32_t stream_id, uint32_t task_id, OpDescInfo &op_desc_info) const {
-    return data_dumper_.GetOpDescInfo(stream_id, task_id, op_desc_info);
+    return exception_dumper_.GetOpDescInfo(stream_id, task_id, op_desc_info);
   }
+  void UpdateOpIOAddrs(uint32_t task_id, uint32_t stream_id, const std::vector<void *> &io_addrs);
+
+  bool GetRunningFlag() const { return running_flg_; }
+  void SetRunningFlag(bool flag) { running_flg_ = flag; }
+  Status SetRunAsyncListenerCallback(const RunAsyncCallback &callback);
 
  private:
   // memory address of weights
@@ -924,6 +939,8 @@ class DavinciModel {
   shared_ptr<ModelListener> listener_;
 
   bool run_flg_;
+  // check whether model is running with data
+  bool running_flg_ = false;
 
   mutex mux_run_flg_;
 
@@ -975,7 +992,7 @@ class DavinciModel {
   vector<uintptr_t> output_mbuf_list_;  // output mbuf created by dequeue task.
 
   uint64_t session_id_;
-  struct ErrorMessage::Context error_context_;
+  struct error_message::Context error_context_;
 
   uint32_t device_id_;
 
@@ -1001,6 +1018,7 @@ class DavinciModel {
   int64_t maxDumpOpNum_;
   // for data dump
   DataDumper data_dumper_;
+  ExceptionDumper exception_dumper_;
   OpdebugRegister opdebug_register_;
   uint64_t iterator_count_;
   bool is_l1_fusion_enable_;

ge/graph/load/model_manager/model_manager.cc

@@ -280,6 +280,7 @@ ModelManager::~ModelManager() {
   model_map_.clear();
   model_aicpu_kernel_.clear();
   cust_aicpu_so_.clear();
+  dump_exception_flag_ = false;
 
   GE_IF_BOOL_EXEC(device_count > 0, GE_CHK_RT(rtDeviceReset(0)));
 }
@@ -330,6 +331,7 @@ Status ModelManager::LoadModelOnline(uint32_t &model_id, const shared_ptr<ge::Ge
   GE_CHK_BOOL_RET_STATUS(listener.get() != nullptr, PARAM_INVALID, "Param incorrect, listener is null");
   if (model_id == INVALID_MODEL_ID) {
     GenModelId(&model_id);
+    GELOGD("Generate new model_id:%u", model_id);
   }
   auto name_to_model = ge_root_model->GetSubgraphInstanceNameToModel();
   string om_name;
@@ -339,11 +341,7 @@ Status ModelManager::LoadModelOnline(uint32_t &model_id, const shared_ptr<ge::Ge
 
   mmTimespec timespec = mmGetTickCount();
   std::shared_ptr<DavinciModel> davinci_model = MakeShared<DavinciModel>(0, listener);
-  if (davinci_model == nullptr) {
-    REPORT_CALL_ERROR("E19999", "New DavinciModel fail, model_id:%u", model_id);
-    GELOGE(FAILED, "davinci_model is nullptr");
-    return FAILED;
-  }
+  GE_CHECK_NOTNULL(davinci_model);
   davinci_model->SetProfileTime(MODEL_LOAD_START, (timespec.tv_sec * kTimeSpecNano +
                                                    timespec.tv_nsec));  // 1000 ^ 3 converts second to nanosecond
   davinci_model->SetId(model_id);
@@ -363,7 +361,18 @@ Status ModelManager::LoadModelOnline(uint32_t &model_id, const shared_ptr<ge::Ge
     GE_IF_BOOL_EXEC(SUCCESS != (ret = davinci_model->Assign(ge_model)), GELOGW("assign model to modeldef failed.");
                     break;);
     GE_TIMESTAMP_END(Assign, "GraphLoader::ModelAssign");
-
+    /// In multi-threaded inference,  using the same session_id among multiple threads may cause some threads to fail.
+    /// These session_ids come from the same model, so the values of session_id are the same.
+    /// Update session_id for infer in load model to avoid the same session_id.
+    if (!ge_root_model->GetTrainFlag()) {
+      uint64_t new_session_id;
+      ret = GenSessionId(new_session_id);
+      GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(ret != SUCCESS, return ret, "Generate session_id for infer failed.");
+      ret = davinci_model->UpdateSessionId(new_session_id);
+      GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(ret != SUCCESS, return ret, "Update session_id for infer failed.");
+      ge_model->InsertSessionMap(model_id, new_session_id);
+      GELOGD("Update new session id: %lu.", new_session_id);
+    }
     GE_TIMESTAMP_START(Init);
     GE_IF_BOOL_EXEC(SUCCESS != (ret = davinci_model->Init()), GELOGW("DavinciInit failed."); break;);
     GE_TIMESTAMP_END(Init, "GraphLoader::ModelInit");
@@ -376,16 +385,16 @@ Status ModelManager::LoadModelOnline(uint32_t &model_id, const shared_ptr<ge::Ge
   return ret;
 }
 
-void ModelManager::InsertModel(uint32_t id, std::shared_ptr<DavinciModel> &davinci_model) {
-  GE_CHK_BOOL_EXEC(davinci_model != nullptr, return, "davinci_model ptr is null, id: %u", id);
+void ModelManager::InsertModel(uint32_t model_id, std::shared_ptr<DavinciModel> &davinci_model) {
+  GE_CHK_BOOL_EXEC(davinci_model != nullptr, return, "davinci_model ptr is null, id: %u", model_id);
   std::lock_guard<std::recursive_mutex> lock(map_mutex_);
-  model_map_[id] = davinci_model;
+  model_map_[model_id] = davinci_model;
 }
 
-void ModelManager::InsertModel(uint32_t id, shared_ptr<hybrid::HybridDavinciModel> &hybrid_model) {
-  GE_CHK_BOOL_EXEC(hybrid_model != nullptr, return, "hybrid_model ptr is null, id: %u", id);
+void ModelManager::InsertModel(uint32_t model_id, shared_ptr<hybrid::HybridDavinciModel> &hybrid_model) {
+  GE_CHK_BOOL_EXEC(hybrid_model != nullptr, return, "hybrid_model ptr is null, id: %u", model_id);
   std::lock_guard<std::recursive_mutex> lock(map_mutex_);
-  hybrid_model_map_[id] = hybrid_model;
+  hybrid_model_map_[model_id] = hybrid_model;
 }
 
 Status ModelManager::DeleteModel(uint32_t id) {
@@ -1083,7 +1092,7 @@ Status ModelManager::GenSessionId(uint64_t &session_id) {
 
   mmTimeval tv;
   if (mmGetTimeOfDay(&tv, nullptr) != 0) {
-    REPORT_CALL_ERROR("E19999", "Call mmGetTimeOfDay fail");
+    REPORT_CALL_ERROR("E19999", "Call mmGetTimeOfDay fail. errmsg:%s", strerror(errno));
     GELOGE(INTERNAL_ERROR, "Failed to get current time.");
     return INTERNAL_ERROR;
   }
@@ -1575,9 +1584,21 @@ Status ModelManager::GetOpDescInfo(uint32_t device_id, uint32_t stream_id, uint3
   for (const auto &model : model_map_) {
     auto davinci_model = model.second;
     if (davinci_model->GetDeviceId() == device_id) {
-      GELOGI("Start to GetOpDescInfo of device_id: %u.", device_id);
+      GELOGI("[Get][OpDescInfo] Start to GetOpDescInfo of device_id: %u in davinci model.", device_id);
       if (davinci_model->GetOpDescInfo(stream_id, task_id, op_desc_info)) {
-        GELOGI("Find specific node of stream_id: %u, task_id: %u.", stream_id, task_id);
+        GELOGI("[Get][OpDescInfo] Find specific node of stream_id: %u, task_id: %u in davinci model.",
+               stream_id, task_id);
+        return SUCCESS;
+      }
+    }
+  }
+  for (const auto &model : hybrid_model_map_) {
+    auto hybrid_model = model.second;
+    if (hybrid_model->GetDeviceId() == device_id) {
+      GELOGI("[Get][OpDescInfo] Start to GetOpDescInfo of device_id: %u in hybrid model.", device_id);
+      if (hybrid_model->GetOpDescInfo(stream_id, task_id, op_desc_info)) {
+        GELOGI("[Get][OpDescInfo] Find specific node of stream_id: %u, task_id: %u in hybrid model.",
+               stream_id, task_id);
         return SUCCESS;
       }
     }
@@ -1590,6 +1611,7 @@ Status ModelManager::EnableExceptionDump(const std::map<string, string> &options
   if (iter != options.end()) {
     GELOGI("Find option enable_exeception_dump is %s", iter->second.c_str());
     if (iter->second == "1") {
+      dump_exception_flag_ = true;
       rtError_t rt_ret = rtSetTaskFailCallback(reinterpret_cast<rtTaskFailCallback>(ExceptionCallback));
       if (rt_ret != RT_ERROR_NONE) {
         REPORT_CALL_ERROR("E19999", "Call rtSetTaskFailCallback fail, ret = 0x%X",

ge/graph/load/model_manager/model_manager.h

@@ -313,6 +313,7 @@ class FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ModelManager {
     instance->AddExceptionInfo(*rt_exception_info);
   }
 
+  bool IsDumpExceptionOpen() { return dump_exception_flag_; }
  private:
   ///
   /// @ingroup domi_ome
@@ -330,8 +331,8 @@ class FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ModelManager {
   /// @ingroup domi_ome
   /// @brief insert new model into model manager set
   ///
-  void InsertModel(uint32_t id, std::shared_ptr<DavinciModel> &davinci_model);
-  void InsertModel(uint32_t id, std::shared_ptr<hybrid::HybridDavinciModel> &hybrid_model);
+  void InsertModel(uint32_t model_id, std::shared_ptr<DavinciModel> &davinci_model);
+  void InsertModel(uint32_t model_id, std::shared_ptr<hybrid::HybridDavinciModel> &hybrid_model);
 
   ///
   /// @ingroup domi_ome
@@ -356,6 +357,7 @@ class FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ModelManager {
   std::map<uintptr_t, std::map<std::string, CustAICPUKernelPtr>> cust_aicpu_so_;
 
   static DumpProperties dump_properties_;
+  bool dump_exception_flag_ = false;
 };
 }  // namespace ge
 

ge/graph/load/model_manager/model_utils.cc

@@ -319,7 +319,7 @@ vector<void *> ModelUtils::GetInputDataAddrs(const RuntimeParam &model_param, Co
     const GeTensorDescPtr tensor_desc = op_desc->MutableInputDesc(static_cast<uint32_t>(i));
     GE_IF_BOOL_EXEC(tensor_desc == nullptr, GELOGD("Op: %s, Index: %zu, has no input", op_desc->GetName().c_str(), i);
                     continue;)
-    if ((i < v_is_input_const.size()) && v_is_input_const[i] && (op_type != NETOUTPUT)) {
+    if ((i < v_is_input_const.size()) && v_is_input_const[i]) {
       // TBE: add weights address to input
       int64_t tensor_size = 0;
       GE_CHK_STATUS(TensorUtils::GetSize(*tensor_desc, tensor_size));

ge/graph/load/model_manager/task_info/kernel_ex_task_info.cc

@@ -357,6 +357,7 @@ void KernelExTaskInfo::SetIoAddrs(const OpDescPtr &op_desc) {
 Status KernelExTaskInfo::UpdateArgs() {
   GELOGI("KernelExTaskInfo::UpdateArgs in.");
   davinci_model_->SetTotalIOAddrs(io_addrs_);
+  davinci_model_->UpdateOpIOAddrs(task_id_, stream_id_, io_addrs_);
   GELOGI("KernelExTaskInfo::UpdateArgs success.");
   return SUCCESS;
 }

ge/graph/load/model_manager/task_info/kernel_task_info.cc

@@ -523,6 +523,7 @@ Status KernelTaskInfo::UpdateArgs() {
       return CopyNoncontinuousArgs(io_addr_offset_);
     }
     davinci_model_->SetTotalIOAddrs(io_addrs_);
+    davinci_model_->UpdateOpIOAddrs(task_id_, stream_id_, io_addrs_);
   } else if (kernel_type_ == ccKernelType::AI_CPU || kernel_type_ == ccKernelType::CUST_AI_CPU) {
     return CopyNoncontinuousArgs(sizeof(aicpu::AicpuParamHead));
   }

ge/graph/load/model_manager/task_info/super_kernel/super_kernel_factory.cc

@@ -31,7 +31,9 @@ Status SuperKernelFactory::Init() {
     std::string skt_bin = "libcce_aicore.so";
     handle_ = mmDlopen(skt_bin.c_str(), MMPA_RTLD_NOW | MMPA_RTLD_GLOBAL);
     if (handle_ == nullptr) {
-      GELOGE(FAILED, "SKT: open skt lib failed, please check LD_LIBRARY_PATH.");
+      const char* error = mmDlerror();
+      GE_IF_BOOL_EXEC(error == nullptr, error = "");
+      GELOGE(FAILED, "SKT: open skt lib failed, please check LD_LIBRARY_PATH. errmsg:%s", error);
     }
     rtError_t rt_ret;
     rt_ret = rtGetFunctionByName(this->sk_stub_name_.c_str(), &this->func_stub_);

ge/graph/manager/graph_manager.h

@@ -184,6 +184,20 @@ class GraphManager {
 
   Status SaveCheckPointResult(const Graph &graph, const std::vector<Tensor> &outputs, map<string, Tensor> &var_results);
 
+  void RemoveGraphCount(GraphId graph_id);
+
+  void IncreaseGraphCount(GraphId graph_id);
+
+  void DecreaseGraphCount(GraphId graph_id);
+
+  Status GetGraphCount(GraphId graph_id, uint32_t &count);
+
+  void SetAddGraphCondition(GraphId graph_id, uint32_t cond);
+
+  uint32_t GetAddGraphCondition(GraphId graph_id);
+
+  void RemoveAddGraphCondition(GraphId graph_id);
+
  private:
   struct CompilerStages {
     GraphPrepare preparer;
@@ -196,7 +210,7 @@ class GraphManager {
     GraphId graph_id;
     std::vector<ge::InputTensorInfo> input_tensor;
     uint64_t session_id;
-    struct ErrorMessage::Context error_context;
+    struct error_message::Context error_context;
     GEThreadLocalContext context;
     RunAsyncCallback callback;
   };
@@ -205,7 +219,7 @@ class GraphManager {
     GraphNodePtr graph_node;
     GraphId graph_id;
     uint64_t session_id;
-    struct ErrorMessage::Context error_context;
+    struct error_message::Context error_context;
     std::vector<ge::InputTensorInfo> input_tensor;
     GeRootModelPtr ge_root_model;
     GEThreadLocalContext context;
@@ -223,7 +237,7 @@ class GraphManager {
                                                 const SubGraphInfoPtr &sub_graph_info_ptr,
                                                 const std::string &root_graph_name,
                                                 uint64_t session_id,
-                                                const struct ErrorMessage::Context &error_context,
+                                                const struct error_message::Context &error_context,
                                                 const GEThreadLocalContext &ge_context);
   Status ParseInputsDims(const std::vector<InputTensorInfo> &input_tensor);
   void ParseInputsDimsForData(const std::vector<InputTensorInfo> &input_tensor);
@@ -381,6 +395,26 @@ class GraphManager {
   CompilerStages &GetCompilerStages(GraphId graph_id);
   void RemoveCompilerStages(GraphId graph_id);
 
+  static Status CheckIncreBuildAndPreRun(GraphManager *graph_manager, const PreRunArgs &args, GraphNodePtr &graph_node,
+                                         GeRootModelPtr &ge_root_model);
+
+  void ReleaseMemory(const GeModelPtr &ge_model, GraphNodePtr &graph_node, const std::vector<uint32_t> &model_ids,
+                     uint32_t graph_id, uint64_t session_id);
+
+  Status CheckRepeatAdd(uint32_t graph_id, bool &is_added);
+
+  Status NotifyWaittingGraph(uint32_t graph_id);
+
+  Status CreateGraphNode(uint32_t graph_id, const Graph &graph, const std::map<std::string, std::string> &options);
+
+  Status SetStagesOptions(uint32_t graph_id, const GraphManagerOptions &options);
+
+  Status UnloadModel(GeRootModelPtr ge_root_model, uint32_t graph_id);
+
+  void SetSessionGraphId(ComputeGraphPtr compute_graph, uint32_t graph_id);
+
+  static Status CheckGraphAdded(const GraphId &graph_id, const Graph &graph);
+
   std::atomic_bool thread_run_flag_;
   BlockingQueue<PreRunArgs> prerun_args_q_{};
   BlockingQueue<RunArgs> run_args_q_{};
@@ -416,6 +450,16 @@ class GraphManager {
 
   std::mutex member_mutex_;
   std::mutex unload_model_mutex_;
+  // avoid repeatively add same graph (owns same graph id)
+  std::mutex add_graph_mutex_;
+  std::mutex add_graph_cond_mutex_;
+  std::condition_variable add_graph_cv_;
+
+  std::map<GraphId, uint32_t> graph_id_to_add_graph_cond_;
+  // use for multi-thread online-infer scenario
+  std::set<GraphId> to_be_deleted_graphs_;
+  std::map<GraphId, uint32_t> graph_count_;
+  std::mutex graph_count_mutex_;
 };
 }  // namespace ge
 

ge/graph/manager/graph_manager_utils.cc

@@ -60,6 +60,15 @@ void GraphNode::Unlock() {
   sem_.Pop(unused);
 }
 
+void GraphNode::IncreaseLoadCount() {
+  std::unique_lock<std::mutex> lock(load_count_mu_);
+  if (load_record_ == kMaxLoadNum) {
+    GELOGW("Reach the maximum of load_count:%u", kMaxLoadNum);
+    return;
+  }
+  ++load_count_;
+}
+
 SubGraphInfo::SubGraphInfo() : subgraph_ptr_(nullptr), ge_model_ptr_(nullptr), malloc_flag_(false) {}
 
 SubGraphInfo::~SubGraphInfo() {

ge/graph/manager/graph_manager_utils.h

@@ -55,6 +55,7 @@ using ConstGraphPtr = std::shared_ptr<const ge::Graph>;
 using GraphPtr = std::shared_ptr<ge::Graph>;
 
 const uint64_t INVALID_SESSION_ID = 0xffffffffffffffffULL;
+const uint32_t kMaxLoadNum = 8;
 
 struct ModelIdInfo {
   uint32_t model_id{INVALID_MODEL_ID};
@@ -162,6 +163,8 @@ class GraphNode {
   bool GetBuildFlag() const { return build_flag_; }
   void SetBuildFlag(bool buildFlag) { build_flag_ = buildFlag; }
   bool GetLoadFlag() const { return load_flag_; }
+  // allow repeatively load graph owns same graph id
+  void UpdateLoadFlag() { load_flag_ = load_count_ == 0 || load_record_ >= kMaxLoadNum; }
   void SetLoadFlag(bool load_flag) { load_flag_ = load_flag; }
   void SetGeModel(const GeModelPtr &ge_model) { ge_model_ = ge_model; }
   GeModelPtr GetGeModel() const { return ge_model_; }
@@ -172,6 +175,16 @@ class GraphNode {
   void Lock();
   void Unlock();
 
+  void SetSemSize(uint32_t size) { sem_.SetMaxSize(size); }
+
+  uint32_t GetLoadCount() const { return load_count_; }
+  void SetLoadCount(uint32_t count) { load_count_ = count; }
+  uint32_t GetLoadRecord() const { return load_record_; }
+  void SetLoadRecord(uint32_t record) { load_record_ = record; }
+  void IncreaseLoadRecord() { ++load_record_; }
+  void IncreaseLoadCount();
+  void DecreaseLoadCount() { --load_count_; }
+
   // run graph asynchronous listener
   std::shared_ptr<RunAsyncListener> graph_run_async_listener_;
 
@@ -184,11 +197,17 @@ class GraphNode {
   GraphPtr graph_;
   ComputeGraphPtr compute_graph_;
   bool build_flag_;
+  // load_flag_ is true if more than 1 model were loaded
   bool load_flag_;
   bool async_;
   GeModelPtr ge_model_;
   GeRootModelPtr ge_root_model_;
   BlockingQueue<uint8_t> sem_;
+  // consist with graph_count of same graph_id in graph_manager
+  uint32_t load_count_ = 0;
+  // total times of loading a graph with same graph_id.
+  uint32_t load_record_ = 0;
+  std::mutex load_count_mu_;
 };
 
 using GraphNodePtr = std::shared_ptr<GraphNode>;

ge/graph/manager/graph_mem_allocator.h

@@ -226,7 +226,7 @@ class MemManager {
 
     // Usually impossible
     if (allocator == nullptr) {
-      GELOGE(ge::INTERNAL_ERROR, "Get allocator failed, memory type is %u.", memory_type);
+      GELOGW("Get allocator failed, memory type is %u.", memory_type);
       static T default_allocator(RT_MEMORY_RESERVED);
       return default_allocator;
     }

ge/graph/manager/host_mem_manager.cc

@@ -45,12 +45,11 @@ Status SharedMemAllocator::Allocate(SharedMemInfo &mem_info) {
     return GE_GRAPH_MEMORY_ALLOC_FAILED;
   }
   mem_info.fd = output_para.fd;
-  mem_info.host_aligned_ptr = AlignedPtr::BuildFromAllocFunc([&output_para](std::unique_ptr<uint8_t[], deleter> &ptr) {
-                                                               ptr.reset(reinterpret_cast<uint8_t *>(output_para.ptr));
-                                                             },
-                                                             [](uint8_t *ptr) {
-                                                               ptr = nullptr;
-                                                             });
+  mem_info.host_aligned_ptr = AlignedPtr::BuildFromAllocFunc(
+    [&output_para](std::unique_ptr<uint8_t[], AlignedPtr::Deleter> &ptr) {
+      ptr.reset(reinterpret_cast<uint8_t *>(output_para.ptr));
+    },
+    [](uint8_t *ptr) { ptr = nullptr; });
   mem_info.device_address = reinterpret_cast<uint8_t *>(output_para.devPtr);
   return SUCCESS;
 }

ge/graph/manager/trans_var_data_utils.cc

@@ -487,7 +487,7 @@ Status TransVarDataUtils::TransAllVarData(const vector<NodePtr> &variable_nodes,
 
     std::future<Status> f = executor.commit(
         [](const ge::NodePtr &node, uint64_t session_id, rtContext_t ctx, uint32_t graph_id,
-           const struct ErrorMessage::Context &error_context) -> Status {
+           const struct error_message::Context &error_context) -> Status {
           ErrorManager::GetInstance().SetErrorContext(error_context);
           rtError_t rt_ret = rtCtxSetCurrent(ctx);
           if (rt_ret != RT_ERROR_NONE) {
@@ -525,7 +525,7 @@ Status TransVarDataUtils::TransAllVarData(const vector<NodePtr> &variable_nodes,
           }
           return SUCCESS;
         },
-        node, session_id, context, graph_id, ErrorManager::GetInstance().GetErrorContext());
+        node, session_id, context, graph_id, ErrorManager::GetInstance().GetErrorManagerContext());
     if (!f.valid()) {
       GELOGE(FAILED, "Future is invalid");
       return FAILED;

ge/graph/manager/util/debug.cc

@@ -35,14 +35,14 @@ void Debug::DumpProto(const Message &proto, const char *file) {
   int fd = mmOpen2(file_path.c_str(), M_WRONLY | M_CREAT | O_TRUNC, M_IRUSR | M_IWUSR | M_UMASK_GRPREAD |
                    M_UMASK_OTHREAD);
   if (fd == -1) {
-    GELOGW("Write %s failed", file_path.c_str());
+    GELOGW("Write %s failed. errmsg:%s", file_path.c_str(), strerror(errno));
     return;
   }
   auto output = ge::MakeShared<FileOutputStream>(fd);
   if (output == nullptr) {
     GELOGW("create output failed.");
     if (mmClose(fd) != 0) {
-      GELOGW("close fd failed.");
+      GELOGW("close fd failed. errmsg:%s", strerror(errno));
     }
     return;
   }
@@ -51,7 +51,7 @@ void Debug::DumpProto(const Message &proto, const char *file) {
     GELOGW("dump proto failed.");
   }
   if (mmClose(fd) != 0) {
-    GELOGW("close fd failed.");
+    GELOGW("close fd failed. errmsg:%s", strerror(errno));
   }
 }
 

ge/graph/passes/attach_stream_label_pass.cc

@@ -24,34 +24,31 @@ namespace ge {
 Status AttachStreamLabelPass::Run(ComputeGraphPtr graph) {
   GELOGD("AttachStreamLabelPass Enter.");
 
-  FindNodes(graph);
-  for (const auto &node : need_label_nodes_) {
-    GE_CHK_STATUS_RET(UpdateCondBranch(node), "Update cond branch failed, start node:%s.", node->GetName().c_str());
+  std::vector<NodePtr> need_label_nodes;
+  std::vector<NodePtr> enter_nodes;
+  std::map<NodePtr, NodePtr> branch_head_nodes;
+  FindNodes(graph, need_label_nodes, enter_nodes, branch_head_nodes);
+  for (const auto &node : need_label_nodes) {
+    GE_CHK_STATUS_RET(UpdateCondBranch(node, branch_head_nodes), "Update cond branch failed, start node:%s.", node->GetName().c_str());
   }
-  GE_CHK_STATUS_RET(UpdateEnterNode(), "UpdateEnterNode failed.");
+  GE_CHK_STATUS_RET(UpdateEnterNode(enter_nodes), "UpdateEnterNode failed.");
 
   GELOGD("AttachStreamLabelPass Leave.");
   return SUCCESS;
 }
 
 ///
-/// @brief Clear Status, used for subgraph pass
-/// @return
-///
-Status AttachStreamLabelPass::ClearStatus() {
-  stream_switch_nodes_.clear();
-  need_label_nodes_.clear();
-  enter_nodes_.clear();
-  branch_head_nodes_.clear();
-  return SUCCESS;
-}
-
-///
 /// @brief Find StreamSwitch / StreamMerge / Enter node
 /// @param [in] graph
+/// @param [out] need_label_nodes
+/// @param [out] enter_nodes
+/// @param [out] branch_head_nodes
 /// @return void
 ///
-void AttachStreamLabelPass::FindNodes(const ComputeGraphPtr &graph) {
+void AttachStreamLabelPass::FindNodes(const ComputeGraphPtr &graph, std::vector<NodePtr> &need_label_nodes,
+                                      std::vector<NodePtr> &enter_nodes,
+                                      std::map<NodePtr, NodePtr> &branch_head_nodes) {
+  std::vector<NodePtr> stream_switch_nodes;
   for (const NodePtr &node : graph->GetDirectNode()) {
     const auto &op_desc = node->GetOpDesc();
     if (op_desc == nullptr) {
@@ -59,29 +56,31 @@ void AttachStreamLabelPass::FindNodes(const ComputeGraphPtr &graph) {
     }
     const std::string &type = op_desc->GetType();
     if ((type == STREAMSWITCH) && op_desc->HasAttr(ATTR_NAME_SWITCH_TRUE_BRANCH_FLAG)) {
-      stream_switch_nodes_.emplace_back(node);
+      stream_switch_nodes.emplace_back(node);
     } else if ((type == STREAMMERGE) && !op_desc->HasAttr(ATTR_NAME_NEXT_ITERATION)) {
-      need_label_nodes_.emplace_back(node);
+      need_label_nodes.emplace_back(node);
     } else if ((type == ENTER) || (type == REFENTER)) {
-      enter_nodes_.emplace_back(node);
+      enter_nodes.emplace_back(node);
     }
   }
 
-  for (const auto &node : stream_switch_nodes_) {
+  for (const auto &node : stream_switch_nodes) {
     for (const auto &out_ctrl_node : node->GetOutControlNodes()) {
       GELOGD("branch_head_node %s of stream_switch %s.", out_ctrl_node->GetName().c_str(), node->GetName().c_str());
-      branch_head_nodes_[out_ctrl_node] = node;
+      branch_head_nodes[out_ctrl_node] = node;
     }
-    need_label_nodes_.emplace_back(node);
+    need_label_nodes.emplace_back(node);
   }
 }
 
 ///
 /// @brief update cond branch
 /// @param [in] node
+/// @param [in] branch_head_nodes
 /// @return Status
 ///
-Status AttachStreamLabelPass::UpdateCondBranch(const NodePtr &node) {
+Status AttachStreamLabelPass::UpdateCondBranch(const NodePtr &node,
+                                               const std::map<NodePtr, NodePtr> &branch_head_nodes) {
   std::string stream_label;
   if (AttachFlag(node, stream_label) != SUCCESS) {
     GELOGE(FAILED, "Attach flag for node %s failed.", node->GetName().c_str());
@@ -103,8 +102,9 @@ Status AttachStreamLabelPass::UpdateCondBranch(const NodePtr &node) {
     const std::string &type = cur_node->GetType();
     for (const auto &out_node : cur_node->GetOutAllNodes()) {
       const std::string &out_type = out_node->GetType();
+      const auto &iter = branch_head_nodes.find(node);
       bool stop_flag = (end_type_set.count(out_type) > 0) ||
-                       ((branch_head_nodes_.count(out_node) > 0) && (branch_head_nodes_[out_node] != node)) ||
+                       ((iter != branch_head_nodes.end()) && (iter->second != node)) ||
                        (((type == ENTER) || (type == REFENTER)) && (out_type != STREAMACTIVE));
       if (!stop_flag) {
         nodes.push(out_node);
@@ -178,11 +178,12 @@ Status AttachStreamLabelPass::AttachFlag(const NodePtr &node, std::string &strea
 
 ///
 /// @brief Update stream_label start with enter nodes
+/// @param [in] enter_nodes
 /// @return Status
 ///
-Status AttachStreamLabelPass::UpdateEnterNode() {
+Status AttachStreamLabelPass::UpdateEnterNode(const std::vector<NodePtr> &enter_nodes) {
   std::unordered_map<NodePtr, std::vector<NodePtr>> enter_active_map;
-  for (const auto &enter_node : enter_nodes_) {
+  for (const auto &enter_node : enter_nodes) {
     for (const auto &out_ctrl_node : enter_node->GetOutControlNodes()) {
       if (out_ctrl_node->GetType() != STREAMACTIVE) {
         continue;
@@ -214,11 +215,11 @@ Status AttachStreamLabelPass::UpdateEnterNode() {
       return INTERNAL_ERROR;
     }
 
-    std::stack<NodePtr> enter_nodes;
+    std::stack<NodePtr> nodes;
     for (const auto &enter_node : pair.second) {
-      enter_nodes.emplace(enter_node);
+      nodes.emplace(enter_node);
     }
-    if (UpdateLoopBranch(enter_nodes, active_label_list[0]) != SUCCESS) {
+    if (UpdateLoopBranch(nodes, active_label_list[0]) != SUCCESS) {
       GELOGE(FAILED, "Update stream_label for loop_branch failed.");
       return FAILED;
     }

ge/graph/passes/attach_stream_label_pass.h

@@ -25,26 +25,25 @@ class AttachStreamLabelPass : public GraphPass {
  public:
   Status Run(ComputeGraphPtr graph);
 
-  ///
-  /// @brief Clear Status, used for subgraph pass
-  /// @return
-  ///
-  Status ClearStatus() override;
-
  private:
   ///
   /// @brief Find StreamSwitch / StreamMerge / Enter node
   /// @param [in] graph
+  /// @param [out] need_label_nodes
+  /// @param [out] enter_nodes
+  /// @param [out] branch_head_nodes
   /// @return void
   ///
-  void FindNodes(const ComputeGraphPtr &graph);
+  void FindNodes(const ComputeGraphPtr &graph, std::vector<NodePtr> &need_label_nodes,
+                 std::vector<NodePtr> &enter_nodes, std::map<NodePtr, NodePtr> &branch_head_nodes);
 
   ///
   /// @brief update cond branch
   /// @param [in] node
+  /// @param [in] branch_head_nodes
   /// @return Status
   ///
-  Status UpdateCondBranch(const NodePtr &node);
+  Status UpdateCondBranch(const NodePtr &node, const std::map<NodePtr, NodePtr> &branch_head_nodes);
 
   ///
   /// @brief attach flag
@@ -64,9 +63,10 @@ class AttachStreamLabelPass : public GraphPass {
 
   ///
   /// @brief Update stream_label start with enter nodes
+  /// @param [in] enter_nodes
   /// @return Status
   ///
-  Status UpdateEnterNode();
+  Status UpdateEnterNode(const std::vector<NodePtr> &enter_nodes);
 
   ///
   /// @brief Set stream_label for enter_nodes
@@ -75,11 +75,6 @@ class AttachStreamLabelPass : public GraphPass {
   /// @return Status
   ///
   static Status SetEnterLabel(const std::vector<NodePtr> &enter_nodes, const NodePtr &active_node);
-
-  std::vector<NodePtr> stream_switch_nodes_;
-  std::vector<NodePtr> need_label_nodes_;
-  std::vector<NodePtr> enter_nodes_;
-  std::unordered_map<NodePtr, NodePtr> branch_head_nodes_;
 };
 }  // namespace ge
 #endif  // GE_GRAPH_PASSES_ATTACH_STREAM_LABEL_PASS_H_

ge/graph/passes/dimension_adjust_pass.cc

@@ -78,7 +78,12 @@ Status DimensionAdjustPass::Run(ge::NodePtr &node) {
     GELOGE(ret, "DimensionAdjustPass compute failed");
     return ret;
   }
+  // Need to handle axis_input of node like ExpandDims
   if (node->GetAllInDataAnchors().size() > static_cast<size_t>(kRemoveInputIndex)) {
+    auto axis_node_out_anchor = node->GetInDataAnchor(kRemoveInputIndex)->GetPeerOutAnchor();
+    GE_CHECK_NOTNULL(axis_node_out_anchor);
+    auto axis_node = axis_node_out_anchor->GetOwnerNode();
+    // 1.Copy control dependency of axis node
     ret = PassUtils::UnlinkNodeWithControlCopy(node, kRemoveInputIndex);
     if (ret != SUCCESS) {
       REPORT_CALL_ERROR("E19999", "Unlink op:%s(%s) data input:%u with control edge copy failed",
@@ -86,6 +91,13 @@ Status DimensionAdjustPass::Run(ge::NodePtr &node) {
       GELOGE(ret, "DimensionAdjustPass unlink node with control copy fail.");
       return ret;
     }
+    // 2.Remove const axis node without any output
+    if ((axis_node->GetType() == CONSTANT || axis_node->GetType() == CONSTANTOP) &&
+        axis_node->GetOutDataNodesSize() == 0) {
+      ret = IsolateAndDeleteNode(axis_node, {});
+      GE_CHK_GRAPH_STATUS_RET(ret, "Fail to remove node %s.", axis_node->GetName().c_str());
+      GELOGI("Remove useless axis input const %s", axis_node->GetName().c_str());
+    }
   }
 
   ret = DealWithInNodes(node);

ge/graph/passes/link_gen_mask_nodes_pass.cc

@@ -107,6 +107,16 @@ void LinkGenMaskNodesPass::GetAllGenMaskNodes(ComputeGraphPtr graph, vector<Node
     auto in_data_nodes = node->GetInDataNodes();
     if (in_data_nodes.size() > kGenMaskInputIndex) {
       NodePtr &gen_mask = in_data_nodes.at(kGenMaskInputIndex);
+      for (auto &in_data_node : in_data_nodes) {
+        // node gen_mask is located at different place in the fused node
+        if (in_data_node->GetName().find(DROPOUTGENMASK) != in_data_node->GetName().npos) {
+          gen_mask = in_data_node;
+          GELOGD("The fused node type [%s], paired with the input node name [%s].",
+                 node->GetType().c_str(), gen_mask->GetName().c_str());
+          break;
+        }
+      }
+
       if ((gen_mask->GetOpDesc() == nullptr) || (gen_mask->GetOpDesc()->HasAttr(ATTR_NAME_STREAM_LABEL))) {
         continue;
       }

ge/graph/passes/net_output_pass.cc

@@ -514,7 +514,7 @@ Status NetOutputPass::Run(ge::ComputeGraphPtr graph) {
     GELOGE(GE_GRAPH_PARAM_NULLPTR, "Compute graph is null.");
     return GE_GRAPH_PARAM_NULLPTR;
   }
-  GELOGI("NetOutputPass Run.graph is [%s]", graph->GetName().c_str());
+  GELOGI("[NETOUTPUT PASS] Run.graph is [%s]", graph->GetName().c_str());
   NodePtr output_node = graph->FindFirstNodeMatchType(NETOUTPUT);
   // save user targets node
   SaveAndRemoveTargets(graph);
@@ -552,10 +552,17 @@ Status NetOutputPass::AddNetOutputNodeToGraph(const ge::ComputeGraphPtr &graph,
   // If user does not set out nodes and targets and no retval node, also add netoutput node
   if ((graph->GetGraphOutNodesInfo().empty()) && (graph->GetGraphTargetNodesInfo().empty()) &&
       !is_include_special_node_) {
-    GELOGI("[NETOUTPUT PASS] output_nodes and target_nodes and special nodes is empty!Add netoutput!");
+    GELOGI("[NETOUTPUT PASS] Both output, target and special nodes are empty! add net output node");
     output_node = graph->AddNode(net_output_desc);
     GE_CHK_STATUS_RET(AddCtrlEdgesBetweenLeafAndNetOutput(graph, output_node),
                       "add ctrl edge between leaf and netoutput failed");
+    if (!ge::AttrUtils::SetInt(output_node->GetOpDesc(), ATTR_NAME_TRUE_BRANCH_STREAM, 0)) {
+      REPORT_CALL_ERROR("E19999", "Set Attr:%s to op:%s(%s) failed", ATTR_NAME_TRUE_BRANCH_STREAM.c_str(),
+                        output_node->GetName().c_str(), output_node->GetType().c_str());
+      GELOGE(INTERNAL_ERROR, "set ATTR_NAME_TRUE_BRANCH_STREAM failed");
+      return INTERNAL_ERROR;
+    }
+    GELOGI("[NETOUTPUT PASS] Add net output node succeed");
     return SUCCESS;
   }
   GELOGI("[NETOUTPUT PASS] Output node size:%lu.", output_nodes_info.size());

ge/graph/passes/pass_utils.cc

@@ -334,6 +334,9 @@ Status PassUtils::UnlinkNodeWithControlCopy(NodePtr &node, int index) {
   auto father_node = out_data_anchor->GetOwnerNode();
   // link father_node's in control nodes to node
   if (GraphUtils::CopyInCtrlEdges(father_node, node) != GRAPH_SUCCESS) {
+    REPORT_CALL_ERROR("E19999", "Copy in control edge from node:%s(%s) to node:%s(%s) failed",
+                      father_node->GetName().c_str(), father_node->GetType().c_str(),
+                      node->GetName().c_str(), node->GetType().c_str());
     return FAILED;
   }
   return SUCCESS;

ge/graph/passes/same_transdata_breadth_fusion_pass.cc

@@ -71,6 +71,7 @@ OpDescPtr SameTransdataBreadthFusionPass::GetCastOp(const GeTensorDesc &in_desc,
   auto cast_op = ge::OpDescUtils::GetOpDescFromOperator(node_op);
   node_op.BreakConnect();
   if (cast_op == nullptr) {
+    REPORT_INNER_ERROR("E19999", "Create Operator:%s(%s) failed", cast_op_name.str().c_str(), CAST);
     GELOGE(INTERNAL_ERROR, "new fusion cast op failed!");
     return nullptr;
   }
@@ -96,6 +97,8 @@ OpDescPtr SameTransdataBreadthFusionPass::GetCastOp(const GeTensorDesc &in_desc,
     }
   }
   if (!AttrUtils::SetInt(cast_op, CAST_ATTR_DST_TYPE, static_cast<int64_t>(out_desc.GetDataType()))) {
+    REPORT_CALL_ERROR("E19999", "Set Attr:%s to op:%s(%s) failed", CAST_ATTR_DST_TYPE.c_str(),
+                      cast_op->GetName().c_str(), cast_op->GetType().c_str());
     GELOGE(INTERNAL_ERROR, "set dst_type attr failed");
     return nullptr;
   }
@@ -204,6 +207,12 @@ graphStatus SameTransdataBreadthFusionPass::ReLinkDataOutput2PreNode(const NodeP
       GELOGI("remove edge.src:%s, dst:%s", out_anchor->GetOwnerNode()->GetName().c_str(),
              transdata_peer_in_anchor->GetOwnerNode()->GetName().c_str());
       if (GraphUtils::RemoveEdge(out_anchor, transdata_peer_in_anchor) != GRAPH_SUCCESS) {
+        REPORT_CALL_ERROR("E19999", "Remove edge between op:%s(%s)(index:%d) and op:%s(%s)(index:%d) failed",
+                          out_anchor->GetOwnerNode()->GetName().c_str(),
+                          out_anchor->GetOwnerNode()->GetType().c_str(), out_anchor->GetIdx(),
+                          transdata_peer_in_anchor->GetOwnerNode()->GetName().c_str(),
+                          transdata_peer_in_anchor->GetOwnerNode()->GetType().c_str(),
+                          transdata_peer_in_anchor->GetIdx());
         GELOGE(GRAPH_FAILED, "remove edge failed!src node:%s, dst node:%s", transdata_node->GetName().c_str(),
                transdata_peer_in_anchor->GetOwnerNode()->GetName().c_str());
         return GRAPH_FAILED;
@@ -211,6 +220,12 @@ graphStatus SameTransdataBreadthFusionPass::ReLinkDataOutput2PreNode(const NodeP
       GELOGI("add edge.src:%s, dst:%s", pre_out_anchor->GetOwnerNode()->GetName().c_str(),
              transdata_peer_in_anchor->GetOwnerNode()->GetName().c_str());
       if (GraphUtils::AddEdge(pre_out_anchor, transdata_peer_in_anchor) != GRAPH_SUCCESS) {
+        REPORT_CALL_ERROR("E19999", "Add edge between op:%s(%s)(index:%d) and op:%s(%s)(index:%d) failed",
+                          pre_out_anchor->GetOwnerNode()->GetName().c_str(),
+                          pre_out_anchor->GetOwnerNode()->GetType().c_str(), pre_out_anchor->GetIdx(),
+                          transdata_peer_in_anchor->GetOwnerNode()->GetName().c_str(),
+                          transdata_peer_in_anchor->GetOwnerNode()->GetType().c_str(),
+                          transdata_peer_in_anchor->GetIdx());
         GELOGE(GRAPH_FAILED, "add edge failed!src node:%s, dst node:%s",
                pre_out_anchor->GetOwnerNode()->GetName().c_str(),
                transdata_peer_in_anchor->GetOwnerNode()->GetName().c_str());
@@ -231,6 +246,11 @@ graphStatus SameTransdataBreadthFusionPass::ReLinkOutDataPeerInControlNodes2PreN
       GELOGD("remove edge.src:%s, dst:%s", out_anchor->GetOwnerNode()->GetName().c_str(),
              transdata_peer_in_control_anchor->GetOwnerNode()->GetName().c_str());
       if (GraphUtils::RemoveEdge(out_anchor, transdata_peer_in_control_anchor) != GRAPH_SUCCESS) {
+        REPORT_CALL_ERROR("E19999", "Remove control edge between op:%s(%s) and op:%s(%s) failed",
+                          out_anchor->GetOwnerNode()->GetName().c_str(),
+                          out_anchor->GetOwnerNode()->GetType().c_str(),
+                          transdata_peer_in_control_anchor->GetOwnerNode()->GetName().c_str(),
+                          transdata_peer_in_control_anchor->GetOwnerNode()->GetType().c_str());
         GELOGE(GRAPH_FAILED, "remove edge failed!src node:%s, dst node:%s", transdata_node->GetName().c_str(),
                transdata_peer_in_control_anchor->GetOwnerNode()->GetName().c_str());
         return GRAPH_FAILED;
@@ -240,6 +260,11 @@ graphStatus SameTransdataBreadthFusionPass::ReLinkOutDataPeerInControlNodes2PreN
         GELOGD("add edge.src:%s, dst:%s", pre_out_anchor->GetOwnerNode()->GetName().c_str(),
                transdata_peer_in_control_anchor->GetOwnerNode()->GetName().c_str());
         if (GraphUtils::AddEdge(pre_out_anchor, transdata_peer_in_control_anchor) != GRAPH_SUCCESS) {
+          REPORT_CALL_ERROR("E19999", "Add control edge between op:%s(%s) and op:%s(%s) failed",
+                            pre_out_anchor->GetOwnerNode()->GetName().c_str(),
+                            pre_out_anchor->GetOwnerNode()->GetType().c_str(),
+                            transdata_peer_in_control_anchor->GetOwnerNode()->GetName().c_str(),
+                            transdata_peer_in_control_anchor->GetOwnerNode()->GetType().c_str());
           GELOGE(GRAPH_FAILED, "add edge failed!src node:%s, dst node:%s",
                  pre_out_anchor->GetOwnerNode()->GetName().c_str(),
                  transdata_peer_in_control_anchor->GetOwnerNode()->GetName().c_str());
@@ -249,6 +274,11 @@ graphStatus SameTransdataBreadthFusionPass::ReLinkOutDataPeerInControlNodes2PreN
         GELOGD("add edge.src node:%s, dst node:%s", pre_out_anchor->GetOwnerNode()->GetName().c_str(),
                transdata_peer_in_control_anchor->GetOwnerNode()->GetName().c_str());
         if (GraphUtils::AddEdge(transdata_peer_out_control_anchor, transdata_peer_in_control_anchor) != GRAPH_SUCCESS) {
+          REPORT_CALL_ERROR("E19999", "Add control edge between op:%s(%s) and op:%s(%s) failed",
+                            transdata_peer_out_control_anchor->GetOwnerNode()->GetName().c_str(),
+                            transdata_peer_out_control_anchor->GetOwnerNode()->GetType().c_str(),
+                            transdata_peer_in_control_anchor->GetOwnerNode()->GetName().c_str(),
+                            transdata_peer_in_control_anchor->GetOwnerNode()->GetType().c_str());
           GELOGE(GRAPH_FAILED, "add edge failed!src node:%s, dst node:%s",
                  pre_out_anchor->GetOwnerNode()->GetName().c_str(),
                  transdata_peer_in_control_anchor->GetOwnerNode()->GetName().c_str());
@@ -290,6 +320,11 @@ graphStatus SameTransdataBreadthFusionPass::ReLinkOutControlPeerInControlAnchors
     GELOGD("remove edge.src:%s, dst:%s", transdata_node_keep->GetName().c_str(),
            transdata_peer_in_control_anchor->GetOwnerNode()->GetName().c_str());
     if (GraphUtils::RemoveEdge(out_control_anchor, transdata_peer_in_control_anchor) != GRAPH_SUCCESS) {
+      REPORT_CALL_ERROR("E19999", "Remove control edge between op:%s(%s) and op:%s(%s) failed",
+                        out_control_anchor->GetOwnerNode()->GetName().c_str(),
+                        out_control_anchor->GetOwnerNode()->GetType().c_str(),
+                        transdata_peer_in_control_anchor->GetOwnerNode()->GetName().c_str(),
+                        transdata_peer_in_control_anchor->GetOwnerNode()->GetType().c_str());
       GELOGE(GRAPH_FAILED, "remove transdata control edge failed!");
       return GRAPH_FAILED;
     }
@@ -298,6 +333,11 @@ graphStatus SameTransdataBreadthFusionPass::ReLinkOutControlPeerInControlAnchors
       GELOGD("add edge.src:%s, dst:%s", pre_out_anchor->GetOwnerNode()->GetName().c_str(),
              transdata_peer_in_control_anchor->GetOwnerNode()->GetName().c_str());
       if (GraphUtils::AddEdge(pre_out_anchor, transdata_peer_in_control_anchor) != GRAPH_SUCCESS) {
+        REPORT_CALL_ERROR("E19999", "Add control edge between op:%s(%s) and op:%s(%s) failed",
+                          pre_out_anchor->GetOwnerNode()->GetName().c_str(),
+                          pre_out_anchor->GetOwnerNode()->GetType().c_str(),
+                          transdata_peer_in_control_anchor->GetOwnerNode()->GetName().c_str(),
+                          transdata_peer_in_control_anchor->GetOwnerNode()->GetType().c_str());
         GELOGE(GRAPH_FAILED, "add control edge failed!");
         return GRAPH_FAILED;
       }
@@ -305,6 +345,11 @@ graphStatus SameTransdataBreadthFusionPass::ReLinkOutControlPeerInControlAnchors
       GELOGD("add edge.src:%s, dst:%s", transdata_peer_out_control_anchor->GetOwnerNode()->GetName().c_str(),
              transdata_peer_in_control_anchor->GetOwnerNode()->GetName().c_str());
       if (GraphUtils::AddEdge(transdata_peer_out_control_anchor, transdata_peer_in_control_anchor) != GRAPH_SUCCESS) {
+        REPORT_CALL_ERROR("E19999", "Add control edge between op:%s(%s) and op:%s(%s) failed",
+                          transdata_peer_out_control_anchor->GetOwnerNode()->GetName().c_str(),
+                          transdata_peer_out_control_anchor->GetOwnerNode()->GetType().c_str(),
+                          transdata_peer_in_control_anchor->GetOwnerNode()->GetName().c_str(),
+                          transdata_peer_in_control_anchor->GetOwnerNode()->GetType().c_str());
         GELOGE(GRAPH_FAILED, "add control edge failed!");
         return GRAPH_FAILED;
       }
@@ -329,6 +374,11 @@ graphStatus SameTransdataBreadthFusionPass::ReLinkOutControlPeerInDataAnchors(
     GELOGD("remove edge.src:%s, dst:%s", transdata_node_keep->GetName().c_str(),
            transdata_peer_in_data_anchor->GetOwnerNode()->GetName().c_str());
     if (GraphUtils::RemoveEdge(out_control_anchor, transdata_peer_in_data_anchor) != GRAPH_SUCCESS) {
+      REPORT_CALL_ERROR("E19999", "Remove control edge between op:%s(%s) and op:%s(%s) failed",
+                        out_control_anchor->GetOwnerNode()->GetName().c_str(),
+                        out_control_anchor->GetOwnerNode()->GetType().c_str(),
+                        transdata_peer_in_data_anchor->GetOwnerNode()->GetName().c_str(),
+                        transdata_peer_in_data_anchor->GetOwnerNode()->GetType().c_str());
       GELOGE(GRAPH_FAILED, "remove transdata control edge failed!");
       return GRAPH_FAILED;
     }
@@ -337,6 +387,12 @@ graphStatus SameTransdataBreadthFusionPass::ReLinkOutControlPeerInDataAnchors(
       GELOGD("add edge.src:%s, dst:%s", pre_out_anchor->GetOwnerNode()->GetName().c_str(),
              transdata_peer_in_data_anchor->GetOwnerNode()->GetName().c_str());
       if (GraphUtils::AddEdge(pre_out_anchor, transdata_peer_in_data_anchor) != GRAPH_SUCCESS) {
+        REPORT_CALL_ERROR("E19999", "Add edge between op:%s(%s)(index:%d) and op:%s(%s)(index:%d) failed",
+                          pre_out_anchor->GetOwnerNode()->GetName().c_str(),
+                          pre_out_anchor->GetOwnerNode()->GetType().c_str(), pre_out_anchor->GetIdx(),
+                          transdata_peer_in_data_anchor->GetOwnerNode()->GetName().c_str(),
+                          transdata_peer_in_data_anchor->GetOwnerNode()->GetType().c_str(),
+                          transdata_peer_in_data_anchor->GetIdx());
         GELOGE(GRAPH_FAILED, "add control edge failed!");
         return GRAPH_FAILED;
       }
@@ -344,6 +400,11 @@ graphStatus SameTransdataBreadthFusionPass::ReLinkOutControlPeerInDataAnchors(
       GELOGD("add edge.src:%s, dst:%s", transdata_peer_out_control_anchor->GetOwnerNode()->GetName().c_str(),
              transdata_peer_in_data_anchor->GetOwnerNode()->GetName().c_str());
       if (GraphUtils::AddEdge(transdata_peer_out_control_anchor, transdata_peer_in_data_anchor) != GRAPH_SUCCESS) {
+        REPORT_CALL_ERROR("E19999", "Add control edge between op:%s(%s) and op:%s(%s) failed",
+                          transdata_peer_out_control_anchor->GetOwnerNode()->GetName().c_str(),
+                          transdata_peer_out_control_anchor->GetOwnerNode()->GetType().c_str(),
+                          transdata_peer_in_data_anchor->GetOwnerNode()->GetName().c_str(),
+                          transdata_peer_in_data_anchor->GetOwnerNode()->GetType().c_str());
         GELOGE(GRAPH_FAILED, "add control edge failed!");
         return GRAPH_FAILED;
       }
@@ -460,6 +521,12 @@ graphStatus SameTransdataBreadthFusionPass::RelinkRemainTransdata(const ComputeG
 
   GELOGI("add edge.out node %s, in node %s", head_node->GetName().c_str(), transdata_node_keep->GetName().c_str());
   if (GraphUtils::AddEdge(head_node_anchor, transdata_in_anchor) != GRAPH_SUCCESS) {
+    REPORT_CALL_ERROR("E19999", "Add edge between op:%s(%s)(index:%d) and op:%s(%s)(index:%d) failed",
+                      head_node_anchor->GetOwnerNode()->GetName().c_str(),
+                      head_node_anchor->GetOwnerNode()->GetType().c_str(), head_node_anchor->GetIdx(),
+                      transdata_in_anchor->GetOwnerNode()->GetName().c_str(),
+                      transdata_in_anchor->GetOwnerNode()->GetType().c_str(),
+                      transdata_in_anchor->GetIdx());
     GELOGE(GRAPH_FAILED, "add edge failed!out node %s, in node %s", head_node->GetName().c_str(),
            transdata_node_keep->GetName().c_str());
     return GRAPH_FAILED;
@@ -545,6 +612,12 @@ graphStatus SameTransdataBreadthFusionPass::ReuseNodesBeforeTransdata(int anchor
   GELOGI("add edge.src:%s, dst:%s", transdata_node_keep->GetName().c_str(),
          head_node_peer_anchor->GetOwnerNode()->GetName().c_str());
   if (GraphUtils::AddEdge(transdata_out_anchor, head_node_peer_anchor) != GRAPH_SUCCESS) {
+    REPORT_CALL_ERROR("E19999", "Add edge between op:%s(%s)(index:%d) and op:%s(%s)(index:%d) failed",
+                      transdata_out_anchor->GetOwnerNode()->GetName().c_str(),
+                      transdata_out_anchor->GetOwnerNode()->GetType().c_str(), transdata_out_anchor->GetIdx(),
+                      head_node_peer_anchor->GetOwnerNode()->GetName().c_str(),
+                      head_node_peer_anchor->GetOwnerNode()->GetType().c_str(),
+                      head_node_peer_anchor->GetIdx());
     GELOGE(GRAPH_FAILED, "add edge.src:%s, dst:%s", transdata_node_keep->GetName().c_str(),
            head_node_peer_anchor->GetOwnerNode()->GetName().c_str());
     return GRAPH_FAILED;
@@ -562,6 +635,8 @@ graphStatus SameTransdataBreadthFusionPass::ReuseNodesBeforeTransdata(int anchor
     auto input_desc = in_op_desc->GetInputDesc(in_data_anchor->GetIdx());
     CopyTensorDesc(transdata_output_desc, input_desc);
     if (in_op_desc->UpdateInputDesc(in_data_anchor->GetIdx(), input_desc) != GRAPH_SUCCESS) {
+      REPORT_CALL_ERROR("E19999", "Update input:%d desc in op:%s(%s) failed", in_data_anchor->GetIdx(),
+                        in_op_desc->GetName().c_str(), in_op_desc->GetType().c_str());
       GELOGE(FAILED, "UpdateInputDesc fail.");
       return FAILED;
     }
@@ -569,6 +644,8 @@ graphStatus SameTransdataBreadthFusionPass::ReuseNodesBeforeTransdata(int anchor
     auto output_desc = in_op_desc->GetOutputDesc(output_idx);
     CopyTensorDesc(transdata_output_desc, output_desc);
     GE_IF_BOOL_EXEC(in_op_desc->UpdateOutputDesc(output_idx, output_desc) != GRAPH_SUCCESS,
+                    REPORT_CALL_ERROR("E19999", "Update output:%d desc in op:%s(%s) failed", output_idx,
+                                      in_op_desc->GetName().c_str(), in_op_desc->GetType().c_str());
                     GELOGE(GRAPH_FAILED, "update input desc failed");
                     return GRAPH_FAILED);
     // relink control edge
@@ -610,6 +687,13 @@ graphStatus SameTransdataBreadthFusionPass::LinkNewCastNode2RemainTransdata(
     GELOGI("remove edge.src:%s, dst:%s", transdata_peer_out_anchor->GetOwnerNode()->GetName().c_str(),
            transdata_remove_in_anchor->GetOwnerNode()->GetName().c_str());
     if (GraphUtils::RemoveEdge(transdata_peer_out_anchor, transdata_remove_in_anchor) != GRAPH_SUCCESS) {
+      REPORT_CALL_ERROR("E19999", "Remove edge between op:%s(%s)(index:%d) and op:%s(%s)(index:%d) failed",
+                        transdata_peer_out_anchor->GetOwnerNode()->GetName().c_str(),
+                        transdata_peer_out_anchor->GetOwnerNode()->GetType().c_str(),
+                        transdata_peer_out_anchor->GetIdx(),
+                        transdata_remove_in_anchor->GetOwnerNode()->GetName().c_str(),
+                        transdata_remove_in_anchor->GetOwnerNode()->GetType().c_str(),
+                        transdata_remove_in_anchor->GetIdx());
       return GRAPH_FAILED;
     }
 
@@ -642,6 +726,9 @@ graphStatus SameTransdataBreadthFusionPass::LinkNewCastNode2RemainTransdata(
     }
 
     if (graph->RemoveNode(transdata_node_remove) != GRAPH_SUCCESS) {
+      REPORT_CALL_ERROR("E19999", "Remove node:%s(%s) from graph:%s failed",
+                        transdata_node_remove->GetName().c_str(), transdata_node_remove->GetType().c_str(),
+                        graph->GetName().c_str());
       GELOGE(GRAPH_FAILED, "remove node %s failed!", transdata_node_remove->GetName().c_str());
       return GRAPH_FAILED;
     }
@@ -660,6 +747,10 @@ graphStatus SameTransdataBreadthFusionPass::RelinkInControlEdge(const NodePtr &n
     GELOGD("remove edge.src:%s, dst:%s", peer_out_control_anchor->GetOwnerNode()->GetName().c_str(),
            node_src->GetName().c_str());
     if (GraphUtils::RemoveEdge(peer_out_control_anchor, node_src->GetInControlAnchor()) != GRAPH_SUCCESS) {
+      REPORT_CALL_ERROR("E19999", "Remove control edge between op:%s(%s) and op:%s(%s) failed",
+                        peer_out_control_anchor->GetOwnerNode()->GetName().c_str(),
+                        peer_out_control_anchor->GetOwnerNode()->GetType().c_str(),
+                        node_src->GetName().c_str(), node_src->GetType().c_str());
       GELOGE(GRAPH_FAILED, "remove edge faliled!src:%s, dst:%s",
              peer_out_control_anchor->GetOwnerNode()->GetName().c_str(), node_src->GetName().c_str());
       return GRAPH_FAILED;
@@ -667,6 +758,10 @@ graphStatus SameTransdataBreadthFusionPass::RelinkInControlEdge(const NodePtr &n
     GELOGD("add edge.src:%s, dst:%s", peer_out_control_anchor->GetOwnerNode()->GetName().c_str(),
            node_dst->GetName().c_str());
     if (GraphUtils::AddEdge(peer_out_control_anchor, node_dst->GetInControlAnchor()) != GRAPH_SUCCESS) {
+      REPORT_CALL_ERROR("E19999", "Add control edge between op:%s(%s) and op:%s(%s) failed",
+                        peer_out_control_anchor->GetOwnerNode()->GetName().c_str(),
+                        peer_out_control_anchor->GetOwnerNode()->GetType().c_str(),
+                        node_dst->GetName().c_str(), node_dst->GetType().c_str());
       GELOGE(GRAPH_FAILED, "add edge failed!src:%s, dst:%s", peer_out_control_anchor->GetOwnerNode()->GetName().c_str(),
              node_dst->GetName().c_str());
       return GRAPH_FAILED;
@@ -713,10 +808,16 @@ graphStatus SameTransdataBreadthFusionPass::AddCastNode(const ComputeGraphPtr &g
 
     auto cast_node = graph->AddNode(cast_op_desc);
     if (cast_node == nullptr) {
+      REPORT_CALL_ERROR("E19999", "Add node:%s(%s) to graph:%s failed",
+                        cast_op_desc->GetName().c_str(), cast_op_desc->GetType().c_str(), graph->GetName().c_str());
       return GRAPH_FAILED;
     }
     GELOGD("add edge.src:%s, dst:%s", pre_out_anchor->GetOwnerNode()->GetName().c_str(), cast_node->GetName().c_str());
     if (GraphUtils::AddEdge(pre_out_anchor, cast_node->GetInDataAnchor(0)) != GRAPH_SUCCESS) {
+      REPORT_CALL_ERROR("E19999", "Add edge between op:%s(%s)(index:%d) and op:%s(%s)(index:0) failed",
+                        pre_out_anchor->GetOwnerNode()->GetName().c_str(),
+                        pre_out_anchor->GetOwnerNode()->GetType().c_str(), pre_out_anchor->GetIdx(),
+                        cast_node->GetName().c_str(), cast_node->GetType().c_str());
       return GRAPH_FAILED;
     }
     if (i == 0) {
@@ -724,6 +825,8 @@ graphStatus SameTransdataBreadthFusionPass::AddCastNode(const ComputeGraphPtr &g
     }
 
     if (!AttrUtils::SetBool(cast_op_desc, ATTR_NEED_COMPILE, true)) {
+      REPORT_CALL_ERROR("E19999", "Set Attr:%s to op:%s(%s) failed", ATTR_NEED_COMPILE.c_str(),
+                        cast_op_desc->GetName().c_str(), cast_op_desc->GetType().c_str());
       GELOGE(FAILED, "SetExtAttr fail.");
       return FAILED;
     }
@@ -738,6 +841,7 @@ graphStatus SameTransdataBreadthFusionPass::GetSubGraphsBetweenNormalAndTransdat
     std::vector<std::pair<OutDataAnchorPtr, InDataAnchorPtr>> &nodes_list) {
   graphStatus ret = GRAPH_SUCCESS;
   if (out_anchor == nullptr) {
+    REPORT_INNER_ERROR("E19999", "Param out_anchor is nullptr, check invalid");
     GELOGE(GRAPH_FAILED, "out data anchor is null!This should not happen!");
     return GRAPH_FAILED;
   }

ge/graph/passes/save_pass.cc

@@ -47,7 +47,9 @@ Status SavePass::Run(ge::ComputeGraphPtr graph) {
             out_index.emplace_back(out_anchor->GetIdx());
             ge::OpDescPtr op_desc = peer_node->GetOpDesc();
             GE_IF_BOOL_EXEC(!ge::AttrUtils::SetStr(op_desc, kVarAttrVarIsSave, kVarIsSave),
-                              GELOGE(INTERNAL_ERROR, "get kVarAttrVarIsSave failed"); return INTERNAL_ERROR);
+                            REPORT_CALL_ERROR("E19999", "Set Attr:%s to op:%s(%s) failed", kVarAttrVarIsSave,
+                                              op_desc->GetName().c_str(), op_desc->GetType().c_str());
+                            GELOGE(INTERNAL_ERROR, "get kVarAttrVarIsSave failed"); return INTERNAL_ERROR);
           }
         }
       }
@@ -65,6 +67,8 @@ Status SavePass::Run(ge::ComputeGraphPtr graph) {
   for (auto &node_ptr : del_nodes) {
     auto ret = graph->RemoveNode(node_ptr);
     if (ret != SUCCESS) {
+      REPORT_CALL_ERROR("E19999", "Remove node:%s(%s) from graph:%s failed",
+                        node_ptr->GetName().c_str(), node_ptr->GetType().c_str(), graph->GetName().c_str());
       GELOGE(ret, "GraphUtils::RemoveNodeWithoutRelink failed.");
       return ret;
     }

ge/graph/passes/set_input_output_offset_pass.cc

@@ -54,6 +54,8 @@ Status SetInputOutputOffsetPass::SetInputOffsetForFusion(const std::vector<int64
       std::vector<int64_t> input_offset_of_node;
       input_offset_of_node = op_desc->GetInputOffset();
       if (input_offset_of_node.size() < i) {
+        REPORT_INNER_ERROR("E19999", "Input offsets size:%zu of node:%s(%s) < index:%zu, check invalid",
+                           input_offset_of_node.size(), op_desc->GetName().c_str(), op_desc->GetType().c_str(), i);
         GELOGE(PARAM_INVALID, "not get input_offset of %zu", i);
         return PARAM_INVALID;
       }
@@ -77,10 +79,15 @@ Status SetInputOutputOffsetPass::SetInputOffsetForFusion(const std::vector<int64
       int64_t relative_offset = input_offset - out_offset;
       zero_copy_relative_offset.emplace_back(relative_offset);
       GE_CHK_BOOL_EXEC(ge::AttrUtils::SetListInt(data_op_desc, ATTR_ZERO_COPY_BASIC_OFFSET, zero_copy_basic_offset),
+                       REPORT_CALL_ERROR("E19999", "Set Attr:%s to op:%s(%s) failed",
+                                         ATTR_ZERO_COPY_BASIC_OFFSET.c_str(),
+                                         data_op_desc->GetName().c_str(), data_op_desc->GetType().c_str());
                        GELOGE(FAILED, "SetListInt of zero_copy_basic_offset failed.");
                        return FAILED);
       GE_CHK_BOOL_EXEC(
           ge::AttrUtils::SetListInt(data_op_desc, ATTR_ZERO_COPY_RELATIVE_OFFSET, zero_copy_relative_offset),
+          REPORT_CALL_ERROR("E19999", "Set Attr:%s to op:%s(%s) failed", ATTR_ZERO_COPY_RELATIVE_OFFSET.c_str(),
+                            data_op_desc->GetName().c_str(), data_op_desc->GetType().c_str());
           GELOGE(FAILED, "SetListInt of zero_copy_relative_offset failed.");
           return FAILED);
     }
@@ -115,10 +122,15 @@ Status SetInputOutputOffsetPass::SetInputOffsetForHcom(const ge::NodePtr &node,
         zero_copy_basic_offset.emplace_back(output_offset);
         zero_copy_relative_offset.emplace_back(relative_offset);
         GE_CHK_BOOL_EXEC(ge::AttrUtils::SetListInt(in_op_desc, ATTR_ZERO_COPY_BASIC_OFFSET, zero_copy_basic_offset),
+                         REPORT_CALL_ERROR("E19999", "Set Attr:%s to op:%s(%s) failed",
+                                           ATTR_ZERO_COPY_BASIC_OFFSET.c_str(),
+                                           in_op_desc->GetName().c_str(), in_op_desc->GetType().c_str());
                          GELOGE(FAILED, "SetListInt of zero_copy_basic_offset failed.");
                          return FAILED);
         GE_CHK_BOOL_EXEC(
             ge::AttrUtils::SetListInt(in_op_desc, ATTR_ZERO_COPY_RELATIVE_OFFSET, zero_copy_relative_offset),
+            REPORT_CALL_ERROR("E19999", "Set Attr:%s to op:%s(%s) failed", ATTR_ZERO_COPY_RELATIVE_OFFSET.c_str(),
+                              in_op_desc->GetName().c_str(), in_op_desc->GetType().c_str());
             GELOGE(FAILED, "SetListInt of zero_copy_relative_offset failed.");
             return FAILED);
       }
@@ -159,6 +171,9 @@ Status SetInputOutputOffsetPass::SetOutputOffsetForConcat(const NodePtr &node) {
   output_offset_of_concat = op_desc->GetOutputOffset();
   // phony_concat has one output
   GE_IF_BOOL_EXEC(output_offset_of_concat.size() != 1,
+                  REPORT_INNER_ERROR("E19999", "Output offsets size:%zu of node:%s(%s) not equal to 1, check invalid",
+                                     output_offset_of_concat.size(),
+                                     op_desc->GetName().c_str(), op_desc->GetType().c_str());
                   GELOGE(PARAM_INVALID, "%s should has one output.", node->GetName().c_str());
                   return PARAM_INVALID);
   NodePtr net_output = node->GetOutDataNodes().at(0);
@@ -186,9 +201,14 @@ Status SetInputOutputOffsetPass::SetOutputOffsetForConcat(const NodePtr &node) {
     zero_copy_relative_offset.emplace_back(relative_offset);
   }
   GE_CHK_BOOL_EXEC(ge::AttrUtils::SetListInt(out_op_desc, ATTR_ZERO_COPY_BASIC_OFFSET, zero_copy_basic_offset),
+                   REPORT_CALL_ERROR("E19999", "Set Attr:%s to op:%s(%s) failed", ATTR_ZERO_COPY_BASIC_OFFSET.c_str(),
+                                     out_op_desc->GetName().c_str(), out_op_desc->GetType().c_str());
                    GELOGE(FAILED, "SetListInt of zero_copy_basic_offset failed.");
                    return FAILED);
   GE_CHK_BOOL_EXEC(ge::AttrUtils::SetListInt(out_op_desc, ATTR_ZERO_COPY_RELATIVE_OFFSET, zero_copy_relative_offset),
+                   REPORT_CALL_ERROR("E19999", "Set Attr:%s to op:%s(%s) failed",
+                                     ATTR_ZERO_COPY_RELATIVE_OFFSET.c_str(),
+                                     out_op_desc->GetName().c_str(), out_op_desc->GetType().c_str());
                    GELOGE(FAILED, "SetListInt of zero_copy_relative_offset failed.");
                    return FAILED);
   return SUCCESS;
@@ -232,9 +252,14 @@ Status SetInputOutputOffsetPass::SetOutputOffsetForHcom(const NodePtr &node, con
   }
 
   GE_CHK_BOOL_EXEC(ge::AttrUtils::SetListInt(out_op_desc, ATTR_ZERO_COPY_BASIC_OFFSET, zero_copy_basic_offset),
+                   REPORT_CALL_ERROR("E19999", "Set Attr:%s to op:%s(%s) failed", ATTR_ZERO_COPY_BASIC_OFFSET.c_str(),
+                                     out_op_desc->GetName().c_str(), out_op_desc->GetType().c_str());
                    GELOGE(FAILED, "SetListInt of zero_copy_basic_offset failed.");
                    return FAILED);
   GE_CHK_BOOL_EXEC(ge::AttrUtils::SetListInt(out_op_desc, ATTR_ZERO_COPY_RELATIVE_OFFSET, zero_copy_relative_offset),
+                   REPORT_CALL_ERROR("E19999", "Set Attr:%s to op:%s(%s) failed",
+                                     ATTR_ZERO_COPY_RELATIVE_OFFSET.c_str(),
+                                     out_op_desc->GetName().c_str(), out_op_desc->GetType().c_str());
                    GELOGE(FAILED, "SetListInt of zero_copy_relative_offset failed.");
                    return FAILED);
   return SUCCESS;

ge/graph/passes/snapshot_pass.cc

@@ -29,6 +29,8 @@ Status SnapshotPass::Run(NodePtr &node) {
   string type;
   Status status_ret = GetOriginalType(node, type);
   if (status_ret != SUCCESS) {
+    REPORT_CALL_ERROR("E19999", "Get OriginalType of op:%s(%s) failed",
+                      node->GetName().c_str(), node->GetType().c_str());
     GELOGE(status_ret, "SnapshotPass get original type failed.");
     return status_ret;
   }

ge/graph/passes/stop_gradient_pass.cc

@@ -20,12 +20,15 @@
 namespace ge {
 Status StopGradientPass::Run(NodePtr &node) {
   if (node == nullptr) {
+    REPORT_INNER_ERROR("E19999", "Param node is nullptr, check invalid");
     GELOGE(FAILED, "parameter is null.");
     return FAILED;
   }
   string type;
   Status status_ret = GetOriginalType(node, type);
   if (status_ret != SUCCESS) {
+    REPORT_CALL_ERROR("E19999", "Get OriginalType of op:%s(%s) failed",
+                      node->GetName().c_str(), node->GetType().c_str());
     GELOGE(status_ret, "StopGradientPass get original type failed.");
     return status_ret;
   }

ge/graph/passes/subexpression_migration_pass.cc

@@ -144,6 +144,8 @@ Status SubexpressionMigrationPass::ClassifyDataNodes(const ComputeGraphPtr &grap
   for (const auto &name : func_desc->GetSubgraphInstanceNames()) {
     const auto &subgraph = graph->GetSubgraph(name);
     if (subgraph == nullptr) {
+      REPORT_INNER_ERROR("E19999", "Get subgraph from graph:%s by name:%s failed",
+                         graph->GetName().c_str(), name.c_str());
       GELOGE(GE_GRAPH_EMPTY_SUBGRAPH, "Subgraph not found, name: %s", name.c_str());
       return GE_GRAPH_EMPTY_SUBGRAPH;
     }
@@ -156,6 +158,8 @@ Status SubexpressionMigrationPass::ClassifyDataNodes(const ComputeGraphPtr &grap
 
       uint32_t parent_index = 0;
       if (!AttrUtils::GetInt(data->GetOpDesc(), ATTR_NAME_PARENT_NODE_INDEX, parent_index)) {
+        REPORT_CALL_ERROR("E19999", "Get Attr:%s from op:%s(%s) failed", ATTR_NAME_PARENT_NODE_INDEX.c_str(),
+                          data->GetName().c_str(), data->GetType().c_str());
         GELOGE(FAILED, "Parent index not found, name: %s", data->GetName().c_str());
         return FAILED;
       }
@@ -229,6 +233,7 @@ bool SubexpressionMigrationPass::IsParallelNodeSame(const map<ComputeGraphPtr, m
     const auto &data_nodes = it->second;
     auto data_it = data_nodes.find(node_idx);
     if (data_it == data_nodes.end()) {
+      REPORT_INNER_ERROR("E19999", "Find node in data_nodes by index:%u failed", node_idx);
       GELOGE(FAILED, "Data: %s not fount, index: %u", base_node->GetName().c_str(), node_idx);
       return false;
     }
@@ -238,12 +243,15 @@ bool SubexpressionMigrationPass::IsParallelNodeSame(const map<ComputeGraphPtr, m
     const auto &in_anchors = out_anchor->GetPeerInDataAnchors();
     const auto &in_anchor = in_anchors.at(anchor_idx);
     if (in_anchor == nullptr) {
+      REPORT_INNER_ERROR("E19999", "Index:%u anchor not exist in out:%u data anchor's peer of node:%s(%s)",
+                         node_idx, kDataOutIndex, work_data->GetName().c_str(), work_data->GetType().c_str());
       GELOGE(FAILED, "Data anchor size: %u, anchor size: %zu", anchor_idx, in_anchors.size());
       return false;
     }
 
     const auto &work_node = in_anchor->GetOwnerNode();
     if (work_node == nullptr) {
+      REPORT_INNER_ERROR("E19999", "Owner node of anchor is nullptr, check invalid");
       GELOGE(FAILED, "Data: %s not found, index: %u", base_node->GetName().c_str(), node_idx);
       return false;
     }
@@ -338,17 +346,22 @@ Status SubexpressionMigrationPass::AppendParallelNode(map<ComputeGraphPtr, map<u
       OpDescBuilder op_builder(data_name, DATA);
       const OpDescPtr op_desc = op_builder.AddInput("x").AddOutput("y").Build();
       if (op_desc == nullptr) {
+        REPORT_CALL_ERROR("E19999", "Build op:%s(%s) failed", data_name.c_str(), DATA);
         GELOGE(OUT_OF_MEMORY, "Create multi-batch case desc failed");
         return OUT_OF_MEMORY;
       }
 
       uint32_t data_index = item.second - kCaseInputBase;
       if (!AttrUtils::SetInt(op_desc, ATTR_NAME_INDEX, data_index)) {
+        REPORT_CALL_ERROR("E19999", "Set Attr:%s to op:%s(%s) failed", ATTR_NAME_INDEX.c_str(),
+                          op_desc->GetName().c_str(), op_desc->GetType().c_str());
         GELOGE(FAILED, "Parent index not found, name: %s", op_desc->GetName().c_str());
         return FAILED;
       }
 
       if (!AttrUtils::SetInt(op_desc, ATTR_NAME_PARENT_NODE_INDEX, item.second)) {
+        REPORT_CALL_ERROR("E19999", "Set Attr:%s to op:%s(%s) failed", ATTR_NAME_PARENT_NODE_INDEX.c_str(),
+                          op_desc->GetName().c_str(), op_desc->GetType().c_str());
         GELOGE(FAILED, "Parent index not found, name: %s", op_desc->GetName().c_str());
         return FAILED;
       }
@@ -392,12 +405,14 @@ Status SubexpressionMigrationPass::DetachParallelNode(const map<uint32_t, NodePt
   for (uint32_t i = 0; i < detach->GetAllOutDataAnchorsSize(); ++i) {
     auto it_idx = outputs.find(i);
     if (it_idx == outputs.end()) {
+      REPORT_INNER_ERROR("E19999", "Node: %s parent index %u not found, check invalid", detach->GetName().c_str(), i);
       GELOGE(FAILED, "Node: %s parent index %u not found", detach->GetName().c_str(), i);
       return FAILED;
     }
 
     auto it_data = graph_datas.find(it_idx->second);
     if (it_data == graph_datas.end()) {
+      REPORT_INNER_ERROR("E19999", "Node: %s parent index %u not found, check invalid", detach->GetName().c_str(), i);
       GELOGE(FAILED, "Node: %s parent index %u not found", detach->GetName().c_str(), i);
       return FAILED;
     }
@@ -444,6 +459,7 @@ Status SubexpressionMigrationPass::AttachParallelNode(const ComputeGraphPtr &gra
   for (uint32_t i = 0; i < attach->GetAllInDataAnchorsSize(); ++i) {
     auto it_idx = inputs.find(i);
     if (it_idx == inputs.end()) {
+      REPORT_INNER_ERROR("E19999", "Node: %s parent index %u not found, check invalid", attach->GetName().c_str(), i);
       GELOGE(FAILED, "Node: %s parent index %u not found", attach->GetName().c_str(), i);
       return FAILED;
     }
@@ -505,6 +521,7 @@ Status SubexpressionMigrationPass::MoveNodeToParent(const ComputeGraphPtr &graph
                                                     uint32_t anchor_idx, const map<uint32_t, uint32_t> &inputs,
                                                     const map<uint32_t, uint32_t> &outputs) {
   if (inputs.empty()) {
+    REPORT_INNER_ERROR("E19999", "Param inputs is empty, check invalid");
     GELOGE(FAILED, "Graph: %s, inputs is empty", graph->GetName().c_str());
     return FAILED;
   }
@@ -516,6 +533,8 @@ Status SubexpressionMigrationPass::MoveNodeToParent(const ComputeGraphPtr &graph
     const auto &subnodes = groups.second;
     auto it = subnodes.find(base_index);
     if (it == subnodes.end()) {
+      REPORT_INNER_ERROR("E19999", "Index:%u data node not found in graph:%s, check invalid",
+                         base_index, subgraph->GetName().c_str());
       GELOGE(FAILED, "Graph: %s, Data: %u node not found", subgraph->GetName().c_str(), base_index);
       return FAILED;
     }
@@ -525,12 +544,15 @@ Status SubexpressionMigrationPass::MoveNodeToParent(const ComputeGraphPtr &graph
     const auto &in_anchors = out_anchor->GetPeerInDataAnchors();
     const auto &in_anchor = in_anchors.at(anchor_idx);
     if (in_anchor == nullptr) {
+      REPORT_INNER_ERROR("E19999", "Index:%u anchor not exist in out:%u data anchor's peer of node:%s(%s)",
+                         anchor_idx, kDataOutIndex, base_data->GetName().c_str(), base_data->GetType().c_str());
       GELOGE(FAILED, "Data anchor index: %u, anchor size: %zu", anchor_idx, in_anchors.size());
       return FAILED;
     }
 
     move_node = in_anchor->GetOwnerNode();
     if (move_node == nullptr) {
+      REPORT_INNER_ERROR("E19999", "Owner node of anchor is nullptr, check invalid");
       GELOGE(FAILED, "Data: %s not found, index: %u", base_data->GetName().c_str(), base_index);
       return FAILED;
     }

ge/graph/passes/subgraph_const_migration_pass.cc

@@ -141,6 +141,8 @@ Status SubgraphConstMigrationPass::ClassifyGraphNodes(const ComputeGraphPtr &gra
   for (const auto &name : func_desc->GetSubgraphInstanceNames()) {
     const auto &subgraph = graph->GetSubgraph(name);
     if (subgraph == nullptr) {
+      REPORT_INNER_ERROR("E19999", "Get subgraph from graph:%s by name:%s failed",
+                         graph->GetName().c_str(), name.c_str());
       GELOGE(GE_GRAPH_EMPTY_SUBGRAPH, "Subgraph not found, name: %s", name.c_str());
       return GE_GRAPH_EMPTY_SUBGRAPH;
     }
@@ -152,6 +154,8 @@ Status SubgraphConstMigrationPass::ClassifyGraphNodes(const ComputeGraphPtr &gra
       if (node->GetType() == DATA) {
         uint32_t parent_index = kInvalidParent;
         if (!AttrUtils::GetInt(node->GetOpDesc(), ATTR_NAME_PARENT_NODE_INDEX, parent_index)) {
+          REPORT_CALL_ERROR("E19999", "Get Attr:%s from op:%s(%s) failed", ATTR_NAME_PARENT_NODE_INDEX.c_str(),
+                            node->GetName().c_str(), node->GetType().c_str());
           return FAILED;
         }
 
@@ -326,17 +330,22 @@ Status SubgraphConstMigrationPass::AppendParallelNode(const NodePtr &func_node,
     OpDescBuilder op_builder(data_name, DATA);
     const auto op_desc = op_builder.AddInput("x").AddOutput("y").Build();
     if (op_desc == nullptr) {
+      REPORT_CALL_ERROR("E19999", "Build op:%s(%s) failed", data_name.c_str(), DATA);
       GELOGE(OUT_OF_MEMORY, "Create multi-batch subgraph data desc failed");
       return OUT_OF_MEMORY;
     }
 
     uint32_t data_index = parent_index - kCaseInputBase;
     if (!AttrUtils::SetInt(op_desc, ATTR_NAME_INDEX, data_index)) {
+      REPORT_CALL_ERROR("E19999", "Set Attr:%s to op:%s(%s) failed", ATTR_NAME_INDEX.c_str(),
+                        op_desc->GetName().c_str(), op_desc->GetType().c_str());
       GELOGE(FAILED, "Parent index not found, name: %s", op_desc->GetName().c_str());
       return FAILED;
     }
 
     if (!AttrUtils::SetInt(op_desc, ATTR_NAME_PARENT_NODE_INDEX, parent_index)) {
+      REPORT_CALL_ERROR("E19999", "Set Attr:%s to op:%s(%s) failed", ATTR_NAME_PARENT_NODE_INDEX.c_str(),
+                        op_desc->GetName().c_str(), op_desc->GetType().c_str());
       GELOGE(FAILED, "Parent index not found, name: %s", op_desc->GetName().c_str());
       return FAILED;
     }
@@ -460,6 +469,8 @@ Status SubgraphConstMigrationPass::MoveNodeToParent(const ComputeGraphPtr &graph
                                                     const map<ComputeGraphPtr, map<uint32_t, NodePtr>> &all_data_nodes,
                                                     const string &node_key, uint32_t parent_index) {
   if (node_key.empty() || parent_index == kInvalidParent) {
+    REPORT_INNER_ERROR("E19999", "Param node_key is empty or param parent_index is 0x%X, check invalid",
+                       kInvalidParent);
     GELOGE(FAILED, "Graph: %s, node key: %s, parent index: %u invalid",
            graph->GetName().c_str(), node_key.c_str(), parent_index);
     return FAILED;
@@ -470,6 +481,8 @@ Status SubgraphConstMigrationPass::MoveNodeToParent(const ComputeGraphPtr &graph
     const auto &subgraph = item.first;
     const auto it_const = item.second.find(node_key);
     if (it_const == item.second.end()) {
+      REPORT_INNER_ERROR("E19999", "Const node name:%s not found in graph:%s, check invalid",
+                         node_key.c_str(), subgraph->GetName().c_str());
       GELOGE(FAILED, "Graph: %s, Const: %s node not found", subgraph->GetName().c_str(), node_key.c_str());
       return FAILED;
     }
@@ -477,11 +490,15 @@ Status SubgraphConstMigrationPass::MoveNodeToParent(const ComputeGraphPtr &graph
 
     const auto it_nodes = all_data_nodes.find(subgraph);
     if (it_nodes == all_data_nodes.end()) {
+      REPORT_INNER_ERROR("E19999", "Const node name:%s not found in graph:%s, check invalid",
+                         node_key.c_str(), subgraph->GetName().c_str());
       GELOGE(FAILED, "Graph: %s, Const: %s node not found", subgraph->GetName().c_str(), node_key.c_str());
       return FAILED;
     }
     const auto it_data = it_nodes->second.find(parent_index);
     if (it_data == it_nodes->second.end()) {
+      REPORT_INNER_ERROR("E19999", "Const node name:%s not found in graph:%s, check invalid",
+                         node_key.c_str(), subgraph->GetName().c_str());
       GELOGE(FAILED, "Graph: %s, Const: %s node not found", subgraph->GetName().c_str(), node_key.c_str());
       return FAILED;
     }

ge/graph/passes/subgraph_pass.cc

@@ -94,6 +94,8 @@ Status SubgraphPass::SubgraphInputNode(const ComputeGraphPtr &graph, const NodeP
 
   uint32_t parent_index = 0;
   if (!AttrUtils::GetInt(node->GetOpDesc(), ATTR_NAME_PARENT_NODE_INDEX, parent_index)) {
+    REPORT_CALL_ERROR("E19999", "Get Attr:%s from op:%s(%s) failed", ATTR_NAME_PARENT_NODE_INDEX.c_str(),
+                      node->GetName().c_str(), node->GetType().c_str());
     GELOGE(FAILED, "Get attr PARENT_NODE_INDEX failed, node:%s.", node->GetName().c_str());
     return FAILED;
   }
@@ -208,6 +210,8 @@ Status SubgraphPass::WhileBodySubgraph(const ComputeGraphPtr &graph, const NodeP
   // index of body_subgraph is 1
   ComputeGraphPtr while_body = NodeUtils::GetSubgraph(*node, 1);
   if (while_body == nullptr) {
+    REPORT_INNER_ERROR("E19999", "While_body of node:%s(%s) is nullptr, check invalid",
+                       node->GetName().c_str(), node->GetType().c_str());
     GELOGE(FAILED, "while_body of %s is NULL.", node->GetName().c_str());
     return FAILED;
   }
@@ -242,12 +246,16 @@ Status SubgraphPass::WhileBodySubgraph(const ComputeGraphPtr &graph, const NodeP
       if (output_node == nullptr) {
         output_node = n;
       } else {
+        REPORT_INNER_ERROR("E19999", "While_body graph:%s exists multi NetOutput nodes, check invalid",
+                           while_body->GetName().c_str());
         GELOGE(FAILED, "while_body %s exists multi NetOutput nodes.", while_body->GetName().c_str());
         return FAILED;
       }
     }
   }
   if (output_node == nullptr) {
+    REPORT_INNER_ERROR("E19999", "While_body graph:%s has no output, check invalid",
+                       while_body->GetName().c_str());
     GELOGE(FAILED, "while_body %s has no output.", while_body->GetName().c_str());
     return FAILED;
   }
@@ -462,6 +470,10 @@ Status SubgraphPass::InsertMemcpyNode(const ComputeGraphPtr &graph, const OutDat
   (void)AttrUtils::SetBool(op_desc, ATTR_NO_NEED_CONSTANT_FOLDING, false);
   (void)AttrUtils::SetBool(op_desc, ATTR_NAME_CANNOT_BE_DELETED, true);
   if (GraphUtils::InsertNodeAfter(out_anchor, in_anchors, graph->AddNode(op_desc)) != GRAPH_SUCCESS) {
+    REPORT_CALL_ERROR("E19999", "Insert Cast node %s(%s) after %s(%s) failed",
+                      op_desc->GetName().c_str(), op_desc->GetType().c_str(),
+                      out_anchor->GetOwnerNode()->GetName().c_str(),
+                      out_anchor->GetOwnerNode()->GetType().c_str());
     GELOGE(FAILED, "Insert IDENTITY node %s after %s failed.", name.c_str(), in_node->GetName().c_str());
     return FAILED;
   }
@@ -481,6 +493,9 @@ Status SubgraphPass::InsertMemcpyNode(const ComputeGraphPtr &graph, const OutDat
 Status SubgraphPass::InsertNodeBetween(const OutDataAnchorPtr &src, const std::vector<InDataAnchorPtr> &dsts,
                                        const NodePtr &insert_node, uint32_t input_index, uint32_t output_index) {
   if (GraphUtils::AddEdge(src, insert_node->GetInDataAnchor(input_index)) != GRAPH_SUCCESS) {
+    REPORT_CALL_ERROR("E19999", "Add edge between op:%s(%s)(index:%d) and op:%s(%s)(index:%u) failed",
+                      src->GetOwnerNode()->GetName().c_str(), src->GetOwnerNode()->GetType().c_str(), src->GetIdx(),
+                      insert_node->GetName().c_str(), insert_node->GetType().c_str(), input_index);
     GELOGE(FAILED, "Add data_edge %s:%d->%s:%u failed.",
            src->GetOwnerNode()->GetName().c_str(), src->GetIdx(), insert_node->GetName().c_str(), input_index);
     return FAILED;
@@ -490,6 +505,12 @@ Status SubgraphPass::InsertNodeBetween(const OutDataAnchorPtr &src, const std::v
            dst->GetOwnerNode()->GetName().c_str());
     if ((GraphUtils::RemoveEdge(src, dst) != GRAPH_SUCCESS) ||
         (GraphUtils::AddEdge(insert_node->GetOutDataAnchor(output_index), dst) != GRAPH_SUCCESS)) {
+      REPORT_CALL_ERROR("E19999", "Remove edge between op:%s(%s)(index:%d) and op:%s(%s)(index:%u) or "
+                        "Add edge between op:%s(%s)(index:%d) and op:%s(%s)(index:%u) failed",
+                        src->GetOwnerNode()->GetName().c_str(), src->GetOwnerNode()->GetType().c_str(), src->GetIdx(),
+                        dst->GetOwnerNode()->GetName().c_str(), dst->GetOwnerNode()->GetType().c_str(), dst->GetIdx(),
+                        insert_node->GetName().c_str(), insert_node->GetType().c_str(), output_index,
+                        dst->GetOwnerNode()->GetName().c_str(), dst->GetOwnerNode()->GetType().c_str(), dst->GetIdx());
       GELOGE(FAILED, "Replace data_edge %s:%d->%s:%d by %s:%u->%s:%d failed.",
              src->GetOwnerNode()->GetName().c_str(), src->GetIdx(),
              dst->GetOwnerNode()->GetName().c_str(), dst->GetIdx(),

ge/graph/passes/switch_data_edges_bypass.cc

@@ -50,6 +50,8 @@ bool IsSwitchInWhileLoop(const NodePtr &node) {
 std::vector<std::pair<NodePtr, InDataAnchorPtr>> GetOutDataNodesByIndex(const NodePtr &node, int index) {
   auto out_anchor = node->GetOutDataAnchor(index);
   if (out_anchor == nullptr) {
+    REPORT_INNER_ERROR("E19999", "Node:%s(%s) has no index:%d out data anchor, check invalid",
+                      node->GetName().c_str(), node->GetType().c_str(), index);
     GELOGE(PARAM_INVALID, "Failed to get out data nodes of index %d from node %s, the anchor does not exists", index,
            node->GetName().c_str());
     return {};
@@ -84,18 +86,23 @@ NodePtr AddIdentityAfterNode(const NodePtr &node, int index) {
 
   auto node_desc = node->GetOpDesc();
   if (node_desc == nullptr) {
+    REPORT_INNER_ERROR("E19999", "OpDesc in node is nullptr, check invalid");
     GELOGE(INTERNAL_ERROR, "Failed to add identity after node %s index %d, the op desc is null",
            node->GetName().c_str(), index);
     return nullptr;
   }
   auto tensor = node_desc->GetOutputDescPtr(index);
   if (tensor == nullptr) {
+    REPORT_INNER_ERROR("E19999", "Node:%s(%s) has no index:%d output tensor, check invalid",
+                       node_desc->GetName().c_str(), node_desc->GetType().c_str(), index);
     GELOGE(INTERNAL_ERROR, "Failed to find the tensor by index %d from node %s, can not add the identity node", index,
            node->GetName().c_str());
     return nullptr;
   }
   auto anchor = node->GetOutDataAnchor(index);
   if (anchor == nullptr) {
+    REPORT_INNER_ERROR("E19999", "Node:%s(%s) has no index:%d out data anchor, check invalid",
+                       node->GetName().c_str(), node->GetType().c_str(), index);
     GELOGE(OUT_OF_MEMORY, "Failed to add identity after node %s index %d, the out anchor does not exists",
            node->GetName().c_str(), index);
     return nullptr;
@@ -104,6 +111,7 @@ NodePtr AddIdentityAfterNode(const NodePtr &node, int index) {
   auto identity_opdesc =
       MakeShared<OpDesc>("SwitchDataEdgesByPass_Identity_" + std::to_string(identity_counter), IDENTITY);
   if (identity_opdesc == nullptr) {
+    REPORT_CALL_ERROR("E19999", "New OpDesc failed");
     GELOGE(OUT_OF_MEMORY, "Failed to add identity after node %s index %d", node->GetName().c_str(), index);
     return nullptr;
   }
@@ -111,6 +119,9 @@ NodePtr AddIdentityAfterNode(const NodePtr &node, int index) {
   auto ret2 = identity_opdesc->AddOutputDesc("y", *tensor);
   auto identity = node->GetOwnerComputeGraph()->AddNode(identity_opdesc);
   if (ret1 != GRAPH_SUCCESS || ret2 != GRAPH_SUCCESS || identity == nullptr) {
+    REPORT_CALL_ERROR("E19999", "Add input ouput desc to op:%s(%s) failed or add it to graph:%s failed",
+                      identity_opdesc->GetName().c_str(), identity_opdesc->GetType().c_str(),
+                      node->GetOwnerComputeGraph()->GetName().c_str());
     GELOGE(OUT_OF_MEMORY, "Failed to add identity after node %s index %d", node->GetName().c_str(), index);
     return nullptr;
   }
@@ -124,18 +135,23 @@ NodePtr AddMemcpyBeforeNode(const NodePtr &node, int index) {
 
   auto node_desc = node->GetOpDesc();
   if (node_desc == nullptr) {
+    REPORT_INNER_ERROR("E19999", "OpDesc in node is nullptr, check invalid");
     GELOGE(INTERNAL_ERROR, "Failed to add memcpy before node %s index %d, null op desc", node->GetName().c_str(),
            index);
     return nullptr;
   }
   auto tensor = node_desc->GetInputDescPtr(index);
   if (tensor == nullptr) {
+    REPORT_INNER_ERROR("E19999", "Node:%s(%s) has no index:%d input tensor, check invalid",
+                       node_desc->GetName().c_str(), node_desc->GetType().c_str(), index);
     GELOGE(INTERNAL_ERROR, "Failed to find the tensor by index %d from node %s, can not add the memcpy node", index,
            node->GetName().c_str());
     return nullptr;
   }
   auto anchor = node->GetInDataAnchor(index);
   if (anchor == nullptr) {
+    REPORT_INNER_ERROR("E19999", "Node:%s(%s) has no index:%d in data anchor, check invalid",
+                       node->GetName().c_str(), node->GetType().c_str(), index);
     GELOGE(INTERNAL_ERROR, "Failed to add memcpy before node %s index %d, the in anchor does not exists",
            node->GetName().c_str(), index);
     return nullptr;
@@ -143,6 +159,7 @@ NodePtr AddMemcpyBeforeNode(const NodePtr &node, int index) {
 
   auto memcpy_opdesc = MakeShared<OpDesc>("SwitchDataEdgesByPass_Memcpy_" + std::to_string(counter), MEMCPYASYNC);
   if (memcpy_opdesc == nullptr) {
+    REPORT_CALL_ERROR("E19999", "New OpDesc failed");
     GELOGE(OUT_OF_MEMORY, "Failed to add memcpy before node %s index %d", node->GetName().c_str(), index);
     return nullptr;
   }
@@ -150,6 +167,9 @@ NodePtr AddMemcpyBeforeNode(const NodePtr &node, int index) {
   auto ret2 = memcpy_opdesc->AddOutputDesc(*tensor);
   auto memcpy_node = node->GetOwnerComputeGraph()->AddNode(memcpy_opdesc);
   if (ret1 != GRAPH_SUCCESS || ret2 != GRAPH_SUCCESS || memcpy_node == nullptr) {
+    REPORT_CALL_ERROR("E19999", "Add input ouput desc to op:%s(%s) failed or add it to graph:%s failed",
+                      memcpy_opdesc->GetName().c_str(), memcpy_opdesc->GetType().c_str(),
+                      node->GetOwnerComputeGraph()->GetName().c_str());
     GELOGE(OUT_OF_MEMORY, "Failed to add memcpy before node %s index %d", node->GetName().c_str(), index);
     return nullptr;
   }

ge/graph/passes/switch_dead_branch_elimination.cc

@@ -31,6 +31,7 @@ const int kDefaultInputIndex = -1;
 
 bool ParsePred(const ConstGeTensorPtr &tensor) {
   if (tensor == nullptr) {
+    REPORT_INNER_ERROR("E19999", "Param tensor is nullptr, check invalid");
     GELOGE(FAILED, "parameter is null.");
     return false;
   }
@@ -65,6 +66,8 @@ bool ParseOutDataAnchors(const NodePtr &node, const NodePtr &pred_node, OutDataA
                          OutDataAnchorPtr &inactive_out_data_anchor) {
   auto tensors = OpDescUtils::MutableWeights(pred_node);
   if (tensors.empty()) {
+    REPORT_INNER_ERROR("E19999", "Node:%s(%s) has no weight, check invalid",
+                       pred_node->GetName().c_str(), pred_node->GetType().c_str());
     return false;
   }
 
@@ -72,6 +75,7 @@ bool ParseOutDataAnchors(const NodePtr &node, const NodePtr &pred_node, OutDataA
   int inactive_output_index = pred_value ? 0 : 1;
 
   if (node == nullptr) {
+    REPORT_INNER_ERROR("E19999", "Param node is nullptr, check invalid");
     GELOGE(FAILED, "parameter is null.");
     return false;
   }
@@ -91,6 +95,7 @@ bool ParseOutDataAnchors(const NodePtr &node, const NodePtr &pred_node, OutDataA
 Status SwitchDeadBranchElimination::DeleteSwitchNode(NodePtr &node, NodePtr &pred_node,
                                                      const OutDataAnchorPtr &active_out_data_anchor) {
   if (node == nullptr || active_out_data_anchor == nullptr) {
+    REPORT_INNER_ERROR("E19999", "Param node or active_out_data_anchor is nullptr, check invalid");
     GELOGE(FAILED, "parameter is null.");
     return FAILED;
   }
@@ -102,6 +107,9 @@ Status SwitchDeadBranchElimination::DeleteSwitchNode(NodePtr &node, NodePtr &pre
 
   // link pred's in control nodes to switch
   if (GraphUtils::CopyInCtrlEdges(pred_node, node) != GRAPH_SUCCESS) {
+    REPORT_CALL_ERROR("E19999", "Copy in control edge from node:%s(%s) to node:%s(%s) failed",
+                      pred_node->GetName().c_str(), pred_node->GetType().c_str(),
+                      node->GetName().c_str(), node->GetType().c_str());
     return FAILED;
   }
   // Remove link between pred and switch
@@ -114,6 +122,8 @@ Status SwitchDeadBranchElimination::DeleteSwitchNode(NodePtr &node, NodePtr &pre
   std::vector<int> switch_io_map = {kDefaultInputIndex, kDefaultInputIndex};
   size_t out_index = static_cast<size_t>(active_out_data_anchor->GetIdx());
   if (out_index >= switch_io_map.size()) {
+    REPORT_INNER_ERROR("E19999", "Out index:%zu of node:%s(%s) >= %zu, check invalid", out_index,
+                       node->GetName().c_str(), node->GetType().c_str(), switch_io_map.size());
     GELOGE(FAILED, "[%s] out index check failed, out_index:%zu.", node->GetName().c_str(), out_index);
     return FAILED;
   }
@@ -123,6 +133,7 @@ Status SwitchDeadBranchElimination::DeleteSwitchNode(NodePtr &node, NodePtr &pre
 
 Status SwitchDeadBranchElimination::Run(NodePtr &node) {
   if (node == nullptr) {
+    REPORT_INNER_ERROR("E19999", "Param node is nullptr, check invalid");
     GELOGE(PARAM_INVALID, "Param [node] must not be null.");
     return PARAM_INVALID;
   }
@@ -168,6 +179,8 @@ Status SwitchDeadBranchElimination::Run(NodePtr &node) {
     std::vector<NodePtr> end_nodes;
     Status ret = PassUtils::RemoveInactiveBranchToMerge(inactive_out_data_anchor, del_nodes, end_nodes);
     if (ret != SUCCESS) {
+      REPORT_CALL_ERROR("E19999", "Remove inactive branch from node:%s(%s) to merge failed",
+                        node->GetName().c_str(), node->GetType().c_str());
       return ret;
     }
 

ge/graph/passes/switch_logic_remove_pass.cc

@@ -45,11 +45,15 @@ Status GetPredNode(const NodePtr &switch_node, PredNodeAndOut &pred_node_index)
   GE_CHECK_NOTNULL(switch_node);
   auto pred_in_anchor = switch_node->GetInDataAnchor(kSwitchPredIndex);
   if (pred_in_anchor == nullptr) {
+    REPORT_INNER_ERROR("E19999", "Node:%s(%s) has no index:%d in data anchor, check invalid",
+                       switch_node->GetName().c_str(), switch_node->GetType().c_str(), kSwitchPredIndex);
     GELOGE(INTERNAL_ERROR, "Failed to get pred node for switch %s, no pred anchor", switch_node->GetName().c_str());
     return INTERNAL_ERROR;
   }
   auto pred_node_anchor = pred_in_anchor->GetPeerOutAnchor();
   if (pred_node_anchor == nullptr) {
+    REPORT_INNER_ERROR("E19999", "Node:%s(%s)'s index:%d in data anchor, its peer anchor is nullptr, check invalid",
+                       switch_node->GetName().c_str(), switch_node->GetType().c_str(), kSwitchPredIndex);
     GELOGE(INTERNAL_ERROR,
            "Failed to get pred node for switch %s, node peer out anchor",
            switch_node->GetName().c_str());
@@ -57,6 +61,8 @@ Status GetPredNode(const NodePtr &switch_node, PredNodeAndOut &pred_node_index)
   }
   auto pred_node = pred_node_anchor->GetOwnerNode();
   if (pred_node == nullptr) {
+    REPORT_INNER_ERROR("E19999", "Node:%s(%s)'s index:%d in data anchor, its peer node is nullptr, check invalid",
+                       switch_node->GetName().c_str(), switch_node->GetType().c_str(), kSwitchPredIndex);
     GELOGE(INTERNAL_ERROR,
            "Failed to get pred node for switch %s, null node",
            switch_node->GetName().c_str());
@@ -89,11 +95,15 @@ Status SwitchLogicRemovePass::Run(NodePtr &node) {
     }
     for (auto &in_anchor : out_anchor->GetPeerInDataAnchors()) {
       if (in_anchor == nullptr) {
+        REPORT_INNER_ERROR("E19999", "Node:%s(%s)'s index:%d out data anchor, its peer anchors has nullptr, "
+                           "check invalid", node->GetName().c_str(), node->GetType().c_str(), i);
         GELOGE(INTERNAL_ERROR, "The in-anchor from out anchor %d node %s is null", i, node->GetName().c_str());
         return INTERNAL_ERROR;
       }
       auto dst_node = in_anchor->GetOwnerNode();
       if (dst_node == nullptr) {
+        REPORT_INNER_ERROR("E19999", "Node:%s(%s)'s index:%d out data anchor, its peer nodes has nullptr, "
+                           "check invalid", node->GetName().c_str(), node->GetType().c_str(), i);
         GELOGE(INTERNAL_ERROR, "The peer node from out anchor %d node %s is null", i, node->GetName().c_str());
         return INTERNAL_ERROR;
       }
@@ -143,6 +153,8 @@ Status SwitchLogicRemovePass::RemoveSwitchNodeLogically(int parent_index, NodePt
     std::vector<NodePtr> end_nodes;
     auto ret = PassUtils::RemoveInactiveBranchToMerge(out_anchor, deleted_nodes, end_nodes);
     if (ret != SUCCESS) {
+      REPORT_CALL_ERROR("E19999", "Remove inactive branch from node:%s(%s) to merge failed",
+                        switch_node->GetName().c_str(), switch_node->GetType().c_str());
       return ret;
     }
 

ge/graph/passes/switch_to_stream_switch_pass.cc

@@ -33,8 +33,14 @@ Status SwitchToStreamSwitchPass::Run(ComputeGraphPtr graph) {
   GE_CHK_STATUS_RET(CombineSwitchNode(graph), "Combine StreamSwitch nodes failed.");
 
   for (const auto &node : bypass_nodes_) {
-    GE_CHK_BOOL_EXEC(graph->IsolateNode(node) == GRAPH_SUCCESS, return FAILED, "Isolate node failed.");
-    GE_CHK_BOOL_EXEC(GraphUtils::RemoveNodeWithoutRelink(graph, node) == GRAPH_SUCCESS, return FAILED,
+    GE_CHK_BOOL_EXEC(graph->IsolateNode(node) == GRAPH_SUCCESS,
+                     REPORT_CALL_ERROR("E19999", "Isolate node:%s(%s) in graph:%s failed",
+                                       node->GetName().c_str(), node->GetType().c_str(), graph->GetName().c_str());
+                     return FAILED, "Isolate node failed.");
+    GE_CHK_BOOL_EXEC(GraphUtils::RemoveNodeWithoutRelink(graph, node) == GRAPH_SUCCESS,
+                     REPORT_CALL_ERROR("E19999", "Remove node:%s(%s) without relink in graph:%s failed",
+                                       node->GetName().c_str(), node->GetType().c_str(), graph->GetName().c_str());
+                     return FAILED,
                      "Remove switch node failed.");
   }
 
@@ -159,7 +165,11 @@ Status SwitchToStreamSwitchPass::ReplaceSwitchNode(const ComputeGraphPtr &graph,
   OpDescPtr cond_desc = peer_cond_anchor->GetOwnerNode()->GetOpDesc();
   GE_CHECK_NOTNULL(cond_desc);
   DataType cond_data_type = cond_desc->GetOutputDesc(peer_cond_anchor->GetIdx()).GetDataType();
-  GE_CHK_BOOL_EXEC(cond_data_type == DT_BOOL, return FAILED,
+  GE_CHK_BOOL_EXEC(cond_data_type == DT_BOOL,
+                   REPORT_INNER_ERROR("E19999", "Pred_input of Switch node:%s(%s) only support DT_BOOL data_type, "
+                                      "but %s exactly", switch_node->GetName().c_str(), switch_node->GetType().c_str(),
+                                      TypeUtils::DataTypeToSerialString(cond_data_type).c_str());
+                   return FAILED,
                    "pred_input of Switch only support DT_BOOL data_type, but %s exactly.",
                    TypeUtils::DataTypeToSerialString(cond_data_type).c_str());
 
@@ -176,6 +186,8 @@ Status SwitchToStreamSwitchPass::ReplaceSwitchNode(const ComputeGraphPtr &graph,
         stream_switch = CreateStreamSwitchNode(graph, switch_node, true_branch_flag ? "_t" : "_f", peer_cond_anchor);
         GE_CHK_BOOL_EXEC(stream_switch != nullptr, return FAILED, "Create stream_switch node failed.");
         if (SetSwitchTrueBranchFlag(stream_switch, true_branch_flag) != SUCCESS) {
+          REPORT_CALL_ERROR("E19999", "Set switch true branch flag from node:%s(%s) failed",
+                            stream_switch->GetName().c_str(), stream_switch->GetType().c_str());
           GELOGE(FAILED, "SetSwitchTrueBranchFlag for node %s failed.", stream_switch->GetName().c_str());
           return FAILED;
         }
@@ -204,6 +216,8 @@ Status SwitchToStreamSwitchPass::ReplaceSwitchNode(const ComputeGraphPtr &graph,
       MoveCtrlEdges(switch_node, stream_switch);
       switch_node_map_[stream_switch] = out_node_list;
       if (SetOriginalNodeName(stream_switch, switch_node->GetName()) != SUCCESS) {
+        REPORT_CALL_ERROR("E19999", "Set original node name:%s to node:%s(%s) failed", switch_node->GetName().c_str(),
+                          stream_switch->GetName().c_str(), stream_switch->GetType().c_str());
         GELOGE(FAILED, "SetOriginalNodeName for node %s failed.", stream_switch->GetName().c_str());
         return FAILED;
       }
@@ -230,6 +244,10 @@ Status SwitchToStreamSwitchPass::BypassSwitchNode(const NodePtr &switch_node, Ou
     GE_CHECK_NOTNULL(peer_out_anchor);
     // Remove Switch data input.
     if (GraphUtils::RemoveEdge(peer_out_anchor, in_data_anchor) != GRAPH_SUCCESS) {
+      REPORT_CALL_ERROR("E19999", "Remove edge between op:%s(%s)(index:%d) and op:%s(%s)(index:%u) failed",
+                        peer_out_anchor->GetOwnerNode()->GetName().c_str(),
+                        peer_out_anchor->GetOwnerNode()->GetType().c_str(), peer_out_anchor->GetIdx(),
+                        switch_node->GetName().c_str(), switch_node->GetType().c_str(), idx);
       GELOGE(FAILED, "Remove data edge %s->%s failed.", peer_out_anchor->GetOwnerNode()->GetName().c_str(),
              switch_node->GetName().c_str());
       return FAILED;
@@ -284,8 +302,13 @@ NodePtr SwitchToStreamSwitchPass::CreateStreamSwitchNode(const ComputeGraphPtr &
                                                          const std::string &suffix,
                                                          const OutDataAnchorPtr &peer_cond_anchor) {
   OpDescPtr switch_op_desc = switch_node->GetOpDesc();
-  GE_CHK_BOOL_EXEC(switch_op_desc != nullptr, return nullptr, "OpDesc of Switch node is invalid.");
+  GE_CHK_BOOL_EXEC(switch_op_desc != nullptr,
+                   REPORT_INNER_ERROR("E19999", "OpDesc in node is nullptr, check invalid");
+                   return nullptr, "OpDesc of Switch node is invalid.");
   GE_IF_BOOL_EXEC(switch_op_desc->GetInputsSize() != SWITCH_INPUT_NUM, {
+    REPORT_INNER_ERROR("E19999", "Input desc size:%zu of node:%s(%s) not equal to %u, check invalid",
+                       switch_op_desc->GetInputsSize(),
+                       switch_op_desc->GetName().c_str(), switch_op_desc->GetType().c_str(), SWITCH_INPUT_NUM);
     GELOGE(FAILED, "Switch input param invalid, input_size=%lu, should be %u.", switch_op_desc->GetInputsSize(),
            SWITCH_INPUT_NUM);
     return nullptr;
@@ -295,6 +318,7 @@ NodePtr SwitchToStreamSwitchPass::CreateStreamSwitchNode(const ComputeGraphPtr &
   GELOGI("Create StreamSwitch, name=%s.", node_name.c_str());
   OpDescPtr op_desc = MakeShared<OpDesc>(node_name, STREAMSWITCH);
   if (op_desc == nullptr) {
+    REPORT_CALL_ERROR("E19999", "New OpDesc failed");
     GELOGE(FAILED, "Create op_desc failed, StreamSwitch:%s.", node_name.c_str());
     return nullptr;
   }
@@ -316,6 +340,9 @@ NodePtr SwitchToStreamSwitchPass::CreateStreamSwitchNode(const ComputeGraphPtr &
 
   if (!AttrUtils::SetInt(op_desc, ATTR_NAME_SWITCH_DATA_TYPE, RT_SWITCH_INT32) ||
       !AttrUtils::SetInt(op_desc, ATTR_NAME_STREAM_SWITCH_COND, (int64_t)RT_EQUAL)) {
+    REPORT_CALL_ERROR("E19999", "Set Attr:%s or Attr:%s to op:%s(%s) failed",
+                      ATTR_NAME_SWITCH_DATA_TYPE.c_str(), ATTR_NAME_STREAM_SWITCH_COND.c_str(),
+                      op_desc->GetName().c_str(), op_desc->GetType().c_str());
     GELOGE(INTERNAL_ERROR, "set int failed");
     return nullptr;
   }
@@ -323,13 +350,22 @@ NodePtr SwitchToStreamSwitchPass::CreateStreamSwitchNode(const ComputeGraphPtr &
   // Already checked, first input is Variable will passed, second is condition will checked.
   GeTensorDesc cond_input_desc = switch_op_desc->GetInputDesc(SWITCH_PRED_INPUT);
   GeTensorDesc input_desc(GeShape(cond_input_desc.GetShape().GetDims()), cond_input_desc.GetFormat(), DT_INT32);
-  GE_CHK_BOOL_EXEC(op_desc->AddInputDesc(input_desc) == GRAPH_SUCCESS, return nullptr,
+  GE_CHK_BOOL_EXEC(op_desc->AddInputDesc(input_desc) == GRAPH_SUCCESS,
+                   REPORT_CALL_ERROR("E19999", "Add input desc to op:%s(%s) failed",
+                                     op_desc->GetName().c_str(), op_desc->GetType().c_str());
+                   return nullptr,
                    "Create StreamSwitch node: add input desc failed.");
-  GE_CHK_BOOL_EXEC(op_desc->AddInputDesc(input_desc) == GRAPH_SUCCESS, return nullptr,
+  GE_CHK_BOOL_EXEC(op_desc->AddInputDesc(input_desc) == GRAPH_SUCCESS,
+                   REPORT_CALL_ERROR("E19999", "Add ouput desc to op:%s(%s) failed",
+                                     op_desc->GetName().c_str(), op_desc->GetType().c_str());
+                   return nullptr,
                    "Create StreamSwitch node: add input desc failed.");
 
   NodePtr stream_switch = graph->AddNode(op_desc);
-  GE_CHK_BOOL_EXEC(stream_switch != nullptr, return nullptr, "Insert StreamSwitch node failed.");
+  GE_CHK_BOOL_EXEC(stream_switch != nullptr,
+                   REPORT_CALL_ERROR("E19999", "Add node:%s(%s) to graph:%s failed",
+                                     op_desc->GetName().c_str(), op_desc->GetType().c_str(), graph->GetName().c_str());
+                   return nullptr, "Insert StreamSwitch node failed.");
   GE_CHK_STATUS(GraphUtils::AddEdge(peer_cond_anchor, stream_switch->GetInDataAnchor(0)),
                 "StreamSwitch node add cond edge failed.");
 
@@ -361,6 +397,8 @@ Status SwitchToStreamSwitchPass::MarkBranches(const OutDataAnchorPtr &peer_cond_
       it->second[switch_group_id] = switch_list;
     } else {
       GE_IF_BOOL_EXEC(switch_group_it->second.size() != SWITCH_OUTPUT_NUM, {
+        REPORT_INNER_ERROR("E19999", "switch group size:%zu not equal to %u, group_id:%ld, check invalid",
+                           switch_group_it->second.size(), SWITCH_OUTPUT_NUM, switch_group_id);
         GELOGE(INTERNAL_ERROR, "Check size failed, node: %s", stream_switch->GetName().c_str());
         return FAILED;
       });
@@ -443,6 +481,8 @@ Status SwitchToStreamSwitchPass::CombineSwitchNode(const ComputeGraphPtr &graph)
       GE_CHK_STATUS(GraphUtils::AddEdge(cast_node->GetOutControlAnchor(), active_node->GetInControlAnchor()),
                     "StreamActive add ctl edge failed.");
       if (SetActiveLabelList(active_node, { cast_node->GetName() }) != SUCCESS) {
+        REPORT_CALL_ERROR("E19999", "Set active label list:%s to op:%s(%s) failed",
+                          cast_node->GetName().c_str(), active_node->GetName().c_str(), active_node->GetType().c_str());
         GELOGE(FAILED, "Set active_label_list attr for node %s failed.", active_node->GetName().c_str());
         return FAILED;
       }
@@ -456,7 +496,13 @@ Status SwitchToStreamSwitchPass::CombineSwitchNode(const ComputeGraphPtr &graph)
         // select first stream_switch
         NodePtr stream_switch = switch_list.front();
         // set stream_label
-        GE_CHK_STATUS_RET(SetStreamLabel(stream_switch, cast_node->GetName()), "Set stream label failed.");
+        if (SetStreamLabel(stream_switch, cast_node->GetName()) != SUCCESS) {
+          REPORT_CALL_ERROR("E19999", "Set stream_label:%s to op:%s(%s) failed",
+                            cast_node->GetName().c_str(), stream_switch->GetName().c_str(),
+                            stream_switch->GetType().c_str());
+          GELOGE(FAILED, "Set stream label failed.");
+          return FAILED;
+        }
         OpDescPtr switch_desc = stream_switch->GetOpDesc();
         GE_CHECK_NOTNULL(switch_desc);
         switch_desc->SetName(CheckDuplicateName(cond_group + "/" + STREAMSWITCH + (true_branch_flag ? "_t" : "_f")));
@@ -497,18 +543,27 @@ NodePtr SwitchToStreamSwitchPass::CreateActiveNode(const ComputeGraphPtr &graph,
   GELOGI("Create StreamActive op:%s.", node_name.c_str());
   OpDescPtr op_desc = MakeShared<OpDesc>(node_name, STREAMACTIVE);
   if (op_desc == nullptr) {
+    REPORT_CALL_ERROR("E19999", "New OpDesc failed");
     GELOGE(FAILED, "Create op_desc failed, StreamActive:%s.", node_name.c_str());
     return nullptr;
   }
 
   NodePtr active_node = graph->AddNode(op_desc);
-  GE_CHK_BOOL_EXEC(active_node != nullptr, return nullptr, "Create StreamActive node failed.");
+  GE_CHK_BOOL_EXEC(active_node != nullptr,
+                   REPORT_CALL_ERROR("E19999", "Add node:%s(%s) to graph:%s failed",
+                                     op_desc->GetName().c_str(), op_desc->GetType().c_str(), graph->GetName().c_str());
+                   return nullptr, "Create StreamActive node failed.");
 
   GE_IF_BOOL_EXEC(GraphUtils::AddEdge(node->GetOutControlAnchor(), active_node->GetInControlAnchor()) != SUCCESS,
+                  REPORT_CALL_ERROR("E19999", "Add control edge between op:%s(%s) and op:%s(%s) failed",
+                                    node->GetName().c_str(), node->GetType().c_str(),
+                                    active_node->GetName().c_str(), active_node->GetType().c_str());
                   GELOGE(INTERNAL_ERROR, "add edge failed");
                   return nullptr);
 
   GE_IF_BOOL_EXEC(SetSwitchBranchNodeLabel(active_node, node_name) != SUCCESS,
+                  REPORT_CALL_ERROR("E19999", "Set switch branch node label:%s to node:%s(%s) failed",
+                                    node_name.c_str(), active_node->GetName().c_str(), active_node->GetType().c_str());
                   GELOGE(INTERNAL_ERROR, "set switch branch node label failed");
                   return nullptr);
 
@@ -529,6 +584,7 @@ NodePtr SwitchToStreamSwitchPass::CreateCastOp(const ComputeGraphPtr &graph, con
   GELOGI("Create cast_node: %s, input datatype:DT_BOOL, out datatype:DT_INT32", cast_name.c_str());
   OpDescPtr cast_desc = MakeShared<OpDesc>(cast_name, CAST);
   if (cast_desc == nullptr) {
+    REPORT_CALL_ERROR("E19999", "New OpDesc failed");
     GELOGE(FAILED, "Create op_desc failed, Cast:%s.", cast_name.c_str());
     return nullptr;
   }
@@ -536,6 +592,10 @@ NodePtr SwitchToStreamSwitchPass::CreateCastOp(const ComputeGraphPtr &graph, con
         AttrUtils::SetInt(cast_desc, CAST_ATTR_DSTT, (int64_t)DT_INT32) &&
         AttrUtils::SetInt(cast_desc, CAST_ATTR_DST_TYPE, (int64_t)DT_INT32) &&
         AttrUtils::SetBool(cast_desc, CAST_ATTR_TRUNCATE, false))) {
+    REPORT_CALL_ERROR("E19999", "Set Attr:%s or %s or %s or %s to op:%s(%s) failed",
+                      CAST_ATTR_SRCT.c_str(), CAST_ATTR_DSTT.c_str(),
+                      CAST_ATTR_DST_TYPE.c_str(), CAST_ATTR_TRUNCATE.c_str(),
+                      cast_desc->GetName().c_str(), cast_desc->GetType().c_str());
     GELOGE(FAILED, "Set CAST_ATTR_SRCT or CAST_ATTR_DSTT or CAST_ATTR_DST_TYPE or CAST_ATTR_TRUNCATE failed, node: %s.",
            cast_name.c_str());
     return nullptr;
@@ -543,14 +603,24 @@ NodePtr SwitchToStreamSwitchPass::CreateCastOp(const ComputeGraphPtr &graph, con
 
   GeTensorDesc tensor_desc = cond_desc->GetOutputDesc(peer_cond_anchor->GetIdx());
   tensor_desc.SetDataType(DT_BOOL);
-  GE_CHK_BOOL_EXEC(cast_desc->AddInputDesc(tensor_desc) == SUCCESS, return nullptr,
+  GE_CHK_BOOL_EXEC(cast_desc->AddInputDesc(tensor_desc) == SUCCESS,
+                   REPORT_CALL_ERROR("E19999", "Add input desc to op:%s(%s) failed",
+                                     cast_desc->GetName().c_str(), cast_desc->GetType().c_str());
+                   return nullptr,
                    "Cast_node add input desc failed.");
   tensor_desc.SetDataType(DT_INT32);
-  GE_CHK_BOOL_EXEC(cast_desc->AddOutputDesc(tensor_desc) == SUCCESS, return nullptr,
+  GE_CHK_BOOL_EXEC(cast_desc->AddOutputDesc(tensor_desc) == SUCCESS,
+                   REPORT_CALL_ERROR("E19999", "Add output desc to op:%s(%s) failed",
+                                     cast_desc->GetName().c_str(), cast_desc->GetType().c_str());
+                   return nullptr,
                    "Cast_node add output desc failed.");
 
   NodePtr cast_node = graph->AddNode(cast_desc);
-  GE_CHK_BOOL_EXEC(cast_node != nullptr, return nullptr, "Create cast_node failed.");
+  GE_CHK_BOOL_EXEC(cast_node != nullptr,
+                   REPORT_CALL_ERROR("E19999", "Add node:%s(%s) to graph:%s failed",
+                                     cast_desc->GetName().c_str(), cast_desc->GetType().c_str(),
+                                     graph->GetName().c_str());
+                   return nullptr, "Create cast_node failed.");
   // Cast node has and only has one input
   GE_CHK_STATUS(GraphUtils::AddEdge(peer_cond_anchor, cast_node->GetInDataAnchor(0)), "Cast add data edge failed.");
 
@@ -567,13 +637,18 @@ Status SwitchToStreamSwitchPass::AddConstNode(const ComputeGraphPtr &graph, cons
   OpDescPtr op_desc = stream_switch->GetOpDesc();
   GE_CHECK_NOTNULL(op_desc);
   bool value = false;
-  GE_CHK_BOOL_EXEC(AttrUtils::GetBool(op_desc, ATTR_NAME_SWITCH_TRUE_BRANCH_FLAG, value), return FAILED,
+  GE_CHK_BOOL_EXEC(AttrUtils::GetBool(op_desc, ATTR_NAME_SWITCH_TRUE_BRANCH_FLAG, value),
+                   REPORT_CALL_ERROR("E19999", "Get Attr:%s from op:%s(%s) failed",
+                                     ATTR_NAME_SWITCH_TRUE_BRANCH_FLAG.c_str(),
+                                     op_desc->GetName().c_str(), op_desc->GetType().c_str());
+                   return FAILED,
                    "StreamSwitch get attr TRUE_BRANCH_STREAM failed.");
 
   const std::string &const_node_name = op_desc->GetName() + "_Constant_" + (value ? "t" : "f");
   GELOGI("Create const op: %s", const_node_name.c_str());
   OpDescPtr const_op_desc = MakeShared<OpDesc>(const_node_name, CONSTANT);
   if (const_op_desc == nullptr) {
+    REPORT_CALL_ERROR("E19999", "New OpDesc failed");
     GELOGE(FAILED, "Create op_desc failed, Constant:%s.", const_node_name.c_str());
     return FAILED;
   }
@@ -583,15 +658,26 @@ Status SwitchToStreamSwitchPass::AddConstNode(const ComputeGraphPtr &graph, cons
   GeTensorPtr const_value =
           MakeShared<GeTensor>(data_desc, reinterpret_cast<uint8_t *>(&resize_value), sizeof(int32_t));
   if (const_value == nullptr) {
+    REPORT_CALL_ERROR("E19999", "New GeTensor failed");
     GELOGE(FAILED, "Create tensor failed.");
     return FAILED;
   }
-  GE_CHK_BOOL_EXEC(AttrUtils::SetTensor(const_op_desc, ATTR_NAME_WEIGHTS, const_value), return FAILED);
-  GE_CHK_BOOL_EXEC(const_op_desc->AddOutputDesc(data_desc) == GRAPH_SUCCESS, return FAILED,
+  GE_CHK_BOOL_EXEC(AttrUtils::SetTensor(const_op_desc, ATTR_NAME_WEIGHTS, const_value),
+                   REPORT_CALL_ERROR("E19999", "Get Attr:%s from op:%s(%s) failed", ATTR_NAME_WEIGHTS.c_str(),
+                                     const_op_desc->GetName().c_str(), const_op_desc->GetType().c_str());
+                   return FAILED);
+  GE_CHK_BOOL_EXEC(const_op_desc->AddOutputDesc(data_desc) == GRAPH_SUCCESS,
+                   REPORT_CALL_ERROR("E19999", "Add output desc to op:%s(%s) failed",
+                                     const_op_desc->GetName().c_str(), const_op_desc->GetType().c_str());
+                   return FAILED,
                    "Create Const op: add output desc failed.");
 
   NodePtr const_node = graph->AddNode(const_op_desc);
-  GE_CHK_BOOL_EXEC(const_node != nullptr, return FAILED, "Insert Const node failed.");
+  GE_CHK_BOOL_EXEC(const_node != nullptr,
+                   REPORT_CALL_ERROR("E19999", "Add node:%s(%s) to graph:%s failed",
+                                     const_op_desc->GetName().c_str(), const_op_desc->GetType().c_str(),
+                                     graph->GetName().c_str());
+                   return FAILED, "Insert Const node failed.");
   GE_CHK_STATUS(GraphUtils::AddEdge(const_node->GetOutDataAnchor(0), stream_switch->GetInDataAnchor(1)),
                 "StreamSwitch node add ctl edge failed.");
 
@@ -613,6 +699,8 @@ Status SwitchToStreamSwitchPass::ModifySwitchInCtlEdges(const NodePtr &switch_no
   OpDescPtr switch_desc = switch_node->GetOpDesc();
   GE_CHECK_NOTNULL(switch_desc);
   if (!AttrUtils::GetStr(switch_desc, ATTR_NAME_ORIG_NODE_NAME, orig_switch_name) || orig_switch_name.empty()) {
+    REPORT_CALL_ERROR("E19999", "Get Attr:%s from op:%s(%s) failed", ATTR_NAME_ORIG_NODE_NAME.c_str(),
+                      switch_desc->GetName().c_str(), switch_desc->GetType().c_str());
     GELOGE(INTERNAL_ERROR, "Get attr ATTR_NAME_ORIG_NODE_NAME failed, node: %s", switch_desc->GetName().c_str());
     return INTERNAL_ERROR;
   }
@@ -634,6 +722,8 @@ Status SwitchToStreamSwitchPass::ModifySwitchInCtlEdges(const NodePtr &switch_no
 
     auto find_res1 = switch_node_map_.find(in_ctrl_node);
     GE_IF_BOOL_EXEC(find_res1 == switch_node_map_.end(), {
+      REPORT_INNER_ERROR("E19999", "Node:%s(%s) can't find in switch_node_map_, check invalid",
+                         in_ctrl_node->GetName().c_str(), in_ctrl_node->GetType().c_str());
       GELOGE(INTERNAL_ERROR, "StreamSwitch node %s not found in switch_node_map_.", in_ctrl_node->GetName().c_str());
       return INTERNAL_ERROR;
     });
@@ -662,10 +752,14 @@ Status SwitchToStreamSwitchPass::ModifySwitchOutCtlEdges(const NodePtr &switch_n
          stream_switch->GetName().c_str(), active_node->GetName().c_str());
   auto find_res = switch_node_map_.find(switch_node);
   GE_IF_BOOL_EXEC(find_res == switch_node_map_.end(), {
+    REPORT_INNER_ERROR("E19999", "Node:%s(%s) can't find in switch_node_map_, check invalid",
+                       switch_node->GetName().c_str(), switch_node->GetType().c_str());
     GELOGE(INTERNAL_ERROR, "StreamSwitch node %s not found in switch_node_map_.", switch_node->GetName().c_str());
     return INTERNAL_ERROR;
   });
   GE_IF_BOOL_EXEC(find_res->second.empty(), {
+    REPORT_INNER_ERROR("E19999", "True_nodes of StreamSwitch node:%s(%s) is empty, check invalid",
+                       switch_node->GetName().c_str(), switch_node->GetType().c_str());
     GELOGE(INTERNAL_ERROR, "true_nodes of StreamSwitch node %s is empty.", switch_node->GetName().c_str());
     return INTERNAL_ERROR;
   });
@@ -678,6 +772,8 @@ Status SwitchToStreamSwitchPass::ModifySwitchOutCtlEdges(const NodePtr &switch_n
     std::string orig_name = op_desc->GetName();
     GE_IF_BOOL_EXEC(op_desc->HasAttr(ATTR_NAME_ORIG_NODE_NAME), {
       if (!AttrUtils::GetStr(op_desc, ATTR_NAME_ORIG_NODE_NAME, orig_name) || orig_name.empty()) {
+        REPORT_CALL_ERROR("E19999", "Get Attr:%s from op:%s(%s) failed", ATTR_NAME_ORIG_NODE_NAME.c_str(),
+                          op_desc->GetName().c_str(), op_desc->GetType().c_str());
         GELOGE(INTERNAL_ERROR, "Get attr ATTR_NAME_ORIG_NODE_NAME failed, node: %s.", op_desc->GetName().c_str());
         return INTERNAL_ERROR;
       }

ge/graph/passes/transop_breadth_fusion_pass.cc

@@ -31,6 +31,7 @@ Status TransOpBreadthFusionPass::Run(ge::ComputeGraphPtr graph) {
   // breadth fusion pass requires new topologic
   Status ret_topo = graph->TopologicalSorting();
   if (ret_topo != SUCCESS) {
+    REPORT_CALL_ERROR("E19999", "Topological sorting for graph:%s failed", graph->GetName().c_str());
     GELOGE(ret_topo, "TopologicalSorting the merged graph failed.");
     return ret_topo;
   }
@@ -60,7 +61,9 @@ std::string TransOpBreadthFusionPass::GetNodeId(const int anchor_index, const No
   bool trans_format = false;
   bool trans_shape = false;
 
-  GE_IF_BOOL_EXEC(node == nullptr || node->GetOpDesc() == nullptr, GELOGE(FAILED, "node is null"); return "");
+  GE_IF_BOOL_EXEC(node == nullptr || node->GetOpDesc() == nullptr,
+                  REPORT_INNER_ERROR("E19999", "Param node or its op_desc is nullptr, check invalid");
+                  GELOGE(FAILED, "node is null"); return "");
   if (node->GetType() == CAST) {
     trans_data_type = true;
   } else if (node->GetType() == TRANSPOSE || node->GetType() == TRANSPOSED || node->GetType() == EXPANDDIMS) {

ge/graph/passes/transop_depth_fusion_pass.cc

@@ -82,6 +82,7 @@ graphStatus TransOpDepthFusionPass::RecursiveInDepth(const InDataAnchorPtr &dst_
 
   if (dst_in_anchor == nullptr || dst_in_anchor->GetOwnerNode() == nullptr ||
       dst_in_anchor->GetOwnerNode()->GetOpDesc() == nullptr) {
+    REPORT_INNER_ERROR("E19999", "Param dst_in_anchor related node info has nullptr, check invalid");
     GELOGE(FAILED, "parameter is null.");
     return GRAPH_FAILED;
   }
@@ -257,11 +258,13 @@ graphStatus TransOpDepthFusionPass::RelinkEdges(const OutDataAnchorPtr &new_out_
                                                 const OutDataAnchorPtr &old_out_anchor,
                                                 const InDataAnchorPtr &in_data_anchor) {
   if (new_out_anchor == nullptr || old_out_anchor == nullptr || in_data_anchor == nullptr) {
+    REPORT_INNER_ERROR("E19999", "Param anchor info has nullptr, check invalid");
     GELOGE(INTERNAL_ERROR, "new_out_anchor or old_out_anchor or in_data_anchor is nullptr");
     return GRAPH_FAILED;
   }
   if (new_out_anchor->GetOwnerNode() == nullptr || old_out_anchor->GetOwnerNode() == nullptr ||
       in_data_anchor->GetOwnerNode() == nullptr) {
+    REPORT_INNER_ERROR("E19999", "Param anchor info owner node has nullptr, check invalid");
     GELOGE(INTERNAL_ERROR, "anchor's owner node is nullptr");
     return GRAPH_FAILED;
   }
@@ -305,11 +308,14 @@ graphStatus TransOpDepthFusionPass::RemoveNode(const NodePtr &node, const ge::Co
     return GRAPH_FAILED;
   }
   if (GraphUtils::IsolateNode(node, {0}) != GRAPH_SUCCESS) {
+    REPORT_CALL_ERROR("E19999", "Isolate node:%s(%s) failed", node->GetName().c_str(), node->GetType().c_str());
     GELOGE(INTERNAL_ERROR, "Isolate removed node: %s, type: %s failed", node->GetName().c_str(),
            node->GetType().c_str());
     return GRAPH_FAILED;
   }
   if (GraphUtils::RemoveNodeWithoutRelink(graph, node) != GRAPH_SUCCESS) {
+    REPORT_CALL_ERROR("E19999", "Remove node:%s(%s) without relink in graph:%s failed",
+                      node->GetName().c_str(), node->GetType().c_str(), graph->GetName().c_str());
     GELOGE(INTERNAL_ERROR, "Remove node: %s, type: %s without relink failed", node->GetName().c_str(),
            node->GetType().c_str());
     return GRAPH_FAILED;

ge/graph/passes/transop_nearby_allreduce_fusion_pass.cc

@@ -99,6 +99,9 @@ Status TransOpNearbyAllreduceFusionPass::RemoveNearbyPairedTransOps(const NodePt
   auto in_data_anchors = node->GetAllInDataAnchors();
   auto out_data_anchors = node->GetAllOutDataAnchors();
   if (in_data_anchors.size() != out_data_anchors.size()) {
+    REPORT_INNER_ERROR("E19999", "In data anchors size:%zu not equal to out data anchors size:%zu in node:%s(%s), "
+                       "check invalid", in_data_anchors.size(), out_data_anchors.size(),
+                       node->GetName().c_str(), node->GetType().c_str());
     GELOGE(FAILED, "in and out data anchor size are not equal, node=%s, in_size=%zu, out_size=%zu",
            node->GetName().c_str(), in_data_anchors.size(), out_data_anchors.size());
     return FAILED;
@@ -143,6 +146,8 @@ Status TransOpNearbyAllreduceFusionPass::RemoveNearbyPairedTransOps(const NodePt
 
     // delete in_node
     if (IsolateAndDeleteNode(in_node, {0}) != SUCCESS) {
+      REPORT_CALL_ERROR("E19999", "Isolate and delete node:%s(%s) failed",
+                        in_node->GetName().c_str(), in_node->GetType().c_str());
       GELOGE(FAILED, "remove node %s failed", in_node->GetName().c_str());
       return FAILED;
     }
@@ -150,6 +155,8 @@ Status TransOpNearbyAllreduceFusionPass::RemoveNearbyPairedTransOps(const NodePt
 
     // delete out_node
     if (IsolateAndDeleteNode(out_node, {0}) != SUCCESS) {
+      REPORT_CALL_ERROR("E19999", "Isolate and delete node:%s(%s) failed",
+                        out_node->GetName().c_str(), out_node->GetType().c_str());
       GELOGE(FAILED, "remove node %s failed", out_node->GetName().c_str());
       return FAILED;
     }
@@ -162,9 +169,13 @@ Status TransOpNearbyAllreduceFusionPass::RemoveNearbyPairedTransOps(const NodePt
     auto input_desc = in_node->GetOpDesc()->GetInputDesc(0);
     auto output_desc = out_node->GetOpDesc()->GetOutputDesc(0);
     if (node->GetOpDesc()->UpdateInputDesc(static_cast<uint32_t>(i), input_desc) != GRAPH_SUCCESS) {
+      REPORT_CALL_ERROR("E19999", "Update input:%zu desc in op:%s(%s) failed",
+                        i, node->GetName().c_str(), node->GetType().c_str());
       GELOGE(FAILED, "UpdateInputDesc fail.");
     }
     if (node->GetOpDesc()->UpdateOutputDesc(static_cast<uint32_t>(i), output_desc) != GRAPH_SUCCESS) {
+      REPORT_CALL_ERROR("E19999", "Update output:%zu desc in op:%s(%s) failed",
+                        i, node->GetName().c_str(), node->GetType().c_str());
       GELOGE(FAILED, "UpdateOutputDesc");
     }
     GELOGI("successfully remove paired transop (%s and %s) for node %s",

ge/graph/passes/transop_symmetry_elimination_pass.cc

@@ -172,6 +172,12 @@ Status TransOpSymmetryEliminationPass::EliminateTransOp(NodePtr &src_node, const
   // 1.Unlink T1->T2
   auto ret = src_out_anchor->Unlink(dst_in_anchor);
   if (ret != GRAPH_SUCCESS) {
+    REPORT_CALL_ERROR("E19999",
+                      "Op:%s(%s) out index:%d unlink from op:%s(%s) in index:%d failed",
+                      src_out_anchor->GetOwnerNode()->GetName().c_str(),
+                      src_out_anchor->GetOwnerNode()->GetType().c_str(), src_out_anchor->GetIdx(),
+                      dst_in_anchor->GetOwnerNode()->GetName().c_str(),
+                      dst_in_anchor->GetOwnerNode()->GetType().c_str(), dst_in_anchor->GetIdx());
     GELOGE(FAILED, "Unlink data anchor from %s to %s.", src_node->GetName().c_str(), dst_node->GetName().c_str());
     return ret;
   }
@@ -183,6 +189,11 @@ Status TransOpSymmetryEliminationPass::EliminateTransOp(NodePtr &src_node, const
   auto pre_normal_node = in_anchor->GetPeerOutAnchor()->GetOwnerNode();
   ret = GraphUtils::AddEdge(in_anchor->GetPeerOutAnchor(), dst_in_anchor);
   if (ret != GRAPH_SUCCESS) {
+    REPORT_CALL_ERROR("E19999", "Add edge between op:%s(%s)(index:%d) and op:%s(%s)(index:%d) failed",
+                      pre_normal_node->GetName().c_str(), pre_normal_node->GetType().c_str(),
+                      in_anchor->GetPeerOutAnchor()->GetIdx(),
+                      dst_in_anchor->GetOwnerNode()->GetName().c_str(),
+                      dst_in_anchor->GetOwnerNode()->GetType().c_str(), dst_in_anchor->GetIdx());
     GELOGE(FAILED, "Add data edge from %s to %s failed.", pre_normal_node->GetName().c_str(),
            dst_node->GetName().c_str());
     return ret;
@@ -190,6 +201,9 @@ Status TransOpSymmetryEliminationPass::EliminateTransOp(NodePtr &src_node, const
   // 3.Copy in-control/data-in-control from T1->T2
   ret = GraphUtils::CopyInCtrlEdges(src_node, dst_node);
   if (ret != GRAPH_SUCCESS) {
+    REPORT_CALL_ERROR("E19999", "Copy in control edge from node:%s(%s) to node:%s(%s) failed",
+                      src_node->GetName().c_str(), src_node->GetType().c_str(),
+                      dst_node->GetName().c_str(), dst_node->GetType().c_str());
     GELOGE(FAILED, "Copy control edge from %s to %s failed.", src_node->GetName().c_str(), dst_node->GetName().c_str());
     return ret;
   }
@@ -198,6 +212,9 @@ Status TransOpSymmetryEliminationPass::EliminateTransOp(NodePtr &src_node, const
     if (in_node->GetName() == pre_normal_node->GetName()) { continue; }
     ret = GraphUtils::AddEdge(in_node->GetOutControlAnchor(), dst_node->GetInControlAnchor());
     if (ret != GRAPH_SUCCESS) {
+      REPORT_CALL_ERROR("E19999", "Add control edge between op:%s(%s) and op:%s(%s) failed",
+                        in_node->GetName().c_str(), in_node->GetType().c_str(),
+                        dst_node->GetName().c_str(), dst_node->GetType().c_str());
       GELOGE(FAILED, "Add control edge from %s to %s failed.", in_node->GetName().c_str(), dst_node->GetName().c_str());
       return ret;
     }
@@ -205,6 +222,8 @@ Status TransOpSymmetryEliminationPass::EliminateTransOp(NodePtr &src_node, const
   // 5.IsolateAndDelete T2, A will link to B automatically, and all control edge will also relink.
   ret = IsolateAndDeleteNode(dst_node, {0});
   if (ret != GRAPH_SUCCESS) {
+    REPORT_CALL_ERROR("E19999", "Isolate and delete node:%s(%s) failed",
+                      dst_node->GetName().c_str(), dst_node->GetType().c_str());
     GELOGE(INTERNAL_ERROR, "Isolate removed node: %s, type: %s failed", dst_node->GetName().c_str(),
            dst_node->GetType().c_str());
     return ret;
@@ -223,6 +242,9 @@ Status TransOpSymmetryEliminationPass::RemoveTransOpWithoutOutput(NodePtr &pre_n
     // 6.1 Copy out control to pre normal node
     Status ret = GraphUtils::CopyOutCtrlEdges(trans_node, pre_node);
     if (ret != GRAPH_SUCCESS) {
+      REPORT_CALL_ERROR("E19999", "Copy out control edge from node:%s(%s) to node:%s(%s) failed",
+                        trans_node->GetName().c_str(), trans_node->GetType().c_str(),
+                        pre_node->GetName().c_str(), pre_node->GetType().c_str());
       GELOGE(FAILED, "Copy control edge from %s to %s failed.", trans_node->GetName().c_str(),
              pre_node->GetName().c_str());
       return ret;
@@ -230,6 +252,8 @@ Status TransOpSymmetryEliminationPass::RemoveTransOpWithoutOutput(NodePtr &pre_n
     // 6.2 Isolate and delete T1
     ret = IsolateAndDeleteNode(trans_node, {});
     if (ret != GRAPH_SUCCESS) {
+      REPORT_CALL_ERROR("E19999", "Isolate and delete node:%s(%s) failed",
+                        trans_node->GetName().c_str(), trans_node->GetType().c_str());
       GELOGE(INTERNAL_ERROR, "Isolate removed node: %s, type: %s failed", trans_node->GetName().c_str(),
              trans_node->GetType().c_str());
       return ret;

ge/graph/passes/transop_without_reshape_fusion_pass.cc

@@ -63,7 +63,10 @@ void TransOpWithoutReshapeFusionPass::SetRemainNode(
       continue;
     }
     GELOGI("SetRemainNode node is %s", op_desc->GetName().c_str());
-    GE_IF_BOOL_EXEC(!op_desc->SetExtAttr(kRemainNode, true), GELOGE(INTERNAL_ERROR, "set ext attr failed"); return);
+    GE_IF_BOOL_EXEC(!op_desc->SetExtAttr(kRemainNode, true),
+                    REPORT_CALL_ERROR("E19999", "Set Attr:%s to op:%s(%s) failed", kRemainNode,
+                                      op_desc->GetName().c_str(), op_desc->GetType().c_str());
+                    GELOGE(INTERNAL_ERROR, "set ext attr failed"); return);
   }
 }
 
@@ -74,17 +77,29 @@ bool TransOpWithoutReshapeFusionPass::FormatContinuousCheck(const OutDataAnchorP
     return false;
   }
   auto in_node = in_anchor->GetOwnerNode();
-  GE_IF_BOOL_EXEC(in_node == nullptr, GELOGE(INTERNAL_ERROR, "in_node is null"); return false);
+  GE_IF_BOOL_EXEC(in_node == nullptr,
+                  REPORT_INNER_ERROR("E19999", "Param in_anchor's owner node is nullptr, check invalid");
+                  GELOGE(INTERNAL_ERROR, "in_node is null"); return false);
   auto in_op = in_node->GetOpDesc();
   auto out_owner_node = out_anchor->GetOwnerNode();
-  GE_IF_BOOL_EXEC(out_owner_node == nullptr, GELOGE(INTERNAL_ERROR, "out_owner_node is null"); return false);
+  GE_IF_BOOL_EXEC(out_owner_node == nullptr,
+                  REPORT_INNER_ERROR("E19999", "Param out_anchor's owner node is nullptr, check invalid");
+                  GELOGE(INTERNAL_ERROR, "out_owner_node is null"); return false);
   auto out_op = out_owner_node->GetOpDesc();
-  GE_IF_BOOL_EXEC(in_op == nullptr, GELOGE(INTERNAL_ERROR, "in_op is null"); return false);
-  GE_IF_BOOL_EXEC(out_op == nullptr, GELOGE(INTERNAL_ERROR, "out_op is null"); return false);
+  GE_IF_BOOL_EXEC(in_op == nullptr,
+                  REPORT_INNER_ERROR("E19999", "Param in_anchor's owner op_desc is nullptr, check invalid");
+                  GELOGE(INTERNAL_ERROR, "in_op is null"); return false);
+  GE_IF_BOOL_EXEC(out_op == nullptr,
+                  REPORT_INNER_ERROR("E19999", "Param out_anchor's owner op_desc is nullptr, check invalid");
+                  GELOGE(INTERNAL_ERROR, "out_op is null"); return false);
   auto in_op_desc = in_op->GetInputDescPtr(in_anchor->GetIdx());
   auto out_op_desc = out_op->GetOutputDescPtr(out_anchor->GetIdx());
-  GE_IF_BOOL_EXEC(in_op_desc == nullptr, GELOGE(INTERNAL_ERROR, "in_op_desc is null"); return false);
-  GE_IF_BOOL_EXEC(out_op_desc == nullptr, GELOGE(INTERNAL_ERROR, "out_op_desc is null"); return false);
+  GE_IF_BOOL_EXEC(in_op_desc == nullptr,
+                  REPORT_INNER_ERROR("E19999", "Param in_anchor corresponding tensor is nullptr, check invalid");
+                  GELOGE(INTERNAL_ERROR, "in_op_desc is null"); return false);
+  GE_IF_BOOL_EXEC(out_op_desc == nullptr,
+                  REPORT_INNER_ERROR("E19999", "Param out_anchor corresponding tensor is nullptr, check invalid");
+                  GELOGE(INTERNAL_ERROR, "out_op_desc is null"); return false);
   if (!ShapeEqualCheck(in_op_desc->GetShape(), out_op_desc->GetShape())) {
     return false;
   }
@@ -357,6 +372,9 @@ graphStatus TransOpWithoutReshapeFusionPass::RelinkSubGraphControlEdges(
     GELOGI("add control edge.src:%s, dst:%s", out_owner_node->GetName().c_str(), in_owner_node->GetName().c_str());
     if (GraphUtils::AddEdge(out_owner_node->GetOutControlAnchor(), in_owner_node->GetInControlAnchor()) !=
         GRAPH_SUCCESS) {
+      REPORT_CALL_ERROR("E19999", "Add control edge between op:%s(%s) and op:%s(%s) failed",
+                        out_owner_node->GetName().c_str(), out_owner_node->GetType().c_str(),
+                        in_owner_node->GetName().c_str(), in_owner_node->GetType().c_str());
       return GRAPH_FAILED;
     }
   }
@@ -365,6 +383,9 @@ graphStatus TransOpWithoutReshapeFusionPass::RelinkSubGraphControlEdges(
     GELOGI("add out data 2 in contorl edge.src:%s, dst:%s", out_owner_node->GetName().c_str(),
            in_owner_node->GetName().c_str());
     if (GraphUtils::AddEdge(out_anchor, in_owner_node->GetInControlAnchor()) != GRAPH_SUCCESS) {
+      REPORT_CALL_ERROR("E19999", "Add control edge between op:%s(%s) and op:%s(%s) failed",
+                        out_owner_node->GetName().c_str(), out_owner_node->GetType().c_str(),
+                        in_owner_node->GetName().c_str(), in_owner_node->GetType().c_str());
       return GRAPH_FAILED;
     }
   }
@@ -392,6 +413,10 @@ graphStatus TransOpWithoutReshapeFusionPass::RelinkControlEdgesWhenDescNotChange
     GELOGI("add control edge.src:%s, dst:%s, dst idx:%d", out_owner_node->GetName().c_str(),
            peer_in_anchor->GetOwnerNode()->GetName().c_str(), peer_in_anchor->GetIdx());
     if (GraphUtils::AddEdge(out_owner_node->GetOutControlAnchor(), peer_in_anchor) != GRAPH_SUCCESS) {
+      REPORT_CALL_ERROR("E19999", "Add control edge between op:%s(%s) and op:%s(%s) failed",
+                        out_owner_node->GetName().c_str(), out_owner_node->GetType().c_str(),
+                        peer_in_anchor->GetOwnerNode()->GetName().c_str(),
+                        peer_in_anchor->GetOwnerNode()->GetType().c_str());
       return GRAPH_FAILED;
     }
   }
@@ -401,6 +426,10 @@ graphStatus TransOpWithoutReshapeFusionPass::RelinkControlEdgesWhenDescNotChange
     GELOGI("add control edge.src:%s, src idx:%d, dst:%s", peer_out_anchor->GetOwnerNode()->GetName().c_str(),
            peer_out_anchor->GetIdx(), in_owner_node->GetName().c_str());
     if (GraphUtils::AddEdge(peer_out_anchor, in_owner_node->GetInControlAnchor()) != GRAPH_SUCCESS) {
+      REPORT_CALL_ERROR("E19999", "Add control edge between op:%s(%s) and op:%s(%s) failed",
+                        peer_out_anchor->GetOwnerNode()->GetName().c_str(),
+                        peer_out_anchor->GetOwnerNode()->GetType().c_str(),
+                        in_owner_node->GetName().c_str(), in_owner_node->GetType().c_str());
       return GRAPH_FAILED;
     }
   }
@@ -410,6 +439,10 @@ graphStatus TransOpWithoutReshapeFusionPass::RelinkControlEdgesWhenDescNotChange
     GELOGI("add out control 2 in data edge.src:%s, dst:%s, dst idx:%d", out_owner_node->GetName().c_str(),
            peer_in_anchor->GetOwnerNode()->GetName().c_str(), peer_in_anchor->GetIdx());
     if (GraphUtils::AddEdge(out_owner_node->GetOutControlAnchor(), peer_in_anchor) != GRAPH_SUCCESS) {
+      REPORT_CALL_ERROR("E19999", "Add control edge between op:%s(%s) and op:%s(%s) failed",
+                        out_owner_node->GetName().c_str(), out_owner_node->GetType().c_str(),
+                        peer_in_anchor->GetOwnerNode()->GetName().c_str(),
+                        peer_in_anchor->GetOwnerNode()->GetType().c_str());
       return GRAPH_FAILED;
     }
   }
@@ -419,6 +452,10 @@ graphStatus TransOpWithoutReshapeFusionPass::RelinkControlEdgesWhenDescNotChange
     GELOGI("add out data 2 in control edge.src:%s, dst:%s, dst idx:%d", out_owner_node->GetName().c_str(),
            peer_in_anchor->GetOwnerNode()->GetName().c_str(), peer_in_anchor->GetIdx());
     if (GraphUtils::AddEdge(out_anchor, peer_in_anchor) != GRAPH_SUCCESS) {
+      REPORT_CALL_ERROR("E19999", "Add control edge between op:%s(%s) and op:%s(%s) failed",
+                        out_owner_node->GetName().c_str(), out_owner_node->GetType().c_str(),
+                        peer_in_anchor->GetOwnerNode()->GetName().c_str(),
+                        peer_in_anchor->GetOwnerNode()->GetType().c_str());
       return GRAPH_FAILED;
     }
   }
@@ -443,6 +480,9 @@ graphStatus TransOpWithoutReshapeFusionPass::RelinkNodesWhenDescNotChanged(
   GELOGI("relink node.src node:%s, src idx:%d, dst node:%s, dst idx:%d", out_owner_node->GetName().c_str(),
          out_anchor->GetIdx(), in_owner_node->GetName().c_str(), in_anchor->GetIdx());
   if (GraphUtils::AddEdge(out_anchor, in_anchor) != GRAPH_SUCCESS) {
+    REPORT_CALL_ERROR("E19999", "Add edge between op:%s(%s)(index:%d) and op:%s(%s)(index:%d) failed",
+                      out_owner_node->GetName().c_str(), out_owner_node->GetType().c_str(), out_anchor->GetIdx(),
+                      in_owner_node->GetName().c_str(), in_owner_node->GetType().c_str(), in_anchor->GetIdx());
     GELOGE(GRAPH_FAILED, "add edge failed!src:%s, src idx:%d, dst:%s, dst idx:%d", out_owner_node->GetName().c_str(),
            out_anchor->GetIdx(), in_owner_node->GetName().c_str(), in_anchor->GetIdx());
     return GRAPH_FAILED;
@@ -466,16 +506,21 @@ OpDescPtr TransOpWithoutReshapeFusionPass::GetFormatTransferOp(const GeTensorDes
   format_transfer_op_name << "fusion_format_transfer_" << fusion_format_transfer_op_count;
   OpDescPtr format_transfer_op = MakeShared<OpDesc>(format_transfer_op_name.str().c_str(), TRANSDATA);
   if (format_transfer_op == nullptr) {
+    REPORT_CALL_ERROR("E19999", "New GeTensor failed");
     GELOGE(INTERNAL_ERROR, "new format transfer op failed!");
     return nullptr;
   }
 
   GE_IF_BOOL_EXEC(!AttrUtils::SetInt(format_transfer_op, ATTR_NAME_INPUT_FORMAT,
                                      static_cast<int64_t>(format_trans_input_desc.GetFormat())),
+                  REPORT_CALL_ERROR("E19999", "Set Attr:%s to op:%s(%s) failed", ATTR_NAME_INPUT_FORMAT.c_str(),
+                                    format_transfer_op->GetName().c_str(), format_transfer_op->GetType().c_str());
                   GELOGE(INTERNAL_ERROR, "set ATTR_NAME_INPUT_FORMAT failed");
                   return nullptr);
   GE_IF_BOOL_EXEC(!AttrUtils::SetInt(format_transfer_op, ATTR_NAME_OUTPUT_FORMAT,
                                      static_cast<int64_t>(format_trans_output_desc.GetFormat())),
+                  REPORT_CALL_ERROR("E19999", "Set Attr:%s to op:%s(%s) failed", ATTR_NAME_OUTPUT_FORMAT.c_str(),
+                                    format_transfer_op->GetName().c_str(), format_transfer_op->GetType().c_str());
                   GELOGE(INTERNAL_ERROR, "set ATTR_NAME_OUTPUT_FORMAT failed");
                   return nullptr);
 
@@ -483,22 +528,32 @@ OpDescPtr TransOpWithoutReshapeFusionPass::GetFormatTransferOp(const GeTensorDes
   string dst_format = TypeUtils::FormatToSerialString(format_trans_output_desc.GetFormat());
 
   GE_IF_BOOL_EXEC(!AttrUtils::SetStr(format_transfer_op, kAttrNameSrcFormat, src_format),
+                  REPORT_CALL_ERROR("E19999", "Set Attr:%s to op:%s(%s) failed", kAttrNameSrcFormat,
+                                    format_transfer_op->GetName().c_str(), format_transfer_op->GetType().c_str());
                   GELOGE(INTERNAL_ERROR, "set kAttrNameSrcFormat failed");
                   return nullptr);
 
   GE_IF_BOOL_EXEC(!AttrUtils::SetStr(format_transfer_op, kAttrNameDstFormat, dst_format),
+                  REPORT_CALL_ERROR("E19999", "Set Attr:%s to op:%s(%s) failed", kAttrNameDstFormat,
+                                    format_transfer_op->GetName().c_str(), format_transfer_op->GetType().c_str());
                   GELOGE(INTERNAL_ERROR, "set kAttrNameDstFormat failed");
                   return nullptr);
 
   GE_IF_BOOL_EXEC(format_transfer_op->AddInputDesc(format_trans_input_desc) != GRAPH_SUCCESS,
+                  REPORT_CALL_ERROR("E19999", "Add input desc to op:%s(%s) failed",
+                                    format_transfer_op->GetName().c_str(), format_transfer_op->GetType().c_str());
                   GELOGE(INTERNAL_ERROR, "add input desc failed");
                   return nullptr);
 
   GE_IF_BOOL_EXEC(format_transfer_op->AddOutputDesc(format_trans_output_desc) != GRAPH_SUCCESS,
+                  REPORT_CALL_ERROR("E19999", "Add ouput desc to op:%s(%s) failed",
+                                    format_transfer_op->GetName().c_str(), format_transfer_op->GetType().c_str());
                   GELOGE(INTERNAL_ERROR, "add output desc failed");
                   return nullptr);
 
   GE_IF_BOOL_EXEC(!ge::AttrUtils::SetBool(format_transfer_op, ATTR_NEED_COMPILE, true),
+                  REPORT_CALL_ERROR("E19999", "Set Attr:%s to op:%s(%s) failed", ATTR_NEED_COMPILE.c_str(),
+                                    format_transfer_op->GetName().c_str(), format_transfer_op->GetType().c_str());
                   GELOGE(INTERNAL_ERROR, "set ext attr failed");
                   return nullptr);
   return format_transfer_op;
@@ -515,6 +570,7 @@ OpDescPtr TransOpWithoutReshapeFusionPass::GetCastOp(const GeTensorDesc &cast_in
   auto cast_op = ge::OpDescUtils::GetOpDescFromOperator(node_op);
   node_op.BreakConnect();
   if (cast_op == nullptr) {
+    REPORT_CALL_ERROR("E19999", "Create operator:%s(%s) failed", cast_op_name.str().c_str(), CAST);
     GELOGE(INTERNAL_ERROR, "new cast op failed!");
     return nullptr;
   }
@@ -522,29 +578,41 @@ OpDescPtr TransOpWithoutReshapeFusionPass::GetCastOp(const GeTensorDesc &cast_in
   const int default_output_index = 0;
   if (cast_op->GetInputsSize() == 0) {
     GE_IF_BOOL_EXEC(cast_op->AddInputDesc(cast_input_desc) != GRAPH_SUCCESS,
+                    REPORT_CALL_ERROR("E19999", "Add input desc to op:%s(%s) failed",
+                                      cast_op->GetName().c_str(), cast_op->GetType().c_str());
                     GELOGE(INTERNAL_ERROR, "add input desc failed");
                     return nullptr);
   } else {
     GE_IF_BOOL_EXEC(cast_op->UpdateInputDesc(default_input_index, cast_input_desc) != GRAPH_SUCCESS,
+                    REPORT_CALL_ERROR("E19999", "Update input:%d desc of op:%s(%s) failed", default_input_index,
+                                      cast_op->GetName().c_str(), cast_op->GetType().c_str());
                     GELOGE(INTERNAL_ERROR, "update input desc failed");
                     return nullptr);
   }
 
   if (cast_op->GetOutputsSize() == 0) {
     GE_IF_BOOL_EXEC(cast_op->AddOutputDesc(cast_output_desc) != GRAPH_SUCCESS,
+                    REPORT_CALL_ERROR("E19999", "Add output desc to op:%s(%s) failed",
+                                      cast_op->GetName().c_str(), cast_op->GetType().c_str());
                     GELOGE(INTERNAL_ERROR, "add output desc failed");
                     return nullptr);
   } else {
     GE_IF_BOOL_EXEC(cast_op->UpdateOutputDesc(default_output_index, cast_output_desc) != GRAPH_SUCCESS,
+                    REPORT_CALL_ERROR("E19999", "Update output:%d desc of op:%s(%s) failed", default_output_index,
+                                      cast_op->GetName().c_str(), cast_op->GetType().c_str());
                     GELOGE(INTERNAL_ERROR, "update output desc failed");
                     return nullptr);
   }
 
   if (!AttrUtils::SetInt(cast_op, CAST_ATTR_DST_TYPE, static_cast<int64_t>(cast_output_desc.GetDataType()))) {
+    REPORT_CALL_ERROR("E19999", "Set Attr:%s to op:%s(%s) failed", CAST_ATTR_DST_TYPE.c_str(),
+                      cast_op->GetName().c_str(), cast_op->GetType().c_str());
     GELOGE(INTERNAL_ERROR, "set dst_type attr failed");
     return nullptr;
   }
   if (!AttrUtils::SetBool(cast_op, ATTR_NEED_COMPILE, true)) {
+    REPORT_CALL_ERROR("E19999", "Set Attr:%s to op:%s(%s) failed", ATTR_NEED_COMPILE.c_str(),
+                      cast_op->GetName().c_str(), cast_op->GetType().c_str());
     GELOGE(INTERNAL_ERROR, "set need_compile attr failed");
     return nullptr;
   }
@@ -879,6 +947,8 @@ graphStatus TransOpWithoutReshapeFusionPass::AddTransNode(const ComputeGraphPtr
 
   trans_node = graph->AddNode(transop);
   if (trans_node == nullptr) {
+    REPORT_CALL_ERROR("E19999", "Add node:%s(%s) to graph:%s failed",
+                      transop->GetName().c_str(), transop->GetType().c_str(), graph->GetName().c_str());
     GELOGE(GRAPH_FAILED, "add node failed!");
     return GRAPH_FAILED;
   }
@@ -945,6 +1015,9 @@ graphStatus TransOpWithoutReshapeFusionPass::InsertNewTransOp(const ComputeGraph
   GELOGI("add edge.src:%s, src idx:%d, dst:%s", out_anchor->GetOwnerNode()->GetName().c_str(), out_anchor->GetIdx(),
          new_trans_nodes.front()->GetName().c_str());
   if (GraphUtils::AddEdge(out_anchor, new_trans_nodes.front()->GetInAnchor(0)) != GRAPH_SUCCESS) {
+    REPORT_CALL_ERROR("E19999", "Add edge between op:%s(%s)(index:%d) and op:%s(%s)(index:0) failed",
+                      out_owner_node->GetName().c_str(), out_owner_node->GetType().c_str(), out_anchor->GetIdx(),
+                      new_trans_nodes.front()->GetName().c_str(), new_trans_nodes.front()->GetType().c_str());
     return GRAPH_FAILED;
   } else {
     auto old_peer_in_anchor = begin_out.second;
@@ -957,6 +1030,9 @@ graphStatus TransOpWithoutReshapeFusionPass::InsertNewTransOp(const ComputeGraph
            new_trans_nodes.back()->GetName().c_str());
     if (GraphUtils::AddEdge(new_trans_nodes.front()->GetOutAnchor(0), new_trans_nodes.back()->GetInAnchor(0)) !=
         GRAPH_SUCCESS) {
+      REPORT_CALL_ERROR("E19999", "Add edge between op:%s(%s)(index:0) and op:%s(%s)(index:0) failed",
+                        new_trans_nodes.front()->GetName().c_str(), new_trans_nodes.front()->GetType().c_str(),
+                        new_trans_nodes.back()->GetName().c_str(), new_trans_nodes.back()->GetType().c_str());
       return GRAPH_FAILED;
     } else {
       auto old_peer_out_anchor = end_in.first;
@@ -967,6 +1043,9 @@ graphStatus TransOpWithoutReshapeFusionPass::InsertNewTransOp(const ComputeGraph
   GELOGI("add edge.src:%s, dst:%s, dst idx:%d", new_trans_nodes.back()->GetName().c_str(),
          in_anchor->GetOwnerNode()->GetName().c_str(), in_anchor->GetIdx());
   if (GraphUtils::AddEdge(new_trans_nodes.back()->GetOutAnchor(0), in_anchor) != GRAPH_SUCCESS) {
+    REPORT_CALL_ERROR("E19999", "Add edge between op:%s(%s)(index:0) and op:%s(%s)(index:%d) failed",
+                      new_trans_nodes.front()->GetName().c_str(), new_trans_nodes.front()->GetType().c_str(),
+                      in_owner_node->GetName().c_str(), in_owner_node->GetType().c_str(), in_anchor->GetIdx());
     return GRAPH_FAILED;
   }
 
@@ -977,6 +1056,7 @@ graphStatus TransOpWithoutReshapeFusionPass::RelinkControlEdge(const int index,
                                                                const vector<NodePtr> &new_trans_nodes) {
   GE_CHECK_NOTNULL(out_anchor);
   if (new_trans_nodes.front() == nullptr || new_trans_nodes.back() == nullptr) {
+    REPORT_INNER_ERROR("E19999", "Param new_trans_nodes front or back is nullptr, check invalid");
     return GRAPH_FAILED;
   }
   if (sub_graph_has_control_edge_[index]) {
@@ -984,6 +1064,9 @@ graphStatus TransOpWithoutReshapeFusionPass::RelinkControlEdge(const int index,
            new_trans_nodes.front()->GetName().c_str());
     if (GraphUtils::AddEdge(out_anchor->GetOwnerNode()->GetOutControlAnchor(),
                             new_trans_nodes.front()->GetInControlAnchor()) != GRAPH_SUCCESS) {
+      REPORT_CALL_ERROR("E19999", "Add control edge between op:%s(%s) and op:%s(%s) failed",
+                        out_anchor->GetOwnerNode()->GetName().c_str(), out_anchor->GetOwnerNode()->GetType().c_str(),
+                        new_trans_nodes.front()->GetName().c_str(), new_trans_nodes.front()->GetType().c_str());
       return GRAPH_FAILED;
     }
   }
@@ -993,6 +1076,10 @@ graphStatus TransOpWithoutReshapeFusionPass::RelinkControlEdge(const int index,
     GELOGI("add control edge.src:%s, dst:%s", new_trans_nodes.back()->GetName().c_str(),
            peer_in_anchor->GetOwnerNode()->GetName().c_str());
     if (GraphUtils::AddEdge(new_trans_nodes.back()->GetOutControlAnchor(), peer_in_anchor) != GRAPH_SUCCESS) {
+      REPORT_CALL_ERROR("E19999", "Add control edge between op:%s(%s) and op:%s(%s) failed",
+                       new_trans_nodes.back()->GetName().c_str(), new_trans_nodes.back()->GetType().c_str(),
+                       peer_in_anchor->GetOwnerNode()->GetName().c_str(),
+                       peer_in_anchor->GetOwnerNode()->GetType().c_str());
       return GRAPH_FAILED;
     }
   }
@@ -1002,6 +1089,10 @@ graphStatus TransOpWithoutReshapeFusionPass::RelinkControlEdge(const int index,
     GELOGI("add control edge.src:%s, dst:%s", peer_out_anchor->GetOwnerNode()->GetName().c_str(),
            new_trans_nodes.front()->GetName().c_str());
     if (GraphUtils::AddEdge(peer_out_anchor, new_trans_nodes.front()->GetInControlAnchor()) != GRAPH_SUCCESS) {
+      REPORT_CALL_ERROR("E19999", "Add control edge between op:%s(%s) and op:%s(%s) failed",
+                        peer_out_anchor->GetOwnerNode()->GetName().c_str(),
+                        peer_out_anchor->GetOwnerNode()->GetType().c_str(),
+                        new_trans_nodes.front()->GetName().c_str(), new_trans_nodes.front()->GetType().c_str());
       return GRAPH_FAILED;
     }
   }
@@ -1011,6 +1102,10 @@ graphStatus TransOpWithoutReshapeFusionPass::RelinkControlEdge(const int index,
     GELOGI("add control edge.src:%s, dst:%s", new_trans_nodes.back()->GetName().c_str(),
            peer_in_anchor->GetOwnerNode()->GetName().c_str());
     if (GraphUtils::AddEdge(new_trans_nodes.back()->GetOutControlAnchor(), peer_in_anchor) != GRAPH_SUCCESS) {
+      REPORT_CALL_ERROR("E19999", "Add control edge between op:%s(%s) and op:%s(%s) failed",
+                       new_trans_nodes.back()->GetName().c_str(), new_trans_nodes.back()->GetType().c_str(),
+                       peer_in_anchor->GetOwnerNode()->GetName().c_str(),
+                       peer_in_anchor->GetOwnerNode()->GetType().c_str());
       return GRAPH_FAILED;
     }
   }
@@ -1020,6 +1115,10 @@ graphStatus TransOpWithoutReshapeFusionPass::RelinkControlEdge(const int index,
     GELOGI("add control edge.src:%s, dst:%s", new_trans_nodes.back()->GetName().c_str(),
            peer_in_anchor->GetOwnerNode()->GetName().c_str());
     if (GraphUtils::AddEdge(new_trans_nodes.back()->GetOutDataAnchor(0), peer_in_anchor) != GRAPH_SUCCESS) {
+      REPORT_CALL_ERROR("E19999", "Add edge between op:%s(%s)(index:0) and op:%s(%s)(index:%d) failed",
+                        new_trans_nodes.back()->GetName().c_str(), new_trans_nodes.back()->GetType().c_str(),
+                        peer_in_anchor->GetOwnerNode()->GetName().c_str(),
+                        peer_in_anchor->GetOwnerNode()->GetType().c_str(), peer_in_anchor->GetIdx());
       return GRAPH_FAILED;
     }
   }
@@ -1081,6 +1180,7 @@ graphStatus TransOpWithoutReshapeFusionPass::GetSubGraphsBetweenNormalNode(
   vector<std::pair<OutDataAnchorPtr, InDataAnchorPtr>> &nodes_list) {
   graphStatus ret = GRAPH_SUCCESS;
   if (out_anchor == nullptr) {
+    REPORT_INNER_ERROR("E19999", "Param out_anchor is nullptr, check invalid");
     return GRAPH_FAILED;
   }
 

ge/graph/passes/transpose_transdata_pass.cc

@@ -34,11 +34,13 @@ const char *const kAttrNameSrcFormat = "src_format";
 namespace ge {
 Status TransposeTransDataPass::Run(NodePtr &node) {
   if (node == nullptr) {
+    REPORT_INNER_ERROR("E19999", "Param node is nullptr, check invalid");
     GELOGE(PARAM_INVALID, "param [node] must not be null.");
     return PARAM_INVALID;
   }
   auto op_desc = node->GetOpDesc();
   if (op_desc == nullptr) {
+    REPORT_INNER_ERROR("E19999", "Param node's op_desc is nullptr, check invalid");
     GELOGE(PARAM_INVALID, "OpDesc of param [node] must not be null.");
     return PARAM_INVALID;
   }
@@ -77,6 +79,7 @@ Status TransposeTransDataPass::Run(NodePtr &node) {
     GE_CHECK_NOTNULL(out_node);
     OpDescPtr out_op_desc = out_node->GetOpDesc();
     if (out_op_desc == nullptr) {
+      REPORT_INNER_ERROR("E19999", "OpDesc in node is nullptr, check invalid");
       GELOGE(FAILED, "OpDesc of out data node of [%s] must not be null.", node->GetName().c_str());
       return FAILED;
     }
@@ -111,6 +114,10 @@ Status TransposeTransDataPass::CheckOneInAndOneOutDataAnchor(NodePtr &node) cons
   // Trans op has one input data node, maybe has N output data nodes
   uint32_t in_data_node_nums = node->GetInDataNodes().size();
   if (in_data_anchor_nums != 1 || out_data_anchor_nums != 1 || in_data_node_nums != 1) {
+    REPORT_INNER_ERROR("E19999", "In data anchor num:%u, out data anchor num:%u, in data node num:%u of node:%s(%s) "
+                       "must be all equal to 1, check invalid",
+                       in_data_anchor_nums, out_data_anchor_nums, in_data_node_nums,
+                       node->GetName().c_str(), node->GetType().c_str());
     GELOGE(FAILED, "[%s] %s has %u in %u out data anchor, has %u in data node.", node->GetType().c_str(),
            node->GetName().c_str(), in_data_anchor_nums, out_data_anchor_nums, in_data_node_nums);
     return FAILED;
@@ -122,6 +129,8 @@ Status TransposeTransDataPass::RemoveTranspose(NodePtr &node) {
   GE_CHECK_NOTNULL(node);
   ComputeGraphPtr graph = node->GetOwnerComputeGraph();
   if (graph == nullptr) {
+    REPORT_INNER_ERROR("E19999", "Owner graph of node:%s(%s) is nullptr, check invalid",
+                       node->GetName().c_str(), node->GetType().c_str());
     GELOGE(FAILED, "[%s] The owner graph must not be null.", node->GetName().c_str());
     return FAILED;
   }
@@ -146,6 +155,8 @@ Status TransposeTransDataPass::RemoveTranspose(NodePtr &node) {
   }
   AddNodeDeleted(node);
   if (GraphUtils::RemoveNodeWithoutRelink(graph, node) != GRAPH_SUCCESS) {
+    REPORT_CALL_ERROR("E19999", "Remove node:%s(%s) without relink in graph:%s failed",
+                      node->GetName().c_str(), node->GetType().c_str(), graph->GetName().c_str());
     GELOGE(FAILED, "[%s] RemoveNodeWithoutRelink failed.", node->GetName().c_str());
     return FAILED;
   }

ge/graph/passes/unused_args_clean_pass.cc

@@ -101,6 +101,8 @@ Status UnusedArgsCleanPass::ClassifyDataNodes(const ComputeGraphPtr &graph, cons
   for (const auto &name : func_desc->GetSubgraphInstanceNames()) {
     const auto &subgraph = graph->GetSubgraph(name);
     if (subgraph == nullptr) {
+      REPORT_CALL_ERROR("E19999", "Get subgraph from graph:%s by name:%s failed",
+                        graph->GetName().c_str(), name.c_str());
       GELOGE(GE_GRAPH_EMPTY_SUBGRAPH, "Subgraph not found, name: %s", name.c_str());
       return GE_GRAPH_EMPTY_SUBGRAPH;
     }
@@ -113,6 +115,8 @@ Status UnusedArgsCleanPass::ClassifyDataNodes(const ComputeGraphPtr &graph, cons
 
       uint32_t parent_index = 0;
       if (!AttrUtils::GetInt(data->GetOpDesc(), ATTR_NAME_PARENT_NODE_INDEX, parent_index)) {
+        REPORT_CALL_ERROR("E19999", "Get Attr:%s from op:%s(%s) failed", ATTR_NAME_PARENT_NODE_INDEX.c_str(),
+                          data->GetName().c_str(), data->GetType().c_str());
         GELOGE(FAILED, "Parent index not found, name: %s", data->GetName().c_str());
         return FAILED;
       }
@@ -150,6 +154,8 @@ Status UnusedArgsCleanPass::UpdateInputTensor(const map<ComputeGraphPtr, map<uin
     const auto data = it->second;
 
     if (!AttrUtils::SetInt(data->GetOpDesc(), ATTR_NAME_PARENT_NODE_INDEX, update_index)) {
+      REPORT_CALL_ERROR("E19999", "Get Attr:%s from op:%s(%s) failed", ATTR_NAME_PARENT_NODE_INDEX.c_str(),
+                        data->GetName().c_str(), data->GetType().c_str());
       GELOGE(FAILED, "Set parent index failed, name: %s", data->GetName().c_str());
       return FAILED;
     }

ge/graph/passes/unused_const_pass.cc

@@ -27,10 +27,12 @@ namespace ge {
 ///
 Status UnusedConstPass::Run(NodePtr &node) {
   if (node == nullptr) {
+    REPORT_INNER_ERROR("E19999", "Param node is nullptr, check invalid");
     GELOGE(FAILED, "parameter is null.");
     return FAILED;
   }
   if (node->GetOpDesc() == nullptr) {
+    REPORT_INNER_ERROR("E19999", "Param node's op_desc is nullptr, check invalid");
     GELOGE(PARAM_INVALID, "param [opDesc] must not be null.");
     return PARAM_INVALID;
   }

ge/graph/passes/var_is_initialized_op_pass.cc

@@ -61,6 +61,8 @@ Status VarIsInitializedOpPass::CheckSrcNode(const NodePtr &node, bool &inited) c
   GE_CHECK_NOTNULL(node);
   auto input_nodes = node->GetInDataNodes();
   if (input_nodes.size() != kVarIsInitializedIOCnt) {
+    REPORT_INNER_ERROR("E19999", "In data node num:%zu of node:%s(%s) not equal to %d, check invalid",
+                       input_nodes.size(), node->GetName().c_str(), node->GetType().c_str(), kVarIsInitializedIOCnt);
     GELOGE(FAILED,
            "[%s] Node input data nodes size [%zu] is not equal 1.",
            node->GetName().c_str(),
@@ -73,6 +75,9 @@ Status VarIsInitializedOpPass::CheckSrcNode(const NodePtr &node, bool &inited) c
   auto input_node_name = input_node->GetName();
   auto input_node_type = input_node->GetType();
   if (input_node_type != VARIABLE) {
+    REPORT_INNER_ERROR("E19999", "Index:%d In data node of node:%s(%s), type:%s not %s, check invalid",
+                       kVarIsInitVarInputIndex, node->GetName().c_str(), node->GetType().c_str(),
+                       input_node_type.c_str(), VARIABLE);
     GELOGE(FAILED, "[%s] Src node %s is not Variable, is %s.", node->GetName().c_str(), input_node_name.c_str(),
            input_node_type.c_str());
     return FAILED;
@@ -95,6 +100,7 @@ Status VarIsInitializedOpPass::CreateConstant(NodePtr &node, OpDescPtr &op_desc,
   // 1. create Constant OpDesc
   op_desc = MakeShared<OpDesc>(node->GetName().c_str(), CONSTANT);
   if (op_desc == nullptr) {
+    REPORT_CALL_ERROR("E19999", "New OpDesc failed");
     GELOGE(FAILED, "[%s] Make shared of Constant op desc failed.", node->GetName().c_str());
     return FAILED;
   }
@@ -102,6 +108,7 @@ Status VarIsInitializedOpPass::CreateConstant(NodePtr &node, OpDescPtr &op_desc,
   // 2. get OpDesc of VarIsInitializedOp
   OpDescPtr original_op_desc = node->GetOpDesc();
   if (original_op_desc == nullptr) {
+    REPORT_INNER_ERROR("E19999", "OpDesc in node is nullptr, check invalid");
     GELOGE(FAILED, "[%s] Op desc must not be null.", node->GetName().c_str());
     return FAILED;
   }
@@ -111,10 +118,13 @@ Status VarIsInitializedOpPass::CreateConstant(NodePtr &node, OpDescPtr &op_desc,
   bool val = inited;
   GeTensorPtr const_tensor_ptr = MakeShared<GeTensor>(original_desc, reinterpret_cast<uint8_t *>(&val), sizeof(bool));
   if (const_tensor_ptr == nullptr) {
+    REPORT_CALL_ERROR("E19999", "New GeTensor failed");
     GELOGE(FAILED, "[%s] Make shared of Constant tensor failed.", node->GetName().c_str());
     return FAILED;
   }
   if (!AttrUtils::SetTensor(op_desc, ATTR_NAME_WEIGHTS, const_tensor_ptr)) {
+    REPORT_CALL_ERROR("E19999", "Set Attr:%s to op:%s(%s) failed", ATTR_NAME_WEIGHTS.c_str(),
+                      op_desc->GetName().c_str(), op_desc->GetType().c_str());
     GELOGE(INTERNAL_ERROR, "get ATTR_NAME_WEIGHTS failed");
     return FAILED;
   }
@@ -131,6 +141,9 @@ Status VarIsInitializedOpPass::ProcessInAnchor(NodePtr &node, NodePtr &new_node)
   auto out_anchors = node->GetAllOutDataAnchors();
   if ((in_anchors.size() != kVarIsInitializedIOCnt) ||
       (out_anchors.size() != kVarIsInitializedIOCnt)) {
+    REPORT_INNER_ERROR("E19999", "In data anchor num:%zu and out data anchor num:%zu of node:%s(%s), "
+                       "must botch equal to %d, check invalid", in_anchors.size(), out_anchors.size(),
+                       node->GetName().c_str(), node->GetType().c_str(), kVarIsInitializedIOCnt);
     GELOGE(FAILED,
            "[%s] Node input/output data anchors"
            " size [%lu][%lu] is not all equal 1.",
@@ -144,22 +157,36 @@ Status VarIsInitializedOpPass::ProcessInAnchor(NodePtr &node, NodePtr &new_node)
   auto peer_out_anchor = in_anchor->GetPeerOutAnchor();
   GE_CHECK_NOTNULL(peer_out_anchor);
   if (GraphUtils::RemoveEdge(in_anchor, peer_out_anchor) != GRAPH_SUCCESS) {
+    REPORT_CALL_ERROR("E19999", "Remove edge between op:%s(%s)(index:%d) and op:%s(%s)(index:%d) failed",
+                      in_anchor->GetOwnerNode()->GetName().c_str(), in_anchor->GetOwnerNode()->GetType().c_str(),
+                      in_anchor->GetIdx(),
+                      peer_out_anchor->GetOwnerNode()->GetName().c_str(),
+                      peer_out_anchor->GetOwnerNode()->GetType().c_str(), peer_out_anchor->GetIdx());
     GELOGE(FAILED, "[%s] Remove in data edge failed.", node->GetName().c_str());
     return FAILED;
   }
   auto src_node = peer_out_anchor->GetOwnerNode();
   if (GraphUtils::AddEdge(src_node->GetOutControlAnchor(), new_node->GetInControlAnchor()) != GRAPH_SUCCESS) {
+    REPORT_CALL_ERROR("E19999", "Add control edge between op:%s(%s) and op:%s(%s) failed",
+                      src_node->GetName().c_str(), src_node->GetType().c_str(),
+                      new_node->GetName().c_str(), new_node->GetType().c_str());
     GELOGE(FAILED, "Failed to link control edges from var %s to new const %s",
            src_node->GetName().c_str(), new_node->GetName().c_str());
     return FAILED;
   }
 
   if (GraphUtils::MoveInCtrlEdges(node, new_node) != GRAPH_SUCCESS) {
+    REPORT_CALL_ERROR("E19999", "Move in control edge from node:%s(%s) to node:%s(%s) failed",
+                      node->GetName().c_str(), node->GetType().c_str(),
+                      new_node->GetName().c_str(), new_node->GetType().c_str());
     GELOGE(FAILED, "Failed to move in ctrl edges from %s to new const", node->GetName().c_str());
     return FAILED;
   }
 
   if (GraphUtils::MoveOutCtrlEdges(node, new_node) != GRAPH_SUCCESS) {
+    REPORT_CALL_ERROR("E19999", "Move out control edge from node:%s(%s) to node:%s(%s) failed",
+                      node->GetName().c_str(), node->GetType().c_str(),
+                      new_node->GetName().c_str(), new_node->GetType().c_str());
     GELOGE(FAILED, "Failed to move out ctrl edges from %s to new const", node->GetName().c_str());
     return FAILED;
   }
@@ -177,6 +204,9 @@ Status VarIsInitializedOpPass::ChangeNodeToConstant(NodePtr &node, bool inited)
 
   NodePtr const_node = graph->AddNodeFront(constant_op_desc);
   if (const_node == nullptr) {
+    REPORT_CALL_ERROR("E19999", "Add node:%s(%s) to graph:%s front failed",
+                      constant_op_desc->GetName().c_str(), constant_op_desc->GetType().c_str(),
+                      graph->GetName().c_str());
     return FAILED;
   }
 
@@ -185,11 +215,16 @@ Status VarIsInitializedOpPass::ChangeNodeToConstant(NodePtr &node, bool inited)
   }
 
   if (NodeUtils::MoveOutputEdges(node, const_node) != GRAPH_SUCCESS) {
+    REPORT_CALL_ERROR("E19999", "Move out edge from node:%s(%s) to node:%s(%s) failed",
+                      node->GetName().c_str(), node->GetType().c_str(),
+                      const_node->GetName().c_str(), const_node->GetType().c_str());
     GELOGE(FAILED, "[%s] Move output edges to new node failed.", node->GetName().c_str());
     return FAILED;
   }
 
   if (GraphUtils::RemoveNodeWithoutRelink(graph, node) != SUCCESS) {
+    REPORT_CALL_ERROR("E19999", "Remove node:%s(%s) without relink in graph:%s failed",
+                      node->GetName().c_str(), node->GetType().c_str(), graph->GetName().c_str());
     GELOGE(FAILED, "[%s] RemoveNodeWithoutRelink failed.", node->GetName().c_str());
     return FAILED;
   }
@@ -263,6 +298,7 @@ Status VarIsInitializedOpPass::UpdateInitedVars(const NodePtr &node) {
 std::set<int64_t> *VarIsInitializedOpPass::CreateInitedVars() {
   std::unique_ptr<std::set<int64_t>> inited_vars_keeper(new(std::nothrow) std::set<int64_t>());
   if (inited_vars_keeper == nullptr) {
+    REPORT_CALL_ERROR("E19999", "New set failed");
     GELOGE(OUT_OF_MEMORY, "Failed to alloc set memory");
     return nullptr;
   }

ge/graph/passes/variable_op_pass.cc

@@ -47,6 +47,9 @@ Status ByPassTransNode(NodePtr &trans_node, NodePtr &ref_node) {
   GELOGD("Begin to bypass trans node %s", trans_node->GetName().c_str());
   auto ret = GraphUtils::CopyInCtrlEdges(trans_node, ref_node);
   if (ret != GRAPH_SUCCESS) {
+    REPORT_CALL_ERROR("E19999", "Copy in control edge from node:%s(%s) to node:%s(%s) failed",
+                      trans_node->GetName().c_str(), trans_node->GetType().c_str(),
+                      ref_node->GetName().c_str(), ref_node->GetType().c_str());
     GELOGE(INTERNAL_ERROR,
            "Failed to move control edges from trans "
            "node %s to var-ref %s",
@@ -55,6 +58,8 @@ Status ByPassTransNode(NodePtr &trans_node, NodePtr &ref_node) {
   }
   auto ref_in_anchor = ref_node->GetInDataAnchor(0);
   if (ref_in_anchor == nullptr) {
+    REPORT_INNER_ERROR("E19999", "Node:%s(%s) has no input anchor, check invalid",
+                       ref_node->GetName().c_str(), ref_node->GetType().c_str());
     GELOGE(INTERNAL_ERROR,
            "The variable ref node %s does not have an "
            "input anchor",
@@ -64,6 +69,8 @@ Status ByPassTransNode(NodePtr &trans_node, NodePtr &ref_node) {
   ref_in_anchor->UnlinkAll();
   auto trans_in_anchor = trans_node->GetInDataAnchor(0);
   if (trans_in_anchor == nullptr) {
+    REPORT_INNER_ERROR("E19999", "Node:%s(%s) has no input anchor, check invalid",
+                       trans_node->GetName().c_str(), trans_node->GetType().c_str());
     GELOGE(INTERNAL_ERROR,
            "Failed to get the in data anchor from trans"
            " node %s type %s",
@@ -79,6 +86,11 @@ Status ByPassTransNode(NodePtr &trans_node, NodePtr &ref_node) {
   } else {
     ret = GraphUtils::AddEdge(prev_trans_node_out_anchor, ref_in_anchor);
     if (ret != GRAPH_SUCCESS) {
+      REPORT_CALL_ERROR("E19999", "Add edge between op:%s(%s)(index:%d) and op:%s(%s)(index:0) failed",
+                      prev_trans_node_out_anchor->GetOwnerNode()->GetName().c_str(),
+                      prev_trans_node_out_anchor->GetOwnerNode()->GetType().c_str(),
+                      prev_trans_node_out_anchor->GetIdx(),
+                      ref_node->GetName().c_str(), ref_node->GetType().c_str());
       GELOGE(INTERNAL_ERROR,
              "Failed to add edge between ref node %s "
              "and the prev node of trans node %s",
@@ -115,6 +127,7 @@ bool IsTransSupport(const TransNodeInfo &trans_info) {
 
 Status VariableOpPass::Run(ge::ComputeGraphPtr graph) {
   if (graph == nullptr) {
+    REPORT_INNER_ERROR("E19999", "Param graph is nullptr, check invalid");
     GELOGE(INTERNAL_ERROR, "Failed to run variable op pass, null graph");
     return INTERNAL_ERROR;
   }
@@ -124,6 +137,7 @@ Status VariableOpPass::Run(ge::ComputeGraphPtr graph) {
          GetContext().SessionId(), graph_id);
 
   if (var_accelerate_ctrl_ == nullptr) {
+    REPORT_INNER_ERROR("E19999", "The variable accelerate control is nullptr, check invalid");
     GELOGE(INTERNAL_ERROR, "Failed to run var op pass, the variable accelerate control is null");
     return INTERNAL_ERROR;
   }
@@ -174,11 +188,15 @@ Status VariableOpPass::Run(ge::ComputeGraphPtr graph) {
 
     ret = VarManager::Instance(graph->GetSessionID())->SetTransRoad(node->GetName(), fusion_road);
     if (ret != SUCCESS) {
+      REPORT_CALL_ERROR("E19999", "Set Trans road for node:%s(%s) failed, session_id:%lu",
+                        node->GetName().c_str(), node->GetType().c_str(), graph->GetSessionID());
       GELOGE(INTERNAL_ERROR, "Failed to update the format fusion road for var %s", node->GetName().c_str());
       return INTERNAL_ERROR;
     }
     ret = VarManager::Instance(graph->GetSessionID())->SetChangedGraphId(node->GetName(), graph_id);
     if (ret != SUCCESS) {
+      REPORT_CALL_ERROR("E19999", "Update graph_id:%u for node:%s(%s) failed, session_id:%lu",
+                        graph_id, node->GetName().c_str(), node->GetType().c_str(), graph->GetSessionID());
       GELOGE(INTERNAL_ERROR, "Failed to update the graph id for var %s", node->GetName().c_str());
       return INTERNAL_ERROR;
     }
@@ -210,10 +228,14 @@ Status VariableOpPass::DealFusion(const ge::NodePtr &var_node) {
            trans_node->GetType().c_str(), var_node->GetName().c_str());
 
     if (GraphUtils::IsolateNode(trans_node, {0}) != SUCCESS) {
+      REPORT_CALL_ERROR("E19999", "Isolate node:%s(%s) failed",
+                        trans_node->GetName().c_str(), trans_node->GetType().c_str());
       return GE_GRAPH_VARIABLE_OP_PASS_FAILED;
     }
 
     if (GraphUtils::RemoveNodeWithoutRelink(graph, trans_node) != SUCCESS) {
+      REPORT_CALL_ERROR("E19999", "Remove node:%s(%s) without relink in graph:%s failed",
+                        trans_node->GetName().c_str(), trans_node->GetType().c_str(), graph->GetName().c_str());
       return GE_GRAPH_VARIABLE_OP_PASS_FAILED;
     }
   }
@@ -245,9 +267,13 @@ Status VariableOpPass::DealFusion(const ge::NodePtr &var_node) {
             " one output data nodes, isolate and remove it.",
             trans_node->GetName().c_str(), trans_node->GetType().c_str(), ref_node->GetName().c_str());
         if (GraphUtils::IsolateNode(trans_node, {0}) != SUCCESS) {
+          REPORT_CALL_ERROR("E19999", "Isolate node:%s(%s) failed",
+                            trans_node->GetName().c_str(), trans_node->GetType().c_str());
           return GE_GRAPH_VARIABLE_OP_PASS_FAILED;
         }
         if (GraphUtils::RemoveNodeWithoutRelink(graph, trans_node) != SUCCESS) {
+          REPORT_CALL_ERROR("E19999", "Remove node:%s(%s) without relink in graph:%s failed",
+                            trans_node->GetName().c_str(), trans_node->GetType().c_str(), graph->GetName().c_str());
           return GE_GRAPH_VARIABLE_OP_PASS_FAILED;
         }
       }
@@ -365,6 +391,7 @@ Status VariableOpPass::CheckVariableRefLegally(const ge::NodePtr &var_node, bool
 
 Status VariableOpPass::UpdateVarAndRefOutputFormatInfo(const GeTensorDesc &final_output, const ge::NodePtr &node) {
   if (node == nullptr || node->GetOpDesc() == nullptr) {
+    REPORT_INNER_ERROR("E19999", "Param node or its op_desc is nullptr, check invalid");
     GELOGE(FAILED, "node or opdesc is nullptr");
     return FAILED;
   }
@@ -377,6 +404,8 @@ Status VariableOpPass::UpdateVarAndRefOutputFormatInfo(const GeTensorDesc &final
   auto node_desc = node->GetOpDesc()->GetOutputDesc(0);
   CopyVariableFormatDataTypeAndShape(final_output, node_desc);
   if (node->GetOpDesc()->UpdateOutputDesc(0, node_desc) != GRAPH_SUCCESS) {
+    REPORT_CALL_ERROR("E19999", "Update ouput:0 desc in op:%s(%s) failed",
+                      node->GetName().c_str(), node->GetType().c_str());
     GELOGE(FAILED, "update output desc fail.");
     return FAILED;
   }
@@ -460,6 +489,10 @@ Status VariableOpPass::CheckVarAndVarRefAreAlike(const NodePtr &var_node, const
   GELOGD("var_ref_node_trans_nodes size is %zu.", var_ref_node_trans_nodes.size());
 
   if (var_ref_node_trans_nodes.size() > 1) {
+    REPORT_INNER_ERROR("E19999", "In data node num:%zu of node:%s(%s) bigger than 1, check invalid",
+                       var_ref_node_trans_nodes.size(),
+                       var_ref_node->GetName().c_str(), var_ref_node->GetType().c_str());
+
     GELOGE(GE_GRAPH_VARIABLE_OP_PASS_FAILED, "var_ref_node_trans_nodes.size() > 1.");
     return GE_GRAPH_VARIABLE_OP_PASS_FAILED;
   }
@@ -525,6 +558,7 @@ void VariableOpPass::CopyVariableFormatDataTypeAndShape(const GeTensorDesc &src_
 
 Status VariableOpPass::CheckIfCouldBeOptimized(const ge::NodePtr &node, bool &flag, VarTransRoad &fusion_road) {
   if (node == nullptr) {
+    REPORT_INNER_ERROR("E19999", "Param node is nullptr, check invalid");
     return FAILED;
   }
   bool is_matched = false;
@@ -602,6 +636,8 @@ Status VariableOpPass::RenewVarDesc(ge::ComputeGraphPtr &graph) {
       GE_CHECK_NOTNULL(node->GetOpDesc());
       ret = ge::VarManager::Instance(graph->GetSessionID())->RenewCurVarDesc(node->GetName(), node->GetOpDesc());
       if (ret != SUCCESS) {
+        REPORT_CALL_ERROR("E19999", "Renew descriptor for node:%s(%s) failed, session_id:%lu",
+                          node->GetName().c_str(), node->GetType().c_str(), graph->GetSessionID());
         GELOGE(FAILED, "var manager renew var[%s] descriptor failed!", node->GetName().c_str());
         return FAILED;
       }
@@ -626,6 +662,8 @@ Status VariableOpPass::RenewVarDesc(uint64_t session_id, const NodePtr &node, co
   GE_CHECK_NOTNULL(node->GetOpDesc());
   Status ret = ge::VarManager::Instance(session_id)->RenewCurVarDesc(node->GetName(), node->GetOpDesc());
   if (ret != SUCCESS) {
+    REPORT_CALL_ERROR("E19999", "Renew descriptor for node:%s(%s) failed, session_id:%lu",
+                      node->GetName().c_str(), node->GetType().c_str(), session_id);
     GELOGE(FAILED, "var manager renew var[%s] descriptor failed!", node->GetName().c_str());
     return FAILED;
   }

ge/graph/passes/variable_ref_delete_op_pass.cc

@@ -35,6 +35,8 @@ Status VariableRefDeleteOpPass::Run(ge::ComputeGraphPtr graph) {
       continue;
     }
     if (all_var_names.count(ref_var_src_var_name) == 0) {
+      REPORT_INNER_ERROR("E19999", "Can not find source variable[%s] of variable ref[%s], check invalid",
+                         ref_var_src_var_name.c_str(), node->GetName().c_str());
       GELOGE(FAILED, "Can not find source variable[%s] of variable ref[%s]", ref_var_src_var_name.c_str(),
              node->GetName().c_str());
       return FAILED;
@@ -53,6 +55,8 @@ Status VariableRefDeleteOpPass::DealVariableRef(ge::ComputeGraphPtr &graph, ge::
   GE_CHECK_NOTNULL(variable_ref);
   auto inAnchor0 = variable_ref->GetInDataAnchor(0);
   if (inAnchor0 == nullptr) {
+    REPORT_INNER_ERROR("E19999", "Node:%s(%s) has no input anchor, check invalid",
+                       variable_ref->GetName().c_str(), variable_ref->GetType().c_str());
     GELOGE(FAILED, "variable_ref [%s] no input", variable_ref->GetName().c_str());
     return FAILED;
   }
@@ -73,17 +77,23 @@ Status VariableRefDeleteOpPass::DealVariableRef(ge::ComputeGraphPtr &graph, ge::
     GELOGI("[%s-%d]: add attr [REF_VAR_SRC_VAR_NAME: %s ] ", peer_node->GetName().c_str(), index,
            ref_var_src_var_name.c_str());
   } else {
+    REPORT_CALL_ERROR("E19999", "Set Attr:%s to output:%d desc of op:%s(%s) failed", REF_VAR_SRC_VAR_NAME.c_str(),
+                      index, op_desc->GetName().c_str(), op_desc->GetType().c_str());
     GELOGE(FAILED, "[%s-%d]: add attr [REF_VAR_SRC_VAR_NAME: %s ] failed", peer_node->GetName().c_str(), index,
            ref_var_src_var_name.c_str());
     return FAILED;
   }
   // remove variable_ref
   if (GraphUtils::IsolateNode(variable_ref, {0}) != GRAPH_SUCCESS) {
+    REPORT_CALL_ERROR("E19999", "Isolate node:%s(%s) failed",
+                      variable_ref->GetName().c_str(), variable_ref->GetType().c_str());
     GELOGE(INTERNAL_ERROR, "Isolate removed node: %s, type: %s failed", variable_ref->GetName().c_str(),
            variable_ref->GetType().c_str());
     return FAILED;
   }
   if (GraphUtils::RemoveNodeWithoutRelink(graph, variable_ref) != GRAPH_SUCCESS) {
+    REPORT_CALL_ERROR("E19999", "Remove node:%s(%s) without relink in graph:%s failed",
+                      variable_ref->GetName().c_str(), variable_ref->GetType().c_str(), graph->GetName().c_str());
     GELOGE(INTERNAL_ERROR, "Remove node: %s, type: %s without relink failed", variable_ref->GetName().c_str(),
            variable_ref->GetType().c_str());
     return FAILED;

ge/graph/preprocess/graph_preprocess.cc

@@ -27,6 +27,7 @@
 #include "common/helper/model_helper.h"
 #include "common/math/math_util.h"
 #include "common/op/ge_op_utils.h"
+#include "ir_build/option_utils.h"
 #include "graph/common/ge_call_wrapper.h"
 #include "graph/common/local_context.h"
 #include "graph/common/transop_util.h"
@@ -991,101 +992,6 @@ Status ProcessNetoutputNodeDynShape(NodePtr &node) {
   }
   return SUCCESS;
 }
-long StringToLongNoThrow(const string &str) {
-  try {
-    return std::stol(str);
-  } catch (const std::invalid_argument) {
-    GELOGE(PARAM_INVALID,
-           "Parse shape range of input failed when transfer from string to int64. Given %s, while correct example:"
-           "\"[1~20,3,3~6,-1],[1~20,3,3~6,-1]\"",
-           str.c_str());
-    return PARAM_INVALID;
-  } catch (const std::out_of_range) {
-    GELOGE(PARAM_INVALID,
-           "Parse shape range of input failed when transfer from string to int64. Given %s, while correct example:"
-           "\"[1~20,3,3~6,-1],[1~20,3,3~6,-1]\"",
-           str.c_str());
-    return PARAM_INVALID;
-  }
-}
-/**
- * Parser shape_range from string to vector
- * shape_range from option normally is "[1~20,3,3~6,-1],[1~20,3,3~6,-1]"
- * @param shape_range
- */
-Status ParseDynamicInputShapeRange(const std::string &shape_range,
-                                   std::vector<std::vector<std::pair<int64_t, int64_t>>> &range) {
-  if (shape_range.size() < 2) {
-    REPORT_INNER_ERROR("E19999", "shape_range.size:%zu < 2, check invalid", shape_range.size());
-    GELOGE(PARAM_INVALID, "Shape range %s is invalid.", shape_range.c_str());
-    return PARAM_INVALID;
-  }
-  // different shape_range of single input are split by ']'
-  vector<string> shape_range_set = ge::StringUtils::Split(shape_range, ']');
-  if (shape_range_set.empty()) {
-    REPORT_INNER_ERROR("E19999", "Shape range %s is not valid. Correct example: \"[1~20,3,3~6,-1],[1~20,3,3~6,-1]\"",
-                       shape_range.c_str());
-    GELOGE(PARAM_INVALID, "Shape range %s is not valid. Correct example: \"[1~20,3,3~6,-1],[1~20,3,3~6,-1]\"",
-           shape_range.c_str());
-    return PARAM_INVALID;
-  }
-  for (auto &shape_range_str : shape_range_set) {
-    if (shape_range_str.size() < 3) {
-      // shape_range_str should be "[2~3,1"
-      // or ",[2~3,1". because we should trim '[' or ',['
-      // so shape_range_str.size() < 3 is invalid
-      continue;
-    }
-    // trim start bytes, after that, single input should be "1~20,3,3~6,-1"
-    if (ge::StringUtils::StartWith(shape_range_str, "[")) {
-      shape_range_str = shape_range_str.substr(1, shape_range_str.size());
-    }
-    if (ge::StringUtils::StartWith(shape_range_str, ",")) {
-      shape_range_str = shape_range_str.substr(2, shape_range_str.size());
-    }
-
-    // parse shape_range of single input. eg. "1~20,3,3~6,-1"
-    std::vector<std::pair<int64_t, int64_t>> range_of_single_input;
-    vector<string> dim_range_set = ge::StringUtils::Split(shape_range_str, ',');
-    for (const auto &range_pair_str : dim_range_set) {
-      vector<string> range_pair_set = ge::StringUtils::Split(range_pair_str, '~');
-      pair<int64_t, int64_t> range_pair;
-      if (range_pair_set.size() == 1) {
-        // fix dim
-        auto range_value = StringToLongNoThrow(range_pair_set.at(0).c_str());
-        if (range_value < 0) {
-          range_pair = std::make_pair(1, range_value);
-        } else {
-          range_pair = std::make_pair(range_value, range_value);
-        }
-      } else if (range_pair_set.size() == 2) {
-        // unknown dim, should get range.
-        auto range_left = StringToLongNoThrow(range_pair_set.at(0).c_str());
-        auto range_right = StringToLongNoThrow(range_pair_set.at(1).c_str());
-        if (range_left < 0 || range_right < 0) {
-          REPORT_INNER_ERROR("E19999", "Shape range of input is invalid. Given range pair [%ld,%ld], "
-                             "while correct example: \"[1~20,3,3~6,-1],[1~20,3,3~6,-1]\"", range_left, range_right);
-          GELOGE(PARAM_INVALID,
-                 "Shape range of input is invalid. Given range pair [%ld,%ld], while correct example: "
-                 "\"[1~20,3,3~6,-1],[1~20,3,3~6,-1]\"",
-                 range_left, range_right);
-          return PARAM_INVALID;
-        }
-        range_pair = std::make_pair(range_left, range_right);
-      } else {
-        REPORT_INNER_ERROR("E19999", "Shape range of input is invalid. Given %s, "
-                           "while correct example: \"[1~20,3,3~6,-1],[1~20,3,3~6,-1]\"", shape_range.c_str());
-        GELOGE(PARAM_INVALID,
-               "Shape range of input is invalid. Given %s, while correct example: \"[1~20,3,3~6,-1],[1~20,3,3~6,-1]\"",
-               shape_range.c_str());
-        return PARAM_INVALID;
-      }
-      range_of_single_input.emplace_back(range_pair);
-    }
-    range.emplace_back(range_of_single_input);
-  }
-  return SUCCESS;
-}
 
 Status GetDynamicInputShapeRange(const std::vector<GeTensor> &user_input, const std::map<string, string> &graph_option,
                                  vector<vector<std::pair<int64_t, int64_t>>> &range_vec) {
@@ -1114,9 +1020,10 @@ Status GetDynamicInputShapeRange(const std::vector<GeTensor> &user_input, const
            OPTION_EXEC_DYNAMIC_EXECUTE_MODE, OPTION_EXEC_DATA_INPUTS_SHAPE_RANGE);
     return PARAM_INVALID;
   }
-
-  auto ret = ParseDynamicInputShapeRange(iter->second, range_vec);
-  GE_CHK_STATUS_RET(ret, "Parse dynamic input shape range failed.");
+  if (ParseInputShapeRange(iter->second, range_vec) != SUCCESS) {
+    GELOGE(PARAM_INVALID, "[Parse][ShapeRange] Parse dynamic input shape range failed.");
+    return PARAM_INVALID;
+  }
   if (range_vec.size() != user_input.size()) {
     GELOGE(PARAM_INVALID, "Dynamic input shape range size is %zu, inputs size is %zu. Not match.", range_vec.size(),
            user_input.size());

ge/hybrid/executor/hybrid_execution_context.cc

@@ -63,5 +63,27 @@ Status GraphExecutionContext::Synchronize(rtStream_t rt_stream) {
   REPORT_CALL_ERROR("E19999", "invoke rtStreamSynchronize failed, ret = %d", rt_ret);
   return RT_FAILED;
 }
+
+Status GraphExecutionContext::DumpExceptionInfo(const std::vector<rtExceptionInfo> &exception_infos) {
+  if (exception_infos.empty()) {
+    GELOGI("[Dump][ExceptionInfo] Exception info is null.");
+    return SUCCESS;
+  }
+  GELOGI("[Dump][ExceptionInfo] Start to search dynamic op info and to dump.");
+  if (exception_dumper.DumpExceptionInfo(exception_infos) != SUCCESS) {
+    GELOGE(FAILED, "[Dump][Exception] Dump dynamic op exception info failed.");
+    return FAILED;
+  }
+  GELOGI("[Dump][ExceptionInfo] Start to search static op info and to dump.");
+  for (const auto &iter : davinci_model) {
+    if (iter != nullptr) {
+      if (iter->DumpExceptionInfo(exception_infos) != SUCCESS) {
+        GELOGE(FAILED, "[Dump][ExceptionInfo] Dump static op exception info failed.");
+        return FAILED;
+      }
+    }
+  }
+  return SUCCESS;
+}
 }  // namespace hybrid
 }  // namespace ge

ge/hybrid/executor/hybrid_execution_context.h

@@ -23,6 +23,7 @@
 #include "common/properties_manager.h"
 #include "framework/common/debug/ge_log.h"
 #include "graph/ge_local_context.h"
+#include "graph/load/model_manager/davinci_model.h"
 #include "hybrid/common/npu_memory_allocator.h"
 #include "hybrid/common/tensor_value.h"
 #include "hybrid/executor/hybrid_profiler.h"
@@ -54,6 +55,7 @@ struct GraphExecutionContext {
   void SetErrorCode(Status error_code);
   Status GetStatus() const;
   Status Synchronize(rtStream_t rt_stream);
+  Status DumpExceptionInfo(const std::vector<rtExceptionInfo> &exception_infos);
 
   uint64_t session_id = 0;
   uint64_t context_id = 0;
@@ -68,7 +70,9 @@ struct GraphExecutionContext {
   DumpProperties dump_properties;
   bool trace_enabled = false;
   bool dump_enabled = false;
-  std::atomic_bool is_eos_;
+  ExceptionDumper exception_dumper;
+  std::vector<std::shared_ptr<ge::DavinciModel>> davinci_model;
+  std::atomic_bool is_eos_{false};
   long profiling_level = 0;
   long iteration = 0;
   void *global_step = nullptr;

ge/hybrid/executor/hybrid_model_async_executor.cc

@@ -19,7 +19,6 @@
 #include "graph/utils/tensor_utils.h"
 #include "graph/utils/type_utils.h"
 #include "graph/ge_context.h"
-#include "omm/csa_interact.h"
 
 namespace ge {
 namespace hybrid {
@@ -144,12 +143,14 @@ Status HybridModelAsyncExecutor::RunInternal() {
   GE_MAKE_GUARD(not_used_var, [&] { GE_CHK_RT(rtDeviceReset(device_id)); });
 
   while (run_flag_) {
+    // Model has not indeedly started running before received data
+    SetRunningFlag(false);
     std::shared_ptr<InputDataWrapper> data_wrapper;
     Status ret = data_inputer_->Pop(data_wrapper);
-    if (data_wrapper == nullptr || ret != SUCCESS) {
-      GELOGI("data_wrapper is null!, ret = %u", ret);
-      continue;
-    }
+    // Model indeedly start running
+    SetRunningFlag(true);
+    GE_IF_BOOL_EXEC(data_wrapper == nullptr || ret != SUCCESS, GELOGI("data_wrapper is null!, ret = %u", ret);
+                    continue);
 
     GELOGI("Getting the input data, model_id:%u", model_id_);
     GE_IF_BOOL_EXEC(!run_flag_, break);
@@ -161,7 +162,6 @@ Status HybridModelAsyncExecutor::RunInternal() {
     ret = PreRun(current_data, args);
     GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(
         ret != SUCCESS, (void) HandleResult(ret, current_data.index, args, data_wrapper->GetOutput());
-        CsaInteract::GetInstance().StoreInternalErrorCode(ret, ERROR_MODULE_FMK, JOBSUBSTATE_GRAPH_EXEC);
         continue, "[Invoke][PreRun] failed, model_id:%u.", model_id_);  // [No need to check value]
 
     if (pipe_executor_ != nullptr) {
@@ -179,16 +179,15 @@ Status HybridModelAsyncExecutor::RunInternal() {
     }
     ret = HandleResult(ret, current_data.index, args, data_wrapper->GetOutput());
     if (ret != SUCCESS) {
-      CsaInteract::GetInstance().StoreInternalErrorCode(ret, ERROR_MODULE_RUNTIME, JOBSUBSTATE_GRAPH_EXEC);
       continue;
     }
 
     RECORD_MODEL_EXECUTION_EVENT(executor_->GetContext(), "[RunInternal] [iteration = %d] End", iterator_count_);
     iterator_count_++;
-    GELOGI("run iterator count is %lu", iterator_count_);
+    SetRunningFlag(false);
+    GELOGI("run iterator count is %lu,  model_id:%u", iterator_count_, model_id_);
   }
 
-  CsaInteract::GetInstance().WriteInternalErrorCode();
   GELOGI("Model run end, model id:%u", model_id_);
   return SUCCESS;
 }

ge/hybrid/executor/hybrid_model_async_executor.h

@@ -55,6 +55,14 @@ class HybridModelAsyncExecutor {
 
   Status EnqueueData(const std::shared_ptr<InputDataWrapper> &data);
 
+  uint32_t GetDataInputerSize() { return data_inputer_->Size(); }
+
+  bool GetRunningFlag() const { return running_flag_; }
+
+  void SetRunningFlag(bool flag) { running_flag_ = flag; }
+
+  const GraphExecutionContext * GeContext() { return executor_->GetContext(); }
+
  private:
   Status InitInputDesc();
 
@@ -84,6 +92,8 @@ class HybridModelAsyncExecutor {
   uint32_t device_id_ = 0U;
   uint32_t model_id_ = 0U;
   std::atomic_bool run_flag_;
+  // check whether model is running with data
+  bool running_flag_ = false;
   std::unique_ptr<DataInputer> data_inputer_;
   std::unique_ptr<HybridModelExecutor> executor_;
   std::unique_ptr<HybridModelPipelineExecutor> pipe_executor_;

ge/hybrid/executor/hybrid_model_executor.cc

@@ -18,6 +18,7 @@
 #include "graph/ge_context.h"
 #include "graph/runtime_inference_context.h"
 #include "graph/utils/tensor_utils.h"
+#include "graph/load/model_manager/model_manager.h"
 #include "common/dump/dump_manager.h"
 #include "common/profiling/profiling_manager.h"
 
@@ -102,7 +103,22 @@ Status HybridModelExecutor::ExecuteGraphInternal(SubgraphExecutor &executor,
   }
 
   if (!model_->IsSingleOp()) {
-    HYBRID_CHK_STATUS_RET(executor.Synchronize(), "Failed to sync root graph.");
+    Status ret = executor.Synchronize();
+    if (ret != ge::SUCCESS) {
+      auto model_manager = ModelManager::GetInstance();
+      GE_CHECK_NOTNULL(model_manager);
+      auto exception_infos = model_manager->GetExceptionInfos();
+      if (!exception_infos.empty()) {
+        HYBRID_CHK_STATUS_RET(context_.DumpExceptionInfo(exception_infos),
+                              "[Execute][GraphInternal] Dump exception info failed.");
+      }
+      if (ret == ge::END_OF_SEQUENCE) {
+        GELOGD("Got end of sequence");
+      } else {
+        GELOGE(ret, "[Execute][GraphInternal] Synchronize failed.");
+      }
+      return ret;
+    }
     RECORD_MODEL_EXECUTION_EVENT(&context_, "[Synchronize] End");
   }
 
@@ -175,19 +191,16 @@ Status HybridModelExecutor::CheckInputShapeByShapeRange(const GraphItem *graph_i
                                                         HybridModelExecutor::ExecuteArgs &args) {
   GE_CHECK_NOTNULL(graph_item);
   auto input_nodes = graph_item->GetInputNodes();
-  if (args.input_desc.size() < input_nodes.size()) {
-    REPORT_INNER_ERROR("E19999", "[%s] Number of inputs [%zu] is not sufficient for graph which needs [%zu] inputs.",
-                       graph_item->GetName().c_str(), args.input_desc.size(), input_nodes.size());
-    GELOGE(INTERNAL_ERROR, "[%s] Number of inputs [%zu] is not sufficient for graph which needs [%zu] inputs.",
-           graph_item->GetName().c_str(), args.input_desc.size(), input_nodes.size());
-    return INTERNAL_ERROR;
-  }
   for (size_t i = 0; i < input_nodes.size(); ++i) {
     auto &input_node = input_nodes[i];
     if (input_node == nullptr) {
       GELOGD("[%s] Input[%zu] is not needed by graph, skip it.", graph_item->GetName().c_str(), i);
       continue;
     }
+    if (!input_node->is_dynamic) {
+      GELOGD("[%s] Input[%zu] is not dynamic, skip it.", graph_item->GetName().c_str(), i);
+      continue;
+    }
     GeTensorDescPtr model_input_desc = input_node->MutableInputDesc(0);
     GE_CHECK_NOTNULL(model_input_desc);
     std::vector<std::pair<int64_t, int64_t>> shape_range;
@@ -200,6 +213,13 @@ Status HybridModelExecutor::CheckInputShapeByShapeRange(const GraphItem *graph_i
       GELOGD("[%s] Input[%zu] shape is not needed to check by shape range, skip it.", graph_item->GetName().c_str(), i);
       continue;
     }
+    if (i >= args.input_desc.size()) {
+      REPORT_INNER_ERROR("E19999", "[%s] Inputs[%zu] is greater than or equal to input desc size[%zu].",
+                         graph_item->GetName().c_str(), i, args.input_desc.size());
+      GELOGE(INTERNAL_ERROR, "[%s] inputs[%zu] is greater than or equal to input desc size[%zu].",
+             graph_item->GetName().c_str(), i, args.input_desc.size());
+      return INTERNAL_ERROR;
+    }
     ConstGeTensorDescPtr args_tensor_desc = args.input_desc[i];
     GE_CHECK_NOTNULL(args_tensor_desc);
     GeShape shape = args_tensor_desc->GetShape();

ge/hybrid/executor/hybrid_model_pipeline_executor.cc

@@ -4,6 +4,7 @@
 #include "common/dump/dump_manager.h"
 #include "graph/ge_context.h"
 #include "graph/runtime_inference_context.h"
+#include "graph/load/model_manager/model_manager.h"
 
 namespace ge {
 namespace hybrid {
@@ -266,6 +267,13 @@ Status HybridModelPipelineExecutor::Execute(HybridModelExecutor::ExecuteArgs &ar
     ret = stage_executors_[i]->Synchronize();
 
     if (ret != SUCCESS) {
+      auto model_manager = ModelManager::GetInstance();
+      GE_CHECK_NOTNULL(model_manager);
+      auto exception_infos = model_manager->GetExceptionInfos();
+      if (!exception_infos.empty()) {
+        HYBRID_CHK_STATUS_RET(context_.DumpExceptionInfo(exception_infos),
+                              "[Execute][GraphInternal] Dump exception info failed.");
+      }
       GELOGE(ret, "[Invoke][Synchronize] failed for [Executor: %zu].", i);
       REPORT_CALL_ERROR("E19999", "[Executor: %zu] failed to Synchronize result.", i);
       has_error = true;

ge/hybrid/executor/worker/execution_engine.cc

@@ -19,6 +19,7 @@
 #include "graph/utils/tensor_utils.h"
 #include "graph/utils/tensor_adapter.h"
 #include "graph/debug/ge_attr_define.h"
+#include "graph/load/model_manager/model_manager.h"
 #include "hybrid/node_executor/node_executor.h"
 #include "hybrid/executor//worker//shape_inference_engine.h"
 #include "common/dump/dump_op.h"
@@ -70,6 +71,7 @@ class NodeDoneCallback {
   Status PrepareConstInputs(const NodeItem &node_item);
   Status DumpDynamicNode();
   Status ProfilingReport();
+  Status SaveDumpOpInfo();
   Status GetTaskDescInfo(const NodePtr node, const HybridModel *model,
                          std::vector<TaskDescInfo> &task_desc_info);
   GraphExecutionContext *graph_context_;
@@ -266,6 +268,40 @@ Status NodeDoneCallback::DumpDynamicNode() {
   return SUCCESS;
 }
 
+Status NodeDoneCallback::SaveDumpOpInfo() {
+  GE_CHECK_NOTNULL(graph_context_);
+  GE_CHECK_NOTNULL(graph_context_->model);
+
+  auto node = context_->GetNodeItem().node;
+  if (node == nullptr) {
+    GELOGE(PARAM_INVALID, "[Save][DumpOpInfo] Get node is nullptr.");
+    return PARAM_INVALID;
+  }
+  auto op_desc = node->GetOpDesc();
+  GE_CHECK_NOTNULL(op_desc);
+
+  vector<void *> input_addrs;
+  vector<void *> output_addrs;
+  for (int i = 0; i < context_->NumInputs(); i++) {
+    auto tensor_value = context_->GetInput(i);
+    GE_CHK_BOOL_RET_STATUS(tensor_value != nullptr, PARAM_INVALID, "[Save][DumpOpInfo] Tensor value is nullptr.");
+    void *input_addr = const_cast<void *>(tensor_value->GetData());
+    input_addrs.emplace_back(input_addr);
+  }
+  for (int j = 0; j < context_->NumOutputs(); j++) {
+    auto tensor_value = context_->GetOutput(j);
+    GE_CHK_BOOL_RET_STATUS(tensor_value != nullptr, PARAM_INVALID, "[Save][DumpOpInfo] Tensor value is nullptr.");
+    void *output_addr = const_cast<void *>(tensor_value->GetData());
+    output_addrs.emplace_back(output_addr);
+  }
+
+  uint32_t stream_id = context_->GetStreamId();
+  uint32_t task_id = context_->GetTaskId();
+  graph_context_->exception_dumper.SaveDumpOpInfo(op_desc, task_id, stream_id, input_addrs, output_addrs);
+
+  return SUCCESS;
+}
+
 Status NodeDoneCallback::OnNodeDone() {
   auto &node_item = context_->GetNodeItem();
   GELOGI("[%s] Start callback process.", node_item.NodeName().c_str());
@@ -278,6 +314,12 @@ Status NodeDoneCallback::OnNodeDone() {
     GE_CHK_STATUS_RET(DumpDynamicNode(), "[Call][DumpDynamicNode] Failed.");
   }
 
+  auto model_manager = ModelManager::GetInstance();
+  GE_CHECK_NOTNULL(model_manager);
+  if (model_manager->IsDumpExceptionOpen()) {
+    GE_CHK_STATUS_RET(SaveDumpOpInfo(), "[Save][DumpOpInfo] Failed to dump op info.");
+  }
+
   if (ProfilingManager::Instance().ProfilingModelExecuteOn()) {
     GE_CHK_STATUS_RET(ProfilingReport(), "[Report][Profiling] of node[%s] failed.", node_item.NodeName().c_str());
   }
@@ -322,20 +364,28 @@ Status ExecutionEngine::ExecuteAsync(NodeState &node_state,
                                      GraphExecutionContext &execution_context) {
   GELOGI("[%s] Node is ready for execution", task_context->GetNodeName());
   RECORD_EXECUTION_EVENT(&execution_context, task_context->GetNodeName(), "Start");
-  auto cb = std::shared_ptr<NodeDoneCallback>(new(std::nothrow) NodeDoneCallback(&execution_context, task_context));
-  GE_CHECK_NOTNULL(cb);
-  auto callback = [task_context, cb]() {
-    auto ret = cb->OnNodeDone();
-    if (ret != SUCCESS) {
-      task_context->OnError(ret);
-    }
-  };
-
+  std::function<void()> callback = nullptr;
+  GE_CHK_STATUS_RET_NOLOG(InitCallback(task_context, execution_context, callback));
   GE_CHK_STATUS_RET_NOLOG(DoExecuteAsync(node_state, *task_context, execution_context, callback));
   GE_CHK_STATUS_RET_NOLOG(PropagateOutputs(*node_state.GetNodeItem(), *task_context, execution_context));
   return SUCCESS;
 }
 
+Status ExecutionEngine::InitCallback(const std::shared_ptr<TaskContext> &task_context,
+                                     GraphExecutionContext &execution_context, std::function<void()> &callback) {
+  if (task_context->NeedCallback()) {
+    auto cb = std::shared_ptr<NodeDoneCallback>(new(std::nothrow) NodeDoneCallback(&execution_context, task_context));
+    GE_CHECK_NOTNULL(cb);
+    callback = [task_context, cb]() {
+      auto ret = cb->OnNodeDone();
+      if (ret != SUCCESS) {
+        task_context->OnError(ret);
+      }
+    };
+  }
+  return SUCCESS;
+}
+
 Status ExecutionEngine::DoExecuteAsync(NodeState &node_state,
                                        TaskContext &task_context,
                                        GraphExecutionContext &context,
@@ -343,7 +393,7 @@ Status ExecutionEngine::DoExecuteAsync(NodeState &node_state,
   const auto &task = node_state.GetKernelTask();
   if (task == nullptr) {
     GELOGE(INTERNAL_ERROR, "[Get][KernelTask] of [%s] is null.", node_state.GetName().c_str());
-    REPORT_INNER_ERROR("E19999", "GetKernelTask of %s is null.", node_state.GetName().c_str());
+    REPORT_INNER_ERROR("E19999", "GetKernelTask of %s failed.", node_state.GetName().c_str());
     return INTERNAL_ERROR;
   }
 

ge/hybrid/executor/worker/execution_engine.h

@@ -35,6 +35,8 @@ class ExecutionEngine {
                                TaskContext &task_context,
                                GraphExecutionContext &context,
                                const std::function<void()> &callback);
+  static Status InitCallback(const std::shared_ptr<TaskContext> &task_context,
+                             GraphExecutionContext &execution_context, std::function<void()> &callback);
 };
 }  // namespace hybrid
 }  // namespace ge

ge/hybrid/hybrid_davinci_model.cc

@@ -19,6 +19,7 @@
 #include "hybrid/model/hybrid_model.h"
 #include "hybrid/executor/hybrid_model_async_executor.h"
 #include "hybrid/node_executor/node_executor.h"
+#include "graph/manager/graph_manager_utils.h"
 
 namespace ge {
 namespace hybrid {
@@ -32,9 +33,10 @@ class HybridDavinciModel::Impl {
   }
 
   Status Init() {
-    GE_CHK_STATUS_RET(NodeExecutorManager::GetInstance().EnsureInitialized(), "Failed to initialize executors");
-    GE_CHK_STATUS_RET(model_.Init(), "Failed to init model.")
-    GE_CHK_STATUS_RET(executor_.Init(), "Failed to init model executor.")
+    GE_CHK_STATUS_RET(NodeExecutorManager::GetInstance().EnsureInitialized(),
+                      "[Initialize][NodeExecutorManager] failed");
+    GE_CHK_STATUS_RET(model_.Init(), "[Init][HybridModel] failed.")
+    GE_CHK_STATUS_RET(executor_.Init(), "[Init][HybridModelAsyncExecutor] failed.")
     return SUCCESS;
   }
 
@@ -80,6 +82,12 @@ class HybridDavinciModel::Impl {
     model_.SetOmName(model_name);
   }
 
+  uint32_t GetDeviceId() {
+    return model_.GetDeviceId();
+  }
+
+  const GraphExecutionContext * GeContext() { return executor_.GeContext(); }
+
   uint64_t GetSessionId() {
     return model_.GetSessionId();
   }
@@ -107,6 +115,17 @@ class HybridDavinciModel::Impl {
     model_.SetModelDescVersion(is_new_model_desc);
   }
 
+  uint32_t GetDataInputerSize() { return executor_.GetDataInputerSize(); }
+
+  bool GetRunningFlag() const { return executor_.GetRunningFlag(); }
+
+  Status SetRunAsyncListenerCallback(const RunAsyncCallback &callback) {
+    auto listener = dynamic_cast<RunAsyncListener *>(listener_.get());
+    GE_CHECK_NOTNULL(listener);
+    listener->SetCallback(callback);
+    return SUCCESS;
+  }
+
  private:
   std::shared_ptr<ModelListener> listener_;
   HybridModel model_;
@@ -186,6 +205,11 @@ void HybridDavinciModel::SetOmName(const string &om_name) {
   }
 }
 
+uint32_t HybridDavinciModel::GetDeviceId() const {
+  GE_CHECK_NOTNULL(impl_);
+  return impl_->GetDeviceId();
+}
+
 Status HybridDavinciModel::GetDynamicBatchInfo(std::vector<std::vector<int64_t>> &batch_info, int32_t &dynamic_type) {
   GE_CHECK_NOTNULL(impl_);
   return impl_->GetDynamicBatchInfo(batch_info, dynamic_type);
@@ -221,5 +245,33 @@ uint64_t HybridDavinciModel::GetSessionId() {
   GE_CHECK_NOTNULL(impl_);
   return impl_->GetSessionId();
 }
+
+uint32_t HybridDavinciModel::GetDataInputerSize() {
+  GE_CHECK_NOTNULL(impl_);
+  return impl_->GetDataInputerSize();
+}
+
+bool HybridDavinciModel::GetRunningFlag() const { return impl_->GetRunningFlag(); }
+
+Status HybridDavinciModel::SetRunAsyncListenerCallback(const RunAsyncCallback &callback) {
+  return impl_->SetRunAsyncListenerCallback(callback);
+}
+
+bool HybridDavinciModel::GetOpDescInfo(uint32_t stream_id, uint32_t task_id, OpDescInfo &op_desc_info) const {
+  if (impl_ == nullptr) {
+    return false;
+  }
+  auto context = impl_->GeContext();
+  GE_CHECK_NOTNULL(context);
+  bool ret = context->exception_dumper.GetOpDescInfo(stream_id, task_id, op_desc_info);
+  if (!ret) {
+    for (const auto &iter : context->davinci_model) {
+      if (iter->GetOpDescInfo(stream_id, task_id, op_desc_info)) {
+        return true;
+      }
+    }
+  }
+  return ret;
+}
 }  // namespace hybrid
 }  // namespace ge

ge/hybrid/hybrid_davinci_model.h

@@ -61,6 +61,8 @@ class HybridDavinciModel {
 
   uint64_t GetSessionId();
 
+  uint32_t GetDeviceId() const;
+
   Status GetDynamicBatchInfo(std::vector<std::vector<int64_t>> &batch_info, int32_t &dynamic_type);
 
   void GetUserDesignateShapeOrder(std::vector<std::string> &user_input_shape_order);
@@ -74,6 +76,14 @@ class HybridDavinciModel {
 
   void SetModelDescVersion(bool is_new_model_desc);
 
+  uint32_t GetDataInputerSize();
+
+  bool GetRunningFlag() const;
+
+  Status SetRunAsyncListenerCallback(const RunAsyncCallback &callback);
+
+  bool GetOpDescInfo(uint32_t stream_id, uint32_t task_id, OpDescInfo &op_desc_info) const;
+
  private:
   HybridDavinciModel() = default;
   class Impl;