!620 check aicpu op type

From: @taoxiangdong
Reviewed-by: 
Signed-off-by:

ge/graph/build/model_builder.cc

@@ -582,9 +582,13 @@ Status ModelBuilder::SaveDataToModel(ge::Model &model, ge::GeModel &ge_model) {
   // Add TBE Kernels and custom aicpu op bin
   std::set<std::string> tbe_name_set;
   std::set<std::string> aicpu_name_set;
+  std::set<std::string> aicpu_op_types;
+  std::set<std::string> aicpu_tf_op_types;
   for (const ge::NodePtr &n : compute_graph_->GetNodes(compute_graph_->GetGraphUnknownFlag())) {
     auto node_op_desc = n->GetOpDesc();
     GE_IF_BOOL_EXEC(node_op_desc == nullptr, continue);
+    // check aicpu op type
+    CollectCheckAicpuAttr(node_op_desc, aicpu_op_types, aicpu_tf_op_types);
     TBEKernelPtr tbe_kernel = node_op_desc->TryGetExtAttr(ge::OP_EXTATTR_NAME_TBE_KERNEL, TBEKernelPtr());
     if (tbe_kernel == nullptr) {
       std::string kernel_name;
@@ -606,6 +610,8 @@ Status ModelBuilder::SaveDataToModel(ge::Model &model, ge::GeModel &ge_model) {
     tbe_kernel_store_.AddTBEKernel(tbe_kernel);
   }
 
+  SetModelCheckAicpuAttr(model, aicpu_op_types, aicpu_tf_op_types);
+
   for (const ge::NodePtr &n : compute_graph_->GetNodes(compute_graph_->GetGraphUnknownFlag())) {
     auto node_op_desc = n->GetOpDesc();
     GE_IF_BOOL_EXEC(node_op_desc == nullptr, continue);
@@ -797,4 +803,51 @@ Status ModelBuilder::CompileSingleOp() {
   GE_TIMESTAMP_CALLNUM_END(BatchCompileOp, "GraphBuild::CompileOp");
   return ge::SUCCESS;
 }
+
+void ModelBuilder::CollectCheckAicpuAttr(const OpDescPtr &op_desc, std::set<std::string> &aicpu_op_types,
+                                           std::set<std::string> &aicpu_tf_op_types) {
+  std::string aicpu_optype;
+  bool has_attr_check_cpu = ge::AttrUtils::GetStr(op_desc, "needCheckCpu", aicpu_optype);
+  std::vector<std::string> tf_optypes;
+  bool has_attr_check_tf = ge::AttrUtils::GetListStr(op_desc, "needCheckTf", tf_optypes);
+  if (has_attr_check_cpu && !aicpu_optype.empty()) {
+    aicpu_op_types.insert(aicpu_optype);
+  }
+
+  if (has_attr_check_tf && !tf_optypes.empty()) {
+    aicpu_tf_op_types.insert(tf_optypes.begin(), tf_optypes.end());
+  }
+
+  return;
+}
+
+void ModelBuilder::SetModelCheckAicpuAttr(ge::Model &model, std::set<std::string> &aicpu_op_types,
+                                            std::set<std::string> &aicpu_tf_op_types) {
+  std::vector<std::string> aicpu_optype_list;
+  std::vector<std::string> aicpu_tf_optype_list;
+  if (ge::AttrUtils::GetListStr(&model, "needCheckCpu", aicpu_optype_list)) {
+    GELOGI("Already have aicpu optype size: %zu", aicpu_optype_list.size());
+    aicpu_op_types.insert(aicpu_optype_list.begin(), aicpu_optype_list.end());
+  }
+
+  if (ge::AttrUtils::GetListStr(&model, "needCheckTf", aicpu_tf_optype_list)) {
+    GELOGI("Already have aicpu tf optype size: %zu", aicpu_tf_optype_list.size());
+    aicpu_tf_op_types.insert(aicpu_tf_optype_list.begin(), aicpu_tf_optype_list.end());
+  }
+
+  // reset list with set
+  aicpu_optype_list.assign(aicpu_op_types.begin(), aicpu_op_types.end());
+  aicpu_tf_optype_list.assign(aicpu_tf_op_types.begin(), aicpu_tf_op_types.end());
+  GELOGI(
+      "Check Aicpu op types ComputeGraph: %s aicpu_op_types: %zu, aicpu_optype_list: %zu, aicpu_tf_op_types: %zu, "
+      "aicpu_tf_optype_list:%zu.",
+      compute_graph_->GetName().c_str(), aicpu_op_types.size(), aicpu_optype_list.size(), aicpu_tf_op_types.size(),
+      aicpu_tf_optype_list.size());
+  GE_CHK_BOOL_EXEC(ge::AttrUtils::SetListStr(&model, "needCheckCpu", aicpu_optype_list), return,
+                   "Set attr needCheckCpu fail.");
+
+  GE_CHK_BOOL_EXEC(ge::AttrUtils::SetListStr(&model, "needCheckTf", aicpu_tf_optype_list), return,
+                   "Set attr needCheckTf fail.");
+  return;
+}
 }  // namespace ge

ge/graph/build/model_builder.h

@@ -83,6 +83,12 @@ class ModelBuilder {
 
   Status CompileSingleOp();
 
+  void CollectCheckAicpuAttr(const OpDescPtr &op_desc, std::set<std::string> &aicpu_op_types,
+                               std::set<std::string> &aicpu_tf_op_types);
+
+  void SetModelCheckAicpuAttr(ge::Model &model, std::set<std::string> &aicpu_op_types,
+                                std::set<std::string> &aicpu_tf_op_types);
+
   uint64_t session_id_;
 
   map<int64_t, size_t> mem_type_to_mem_offset_;

ge/graph/load/new_model_manager/davinci_model.cc

@@ -487,6 +487,8 @@ Status DavinciModel::DoTaskSink() {
 
   GE_CHK_STATUS_RET(ModelManager::GetInstance()->LaunchCustAicpuSo(), "Launch cust aicpu so failed.");
 
+  GE_CHK_STATUS_RET(ModelManager::GetInstance()->CheckAicpuOpList(ge_model_), "Check aicpu op type failed.");
+
   GE_CHK_STATUS_RET(InitEntryTask(), "InitEntryTask failed.");
 
   GE_CHK_STATUS_RET(DistributeTask(), "Distribute failed.");

ge/graph/load/new_model_manager/model_manager.cc

@@ -18,6 +18,7 @@
 
 #include <string>
 
+#include "aicpu/aicpu_schedule/aicpu_op_type_list.h"
 #include "common/dump/dump_manager.h"
 #include "common/l2_cache_optimize.h"
 #include "common/profiling/profiling_manager.h"
@@ -30,6 +31,7 @@
 #include "graph/load/new_model_manager/davinci_model_parser.h"
 #include "model/ge_root_model.h"
 #include "graph/common/local_context.h"
+#include "graph/utils/attr_utils.h"
 #include "common/formats/utils/formats_trans_utils.h"
 #include "hybrid/hybrid_davinci_model.h"
 
@@ -52,6 +54,7 @@ const char *const kDeleteCustOp = "deleteCustOp";
 const int kTimeSpecNano = 1000000000;
 const int kTimeSpecMiro = 1000000;
 const int kSessionMaxBias = 100;
+const int kOpNameMaxSize = 100;
 struct CustAicpuSoBuf {
   uint64_t kernelSoBuf;
   uint32_t kernelSoBufLen;
@@ -1534,4 +1537,200 @@ Status ModelManager::EnableExceptionDump(const std::map<string, string> &options
   return SUCCESS;
 }
 
+Status ModelManager::LaunchKernelCheckAicpuOp(std::vector<std::string> &aicpu_optype_list,
+                                              std::vector<std::string> &aicpu_tf_optype_list) {
+  std::string kernel_name = "checkOpType";
+  GELOGI("LaunchKernelCheckAicpuOpType in, kernel name %s", kernel_name.c_str());
+  std::lock_guard<std::mutex> lock(cust_aicpu_mutex_);
+  std::vector<SysOpInfo> req_aicpu_op_info_list;
+  std::vector<SysOpInfo> res_aicpu_op_info_list;
+  std::vector<ReturnCode> res_ret_code_list;
+
+  if (aicpu_optype_list.empty() && aicpu_tf_optype_list.empty()) {
+    GELOGI("No need to check aicpu op type.");
+    return SUCCESS;
+  }
+
+  vector<void *> allocated_mem;
+  rtError_t status;
+  rtStream_t stream = nullptr;
+  void *args = nullptr;
+
+  void *d_req_op_list = nullptr;
+  void *d_res_op_list = nullptr;
+  void *d_ret_code_list = nullptr;
+
+  size_t aicpu_op_nums = aicpu_optype_list.size();
+  size_t tf_op_nums = aicpu_tf_optype_list.size();
+  size_t op_nums = aicpu_op_nums + tf_op_nums;
+  // malloc sysOpInfoList in SysOpCheckInfo
+  status = rtMalloc(&d_req_op_list, op_nums * sizeof(SysOpInfo), RT_MEMORY_HBM);
+  if (status != RT_ERROR_NONE) {
+    GELOGE(RT_FAILED, "Call rt failed, status: 0x%x", status);
+    return RT_ERROR_TO_GE_STATUS(status);
+  }
+  allocated_mem.push_back(d_req_op_list);
+
+  // malloc sysOpInfoList in SysOpCheckResp
+  status = rtMalloc(&d_res_op_list, op_nums * sizeof(SysOpInfo), RT_MEMORY_HBM);
+  if (status != RT_ERROR_NONE) {
+    GELOGE(RT_FAILED, "Call rt failed, status: 0x%x", status);
+    return RT_ERROR_TO_GE_STATUS(status);
+  }
+  allocated_mem.push_back(d_res_op_list);
+
+  // malloc returnCodeList in SysOpCheckResp
+  status = rtMalloc(&d_ret_code_list, op_nums * sizeof(ReturnCode), RT_MEMORY_HBM);
+  if (status != RT_ERROR_NONE) {
+    GELOGE(RT_FAILED, "Call rt failed, status: 0x%x", status);
+    return RT_ERROR_TO_GE_STATUS(status);
+  }
+  allocated_mem.push_back(d_ret_code_list);
+
+  for (const auto &op_type : aicpu_optype_list) {
+    SysOpInfo op_info;
+    // malloc op_type name in SysOpInfo
+    void *d_op_type_name = nullptr;
+    status = rtMalloc(&d_op_type_name, op_type.length(), RT_MEMORY_HBM);
+    if (status != RT_ERROR_NONE) {
+      GELOGE(RT_FAILED, "Call rt failed, status: 0x%x", status);
+      return RT_ERROR_TO_GE_STATUS(status);
+    }
+    allocated_mem.push_back(d_op_type_name);
+    GE_CHK_RT(rtMemcpy(d_op_type_name, op_type.length(), op_type.c_str(), op_type.length(), RT_MEMCPY_HOST_TO_DEVICE));
+    op_info.opType = static_cast<uint64_t>(reinterpret_cast<uintptr_t>(d_op_type_name));
+    op_info.opLen = op_type.length();
+    op_info.kernelsType = CPU_KERNEL;
+    req_aicpu_op_info_list.emplace_back(op_info);
+  }
+
+  for (const auto &op_type : aicpu_tf_optype_list) {
+    SysOpInfo op_info;
+    // malloc op_type name in SysOpInfo
+    void *d_op_type_name = nullptr;
+    status = rtMalloc(&d_op_type_name, op_type.size(), RT_MEMORY_HBM);
+    if (status != RT_ERROR_NONE) {
+      GELOGE(RT_FAILED, "Call rt failed, status: 0x%x", status);
+      return RT_ERROR_TO_GE_STATUS(status);
+    }
+    allocated_mem.push_back(d_op_type_name);
+    GE_CHK_RT(rtMemcpy(d_op_type_name, op_type.size(), op_type.c_str(), op_type.size(), RT_MEMCPY_HOST_TO_DEVICE));
+    op_info.opType = static_cast<uint64_t>(reinterpret_cast<uintptr_t>(d_op_type_name));
+    op_info.opLen = op_type.size();
+    op_info.kernelsType = TF_KERNEL;
+    req_aicpu_op_info_list.emplace_back(op_info);
+  }
+  GELOGI("Check aicpu op all attr size: %zu, real attr size: %zu.", op_nums, req_aicpu_op_info_list.size());
+  GE_CHK_RT(rtMemcpy(d_req_op_list, sizeof(SysOpInfo) * req_aicpu_op_info_list.size(), req_aicpu_op_info_list.data(),
+                     sizeof(SysOpInfo) * req_aicpu_op_info_list.size(), RT_MEMCPY_HOST_TO_DEVICE));
+
+  SysOpCheckInfo op_check_info_req = { 0 };
+  SysOpCheckResp op_check_info_res = { 0 };
+  op_check_info_req.opListNum = op_nums;
+  op_check_info_req.offSetLen = sizeof(SysOpCheckInfo);
+  op_check_info_req.sysOpInfoList = static_cast<uint64_t>(reinterpret_cast<uintptr_t>(d_req_op_list));
+
+  op_check_info_res.opListNum = 0;
+  op_check_info_res.isWithoutJson = 0;
+  op_check_info_res.returnCodeList = static_cast<uint64_t>(reinterpret_cast<uintptr_t>(d_ret_code_list));
+  op_check_info_res.sysOpInfoList = static_cast<uint64_t>(reinterpret_cast<uintptr_t>(d_res_op_list));
+
+  uint32_t args_size = sizeof(SysOpCheckInfo) + sizeof(SysOpCheckResp);
+  status = rtMalloc(&args, args_size, RT_MEMORY_HBM);
+  if (status != RT_ERROR_NONE) {
+    GELOGE(RT_FAILED, "Call rt failed, status: 0x%x", status);
+    return RT_ERROR_TO_GE_STATUS(status);
+  }
+  allocated_mem.push_back(args);
+  GE_CHK_RT(
+      rtMemcpy(args, sizeof(SysOpCheckInfo), reinterpret_cast<void *>(&op_check_info_req), sizeof(SysOpCheckInfo), RT_MEMCPY_HOST_TO_DEVICE));
+  GE_CHK_RT(rtMemcpy(reinterpret_cast<void *>(static_cast<uintptr_t>(static_cast<uint64_t>(reinterpret_cast<uintptr_t>(args)) + op_check_info_req.offSetLen)),
+                     sizeof(SysOpCheckResp), reinterpret_cast<void *>(&op_check_info_res), sizeof(SysOpCheckResp), RT_MEMCPY_HOST_TO_DEVICE));
+  GE_CHK_RT(rtStreamCreate(&stream, 0));
+  GE_CHK_RT(rtCpuKernelLaunch(nullptr, kernel_name.c_str(), 1, args, args_size, nullptr, stream));
+
+  status = rtStreamSynchronize(stream);
+  if (status != RT_ERROR_NONE) {
+    GELOGE(RT_FAILED, "Call rt stream sync failed, status: 0x%x", status);
+    return RT_ERROR_TO_GE_STATUS(status);
+  }
+
+  // Check the response
+  SysOpCheckResp *d_op_check_info_res = reinterpret_cast<SysOpCheckResp *>(reinterpret_cast<void *>(static_cast<uintptr_t>(static_cast<uint64_t>(reinterpret_cast<uintptr_t>(args)) + op_check_info_req.offSetLen)));
+  (void)memset_s(&op_check_info_res, sizeof(SysOpCheckResp), 0, sizeof(SysOpCheckResp));
+  GE_CHK_RT(rtMemcpy(&op_check_info_res, sizeof(SysOpCheckResp), d_op_check_info_res, sizeof(SysOpCheckResp),
+                     RT_MEMCPY_DEVICE_TO_HOST));
+  std::function<void()> callback = [&]() {
+    for (auto mem : allocated_mem) {
+      GE_CHK_RT(rtFree(mem));
+    }
+    GE_CHK_RT(rtStreamDestroy(stream));
+  };
+
+  if (op_check_info_res.isWithoutJson) {
+    GELOGI("No need to check aicpu in this scenoria.");
+    GE_MAKE_GUARD(release, callback);
+    return SUCCESS;
+  }
+  uint64_t res_op_nums = op_check_info_res.opListNum;
+  GELOGI("Check aicpu type, is without json: %d, res op num: %lu.", op_check_info_res.isWithoutJson, res_op_nums);
+  if (res_op_nums != 0) {
+    res_ret_code_list.clear();
+    res_ret_code_list.resize(res_op_nums);
+    res_aicpu_op_info_list.clear();
+    res_aicpu_op_info_list.resize(res_op_nums);
+    GE_CHK_RT(rtMemcpy(res_ret_code_list.data(), sizeof(ReturnCode) * res_op_nums,
+                       reinterpret_cast<void *>(static_cast<uintptr_t>(op_check_info_res.returnCodeList)),
+                       sizeof(ReturnCode) * res_op_nums, RT_MEMCPY_DEVICE_TO_HOST));
+    GE_CHK_RT(rtMemcpy(res_aicpu_op_info_list.data(), sizeof(SysOpInfo) * res_op_nums,
+                       reinterpret_cast<void *>(static_cast<uintptr_t>(op_check_info_res.sysOpInfoList)),
+                       sizeof(SysOpInfo) * res_op_nums, RT_MEMCPY_DEVICE_TO_HOST));
+    if (res_ret_code_list.size() != res_aicpu_op_info_list.size() || res_ret_code_list.size() != res_op_nums) {
+      GELOGE(FAILED, "Number of retcode is not equal to number of op type.");
+      GE_MAKE_GUARD(release, callback);
+      return FAILED;
+    }
+    std::string fail_reason;
+    for (uint32_t i = 0; i < res_op_nums; i++) {
+      ReturnCode ret_code = res_ret_code_list.at(i);
+      SysOpInfo aicpu_info = res_aicpu_op_info_list.at(i);
+      GELOGI("Not support aicpu op type: %lu, kernel_type:%d, opLen:%d, ret_code:%d", aicpu_info.opType,
+             aicpu_info.kernelsType, aicpu_info.opLen, ret_code);
+      std::vector<char> op_name;
+      op_name.clear();
+      op_name.resize(kOpNameMaxSize);
+      GE_CHK_RT(rtMemcpy(op_name.data(), aicpu_info.opLen, reinterpret_cast<void *>(aicpu_info.opType),
+                         aicpu_info.opLen, RT_MEMCPY_DEVICE_TO_HOST));
+      std::string kernel_type =
+          (static_cast<OpKernelType>(aicpu_info.kernelsType) == TF_KERNEL) ? "TF_KERNEL" : "CPU_KERNEL";
+      string op_name_str(op_name.data());
+      fail_reason += "op_type: " + op_name_str + " kernel_type: " + kernel_type +
+                     "  ret code:" + std::to_string(static_cast<int>(ret_code)) +
+                     "<0: op_type, 1: format, 2: datatype> \n";
+    }
+    fail_reason += "not support.";
+    GELOGE(FAILED, "Check aicpu op_type failed. details: %s", fail_reason.c_str());
+    GE_MAKE_GUARD(release, callback);
+    return FAILED;
+  }
+
+  GE_MAKE_GUARD(release, callback);
+  GELOGI("Cpu kernel launch check optype task success.");
+  return SUCCESS;
+}
+
+Status ModelManager::CheckAicpuOpList(GeModelPtr ge_model) {
+  std::vector<std::string> aicpu_optype_list;
+  std::vector<std::string> aicpu_tf_optype_list;
+  bool aicpu_need_check = ge::AttrUtils::GetListStr(ge_model, "needCheckCpu", aicpu_optype_list);
+  bool tf_need_check = ge::AttrUtils::GetListStr(ge_model, "needCheckTf", aicpu_tf_optype_list);
+  if (!aicpu_need_check && !tf_need_check) {
+    GELOGI("Graph:%s No need to check aicpu optype.", ge_model->GetGraph().GetName().c_str());
+    return SUCCESS;
+  }
+  GE_CHK_STATUS_RET(LaunchKernelCheckAicpuOp(aicpu_optype_list, aicpu_tf_optype_list),
+                    "Launch check aicpu op type failed.");
+  return SUCCESS;
+}
+
 }  // namespace ge

ge/graph/load/new_model_manager/model_manager.h

@@ -295,6 +295,11 @@ class FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ModelManager {
 
   ge::Status LaunchKernelCustAicpuSo(const string &kernel_name);
 
+  ge::Status LaunchKernelCheckAicpuOp(std::vector<std::string> &aicpu_optype_list,
+                                      std::vector<std::string> &aicpu_tf_optype_list);
+
+  ge::Status CheckAicpuOpList(GeModelPtr ge_model);
+
   ge::Status GetOrigInputInfo(uint32_t model_id, uint32_t index, OriginInputInfo &orig_input_info);
 
   ge::Status GenSessionId(uint64_t &session_id);

ge/hybrid/model/hybrid_model_builder.cc

@@ -21,6 +21,7 @@
 #include "graph/build/memory/var_mem_assign_util.h"
 #include "graph/debug/ge_attr_define.h"
 #include "graph/load/new_model_manager/model_utils.h"
+#include "graph/load/new_model_manager/model_manager.h"
 #include "graph/manager/graph_var_manager.h"
 #include "graph/manager/host_mem_manager.h"
 #include "graph/manager/trans_var_data_utils.h"
@@ -924,6 +925,7 @@ Status HybridModelBuilder::InitWeights() {
 }
 
 Status HybridModelBuilder::LoadTasks() {
+  GE_CHK_STATUS_RET(CheckAicpuOpList(), "Check Aicpu op failed.");
   for (auto &it : hybrid_model_.node_items_) {
     auto &node_item = it.second;
     auto &node_ptr = node_item->node;
@@ -1560,5 +1562,29 @@ Status HybridModelBuilder::BuildInputMapping(GraphItem &graph_item,
 
   return SUCCESS;
 }
+
+Status HybridModelBuilder::CheckAicpuOpList() {
+  std::vector<std::string> aicpu_optype_list;
+  std::vector<std::string> aicpu_tf_optype_list;
+  std::set<std::string> aicpu_optype_set;
+  std::set<std::string> aicpu_tf_optype_set;
+  for (auto &it : ge_root_model_->GetSubgraphInstanceNameToModel()) {
+    auto &ge_model = it.second;
+    GE_CHECK_NOTNULL(ge_model);
+    if (ge::AttrUtils::GetListStr(*ge_model, "needCheckCpu", aicpu_optype_list)) {
+      aicpu_optype_set.insert(aicpu_optype_list.begin(), aicpu_optype_list.end());
+    }
+
+    if (ge::AttrUtils::GetListStr(*ge_model, "needCheckTf", aicpu_tf_optype_list)) {
+      aicpu_tf_optype_set.insert(aicpu_tf_optype_list.begin(), aicpu_tf_optype_list.end());
+    }
+  }
+  // reset list with set
+  aicpu_optype_list.assign(aicpu_optype_set.begin(), aicpu_optype_set.end());
+  aicpu_tf_optype_list.assign(aicpu_tf_optype_set.begin(), aicpu_tf_optype_set.end());
+  GE_CHK_STATUS_RET(ModelManager::GetInstance()->LaunchKernelCheckAicpuOp(aicpu_optype_list, aicpu_tf_optype_list),
+                    "Launch check aicpu op type failed.");
+  return SUCCESS;
+}
 }  // namespace hybrid
 }  // namespace ge

ge/hybrid/model/hybrid_model_builder.h

@@ -78,6 +78,7 @@ class HybridModelBuilder {
   Status ParseVarOutputs(NodeItem &node_item);
   Status LoadKnownShapedSubgraph(ComputeGraph &graph, NodeItem *parent_node_item);
   Status RecoverGraphUnknownFlag();
+  Status CheckAicpuOpList();
 
   const char* GetGraphName() const {
     return hybrid_model_.model_name_.c_str();