From 139d3f6147b1a0e4702f5d03572cba97d72a9c07 Mon Sep 17 00:00:00 2001 From: zhangzhenghai Date: Tue, 4 Aug 2020 15:54:54 +0800 Subject: [PATCH] modify ge_common, update libge_common.so --- src/ge/common/convert/pb2json.cc | 11 +- .../formats/format_transfers/datatype_transfer.cc | 1 + .../formats/format_transfers/datatype_transfer.h | 1 + .../format_transfer_dhwcn_fracz3D.cc | 1 + .../format_transfer_dhwnc_fracz3D_transpose.cc | 1 + .../format_transfers/format_transfer_fractal_nz.cc | 2 +- .../format_transfers/format_transfer_fractal_z.cc | 95 +++--- .../format_transfer_nchw_fz_c04.cc | 4 +- src/ge/common/formats/utils/formats_definitions.h | 2 + src/ge/common/formats/utils/formats_trans_utils.h | 1 + src/ge/common/fp16_t.h | 2 +- src/ge/common/ge/op_tiling_manager.cc | 81 ------ src/ge/common/ge/op_tiling_manager.h | 38 --- src/ge/common/helper/model_helper.cc | 103 ++++++- src/ge/common/math/fp16_math.h | 2 +- src/ge/common/math_util.h | 2 + src/ge/common/model_saver.cc | 7 +- src/ge/common/profiling/profiling_manager.cc | 75 +++-- src/ge/common/profiling/profiling_manager.h | 5 + src/ge/common/properties_manager.cc | 320 ++++++++------------- src/ge/common/properties_manager.h | 68 ++--- src/ge/common/tbe_kernel_store.h | 1 + src/ge/common/types.cc | 195 ++++++------- src/ge/common/util.cc | 66 +++-- 24 files changed, 488 insertions(+), 596 deletions(-) delete mode 100644 src/ge/common/ge/op_tiling_manager.cc delete mode 100644 src/ge/common/ge/op_tiling_manager.h diff --git a/src/ge/common/convert/pb2json.cc b/src/ge/common/convert/pb2json.cc index 0a5d24ee..832a8278 100644 --- a/src/ge/common/convert/pb2json.cc +++ b/src/ge/common/convert/pb2json.cc @@ -72,6 +72,9 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void Pb2Json::Message2Json(cons void Pb2Json::OneField2Json(const ProtobufMsg &message, const ProtobufFieldDescriptor *field, const ProtobufReflection *reflection, const set &black_fields, Json &json, bool enum2str) { + if (field == nullptr || reflection == nullptr) { + return; + } switch (field->type()) { case ProtobufFieldDescriptor::TYPE_MESSAGE: { const ProtobufMsg &tmp_message = reflection->GetMessage(message, field); @@ -115,12 +118,8 @@ void Pb2Json::OneField2Json(const ProtobufMsg &message, const ProtobufFieldDescr case ProtobufFieldDescriptor::TYPE_FLOAT: char str[kSignificantDigits]; - if (sprintf_s(str, kSignificantDigits, "%g", reflection->GetFloat(message, field)) != -1) { - json[field->name()] = str; - } else { - json[field->name()] = reflection->GetFloat(message, field); - } - + sprintf_s(str, kSignificantDigits, "%g", reflection->GetFloat(message, field)); + json[field->name()] = str; break; case ProtobufFieldDescriptor::TYPE_STRING: diff --git a/src/ge/common/formats/format_transfers/datatype_transfer.cc b/src/ge/common/formats/format_transfers/datatype_transfer.cc index 08c6889f..0bd4b8e5 100644 --- a/src/ge/common/formats/format_transfers/datatype_transfer.cc +++ b/src/ge/common/formats/format_transfers/datatype_transfer.cc @@ -29,6 +29,7 @@ namespace ge { namespace formats { + namespace { enum DataTypeTransMode { kTransferWithDatatypeFloatToFloat16, diff --git a/src/ge/common/formats/format_transfers/datatype_transfer.h b/src/ge/common/formats/format_transfers/datatype_transfer.h index 4d93fd6c..0702592f 100644 --- a/src/ge/common/formats/format_transfers/datatype_transfer.h +++ b/src/ge/common/formats/format_transfers/datatype_transfer.h @@ -27,6 +27,7 @@ namespace ge { namespace formats { + struct CastArgs { const uint8_t *data; size_t src_data_size; diff --git a/src/ge/common/formats/format_transfers/format_transfer_dhwcn_fracz3D.cc b/src/ge/common/formats/format_transfers/format_transfer_dhwcn_fracz3D.cc index 76d8696a..dc8e1033 100644 --- a/src/ge/common/formats/format_transfers/format_transfer_dhwcn_fracz3D.cc +++ b/src/ge/common/formats/format_transfers/format_transfer_dhwcn_fracz3D.cc @@ -179,5 +179,6 @@ Status FormatTransferDhwcnFractalZ3D::TransShape(Format src_format, const std::v } REGISTER_FORMAT_TRANSFER(FormatTransferDhwcnFractalZ3D, FORMAT_DHWCN, FORMAT_FRACTAL_Z_3D) + } // namespace formats } // namespace ge diff --git a/src/ge/common/formats/format_transfers/format_transfer_dhwnc_fracz3D_transpose.cc b/src/ge/common/formats/format_transfers/format_transfer_dhwnc_fracz3D_transpose.cc index 9de2e3a0..11e3d270 100644 --- a/src/ge/common/formats/format_transfers/format_transfer_dhwnc_fracz3D_transpose.cc +++ b/src/ge/common/formats/format_transfers/format_transfer_dhwnc_fracz3D_transpose.cc @@ -180,5 +180,6 @@ Status FormatTransferDhwncFractalZ3DTranspose::TransShape(Format src_format, con } REGISTER_FORMAT_TRANSFER(FormatTransferDhwncFractalZ3DTranspose, FORMAT_DHWNC, FORMAT_FRACTAL_Z_3D_TRANSPOSE) + } // namespace formats } // namespace ge diff --git a/src/ge/common/formats/format_transfers/format_transfer_fractal_nz.cc b/src/ge/common/formats/format_transfers/format_transfer_fractal_nz.cc index 65798f29..ff7b84a4 100644 --- a/src/ge/common/formats/format_transfers/format_transfer_fractal_nz.cc +++ b/src/ge/common/formats/format_transfers/format_transfer_fractal_nz.cc @@ -56,7 +56,7 @@ Status TransShapeToFracNz(const ShapeVector &src_shape, DataType data_type, Shap dst_shape.clear(); hw_shape.clear(); auto w0 = GetCubeSizeByDataType(data_type); - int64_t h0 = kCubeSize; + auto h0 = GetCubeSizeByDataType(data_type); switch (src_shape.size()) { case 1: dst_shape.push_back(Ceil(src_shape[0], w0)); diff --git a/src/ge/common/formats/format_transfers/format_transfer_fractal_z.cc b/src/ge/common/formats/format_transfers/format_transfer_fractal_z.cc index f2ec29da..f3d06496 100644 --- a/src/ge/common/formats/format_transfers/format_transfer_fractal_z.cc +++ b/src/ge/common/formats/format_transfers/format_transfer_fractal_z.cc @@ -19,7 +19,6 @@ #include #include -#include "common/debug/log.h" #include "common/formats/utils/formats_definitions.h" #include "common/formats/utils/formats_trans_utils.h" #include "framework/common/debug/ge_log.h" @@ -108,8 +107,8 @@ Status TransFormatFromNchwToFz(const TransArgs &args, TransResult &result) { int64_t hw = h * w; int64_t chw = c * hw; - int64_t nchw = n * chw; int64_t hwc0 = hw * c0; + int64_t nchw = n * chw; // horizontal fractal matrix count (N) int64_t hf_cnt = Ceil(n, static_cast(kNiSize)); @@ -120,15 +119,18 @@ Status TransFormatFromNchwToFz(const TransArgs &args, TransResult &result) { int64_t total_ele_cnt = hf_cnt * vf_cnt * fractal_ele_cnt; int size = GetSizeByDataType(args.src_data_type); int64_t dst_size = total_ele_cnt * size; - GE_CHK_BOOL_EXEC_NOLOG(dst_size != 0, result.length = static_cast(dst_size); return SUCCESS;); + if (dst_size == 0) { + result.length = static_cast(dst_size); + return SUCCESS; + } std::shared_ptr dst(new (std::nothrow) uint8_t[dst_size], std::default_delete()); - GE_CHK_BOOL_TRUE_EXEC_WITH_LOG( - dst == nullptr, + if (dst == nullptr) { GELOGE(OUT_OF_MEMORY, "Failed to trans format from %s to %s, can not alloc the memory for dst buf %ld", TypeUtils::FormatToSerialString(args.src_format).c_str(), TypeUtils::FormatToSerialString(args.dst_format).c_str(), dst_size); - return OUT_OF_MEMORY;); + return OUT_OF_MEMORY; + } for (int64_t vfi = 0; vfi < vf_cnt; vfi++) { // vertical fractal matrix base index @@ -154,20 +156,12 @@ Status TransFormatFromNchwToFz(const TransArgs &args, TransResult &result) { auto protected_size = dst_size - offset < static_cast(SECUREC_MEM_MAX_LEN) ? dst_size - offset : static_cast(SECUREC_MEM_MAX_LEN); - errno_t ret = EOK; + errno_t ret; if (need_pad_zero) { ret = memset_s(dst.get() + offset, static_cast(protected_size), 0, static_cast(size)); } else { - if (protected_size < size) { - GELOGE(INTERNAL_ERROR, "Failed to operate the dst memory, protected_size is %ld and size is %ld", - protected_size, size); - return INTERNAL_ERROR; - } - char *dst_data = reinterpret_cast(dst.get() + offset); - const char *src_data = reinterpret_cast(args.data + src_offset * size); - for (int64_t index = 0; index < size; index++) { - *dst_data++ = *src_data++; - } + ret = memcpy_s(dst.get() + offset, static_cast(protected_size), args.data + src_offset * size, + static_cast(size)); } if (ret != EOK) { GELOGE(INTERNAL_ERROR, "Failed to operate the dst memory at offset %ld, error-code %d pad mode %d", offset, @@ -205,15 +199,18 @@ Status TransFormatHwcnToFz(const TransArgs &args, TransResult &result) { dst_size *= dim; } dst_size *= data_size; - GE_CHK_BOOL_EXEC_NOLOG(dst_size != 0, result.length = static_cast(dst_size); return SUCCESS;); + if (dst_size == 0) { + result.length = static_cast(dst_size); + return SUCCESS; + } std::shared_ptr dst(new (std::nothrow) uint8_t[dst_size], std::default_delete()); - GE_CHK_BOOL_TRUE_EXEC_WITH_LOG( - dst == nullptr, + if (dst == nullptr) { GELOGE(OUT_OF_MEMORY, "Failed to trans format from %s to %s, can not alloc the memory for dst buf %ld", TypeUtils::FormatToSerialString(args.src_format).c_str(), TypeUtils::FormatToSerialString(args.dst_format).c_str(), dst_size); - return OUT_OF_MEMORY;); + return OUT_OF_MEMORY; + } for (int64_t c1i = 0; c1i < c1; c1i++) { for (int64_t hi = 0; hi < h; hi++) { @@ -226,22 +223,14 @@ Status TransFormatHwcnToFz(const TransArgs &args, TransResult &result) { ? dst_size - dst_offset : static_cast(SECUREC_MEM_MAX_LEN); auto pad_zero = ((c1i * c0 + c0i) >= c) || (n1n0i >= n); - errno_t ret = EOK; + errno_t ret; if (pad_zero) { ret = memset_s(dst.get() + dst_offset, static_cast(protected_size), 0, static_cast(data_size)); } else { - if (protected_size < data_size) { - GELOGE(INTERNAL_ERROR, "Failed to operate the dst memory, protected_size is %ld and size is %ld", - protected_size, data_size); - return INTERNAL_ERROR; - } int64_t src_idx = hi * wcn + wi * cn + (c1i * c0 + c0i) * n + n1n0i; - char *dst_data = reinterpret_cast(dst.get() + dst_offset); - const char *src_data = reinterpret_cast(args.data + src_idx * data_size); - for (int64_t index = 0; index < data_size; index++) { - *dst_data++ = *src_data++; - } + ret = memcpy_s(dst.get() + dst_offset, static_cast(protected_size), + args.data + src_idx * data_size, static_cast(data_size)); } if (ret != EOK) { GELOGE(INTERNAL_ERROR, "Failed to operate the dst memory at offset %ld, error-code %d, pad mode %d", @@ -280,15 +269,18 @@ Status TransFormatNhwcToFz(const TransArgs &args, TransResult &result) { dst_size *= dim; } dst_size *= data_size; - GE_CHK_BOOL_EXEC_NOLOG(dst_size != 0, result.length = static_cast(dst_size); return SUCCESS;); + if (dst_size == 0) { + result.length = static_cast(dst_size); + return SUCCESS; + } std::shared_ptr dst(new (std::nothrow) uint8_t[dst_size], std::default_delete()); - GE_CHK_BOOL_TRUE_EXEC_WITH_LOG( - dst == nullptr, + if (dst == nullptr) { GELOGE(OUT_OF_MEMORY, "Failed to trans format from %s to %s, can not alloc the memory for dst buf %ld", TypeUtils::FormatToSerialString(args.src_format).c_str(), TypeUtils::FormatToSerialString(args.dst_format).c_str(), dst_size); - return OUT_OF_MEMORY;); + return OUT_OF_MEMORY; + } for (int64_t c1i = 0; c1i < c1; c1i++) { for (int64_t hi = 0; hi < h; hi++) { @@ -301,22 +293,14 @@ Status TransFormatNhwcToFz(const TransArgs &args, TransResult &result) { ? dst_size - dst_offset : static_cast(SECUREC_MEM_MAX_LEN); auto pad_zero = ((c1i * c0 + c0i) >= c) || (n1n0i >= n); - errno_t ret = EOK; + errno_t ret; if (pad_zero) { ret = memset_s(dst.get() + dst_offset, static_cast(protected_size), 0, static_cast(data_size)); } else { - if (protected_size < data_size) { - GELOGE(INTERNAL_ERROR, "Failed to operate the dst memory, protected_size is %ld and size is %ld", - protected_size, data_size); - return INTERNAL_ERROR; - } int64_t src_idx = n1n0i * hwc + hi * wc + wi * c + (c1i * c0 + c0i); - char *dst_data = reinterpret_cast(dst.get() + dst_offset); - const char *src_data = reinterpret_cast(args.data + src_idx * data_size); - for (int64_t index = 0; index < data_size; index++) { - *dst_data++ = *src_data++; - } + ret = memcpy_s(dst.get() + dst_offset, static_cast(protected_size), + args.data + src_idx * data_size, static_cast(data_size)); } if (ret != EOK) { GELOGE(INTERNAL_ERROR, "Failed to operate the dst memory at offset %ld, error-code %d, pad mode %d", @@ -353,16 +337,16 @@ Status FormatTransferFractalZ::TransFormat(const TransArgs &args, TransResult &r return PARAM_INVALID; } - if (args.src_format == FORMAT_NHWC && args.dst_format == FORMAT_FRACTAL_Z) { - return TransFormatNhwcToFz(args, result); + if (args.src_format == FORMAT_NCHW && args.dst_format == FORMAT_FRACTAL_Z) { + return TransFormatFromNchwToFz(args, result); } if (args.src_format == FORMAT_HWCN && args.dst_format == FORMAT_FRACTAL_Z) { return TransFormatHwcnToFz(args, result); } - if (args.src_format == FORMAT_NCHW && args.dst_format == FORMAT_FRACTAL_Z) { - return TransFormatFromNchwToFz(args, result); + if (args.src_format == FORMAT_NHWC && args.dst_format == FORMAT_FRACTAL_Z) { + return TransFormatNhwcToFz(args, result); } return UNSUPPORTED; @@ -374,14 +358,14 @@ Status FormatTransferFractalZ::TransShape(Format src_format, const std::vector 0 ? SUCCESS : UNSUPPORTED; } @@ -108,7 +109,7 @@ Status TransFormatFromNchwToFzC04(const TransArgs &args, TransResult &result) { return NOT_CHANGED; } - // prepare for padding in chw + /* prepare for padding in chw*/ int64_t tmp = h * w * c; int64_t n_o = Ceil(n, static_cast(c0)); int64_t c_o = c0; @@ -308,5 +309,6 @@ Status FormatTransferNchwToFZC04::TransShape(Format src_format, const std::vecto } REGISTER_FORMAT_TRANSFER(FormatTransferNchwToFZC04, FORMAT_NCHW, FORMAT_FRACTAL_Z_C04) + } // namespace formats } // namespace ge diff --git a/src/ge/common/formats/utils/formats_definitions.h b/src/ge/common/formats/utils/formats_definitions.h index 2faa60e1..d889c33c 100644 --- a/src/ge/common/formats/utils/formats_definitions.h +++ b/src/ge/common/formats/utils/formats_definitions.h @@ -19,6 +19,7 @@ namespace ge { namespace formats { + static const int kCubeSize = 16; static const int kNiSize = 16; static const int64_t kShapeItemNumMAX = 1024UL * 1024UL * 1024UL * 1024UL; @@ -46,6 +47,7 @@ enum FracZDimIndex { kFracZHWC1, kFracZN0, kFracZNi, kFracZC0, kFracZDimsNum }; enum DhwcnDimIndex { kDhwcnD, kDhwcnH, kDhwcnW, kDhwcnC, kDhwcnN, kDhwcnDimsNum }; enum DhwncDimIndex { kDhwncD, kDhwncH, kDhwncW, kDhwncN, kDhwncC, kDhwncDimsNum }; + } // namespace formats } // namespace ge #endif // GE_COMMON_FORMATS_UTILS_FORMATS_DEFINITIONS_H_ diff --git a/src/ge/common/formats/utils/formats_trans_utils.h b/src/ge/common/formats/utils/formats_trans_utils.h index 7b902c3e..a8fbd09b 100644 --- a/src/ge/common/formats/utils/formats_trans_utils.h +++ b/src/ge/common/formats/utils/formats_trans_utils.h @@ -69,6 +69,7 @@ T Ceil(T n1, T n2) { } return (n2 != 0) ? (n1 - 1) / n2 + 1 : 0; } + } // namespace formats } // namespace ge #endif // GE_COMMON_FORMATS_UTILS_FORMATS_TRANS_UTILS_H_ diff --git a/src/ge/common/fp16_t.h b/src/ge/common/fp16_t.h index 0fda2cd2..34908b95 100644 --- a/src/ge/common/fp16_t.h +++ b/src/ge/common/fp16_t.h @@ -600,5 +600,5 @@ int16_t GetManBitLength(T man) { } return len; } -} // namespace ge +}; // namespace ge #endif // GE_COMMON_FP16_T_H_ diff --git a/src/ge/common/ge/op_tiling_manager.cc b/src/ge/common/ge/op_tiling_manager.cc deleted file mode 100644 index 7fb7a8fc..00000000 --- a/src/ge/common/ge/op_tiling_manager.cc +++ /dev/null @@ -1,81 +0,0 @@ -/** - * Copyright 2019-2020 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "common/ge/op_tiling_manager.h" -#include "framework/common/debug/log.h" -#include - -namespace { -const char *const kEnvName = "ASCEND_OPP_PATH"; -const std::string kDefaultPath = "/usr/local/Ascend/opp"; -const std::string kDefaultBuiltInTilingPath = "/op_impl/built-in/liboptiling.so"; -const std::string kDefaultCustomTilingPath = "/op_impl/custom/liboptiling.so"; -const uint8_t kPrefixIndex = 9; -} // namespace - -namespace ge { -void OpTilingManager::ClearHandles() noexcept { - for (const auto &handle : handles_) { - if (dlclose(handle.second) != 0) { - GELOGE(FAILED, "Failed to close handle of %s: %s", handle.first.c_str(), dlerror()); - } - } - handles_.clear(); -} - -OpTilingManager::~OpTilingManager() { ClearHandles(); } - -std::string OpTilingManager::GetPath() { - const char *opp_path_env = std::getenv(kEnvName); - std::string opp_path = kDefaultPath; - if (opp_path_env != nullptr) { - char resolved_path[PATH_MAX]; - if (realpath(opp_path_env, resolved_path) == NULL) { - GELOGE(PARAM_INVALID, "Failed load tiling lib as env 'ASCEND_OPP_PATH'(%s) is invalid path.", opp_path_env); - return std::string(); - } - opp_path = resolved_path; - } - return opp_path; -} - -void OpTilingManager::LoadSo() { - std::string opp_path = GetPath(); - if (opp_path.empty()) { - GELOGW("Skip load tiling lib."); - return; - } - std::string built_in_tiling_lib = opp_path + kDefaultBuiltInTilingPath; - std::string custom_tiling_lib = opp_path + kDefaultCustomTilingPath; - std::string built_in_name = kDefaultBuiltInTilingPath.substr(kPrefixIndex); - std::string custom_name = kDefaultCustomTilingPath.substr(kPrefixIndex); - - void *handle_bi = dlopen(built_in_tiling_lib.c_str(), RTLD_NOW | RTLD_GLOBAL); - if (handle_bi == nullptr) { - GELOGW("Failed to dlopen %s!", dlerror()); - } else { - handles_[built_in_name] = handle_bi; - } - - void *handle_ct = dlopen(custom_tiling_lib.c_str(), RTLD_NOW | RTLD_GLOBAL); - if (handle_ct == nullptr) { - GELOGW("Failed to dlopen %s!", dlerror()); - } else { - handles_[custom_name] = handle_ct; - } -} - -} // namespace ge diff --git a/src/ge/common/ge/op_tiling_manager.h b/src/ge/common/ge/op_tiling_manager.h deleted file mode 100644 index 320e1411..00000000 --- a/src/ge/common/ge/op_tiling_manager.h +++ /dev/null @@ -1,38 +0,0 @@ -/** - * Copyright 2019-2020 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef GE_COMMON_GE_OP_TILING_MANAGER_H_ -#define GE_COMMON_GE_OP_TILING_MANAGER_H_ - -#include - -namespace ge { -using SoToHandleMap = std::map; - -class OpTilingManager { - public: - OpTilingManager() = default; - ~OpTilingManager(); - void LoadSo(); - - private: - static std::string GetPath(); - void ClearHandles() noexcept; - SoToHandleMap handles_; -}; -} // namespace ge - -#endif // GE_COMMON_GE_OP_TILING_MANAGER_H_ diff --git a/src/ge/common/helper/model_helper.cc b/src/ge/common/helper/model_helper.cc index e1f7c75f..2f95cbb1 100644 --- a/src/ge/common/helper/model_helper.cc +++ b/src/ge/common/helper/model_helper.cc @@ -17,7 +17,6 @@ #include "framework/common/helper/model_helper.h" #include "common/ge/ge_util.h" -#include "common/util/error_manager/error_manager.h" #include "framework/common/debug/log.h" #include "framework/common/util.h" #include "framework/common/debug/ge_log.h" @@ -268,7 +267,6 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ModelHelper::LoadModel(c } auto partition_table = reinterpret_cast(model_addr_tmp_); if (partition_table->num == kOriginalOmPartitionNum) { - model_addr_tmp_ = nullptr; GELOGE(FAILED, "om model is error,please use executable om model"); return FAILED; } @@ -392,6 +390,107 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY GeModelPtr ModelHelper::GetGeMo return out_model; } +// Transit func for model to ge_model. It will be removed when load and build support ge_model in future +FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ModelHelper::TransModelToGeModel(const ModelPtr &model, + GeModelPtr &ge_model) { + if (model == nullptr) { + GELOGE(FAILED, "Model is null"); + return FAILED; + } + ge_model = ge::MakeShared(); + GE_CHECK_NOTNULL(ge_model); + ge_model->SetGraph(model->GetGraph()); + ge_model->SetName(model->GetName()); + ge_model->SetVersion(model->GetVersion()); + ge_model->SetPlatformVersion(model->GetPlatformVersion()); + ge_model->SetAttr(model->MutableAttrMap()); + + // Copy weight info + auto compute_graph = ge::GraphUtils::GetComputeGraph(model->GetGraph()); + // ge::Buffer weight; + ge::Buffer weight; + (void)ge::AttrUtils::GetZeroCopyBytes(compute_graph, ge::ATTR_NAME_WEIGHTS_DATA, weight); + ge_model->SetWeight(weight); + // Copy task info + if (model->HasAttr(MODEL_ATTR_TASKS)) { + ge::Buffer task_buffer; + GE_CHK_BOOL_RET_STATUS(ge::AttrUtils::GetZeroCopyBytes(model, MODEL_ATTR_TASKS, task_buffer), FAILED, + "Get bytes failed."); + + std::shared_ptr task = ge::MakeShared(); + GE_CHECK_NOTNULL(task); + GE_IF_BOOL_EXEC(task_buffer.GetData() == nullptr, GELOGE(FAILED, "Get data fail"); return FAILED); + GE_IF_BOOL_EXEC(task_buffer.GetSize() == 0, GELOGE(FAILED, "Get size fail"); return FAILED); + + GE_CHK_BOOL_EXEC(ReadProtoFromArray(task_buffer.GetData(), static_cast(task_buffer.GetSize()), task.get()), + return INTERNAL_ERROR, "ReadProtoFromArray failed."); + + ge_model->SetModelTaskDef(task); + } + // Copy tbe kernel info + // TBEKernelStore kernel_store; + TBEKernelStore kernel_store; + if (compute_graph != nullptr && compute_graph->GetDirectNodesSize() != 0) { + for (const ge::NodePtr &n : compute_graph->GetDirectNode()) { + auto node_op_desc = n->GetOpDesc(); + GE_IF_BOOL_EXEC(node_op_desc == nullptr, continue); + TBEKernelPtr tbe_kernel = node_op_desc->TryGetExtAttr(ge::OP_EXTATTR_NAME_TBE_KERNEL, TBEKernelPtr()); + GE_IF_BOOL_EXEC(tbe_kernel == nullptr, continue); + kernel_store.AddTBEKernel(tbe_kernel); + GELOGI("Add tbe kernel bin %s", tbe_kernel->GetName().c_str()); + } + } + if (!kernel_store.Build()) { + GELOGE(FAILED, "TBE Kernels store build failed!"); + return FAILED; + } + ge_model->SetTBEKernelStore(kernel_store); + + return SUCCESS; +} + +// trasit func for ge_model to Model. will be removed when load and build support ge_model in future +FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ModelHelper::TransGeModelToModel(const GeModelPtr &ge_model, + ModelPtr &model) { + if (ge_model == nullptr) { + GELOGE(FAILED, "Ge_model is null"); + return FAILED; + } + model = ge::MakeShared(); + GE_CHECK_NOTNULL(model); + model->SetGraph(ge_model->GetGraph()); + model->SetName(ge_model->GetName()); + model->SetVersion(ge_model->GetVersion()); + model->SetPlatformVersion(ge_model->GetPlatformVersion()); + model->SetAttr(ge_model->MutableAttrMap()); + // Copy weight info + auto compute_graph = ge::GraphUtils::GetComputeGraph(model->GetGraph()); + bool ret = ge::AttrUtils::SetZeroCopyBytes(compute_graph, ge::ATTR_NAME_WEIGHTS_DATA, ge_model->GetWeight()); + if (!ret) { + GELOGE(FAILED, "Copy weight buffer failed!"); + return FAILED; + } + // Copy task info + std::shared_ptr model_task = ge_model->GetModelTaskDefPtr(); + + if (model_task != nullptr) { + int size = model_task->ByteSize(); + ge::Buffer buffer(static_cast(size)); + if (buffer.GetSize() == 0) { + GELOGE(MEMALLOC_FAILED, "alloc model attr task buffer failed!"); + return MEMALLOC_FAILED; + } + // no need to check value + (void)model_task->SerializePartialToArray(buffer.GetData(), size); + ret = ge::AttrUtils::SetZeroCopyBytes(model, MODEL_ATTR_TASKS, std::move(buffer)); + if (!ret) { + GELOGE(FAILED, "Copy task buffer failed!"); + return FAILED; + } + } + return SUCCESS; +} + Status ModelHelper::ReleaseLocalModelData() noexcept { Status result = SUCCESS; if (model_addr_tmp_ != nullptr) { diff --git a/src/ge/common/math/fp16_math.h b/src/ge/common/math/fp16_math.h index c3a4eb28..5bc9ac6d 100644 --- a/src/ge/common/math/fp16_math.h +++ b/src/ge/common/math/fp16_math.h @@ -92,5 +92,5 @@ fp16_t max(fp16_t fp1, fp16_t fp2); /// @brief Calculate the minimum fp16_t of fp1 and fp2 /// @return Returns minimum fp16_t of fp1 and fp2 fp16_t min(fp16_t fp1, fp16_t fp2); -} // namespace ge +}; // namespace ge #endif // GE_COMMON_MATH_FP16_MATH_H_ \ No newline at end of file diff --git a/src/ge/common/math_util.h b/src/ge/common/math_util.h index a12be9e0..5e783e81 100644 --- a/src/ge/common/math_util.h +++ b/src/ge/common/math_util.h @@ -27,6 +27,7 @@ #include "mmpa/mmpa_api.h" namespace ge { + /** * @ingroup domi_calibration * @brief Initializes an input array to a specified value @@ -66,6 +67,7 @@ Status NnSet(const int32_t n, const Dtype alpha, Dtype *output) { } return SUCCESS; } + } // end namespace ge #endif // GE_COMMON_MATH_UTIL_H_ diff --git a/src/ge/common/model_saver.cc b/src/ge/common/model_saver.cc index 821fde60..11d9e804 100644 --- a/src/ge/common/model_saver.cc +++ b/src/ge/common/model_saver.cc @@ -60,8 +60,8 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ModelSaver::SaveJsonToFi mode_t mode = S_IRUSR | S_IWUSR; int32_t fd = mmOpen2(real_path, O_RDWR | O_CREAT | O_TRUNC, mode); if (fd == EN_ERROR || fd == EN_INVALID_PARAM) { - ErrorManager::GetInstance().ATCReportErrMessage("E19001", {"file", "errmsg"}, {file_path, strerror(errno)}); - GELOGE(FAILED, "Open file[%s] failed. %s", file_path, strerror(errno)); + ErrorManager::GetInstance().ATCReportErrMessage("E19001", {"filepath", "errMsg"}, {file_path, strerror(errno)}); + GELOGE(FAILED, "Open file failed. file path : %s, %s", file_path, strerror(errno)); return FAILED; } const char *model_char = model_str.c_str(); @@ -69,7 +69,8 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ModelSaver::SaveJsonToFi // Write data to file mmSsize_t mmpa_ret = mmWrite(fd, const_cast((const void *)model_char), len); if (mmpa_ret == EN_ERROR || mmpa_ret == EN_INVALID_PARAM) { - ErrorManager::GetInstance().ATCReportErrMessage("E19004", {"file", "errmsg"}, {file_path, strerror(errno)}); + ErrorManager::GetInstance().ATCReportErrMessage("E19003", {"mmpa_ret", "errMsg"}, + {std::to_string(mmpa_ret), strerror(errno)}); // Need to both print the error info of mmWrite and mmClose, so return ret after mmClose GELOGE(FAILED, "Write to file failed. errno = %d, %s", mmpa_ret, strerror(errno)); ret = FAILED; diff --git a/src/ge/common/profiling/profiling_manager.cc b/src/ge/common/profiling/profiling_manager.cc index 04d23546..ecbbf5f2 100644 --- a/src/ge/common/profiling/profiling_manager.cc +++ b/src/ge/common/profiling/profiling_manager.cc @@ -16,15 +16,12 @@ #include "common/profiling/profiling_manager.h" -#include #include "framework/common/debug/ge_log.h" #include "framework/common/debug/log.h" #include "framework/common/string_util.h" #include "graph/ge_context.h" #include "runtime/base.h" -using Json = nlohmann::json; - namespace { const char *const kJobID = "jobID"; const char *const kDeviceID = "deviceID"; @@ -35,6 +32,7 @@ const char *const kEvents = "events"; const char *const kAiCoreEvents = "ai_core_events"; const char *const kName = "name"; const char *const kTraceID = "traceId"; +const char *const kProfDir = "resultPath"; const size_t kReportMaxLen = 2048; } // namespace @@ -100,6 +98,10 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ge::Status ProfilingManager::In Json start_prof_conf = Json::parse(config); Json &prof_conf = start_prof_conf[kStartCfg][0]; job_id_ = prof_conf[kJobID]; + auto iter = prof_conf.find(kProfDir); + if (iter != prof_conf.end()) { + prof_dir_ = prof_conf[kProfDir]; + } Json &device_id = prof_conf[kDeviceID]; if (device_id.size() != 0) { vector().swap(device_id_); @@ -126,23 +128,36 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ge::Status ProfilingManager::In } } - GELOGI("Profiling json config from acl:%s", config.c_str()); Json &features = prof_conf[kFeatures]; + if (ParseFeaturesFromAclCfg(features) != SUCCESS) { + GELOGE(FAILED, "Parse feature from acl cfg failed."); + return FAILED; + } + is_profiling_ = true; + } catch (...) { + GELOGE(FAILED, "Json conf is not invalid !"); + return ge::PARAM_INVALID; + } +#endif + return ge::SUCCESS; +} + +FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ge::Status ProfilingManager::ParseFeaturesFromAclCfg( + const Json &features) { +#ifdef DAVINCI_SUPPORT_PROFILING + try { for (size_t i = 0; i < features.size(); ++i) { - Json &feature = features[i]; + const Json &feature = features[i]; if ((feature.find(kName) == feature.end()) || feature[kName].is_null()) { continue; } - const std::string &name = feature[kName]; if (name == "op_trace") { - GELOGI("Op trace config from acl"); - Json &conf = feature[kConf]; - Json &events = conf[0][kEvents]; + const Json &conf = feature[kConf]; + const Json &events = conf[0][kEvents]; const std::string &ai_core_events = events[0][kAiCoreEvents]; GELOGI("Op trace config from acl ai_core_events:%s", ai_core_events.c_str()); is_op_trace_ = true; - // op trace get conf ProfMgrConf prof_mgr_conf; int result = ProfMgrGetConf(ai_core_events, &prof_mgr_conf); if (result != 0) { @@ -154,10 +169,16 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ge::Status ProfilingManager::In GELOGI("Op trace profiling iter num %d,", op_trace_iter_num_); } else if (name == "task_trace") { is_op_trace_ = false; + if (feature.find(kConf) != feature.end()) { + const Json &conf = feature[kConf]; + std::stringstream task_trace_conf; + task_trace_conf << conf; + task_trace_conf_ = task_trace_conf.str(); + } GELOGI("Task trace config from acl"); } else if (name == "system_trace") { is_op_trace_ = false; - Json &conf = feature[kConf]; + const Json &conf = feature[kConf]; std::stringstream system_trace_conf; system_trace_conf << conf; system_trace_conf_ = system_trace_conf.str(); @@ -165,10 +186,8 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ge::Status ProfilingManager::In } profiling_opts_.push_back(name); } - - is_profiling_ = true; } catch (...) { - GELOGE(FAILED, "Json conf is not invalid !"); + GELOGE(ge::PARAM_INVALID, "Json conf feature is not invalid !"); return ge::PARAM_INVALID; } #endif @@ -235,6 +254,10 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ge::Status ProfilingManager::St p_device[kDeviceID] = std::to_string(device_id); p_device[kJobID] = job_id_; p_device[kTraceID] = std::to_string(GetContext().TraceId()); + if (!prof_dir_.empty()) { + p_device[kProfDir] = prof_dir_; + GELOGI("Prof dir: %s.", prof_dir_.c_str()); + } Json features; if (is_op_trace_) { @@ -258,6 +281,10 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ge::Status ProfilingManager::St Json f; if (profiling_opts_[i] == "system_trace") { f[kConf] = nlohmann::json::parse(system_trace_conf_); + } else if (profiling_opts_[i] == "task_trace") { + if (!task_trace_conf_.empty()) { + f[kConf] = nlohmann::json::parse(task_trace_conf_); + } } f[kName] = profiling_opts_[i]; features[i] = f; @@ -336,17 +363,16 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::Profilin std::string data; for (const auto &task : task_desc_info) { - std::string model_name = task.model_name; std::string op_name = task.op_name; uint32_t block_dim = task.block_dim; uint32_t task_id = task.task_id; uint32_t stream_id = task.stream_id; - data = model_name.append(" ").append(op_name).append(" ").append(std::to_string(block_dim) - .append(" ") - .append(std::to_string(task_id)) - .append(" ") - .append(std::to_string(stream_id)) - .append("\n")); + data = op_name.append(" ").append(std::to_string(block_dim) + .append(" ") + .append(std::to_string(task_id)) + .append(" ") + .append(std::to_string(stream_id)) + .append("\n")); Msprof::Engine::ReporterData reporter_data{}; reporter_data.deviceId = device_id; @@ -377,12 +403,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::Profilin std::string data; for (const auto &graph : compute_graph_desc_info) { - data.append("model_name:") - .append(graph.model_name) - .append(" op_name:") - .append(graph.op_name) - .append(" op_type:") - .append(graph.op_type); + data.append("op_name:").append(graph.op_name).append(" op_type:").append(graph.op_type); for (size_t i = 0; i < graph.input_format.size(); ++i) { data.append(" input_id:") .append(std::to_string(i)) diff --git a/src/ge/common/profiling/profiling_manager.h b/src/ge/common/profiling/profiling_manager.h index 2dc0b407..26ee84ca 100644 --- a/src/ge/common/profiling/profiling_manager.h +++ b/src/ge/common/profiling/profiling_manager.h @@ -17,6 +17,7 @@ #ifndef GE_COMMON_PROFILING_PROFILING_MANAGER_H_ #define GE_COMMON_PROFILING_PROFILING_MANAGER_H_ +#include #include #include #include @@ -30,6 +31,7 @@ using std::map; using std::string; using std::vector; +using Json = nlohmann::json; namespace ge { const std::string GE_PROFILING_MODULE = "Framework"; @@ -84,11 +86,13 @@ class FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ProfilingManager { void PluginUnInit(const std::string &module) const; private: + ge::Status ParseFeaturesFromAclCfg(const Json &feature); bool is_profiling_ = false; bool is_op_trace_ = false; bool is_load_ = false; int32_t op_trace_iter_num_ = 0; string job_id_; + string prof_dir_; vector device_id_; vector op_trace_conf_; vector profiling_opts_; @@ -96,6 +100,7 @@ class FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ProfilingManager { string recv_profiling_config_; string send_profiling_config_; string system_trace_conf_; + string task_trace_conf_; const ProfilingEngineImpl engine_; }; } // namespace ge diff --git a/src/ge/common/properties_manager.cc b/src/ge/common/properties_manager.cc index 0c2b1db6..cf1ada05 100644 --- a/src/ge/common/properties_manager.cc +++ b/src/ge/common/properties_manager.cc @@ -20,204 +20,15 @@ #include #include -#include "common/ge/ge_util.h" #include "common/util.h" #include "framework/common/debug/ge_log.h" #include "framework/common/debug/log.h" #include "framework/common/ge_types.h" #include "framework/common/types.h" #include "graph/debug/ge_attr_define.h" -#include "graph/ge_context.h" #include "graph/utils/attr_utils.h" namespace ge { -namespace { -const string kEnableFlag = "1"; - -const uint32_t kAicoreOverflow = (0x1 << 0); -const uint32_t kAtomicOverflow = (0x1 << 1); -const uint32_t kAllOverflow = (kAicoreOverflow | kAtomicOverflow); -} // namespace - -FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY DumpProperties::DumpProperties(const DumpProperties &other) { - CopyFrom(other); -} - -FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY DumpProperties &DumpProperties::operator=( - const DumpProperties &other) { - CopyFrom(other); - return *this; -} - -FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void DumpProperties::InitByOptions() { - enable_dump_.clear(); - enable_dump_debug_.clear(); - dump_path_.clear(); - dump_step_.clear(); - dump_mode_.clear(); - is_op_debug_ = false; - op_debug_mode_ = 0; - - string enable_dump; - (void)GetContext().GetOption(OPTION_EXEC_ENABLE_DUMP, enable_dump); - enable_dump_ = enable_dump; - - string enable_dump_debug; - (void)GetContext().GetOption(OPTION_EXEC_ENABLE_DUMP_DEBUG, enable_dump_debug); - enable_dump_debug_ = enable_dump_debug; - - if ((enable_dump_ == kEnableFlag) || (enable_dump_debug_ == kEnableFlag)) { - string dump_path; - if (GetContext().GetOption(OPTION_EXEC_DUMP_PATH, dump_path) == GRAPH_SUCCESS) { - if (!dump_path.empty() && dump_path[dump_path.size() - 1] != '/') { - dump_path = dump_path + "/"; - } - dump_path = dump_path + CurrentTimeInStr() + "/"; - GELOGI("Get dump path %s successfully", dump_path.c_str()); - SetDumpPath(dump_path); - } else { - GELOGW("DUMP_PATH is not set"); - } - } - - if (enable_dump_ == kEnableFlag) { - string dump_step; - if (GetContext().GetOption(OPTION_EXEC_DUMP_STEP, dump_step) == GRAPH_SUCCESS) { - GELOGD("Get dump step %s successfully", dump_step.c_str()); - SetDumpStep(dump_step); - } - string dump_mode; - if (GetContext().GetOption(OPTION_EXEC_DUMP_MODE, dump_mode) == GRAPH_SUCCESS) { - GELOGD("Get dump mode %s successfully", dump_mode.c_str()); - SetDumpMode(dump_mode); - } - AddPropertyValue(DUMP_ALL_MODEL, {}); - } - - SetDumpDebugOptions(); -} - -// The following is the new dump scenario of the fusion operator -FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void DumpProperties::AddPropertyValue( - const std::string &model, const std::set &layers) { - for (const std::string &layer : layers) { - GELOGI("This model %s config to dump layer %s", model.c_str(), layer.c_str()); - } - - model_dump_properties_map_[model] = layers; -} - -FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void DumpProperties::DeletePropertyValue(const std::string &model) { - auto iter = model_dump_properties_map_.find(model); - if (iter != model_dump_properties_map_.end()) { - model_dump_properties_map_.erase(iter); - } -} - -FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY std::set DumpProperties::GetAllDumpModel() const { - std::set model_list; - for (auto &iter : model_dump_properties_map_) { - model_list.insert(iter.first); - } - - return model_list; -} - -FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY std::set DumpProperties::GetPropertyValue( - const std::string &model) const { - auto iter = model_dump_properties_map_.find(model); - if (iter != model_dump_properties_map_.end()) { - return iter->second; - } - return {}; -} - -FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY bool DumpProperties::IsLayerNeedDump( - const std::string &model, const std::string &om_name, const std::string &op_name) const { - // if dump all - if (model_dump_properties_map_.find(DUMP_ALL_MODEL) != model_dump_properties_map_.end()) { - return true; - } - - // if this model need dump - auto om_name_iter = model_dump_properties_map_.find(om_name); - auto model_name_iter = model_dump_properties_map_.find(model); - if (om_name_iter != model_dump_properties_map_.end() || model_name_iter != model_dump_properties_map_.end()) { - // if no dump layer info, dump all layer in this model - auto model_iter = om_name_iter != model_dump_properties_map_.end() ? om_name_iter : model_name_iter; - if (model_iter->second.empty()) { - return true; - } - - return model_iter->second.find(op_name) != model_iter->second.end(); - } - - GELOGD("Model %s is not seated to be dump.", model.c_str()); - return false; -} - -FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void DumpProperties::SetDumpPath(const std::string &path) { - dump_path_ = path; -} - -FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY std::string DumpProperties::GetDumpPath() const { return dump_path_; } - -FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void DumpProperties::SetDumpStep(const std::string &step) { - dump_step_ = step; -} - -FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY std::string DumpProperties::GetDumpStep() const { return dump_step_; } - -FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void DumpProperties::SetDumpMode(const std::string &mode) { - dump_mode_ = mode; -} - -FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY std::string DumpProperties::GetDumpMode() const { return dump_mode_; } - -void DumpProperties::CopyFrom(const DumpProperties &other) { - if (&other != this) { - enable_dump_ = other.enable_dump_; - enable_dump_debug_ = other.enable_dump_debug_; - dump_path_ = other.dump_path_; - dump_step_ = other.dump_step_; - dump_mode_ = other.dump_mode_; - - model_dump_properties_map_ = other.model_dump_properties_map_; - is_op_debug_ = other.is_op_debug_; - op_debug_mode_ = other.op_debug_mode_; - } -} - -void DumpProperties::SetDumpDebugOptions() { - if (enable_dump_debug_ == kEnableFlag) { - string dump_debug_mode; - if (GetContext().GetOption(OPTION_EXEC_DUMP_DEBUG_MODE, dump_debug_mode) == GRAPH_SUCCESS) { - GELOGD("Get dump debug mode %s successfully", dump_debug_mode.c_str()); - } else { - GELOGW("Dump debug mode is not set."); - return; - } - - if (dump_debug_mode == OP_DEBUG_AICORE) { - GELOGD("ge.exec.dumpDebugMode=aicore_overflow, op debug is open."); - is_op_debug_ = true; - op_debug_mode_ = kAicoreOverflow; - } else if (dump_debug_mode == OP_DEBUG_ATOMIC) { - GELOGD("ge.exec.dumpDebugMode=atomic_overflow, op debug is open."); - is_op_debug_ = true; - op_debug_mode_ = kAtomicOverflow; - } else if (dump_debug_mode == OP_DEBUG_ALL) { - GELOGD("ge.exec.dumpDebugMode=all, op debug is open."); - is_op_debug_ = true; - op_debug_mode_ = kAllOverflow; - } else { - GELOGW("ge.exec.dumpDebugMode is invalid."); - } - } else { - GELOGI("ge.exec.enableDumpDebug is false or is not set."); - } -} - PropertiesManager::PropertiesManager() : is_inited_(false), delimiter("=") {} PropertiesManager::~PropertiesManager() {} @@ -348,22 +159,131 @@ PropertiesManager::GetPropertyMap() { // Set separator FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void PropertiesManager::SetPropertyDelimiter(const std::string &de) { - std::lock_guard lock(mutex_); delimiter = de; } -FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY DumpProperties &PropertiesManager::GetDumpProperties( - uint64_t session_id) { - std::lock_guard lock(mutex_); - // If session_id is not found in dump_properties_map_, operator[] will insert one. - return dump_properties_map_[session_id]; +// The following is the new dump scenario of the fusion operator +FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void PropertiesManager::AddDumpPropertyValue( + const std::string &model, const std::set &layers) { + for (const std::string &layer : layers) { + GELOGI("This model %s config to dump layer %s", model.c_str(), layer.c_str()); + } + + std::lock_guard lock(dump_mutex_); + model_dump_properties_map_[model] = layers; } -FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void PropertiesManager::RemoveDumpProperties(uint64_t session_id) { - std::lock_guard lock(mutex_); - auto iter = dump_properties_map_.find(session_id); - if (iter != dump_properties_map_.end()) { - dump_properties_map_.erase(iter); +FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void PropertiesManager::DeleteDumpPropertyValue( + const std::string &model) { + std::lock_guard lock(dump_mutex_); + auto iter = model_dump_properties_map_.find(model); + if (iter != model_dump_properties_map_.end()) { + model_dump_properties_map_.erase(iter); + } +} + +FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void PropertiesManager::ClearDumpPropertyValue() { + std::lock_guard lock(dump_mutex_); + model_dump_properties_map_.clear(); +} + +FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY std::set PropertiesManager::GetAllDumpModel() { + std::set model_list; + std::lock_guard lock(dump_mutex_); + for (auto &iter : model_dump_properties_map_) { + model_list.insert(iter.first); + } + + return model_list; +} + +FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY std::set PropertiesManager::GetDumpPropertyValue( + const std::string &model) { + std::lock_guard lock(dump_mutex_); + auto iter = model_dump_properties_map_.find(model); + if (iter != model_dump_properties_map_.end()) { + return iter->second; + } + return {}; +} + +FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY bool PropertiesManager::IsLayerNeedDump(const std::string &model, + const std::string &om_name, + const std::string &op_name) { + std::lock_guard lock(dump_mutex_); + // if dump all + if (model_dump_properties_map_.find(ge::DUMP_ALL_MODEL) != model_dump_properties_map_.end()) { + return true; } + + // if this model need dump + auto om_name_iter = model_dump_properties_map_.find(om_name); + auto model_name_iter = model_dump_properties_map_.find(model); + if (om_name_iter != model_dump_properties_map_.end() || model_name_iter != model_dump_properties_map_.end()) { + // if no dump layer info, dump all layer in this model + auto model_iter = om_name_iter != model_dump_properties_map_.end() ? om_name_iter : model_name_iter; + if (model_iter->second.empty()) { + return true; + } + + return model_iter->second.find(op_name) != model_iter->second.end(); + } + + GELOGD("Model %s is not seated to be dump.", model.c_str()); + return false; +} + +FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY bool PropertiesManager::QueryModelDumpStatus( + const std::string &model) { + std::lock_guard lock(dump_mutex_); + auto iter = model_dump_properties_map_.find(model); + if (iter != model_dump_properties_map_.end()) { + return true; + } else if (model_dump_properties_map_.find(ge::DUMP_ALL_MODEL) != model_dump_properties_map_.end()) { + return true; + } + return false; +} + +FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void PropertiesManager::SetDumpOutputModel( + const std::string &output_mode) { + std::lock_guard lock(dump_mutex_); + this->output_mode_ = output_mode; +} + +FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY std::string PropertiesManager::GetDumpOutputModel() { + std::lock_guard lock(dump_mutex_); + return this->output_mode_; +} + +FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void PropertiesManager::SetDumpOutputPath( + const std::string &output_path) { + std::lock_guard lock(dump_mutex_); + this->output_path_ = output_path; +} + +FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY std::string PropertiesManager::GetDumpOutputPath() { + std::lock_guard lock(dump_mutex_); + return this->output_path_; +} + +FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void PropertiesManager::SetDumpStep(const std::string &dump_step) { + std::lock_guard lock(dump_mutex_); + this->dump_step_ = dump_step; +} + +FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY std::string PropertiesManager::GetDumpStep() { + std::lock_guard lock(dump_mutex_); + return this->dump_step_; +} + +FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void PropertiesManager::SetDumpMode(const std::string &dump_mode) { + std::lock_guard lock(dump_mutex_); + this->dump_mode_ = dump_mode; +} + +FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY std::string PropertiesManager::GetDumpMode() { + std::lock_guard lock(dump_mutex_); + return this->dump_mode_; } } // namespace ge diff --git a/src/ge/common/properties_manager.h b/src/ge/common/properties_manager.h index 3b1547f5..7cbb5949 100644 --- a/src/ge/common/properties_manager.h +++ b/src/ge/common/properties_manager.h @@ -32,50 +32,6 @@ static const char *USE_FUSION __attribute__((unused)) = "FMK_USE_FUSION"; static const char *TIMESTAT_ENABLE __attribute__((unused)) = "DAVINCI_TIMESTAT_ENABLE"; static const char *ANNDROID_DEBUG __attribute__((unused)) = "ANNDROID_DEBUG"; -class DumpProperties { - public: - DumpProperties() = default; - ~DumpProperties() = default; - DumpProperties(const DumpProperties &dump); - DumpProperties &operator=(const DumpProperties &dump); - - void InitByOptions(); - - void AddPropertyValue(const std::string &model, const std::set &layers); - void DeletePropertyValue(const std::string &model); - - std::set GetAllDumpModel() const; - std::set GetPropertyValue(const std::string &model) const; - bool IsLayerNeedDump(const std::string &model, const std::string &om_name, const std::string &op_name) const; - - void SetDumpPath(const std::string &path); - std::string GetDumpPath() const; - - void SetDumpStep(const std::string &step); - std::string GetDumpStep() const; - - void SetDumpMode(const std::string &mode); - std::string GetDumpMode() const; - - bool IsOpDebugOpen() const { return is_op_debug_; } - uint32_t GetOpDebugMode() const { return op_debug_mode_; } - - private: - void CopyFrom(const DumpProperties &other); - void SetDumpDebugOptions(); - - string enable_dump_; - string enable_dump_debug_; - - std::string dump_path_; - std::string dump_step_; - std::string dump_mode_; - std::map> model_dump_properties_map_; - - bool is_op_debug_ = false; - uint32_t op_debug_mode_ = 0; -}; - class PropertiesManager { public: // Singleton @@ -125,8 +81,21 @@ class PropertiesManager { */ void SetPropertyDelimiter(const std::string &de); - DumpProperties &GetDumpProperties(uint64_t session_id); - void RemoveDumpProperties(uint64_t session_id); + void AddDumpPropertyValue(const std::string &model, const std::set &layers); + std::set GetAllDumpModel(); + std::set GetDumpPropertyValue(const std::string &model); + bool IsLayerNeedDump(const std::string &model, const std::string &om_name, const std::string &op_name); + void DeleteDumpPropertyValue(const std::string &model); + void ClearDumpPropertyValue(); + bool QueryModelDumpStatus(const std::string &model); + void SetDumpOutputModel(const std::string &output_model); + std::string GetDumpOutputModel(); + void SetDumpOutputPath(const std::string &output_path); + std::string GetDumpOutputPath(); + void SetDumpStep(const std::string &dump_step); + std::string GetDumpStep(); + void SetDumpMode(const std::string &dump_mode); + std::string GetDumpMode(); private: // Private construct, destructor @@ -150,7 +119,12 @@ class PropertiesManager { std::map properties_map_; std::mutex mutex_; - std::map dump_properties_map_; + std::string output_mode_; + std::string output_path_; + std::string dump_step_; + std::string dump_mode_; + std::map> model_dump_properties_map_; // model_dump_layers_map_ + std::mutex dump_mutex_; }; } // namespace ge diff --git a/src/ge/common/tbe_kernel_store.h b/src/ge/common/tbe_kernel_store.h index 51d69af2..da231358 100644 --- a/src/ge/common/tbe_kernel_store.h +++ b/src/ge/common/tbe_kernel_store.h @@ -28,6 +28,7 @@ #include "graph/op_kernel_bin.h" namespace ge { + using TBEKernel = ge::OpKernelBin; using TBEKernelPtr = std::shared_ptr; diff --git a/src/ge/common/types.cc b/src/ge/common/types.cc index 80dea8a0..97761dea 100644 --- a/src/ge/common/types.cc +++ b/src/ge/common/types.cc @@ -26,11 +26,6 @@ const std::string DUMP_LAYER = "layer"; const std::string DUMP_FILE_PATH = "path"; const std::string DUMP_MODE = "dump_mode"; -// op debug mode -const std::string OP_DEBUG_AICORE = "aicore_overflow"; -const std::string OP_DEBUG_ATOMIC = "atomic_overflow"; -const std::string OP_DEBUG_ALL = "all"; - const int DEFAULT_FORMAT = static_cast(ge::FORMAT_NCHW); // Supported public property names const std::string PROP_OME_START_TIME = "ome_start_time"; // start time @@ -282,8 +277,8 @@ REGISTER_OPTYPE_DEFINE(GETSPAN, "GetSpan"); REGISTER_OPTYPE_DEFINE(STOPGRADIENT, "StopGradient"); REGISTER_OPTYPE_DEFINE(PREVENTGRADIENT, "PreventGradient"); REGISTER_OPTYPE_DEFINE(GUARANTEECONST, "GuaranteeConst"); -REGISTER_OPTYPE_DEFINE(BROADCASTGRADIENTARGS, "BroadcastGradientArgs"); -REGISTER_OPTYPE_DEFINE(BROADCASTARGS, "BroadcastArgs"); +REGISTER_OPTYPE_DEFINE(BROADCASTGRADIENTARGS, "BroadcastGradientArgs") +REGISTER_OPTYPE_DEFINE(BROADCASTARGS, "BroadcastArgs") REGISTER_OPTYPE_DEFINE(CONFUSIONMATRIX, "ConfusionMatrix"); REGISTER_OPTYPE_DEFINE(RANK, "Rank"); REGISTER_OPTYPE_DEFINE(PLACEHOLDER, "PlaceHolder"); @@ -291,7 +286,6 @@ REGISTER_OPTYPE_DEFINE(END, "End"); REGISTER_OPTYPE_DEFINE(BASICLSTMCELL, "BasicLSTMCell"); REGISTER_OPTYPE_DEFINE(GETNEXT, "GetNext"); REGISTER_OPTYPE_DEFINE(INITDATA, "InitData"); -REGISTER_OPTYPE_DEFINE(REFIDENTITY, "RefIdentity"); /***************Ann special operator*************************/ REGISTER_OPTYPE_DEFINE(ANN_MEAN, "AnnMean"); @@ -485,72 +479,72 @@ const uint64_t ALLOC_MEMORY_MAX_SIZE = 536870912; // Max size of 512M. #endif /// -/// @brief Magic number of model file +///@brief Magic number of model file /// const uint32_t MODEL_FILE_MAGIC_NUM = 0x444F4D49; // magic number /// -/// @brief Model head length +///@brief Model head length /// const uint32_t MODEL_FILE_HEAD_LEN = 256; /// -/// @ingroup domi_omg -/// @brief Input node type +///@ingroup domi_omg +///@brief Input node type /// const std::string INPUT_TYPE = "Input"; /// -/// @ingroup domi_omg -/// @brief AIPP label, label AIPP conv operator +///@ingroup domi_omg +///@brief AIPP label, label AIPP conv operator /// const std::string AIPP_CONV_FLAG = "Aipp_Conv_Flag"; /// -/// @ingroup domi_omg -/// @brief AIPP label, label aipp data operator +///@ingroup domi_omg +///@brief AIPP label, label aipp data operator /// const std::string AIPP_DATA_FLAG = "Aipp_Data_Flag"; /// -/// @ingroup domi_omg -/// @brief Record the w dimension of model input corresponding to dynamic AIPP +///@ingroup domi_omg +///@brief Record the w dimension of model input corresponding to dynamic AIPP /// const std::string AIPP_RELATED_DATA_DIM_W = "aipp_related_data_dim_w"; /// -/// @ingroup domi_omg -/// @brief Record the H dimension of model input corresponding to dynamic AIPP +///@ingroup domi_omg +///@brief Record the H dimension of model input corresponding to dynamic AIPP /// const std::string AIPP_RELATED_DATA_DIM_H = "aipp_related_data_dim_h"; /// -/// @ingroup domi_omg -/// @brief The tag of the data operator. Mark this input to the dynamic AIPP operator +///@ingroup domi_omg +///@brief The tag of the data operator. Mark this input to the dynamic AIPP operator /// const std::string INPUT_TO_DYNAMIC_AIPP = "input_to_dynamic_aipp"; /// -/// @ingroup domi_omg -/// @brief DATA node type +///@ingroup domi_omg +///@brief DATA node type /// const std::string DATA_TYPE = "Data"; /// -/// @ingroup domi_omg -/// @brief DATA node type +///@ingroup domi_omg +///@brief DATA node type /// const std::string AIPP_DATA_TYPE = "AippData"; /// -/// @ingroup domi_omg -/// @brief Frame operator type +///@ingroup domi_omg +///@brief Frame operator type /// const std::string FRAMEWORK_OP_TYPE = "FrameworkOp"; /// -/// @ingroup domi_omg -/// @brief Data node type +///@ingroup domi_omg +///@brief Data node type /// const std::string ANN_DATA_TYPE = "AnnData"; const std::string ANN_NETOUTPUT_TYPE = "AnnNetOutput"; @@ -558,139 +552,136 @@ const std::string ANN_DEPTHCONV_TYPE = "AnnDepthConv"; const std::string ANN_CONV_TYPE = "AnnConvolution"; const std::string ANN_FC_TYPE = "AnnFullConnection"; /// -/// @ingroup domi_omg -/// @brief Convolution node type +///@ingroup domi_omg +///@brief Convolution node type /// const std::string NODE_NAME_NET_OUTPUT = "Node_Output"; const std::string NODE_NAME_END_GRAPH = "Node_EndGraph"; -const std::string NODE_NAME_OP_DEBUG = "Node_OpDebug"; -const std::string OP_TYPE_OP_DEBUG = "Opdebug"; - /// -/// @ingroup domi_omg -/// @brief Convolution node type +///@ingroup domi_omg +///@brief Convolution node type /// const std::string OP_TYPE_CONVOLUTION = "Convolution"; /// -/// @ingroup domi_omg -/// @brief Add convolution node name to AIPP +///@ingroup domi_omg +///@brief Add convolution node name to AIPP /// const std::string AIPP_CONV_OP_NAME = "aipp_conv_op"; /// -/// @ingroup domi_omg -/// @brief Operator configuration item separator +///@ingroup domi_omg +///@brief Operator configuration item separator /// const std::string OP_CONF_DELIMITER = ":"; /// -/// @ingroup domi_omg -/// @brief attr value name +///@ingroup domi_omg +///@brief attr value name /// const std::string ATTR_NAME_VALUE1 = "value1"; /// -/// @ingroup domi_omg -/// @brief attr value name, 6d_2_4d C +///@ingroup domi_omg +///@brief attr value name, 6d_2_4d C /// const std::string ATTR_NAME_INPUT_CVALUE = "input_cvalue"; /// -/// @ingroup domi_omg -/// @brief alpha default value +///@ingroup domi_omg +///@brief alpha default value /// const float ALPHA_DEFAULT_VALUE = 1.0; /// -/// @ingroup domi_omg -/// @brief beta default value +///@ingroup domi_omg +///@brief beta default value /// const float BETA_DEFAULT_VALUE = 0.0; /// -/// @ingroup domi_omg -/// @brief coef default value +///@ingroup domi_omg +///@brief coef default value /// const float COEF_DEFAULT_VALUE = 0.0; /// -/// @ingroup domi_omg -/// @brief Relu6 coef value +///@ingroup domi_omg +///@brief Relu6 coef value /// const float RELU6_COEF = 6.0; /// -/// @ingroup domi_omg -/// @brief stride default value +///@ingroup domi_omg +///@brief stride default value /// const uint32_t STRIDE_DEFAULT_VALUE = 1; /// -/// @ingroup domi_omg -/// @brief pad default value +///@ingroup domi_omg +///@brief pad default value /// const uint32_t PAD_DEFAULT_VALUE = 0; /// -/// @ingroup domi_omg -/// @brief dilation default value +///@ingroup domi_omg +///@brief dilation default value /// const int DILATION_DEFAULT_VALUE = 1; /// -/// @ingroup domi_omg -/// @brief kernel default value +///@ingroup domi_omg +///@brief kernel default value /// const uint32_t KERNEL_DEFAULT_VALUE = 0; /// -/// @ingroup domi_omg -/// @brief defaule convolution group size +///@ingroup domi_omg +///@brief defaule convolution group size /// const uint32_t DEFAULT_CONV_GROUP = 1; /// -/// @ingroup domi_omg -/// @brief Default deconvolution adj +///@ingroup domi_omg +///@brief Default deconvolution adj /// const uint32_t DEFAULT_DECONV_ADJ = 0; /// -/// @ingroup domi_omg -/// @brief Represents value 1 +///@ingroup domi_omg +///@brief Represents value 1 /// const uint32_t NUM_ONE = 1; /// -/// @ingroup domi_omg -/// @brief spatial dim size default value +///@ingroup domi_omg +///@brief spatial dim size default value /// const int32_t SPATIAL_DIM_DEFAULT_SIZE = 2; /// -/// @ingroup domi_omg -/// @brief dim extended default value +///@ingroup domi_omg +///@brief dim extended default value /// const int32_t DIM_DEFAULT_VALUE = 1; /// -/// @ingroup domi_omg -/// @brief The first weight list in opdef is filter +///@ingroup domi_omg +///@brief The first weight list in opdef is filter /// const int32_t WEIGHT_FILTER_INDEX = 0; /// -/// @ingroup domi_omg -/// @brief The second weight list in opdef is bias +///@ingroup domi_omg +///@brief The second weight list in opdef is bias /// const int32_t WEIGHT_BIAS_INDEX = 1; const int32_t TENSOR_ND_SUPPORT_SIZE = 8; /// -/// @ingroup domi_omg -/// @brief NCHW index default value +///@ingroup domi_omg +///@brief NCHW index default value /// const uint32_t NCHW_DIM_N = 0; const uint32_t NCHW_DIM_C = 1; @@ -698,8 +689,8 @@ const uint32_t NCHW_DIM_H = 2; const uint32_t NCHW_DIM_W = 3; /// -/// @ingroup domi_omg -/// @brief KCHW index default value +///@ingroup domi_omg +///@brief KCHW index default value /// const uint32_t KCHW_DIM_K = 0; const uint32_t KCHW_DIM_C = 1; @@ -707,8 +698,8 @@ const uint32_t KCHW_DIM_H = 2; const uint32_t KCHW_DIM_W = 3; /// -/// @ingroup domi_omg -/// @brief HWCK index default value +///@ingroup domi_omg +///@brief HWCK index default value /// const uint32_t HWCK_DIM_H = 0; const uint32_t HWCK_DIM_W = 1; @@ -716,8 +707,8 @@ const uint32_t HWCK_DIM_C = 2; const uint32_t HWCK_DIM_K = 3; /// -/// @ingroup domi_omg -/// @brief NHWC index default value +///@ingroup domi_omg +///@brief NHWC index default value /// const uint32_t NHWC_DIM_N = 0; const uint32_t NHWC_DIM_H = 1; @@ -725,8 +716,8 @@ const uint32_t NHWC_DIM_W = 2; const uint32_t NHWC_DIM_C = 3; /// -/// @ingroup domi_omg -/// @brief CHWN index default value +///@ingroup domi_omg +///@brief CHWN index default value /// const uint32_t CHWN_DIM_N = 3; const uint32_t CHWN_DIM_C = 0; @@ -734,23 +725,23 @@ const uint32_t CHWN_DIM_H = 1; const uint32_t CHWN_DIM_W = 2; /// -/// @ingroup domi_omg -/// @brief CHW index default value +///@ingroup domi_omg +///@brief CHW index default value /// const uint32_t CHW_DIM_C = 0; const uint32_t CHW_DIM_H = 1; const uint32_t CHW_DIM_W = 2; /// -/// @ingroup domi_omg -/// @brief HWC index default value +///@ingroup domi_omg +///@brief HWC index default value /// const uint32_t HWC_DIM_H = 0; const uint32_t HWC_DIM_W = 1; const uint32_t HWC_DIM_C = 2; /// -/// @ingroup domi_omg -/// @brief Pad index default value +///@ingroup domi_omg +///@brief Pad index default value /// const uint32_t PAD_H_HEAD = 0; const uint32_t PAD_H_TAIL = 1; @@ -758,35 +749,35 @@ const uint32_t PAD_W_HEAD = 2; const uint32_t PAD_W_TAIL = 3; /// -/// @ingroup domi_omg -/// @brief window index default value +///@ingroup domi_omg +///@brief window index default value /// const uint32_t WINDOW_H = 0; const uint32_t WINDOW_W = 1; /// -/// @ingroup domi_omg -/// @brief stride index default value +///@ingroup domi_omg +///@brief stride index default value /// const uint32_t STRIDE_H = 0; const uint32_t STRIDE_W = 1; /// -/// @ingroup domi_omg -/// @brief dilation index default value +///@ingroup domi_omg +///@brief dilation index default value /// const uint32_t DILATION_H = 0; const uint32_t DILATION_W = 1; /// -/// @ingroup domi_omg -/// @brief the num of XRBG channel +///@ingroup domi_omg +///@brief the num of XRBG channel /// const uint32_t XRGB_CHN_NUM = 4; /// -/// @ingroup domi_omg -/// @brief global pooling default value +///@ingroup domi_omg +///@brief global pooling default value /// const bool DEFAULT_GLOBAL_POOLING = false; @@ -810,4 +801,4 @@ const uint32_t STREAM_SWITCH_INPUT_NUM = 2; const std::string NODE_NAME_GLOBAL_STEP = "ge_global_step"; const std::string NODE_NAME_GLOBAL_STEP_ASSIGNADD = "global_step_assignadd"; -} // namespace ge +}; // namespace ge diff --git a/src/ge/common/util.cc b/src/ge/common/util.cc index 69dc7442..50ed2f33 100644 --- a/src/ge/common/util.cc +++ b/src/ge/common/util.cc @@ -56,7 +56,6 @@ const int kWarningThreshold = 536870912 * 2; // 536870912 represent 512M /// The maximum length of the file. /// Based on the security coding specification and the current actual (protobuf) model size, it is determined as 2G-1 const int kMaxFileSizeLimit = INT_MAX; -const char *const kPathValidReason = "The path can only contains 'a-z' 'A-Z' '0-9' '-' '.' '_' and chinese character"; } // namespace namespace ge { @@ -78,7 +77,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY bool ReadProtoFromBinaryFile(co std::ifstream fs(real_path, std::ifstream::in | std::ifstream::binary); if (!fs.is_open()) { - ErrorManager::GetInstance().ATCReportErrMessage("E19001", {"file", "errmsg"}, {file, "ifstream is_open failed"}); + ErrorManager::GetInstance().ATCReportErrMessage("E19004", {"realpath"}, {file}); GELOGE(ge::FAILED, "Open real path[%s] failed.", file); return false; } @@ -91,7 +90,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY bool ReadProtoFromBinaryFile(co fs.close(); if (!ret) { - ErrorManager::GetInstance().ATCReportErrMessage("E19005", {"file"}, {file}); + ErrorManager::GetInstance().ATCReportErrMessage("E19005", {"filepath"}, {file}); GELOGE(ge::FAILED, "Parse file[%s] failed.", file); return ret; } @@ -115,18 +114,17 @@ long GetFileLength(const std::string &input_file) { GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(real_path.empty(), return -1, "input_file path '%s' not valid", input_file.c_str()); unsigned long long file_length = 0; - GE_CHK_BOOL_TRUE_EXEC_WITH_LOG( - mmGetFileSize(input_file.c_str(), &file_length) != EN_OK, - ErrorManager::GetInstance().ATCReportErrMessage("E19001", {"file", "errmsg"}, {input_file, strerror(errno)}); - return -1, "Open file[%s] failed. %s", input_file.c_str(), strerror(errno)); + GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(mmGetFileSize(input_file.c_str(), &file_length) != EN_OK, + ErrorManager::GetInstance().ATCReportErrMessage("E10037", {"filepath"}, {input_file}); + return -1, "Open file[%s] failed", input_file.c_str()); GE_CHK_BOOL_TRUE_EXEC_WITH_LOG((file_length == 0), - ErrorManager::GetInstance().ATCReportErrMessage("E19015", {"filepath"}, {input_file}); + ErrorManager::GetInstance().ATCReportErrMessage("E10038", {"filepath"}, {input_file}); return -1, "File[%s] size is 0, not valid.", input_file.c_str()); GE_CHK_BOOL_TRUE_EXEC_WITH_LOG( file_length > kMaxFileSizeLimit, ErrorManager::GetInstance().ATCReportErrMessage( - "E19016", {"filepath", "filesize", "maxlen"}, + "E10039", {"filepath", "filesize", "maxlen"}, {input_file, std::to_string(file_length), std::to_string(kMaxFileSizeLimit)}); return -1, "File[%s] size %lld is out of limit: %d.", input_file.c_str(), file_length, kMaxFileSizeLimit); return static_cast(file_length); @@ -221,7 +219,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY int CreateDirectory(const std:: if (ret != 0) { if (errno != EEXIST) { ErrorManager::GetInstance().ATCReportErrMessage("E19006", {"path"}, {directory_path}); - GELOGW("Can not create directory %s. Make sure the directory exists and writable.", directory_path.c_str()); + GELOGW("Cannot create directory %s. Make sure the directory exists and writable.", directory_path.c_str()); return ret; } } @@ -232,7 +230,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY int CreateDirectory(const std:: if (ret != 0) { if (errno != EEXIST) { ErrorManager::GetInstance().ATCReportErrMessage("E19006", {"path"}, {directory_path}); - GELOGW("Can not create directory %s. Make sure the directory exists and writable.", directory_path.c_str()); + GELOGW("Cannot create directory %s. Make sure the directory exists and writable.", directory_path.c_str()); return ret; } } @@ -260,16 +258,16 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY bool ReadProtoFromText(const ch "incorrect parameter. nullptr == file || nullptr == message"); std::string real_path = RealPath(file); - GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(real_path.empty(), ErrorManager::GetInstance().ATCReportErrMessage( - "E19000", {"path", "errmsg"}, {file, strerror(errno)}); - return false, "Path[%s]'s realpath is empty, errmsg[%s]", file, strerror(errno)); + GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(real_path.empty(), + ErrorManager::GetInstance().ATCReportErrMessage("E10036", {"filepath"}, {file}); + return false, "Get path[%s]'s real path failed", file); GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(GetFileLength(real_path) == -1, return false, "file size not valid."); std::ifstream fs(real_path.c_str(), std::ifstream::in); if (!fs.is_open()) { - ErrorManager::GetInstance().ATCReportErrMessage("E19017", {"realpth", "protofile"}, {real_path, file}); + ErrorManager::GetInstance().ATCReportErrMessage("E10040", {"realpth", "protofile"}, {real_path, file}); GELOGE(ge::FAILED, "Fail to open proto file real path is '%s' when orginal file path is '%s'.", real_path.c_str(), file); return false; @@ -277,7 +275,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY bool ReadProtoFromText(const ch google::protobuf::io::IstreamInputStream input(&fs); bool ret = google::protobuf::TextFormat::Parse(&input, message); - GE_IF_BOOL_EXEC(!ret, ErrorManager::GetInstance().ATCReportErrMessage("E19018", {"protofile"}, {file}); + GE_IF_BOOL_EXEC(!ret, ErrorManager::GetInstance().ATCReportErrMessage("E10041", {"protofile"}, {file}); GELOGE(ret, "Parse file[%s] through [google::protobuf::TextFormat::Parse] failed, " "please check whether the file is a valid protobuf format file.", @@ -362,14 +360,14 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY bool CheckInputPathValid(const // The specified path is empty std::map args_map; if (file_path.empty()) { - ErrorManager::GetInstance().ATCReportErrMessage("E10004", {"parameter"}, {atc_param}); + ErrorManager::GetInstance().ATCReportErrMessage("E10000", {"parameter"}, {atc_param}); GELOGW("Input parameter's value is empty."); return false; } std::string real_path = RealPath(file_path.c_str()); // Unable to get absolute path (does not exist or does not have permission to access) if (real_path.empty()) { - ErrorManager::GetInstance().ATCReportErrMessage("E19000", {"path", "errmsg"}, {file_path, strerror(errno)}); + ErrorManager::GetInstance().ATCReportErrMessage("E10002", {"path", "errmsg"}, {file_path.c_str(), strerror(errno)}); GELOGW("Path[%s]'s realpath is empty, errmsg[%s]", file_path.c_str(), strerror(errno)); return false; } @@ -382,14 +380,16 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY bool CheckInputPathValid(const GE_CHK_BOOL_TRUE_EXEC_WITH_LOG( !ValidateStr(real_path, mode), - ErrorManager::GetInstance().ATCReportErrMessage("E10001", {"parameter", "value", "reason"}, - {atc_param, real_path, kPathValidReason}); - return false, "Invalid value for %s[%s], %s.", atc_param.c_str(), real_path.c_str(), kPathValidReason); + ErrorManager::GetInstance().ATCReportErrMessage("E10001", {"parameter", "path"}, {atc_param, real_path}); + return false, + "Input parameter[--%s]'s value[%s] is illegal. The path can only contains 'a-z' 'A-Z' '0-9' '-' '.' '_' " + "and chinese character.", + atc_param.c_str(), real_path.c_str()); // The absolute path points to a file that is not readable if (access(real_path.c_str(), R_OK) != 0) { - ErrorManager::GetInstance().ATCReportErrMessage("E19003", {"file", "errmsg"}, {file_path.c_str(), strerror(errno)}); - GELOGW("Read file[%s] failed, errmsg[%s]", file_path.c_str(), strerror(errno)); + ErrorManager::GetInstance().ATCReportErrMessage("E10003", {"path", "errmsg"}, {file_path.c_str(), strerror(errno)}); + GELOGW("Read path[%s] failed, errmsg[%s]", file_path.c_str(), strerror(errno)); return false; } @@ -400,7 +400,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY bool CheckOutputPathValid(const const std::string &atc_param) { // The specified path is empty if (file_path.empty()) { - ErrorManager::GetInstance().ATCReportErrMessage("E10004", {"parameter"}, {atc_param}); + ErrorManager::GetInstance().ATCReportErrMessage("E10000", {"parameter"}, {atc_param}); GELOGW("Input parameter's value is empty."); return false; } @@ -416,14 +416,18 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY bool CheckOutputPathValid(const GE_CHK_BOOL_TRUE_EXEC_WITH_LOG( !ValidateStr(real_path, mode), - ErrorManager::GetInstance().ATCReportErrMessage("E10001", {"parameter", "value", "reason"}, - {atc_param, real_path, kPathValidReason}); - return false, "Invalid value for %s[%s], %s.", atc_param.c_str(), real_path.c_str(), kPathValidReason); + ErrorManager::GetInstance().ATCReportErrMessage("E10001", {"parameter", "path"}, {atc_param, real_path}); + return false, + "Input parameter[--%s]'s value[%s] is illegal. The path can only contains 'a-z' 'A-Z' '0-9' '-' '.' '_' " + "and chinese character.", + atc_param.c_str(), real_path.c_str()); // File is not readable or writable if (access(real_path.c_str(), W_OK | F_OK) != 0) { - ErrorManager::GetInstance().ATCReportErrMessage("E19004", {"file", "errmsg"}, {real_path, strerror(errno)}); - GELOGW("Write file[%s] failed, errmsg[%s]", real_path.c_str(), strerror(errno)); + ErrorManager::GetInstance().ATCReportErrMessage("E10004", {"realpath", "path", "errmsg"}, + {real_path, file_path, strerror(errno)}); + GELOGW("Write file[%s] failed, input path is %s, errmsg[%s]", real_path.c_str(), file_path.c_str(), + strerror(errno)); return false; } } else { @@ -441,8 +445,8 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY bool CheckOutputPathValid(const std::string prefix_path = std::string(file_path).substr(0, static_cast(path_split_pos)); // Determine whether the specified path is valid by creating the path if (CreateDirectory(prefix_path) != 0) { - ErrorManager::GetInstance().ATCReportErrMessage("E19006", {"path"}, {file_path}); - GELOGW("Can not create directory[%s].", file_path.c_str()); + ErrorManager::GetInstance().ATCReportErrMessage("E10005", {"path"}, {file_path}); + GELOGW("Can not create prefix path for path[%s].", file_path.c_str()); return false; } }