From 950558cab5b61913d94b94bdc8ef06370345ecc1 Mon Sep 17 00:00:00 2001 From: wxl Date: Sat, 5 Dec 2020 11:58:11 +0800 Subject: [PATCH 1/9] single op add check json file --- ge/offline/single_op_parser.cc | 32 ++++++++++++++++++++++++++++++++ ge/offline/single_op_parser.h | 11 +++++++++++ 2 files changed, 43 insertions(+) diff --git a/ge/offline/single_op_parser.cc b/ge/offline/single_op_parser.cc index d4b9c1c9..767634b5 100644 --- a/ge/offline/single_op_parser.cc +++ b/ge/offline/single_op_parser.cc @@ -28,6 +28,7 @@ #include "framework/common/util.h" #include "graph/utils/tensor_utils.h" #include "graph/utils/op_desc_utils.h" +#include "graph/utils/type_utils.h" #include "graph/operator_factory_impl.h" using Json = nlohmann::json; @@ -176,6 +177,7 @@ T GetValue(const map &dict, string &key, T default_val) { } void from_json(const Json &j, SingleOpTensorDesc &desc) { + JsonTensorVeriry tensor_verify_result; desc.dims = j.at(kKeyShape).get>(); auto it = j.find(kKeyShapeRange); if (it != j.end()) { @@ -187,6 +189,11 @@ void from_json(const Json &j, SingleOpTensorDesc &desc) { } string format_str = j.at(kKeyFormat).get(); string type_str = j.at(kKeyType).get(); + tensor_verify_result.is_format_valid = TypeUtils::IsFormatValid(format_str); + tensor_verify_result.format = format_str; + tensor_verify_result.is_dtype_valid = TypeUtils::IsFormatValid(type_str); + tensor_verify_result.dtye = type_str; + desc.format = GetValue(kFormatDict, format_str, FORMAT_RESERVED); desc.type = GetValue(kDataTypeDict, type_str, DT_UNDEFINED); it = j.find(kKeyOriginFormat); @@ -198,10 +205,12 @@ void from_json(const Json &j, SingleOpTensorDesc &desc) { if (tensor_name != j.end()) { desc.name = tensor_name->get(); } + tensor_verify_result.tensor_name = desc.name; auto dynamic_input_name = j.find(kKeyDynamicInput); if (dynamic_input_name != j.end()) { desc.dynamic_input_name = dynamic_input_name->get(); } + json_op_valid_result_.emplace_back(tensor_verify_result); } void from_json(const Json &j, SingleOpAttr &attr) { @@ -350,6 +359,22 @@ bool SingleOpParser::Validate(const SingleOpDesc &op_desc) { return true; } +Status ValidateSingleOpJson() { + for (const auto &r : json_op_valid_result_) { + if (!r.is_format_valid) { + string err_str = "json tensor format invalid.Tensor name is [" + r.tensor_name + "], format is " + r.format; + GE_ERRORLOG_AND_ERRORMSG(PARAM_INVALID, err_str); + return PARAM_INVALID; + } + if (!r.is_dtyep_valid) { + string err_str = "json tensor datatype invalid.Tensor name is [" + r.tensor_name + "], datatype is " + r.dtype; + GE_ERRORLOG_AND_ERRORMSG(PARAM_INVALID, err_str); + return PARAM_INVALID; + } + } + return SUCCESS; +} + std::unique_ptr SingleOpParser::CreateOpDesc(const string &op_type) { return std::unique_ptr(new(std::nothrow) OpDesc(op_type, op_type)); } @@ -556,6 +581,13 @@ Status SingleOpParser::ParseSingleOpList(const std::string &file, std::vector &op_list); private: + struct JsonTensorVeriry { + bool is_format_valid = true; + bool is_dtype_valid = true; + std::string tensor_name; + std::string format; + std::string dtype; + } + static Status ReadJsonFile(const std::string &file, nlohmann::json &json_obj); static bool Validate(const SingleOpDesc &op_desc); + static Status ValidateSingleOpJson(); static std::unique_ptr CreateOpDesc(const std::string &op_type); static Status ConvertToBuildParam(int index, const SingleOpDesc &single_op_desc, SingleOpBuildParam &build_param); static Status UpdateDynamicTensorName(std::vector &desc); @@ -78,6 +87,8 @@ class SingleOpParser { static Status SetShapeRange(const std::string &op_name, const SingleOpTensorDesc &tensor_desc, GeTensorDesc &ge_tensor_desc); + + static std::vector json_op_valid_result_; }; } // namespace ge From 4bb21fcde28dfa98d2a97fd207cdec6040ed99ad Mon Sep 17 00:00:00 2001 From: wxl Date: Sat, 5 Dec 2020 12:34:52 +0800 Subject: [PATCH 2/9] single op add check json file --- ge/offline/single_op_parser.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ge/offline/single_op_parser.h b/ge/offline/single_op_parser.h index c64492b4..4da25789 100644 --- a/ge/offline/single_op_parser.h +++ b/ge/offline/single_op_parser.h @@ -75,7 +75,7 @@ class SingleOpParser { std::string tensor_name; std::string format; std::string dtype; - } + }; static Status ReadJsonFile(const std::string &file, nlohmann::json &json_obj); static bool Validate(const SingleOpDesc &op_desc); From 799344f71ca8b983bdb6f3a3b0ab5df314cb66f4 Mon Sep 17 00:00:00 2001 From: wxl Date: Sat, 5 Dec 2020 14:03:51 +0800 Subject: [PATCH 3/9] single op add check json file --- ge/offline/single_op_parser.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ge/offline/single_op_parser.cc b/ge/offline/single_op_parser.cc index 767634b5..c5ad23bb 100644 --- a/ge/offline/single_op_parser.cc +++ b/ge/offline/single_op_parser.cc @@ -177,7 +177,7 @@ T GetValue(const map &dict, string &key, T default_val) { } void from_json(const Json &j, SingleOpTensorDesc &desc) { - JsonTensorVeriry tensor_verify_result; + SingleOpParser::JsonTensorVeriry tensor_verify_result; desc.dims = j.at(kKeyShape).get>(); auto it = j.find(kKeyShapeRange); if (it != j.end()) { @@ -210,7 +210,7 @@ void from_json(const Json &j, SingleOpTensorDesc &desc) { if (dynamic_input_name != j.end()) { desc.dynamic_input_name = dynamic_input_name->get(); } - json_op_valid_result_.emplace_back(tensor_verify_result); + SingleOpParser::json_op_valid_result_.emplace_back(tensor_verify_result); } void from_json(const Json &j, SingleOpAttr &attr) { @@ -372,6 +372,7 @@ Status ValidateSingleOpJson() { return PARAM_INVALID; } } + json_op_valid_result_.clear(); return SUCCESS; } From e9733134196e66f763df4e76bbb03cd9de24d88d Mon Sep 17 00:00:00 2001 From: wxl Date: Sat, 5 Dec 2020 15:01:53 +0800 Subject: [PATCH 4/9] single op add check json file --- ge/offline/single_op_parser.cc | 6 +++--- ge/offline/single_op_parser.h | 3 +++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/ge/offline/single_op_parser.cc b/ge/offline/single_op_parser.cc index c5ad23bb..4e05ff88 100644 --- a/ge/offline/single_op_parser.cc +++ b/ge/offline/single_op_parser.cc @@ -210,7 +210,7 @@ void from_json(const Json &j, SingleOpTensorDesc &desc) { if (dynamic_input_name != j.end()) { desc.dynamic_input_name = dynamic_input_name->get(); } - SingleOpParser::json_op_valid_result_.emplace_back(tensor_verify_result); + SingleOpParser::AppendJsonTensorVerifyResult(tensor_verify_result); } void from_json(const Json &j, SingleOpAttr &attr) { @@ -360,7 +360,7 @@ bool SingleOpParser::Validate(const SingleOpDesc &op_desc) { } Status ValidateSingleOpJson() { - for (const auto &r : json_op_valid_result_) { + for (const auto &r : GetJsonTensorVerifyResult()) { if (!r.is_format_valid) { string err_str = "json tensor format invalid.Tensor name is [" + r.tensor_name + "], format is " + r.format; GE_ERRORLOG_AND_ERRORMSG(PARAM_INVALID, err_str); @@ -372,7 +372,7 @@ Status ValidateSingleOpJson() { return PARAM_INVALID; } } - json_op_valid_result_.clear(); + ClearJsonTensorVerifyResult(); return SUCCESS; } diff --git a/ge/offline/single_op_parser.h b/ge/offline/single_op_parser.h index 4da25789..59bab269 100644 --- a/ge/offline/single_op_parser.h +++ b/ge/offline/single_op_parser.h @@ -80,6 +80,9 @@ class SingleOpParser { static Status ReadJsonFile(const std::string &file, nlohmann::json &json_obj); static bool Validate(const SingleOpDesc &op_desc); static Status ValidateSingleOpJson(); + static void AppendJsonTensorVerifyResult(JsonTensorVeriry result) { json_op_valid_result_.emplace_back(result); } + static void ClearJsonTensorVerifyResult() { json_op_valid_result_.clear(); } + static std::vector GetJsonTensorVerifyResult() { return json_op_valid_result_; } static std::unique_ptr CreateOpDesc(const std::string &op_type); static Status ConvertToBuildParam(int index, const SingleOpDesc &single_op_desc, SingleOpBuildParam &build_param); static Status UpdateDynamicTensorName(std::vector &desc); From 8963e822bb8ebbcba023b996b375f704f712acef Mon Sep 17 00:00:00 2001 From: wxl Date: Sat, 5 Dec 2020 15:57:36 +0800 Subject: [PATCH 5/9] single op add check json file --- ge/offline/single_op_parser.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ge/offline/single_op_parser.cc b/ge/offline/single_op_parser.cc index 4e05ff88..13641602 100644 --- a/ge/offline/single_op_parser.cc +++ b/ge/offline/single_op_parser.cc @@ -363,12 +363,12 @@ Status ValidateSingleOpJson() { for (const auto &r : GetJsonTensorVerifyResult()) { if (!r.is_format_valid) { string err_str = "json tensor format invalid.Tensor name is [" + r.tensor_name + "], format is " + r.format; - GE_ERRORLOG_AND_ERRORMSG(PARAM_INVALID, err_str); + GE_ERRORLOG_AND_ERRORMSG(PARAM_INVALID, err_str.c_str()); return PARAM_INVALID; } if (!r.is_dtyep_valid) { string err_str = "json tensor datatype invalid.Tensor name is [" + r.tensor_name + "], datatype is " + r.dtype; - GE_ERRORLOG_AND_ERRORMSG(PARAM_INVALID, err_str); + GE_ERRORLOG_AND_ERRORMSG(PARAM_INVALID, err_str.c_str()); return PARAM_INVALID; } } From 6a72f5aa8497bbb69dd80091df8bb06051abd8e9 Mon Sep 17 00:00:00 2001 From: wxl Date: Sat, 5 Dec 2020 17:35:22 +0800 Subject: [PATCH 6/9] single op add check json file --- ge/offline/single_op_parser.cc | 2 +- ge/offline/single_op_parser.h | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ge/offline/single_op_parser.cc b/ge/offline/single_op_parser.cc index 13641602..7e6fe1a9 100644 --- a/ge/offline/single_op_parser.cc +++ b/ge/offline/single_op_parser.cc @@ -192,7 +192,7 @@ void from_json(const Json &j, SingleOpTensorDesc &desc) { tensor_verify_result.is_format_valid = TypeUtils::IsFormatValid(format_str); tensor_verify_result.format = format_str; tensor_verify_result.is_dtype_valid = TypeUtils::IsFormatValid(type_str); - tensor_verify_result.dtye = type_str; + tensor_verify_result.dtype = type_str; desc.format = GetValue(kFormatDict, format_str, FORMAT_RESERVED); desc.type = GetValue(kDataTypeDict, type_str, DT_UNDEFINED); diff --git a/ge/offline/single_op_parser.h b/ge/offline/single_op_parser.h index 59bab269..3752c20f 100644 --- a/ge/offline/single_op_parser.h +++ b/ge/offline/single_op_parser.h @@ -66,9 +66,6 @@ void from_json(const nlohmann::json &json, SingleOpDesc &desc); class SingleOpParser { public: - static Status ParseSingleOpList(const std::string &file, std::vector &op_list); - - private: struct JsonTensorVeriry { bool is_format_valid = true; bool is_dtype_valid = true; @@ -76,7 +73,10 @@ class SingleOpParser { std::string format; std::string dtype; }; + public: + static Status ParseSingleOpList(const std::string &file, std::vector &op_list); + private: static Status ReadJsonFile(const std::string &file, nlohmann::json &json_obj); static bool Validate(const SingleOpDesc &op_desc); static Status ValidateSingleOpJson(); From 9d752954dd9c30330d13d1971bd30f8c167ba129 Mon Sep 17 00:00:00 2001 From: wxl Date: Sat, 5 Dec 2020 17:37:40 +0800 Subject: [PATCH 7/9] single op add check json file --- ge/offline/single_op_parser.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ge/offline/single_op_parser.h b/ge/offline/single_op_parser.h index 3752c20f..318f728e 100644 --- a/ge/offline/single_op_parser.h +++ b/ge/offline/single_op_parser.h @@ -75,14 +75,14 @@ class SingleOpParser { }; public: static Status ParseSingleOpList(const std::string &file, std::vector &op_list); + static void AppendJsonTensorVerifyResult(JsonTensorVeriry result) { json_op_valid_result_.emplace_back(result); } + static void ClearJsonTensorVerifyResult() { json_op_valid_result_.clear(); } + static std::vector GetJsonTensorVerifyResult() { return json_op_valid_result_; } private: static Status ReadJsonFile(const std::string &file, nlohmann::json &json_obj); static bool Validate(const SingleOpDesc &op_desc); static Status ValidateSingleOpJson(); - static void AppendJsonTensorVerifyResult(JsonTensorVeriry result) { json_op_valid_result_.emplace_back(result); } - static void ClearJsonTensorVerifyResult() { json_op_valid_result_.clear(); } - static std::vector GetJsonTensorVerifyResult() { return json_op_valid_result_; } static std::unique_ptr CreateOpDesc(const std::string &op_type); static Status ConvertToBuildParam(int index, const SingleOpDesc &single_op_desc, SingleOpBuildParam &build_param); static Status UpdateDynamicTensorName(std::vector &desc); From 62bbfd9d3b44c5870255a52ee5dabad40e915d39 Mon Sep 17 00:00:00 2001 From: wxl Date: Sat, 5 Dec 2020 18:26:31 +0800 Subject: [PATCH 8/9] single op add check json file --- ge/offline/single_op_parser.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ge/offline/single_op_parser.cc b/ge/offline/single_op_parser.cc index 7e6fe1a9..38374f32 100644 --- a/ge/offline/single_op_parser.cc +++ b/ge/offline/single_op_parser.cc @@ -360,7 +360,7 @@ bool SingleOpParser::Validate(const SingleOpDesc &op_desc) { } Status ValidateSingleOpJson() { - for (const auto &r : GetJsonTensorVerifyResult()) { + for (const auto &r : SingleOpParser::GetJsonTensorVerifyResult()) { if (!r.is_format_valid) { string err_str = "json tensor format invalid.Tensor name is [" + r.tensor_name + "], format is " + r.format; GE_ERRORLOG_AND_ERRORMSG(PARAM_INVALID, err_str.c_str()); From f3690f4f2cfa09743ae1ea57ac152a89b92b9bc8 Mon Sep 17 00:00:00 2001 From: wxl Date: Sat, 5 Dec 2020 19:02:03 +0800 Subject: [PATCH 9/9] single op add check json file --- ge/offline/single_op_parser.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ge/offline/single_op_parser.cc b/ge/offline/single_op_parser.cc index 38374f32..9e4c9034 100644 --- a/ge/offline/single_op_parser.cc +++ b/ge/offline/single_op_parser.cc @@ -366,13 +366,13 @@ Status ValidateSingleOpJson() { GE_ERRORLOG_AND_ERRORMSG(PARAM_INVALID, err_str.c_str()); return PARAM_INVALID; } - if (!r.is_dtyep_valid) { + if (!r.is_dtype_valid) { string err_str = "json tensor datatype invalid.Tensor name is [" + r.tensor_name + "], datatype is " + r.dtype; GE_ERRORLOG_AND_ERRORMSG(PARAM_INVALID, err_str.c_str()); return PARAM_INVALID; } } - ClearJsonTensorVerifyResult(); + SingleOpParser::ClearJsonTensorVerifyResult(); return SUCCESS; }