You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.

util.h 16 kB

5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
4 years ago
5 years ago
4 years ago
5 years ago
4 years ago
5 years ago
4 years ago
5 years ago
4 years ago
5 years ago
4 years ago
5 years ago
4 years ago
5 years ago
4 years ago
5 years ago
5 years ago
4 years ago
5 years ago
5 years ago
4 years ago
5 years ago
4 years ago
5 years ago
4 years ago
5 years ago
4 years ago
5 years ago
4 years ago
5 years ago
4 years ago
5 years ago
4 years ago
4 years ago
5 years ago
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400
  1. /**
  2. * Copyright 2019-2020 Huawei Technologies Co., Ltd
  3. *
  4. * Licensed under the Apache License, Version 2.0 (the "License");
  5. * you may not use this file except in compliance with the License.
  6. * You may obtain a copy of the License at
  7. *
  8. * http://www.apache.org/licenses/LICENSE-2.0
  9. *
  10. * Unless required by applicable law or agreed to in writing, software
  11. * distributed under the License is distributed on an "AS IS" BASIS,
  12. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13. * See the License for the specific language governing permissions and
  14. * limitations under the License.
  15. */
  16. #ifndef INC_FRAMEWORK_COMMON_UTIL_H_
  17. #define INC_FRAMEWORK_COMMON_UTIL_H_
  18. #include <google/protobuf/text_format.h>
  19. #include <climits>
  20. #include <cmath>
  21. #include <sstream>
  22. #include <string>
  23. #include <vector>
  24. #include "framework/common/debug/ge_log.h"
  25. #include "framework/common/debug/log.h"
  26. #include "framework/common/scope_guard.h"
  27. #include "framework/common/ge_inner_error_codes.h"
  28. #include "mmpa/mmpa_api.h"
  29. #define GE_CHECK_POSITIVE_SIZE_RANGE(size) \
  30. do { \
  31. if (size <= 0) { \
  32. GELOGE(ge::FAILED, "param[%s] is not a positive number", #size); \
  33. return PARAM_INVALID; \
  34. } \
  35. } while (false)
  36. #define CHECK_FALSE_EXEC(expr, exec_expr, ...) \
  37. { \
  38. const bool b = (expr); \
  39. if (!b) { \
  40. exec_expr; \
  41. } \
  42. }
  43. // new ge marco
  44. // Encapsulate common resource releases
  45. #define GE_MAKE_GUARD_RTMEM(var) \
  46. GE_MAKE_GUARD(var, [&] { \
  47. if (var) GE_CHK_RT(rtFreeHost(var)); \
  48. });
  49. #define GE_MAKE_GUARD_RTSTREAM(var) \
  50. GE_MAKE_GUARD(var, [&] { \
  51. if (var) GE_CHK_RT(rtStreamDestroy(var)); \
  52. });
  53. // For propagating errors when calling a function.
  54. #define GE_RETURN_IF_ERROR(expr) \
  55. do { \
  56. const ge::Status _chk_status = (expr); \
  57. if (_chk_status != ge::SUCCESS) { \
  58. return _chk_status; \
  59. } \
  60. } while (false)
  61. #define GE_RETURN_WITH_LOG_IF_ERROR(expr, ...) \
  62. do { \
  63. const ge::Status _chk_status = (expr); \
  64. if (_chk_status != ge::SUCCESS) { \
  65. GELOGE(ge::FAILED, __VA_ARGS__); \
  66. return _chk_status; \
  67. } \
  68. } while (false)
  69. // check whether the parameter is true. If it is, return FAILED and record the error log
  70. #define GE_RETURN_WITH_LOG_IF_TRUE(condition, ...) \
  71. do { \
  72. if (condition) { \
  73. GELOGE(ge::FAILED, __VA_ARGS__); \
  74. return ge::FAILED; \
  75. } \
  76. } while (false)
  77. // Check if the parameter is false. If yes, return FAILED and record the error log
  78. #define GE_RETURN_WITH_LOG_IF_FALSE(condition, ...) \
  79. do { \
  80. const bool _condition = (condition); \
  81. if (!_condition) { \
  82. GELOGE(ge::FAILED, __VA_ARGS__); \
  83. return ge::FAILED; \
  84. } \
  85. } while (false)
  86. // Checks whether the parameter is true. If so, returns PARAM_INVALID and records the error log
  87. #define GE_RT_PARAM_INVALID_WITH_LOG_IF_TRUE(condition, ...) \
  88. do { \
  89. if (condition) { \
  90. GELOGE(ge::FAILED, __VA_ARGS__); \
  91. return ge::PARAM_INVALID; \
  92. } \
  93. } while (false)
  94. // Check if the parameter is false. If yes, return PARAM_INVALID and record the error log
  95. #define GE_RT_PARAM_INVALID_WITH_LOG_IF_FALSE(condition, ...) \
  96. do { \
  97. const bool _condition = (condition); \
  98. if (!_condition) { \
  99. GELOGE(ge::FAILED, __VA_ARGS__); \
  100. return ge::PARAM_INVALID; \
  101. } \
  102. } while (false)
  103. // Check if the parameter is null. If yes, return PARAM_INVALID and record the error
  104. #define GE_CHECK_NOTNULL(val) \
  105. do { \
  106. if (val == nullptr) { \
  107. REPORT_INNER_ERROR("E19999", "Param:%s is nullptr, check invalid", #val); \
  108. GELOGE(ge::FAILED, "[Check][Param:%s]null is invalid.", #val); \
  109. return ge::PARAM_INVALID; \
  110. } \
  111. } while (false)
  112. // Check if the parameter is null. If yes, just return and record the error
  113. #define GE_CHECK_NOTNULL_JUST_RETURN(val) \
  114. do { \
  115. if (val == nullptr) { \
  116. GELOGE(ge::FAILED, "param[%s] must not be null.", #val); \
  117. return; \
  118. } \
  119. } while (false)
  120. // Check whether the parameter is null. If so, execute the exec_expr expression and record the error log
  121. #define GE_CHECK_NOTNULL_EXEC(val, exec_expr) \
  122. do { \
  123. if (val == nullptr) { \
  124. GELOGE(ge::FAILED, "param[%s] must not be null.", #val); \
  125. exec_expr; \
  126. } \
  127. } while (false)
  128. // Check whether the parameter is null. If yes, return directly and record the error log
  129. #define GE_RT_VOID_CHECK_NOTNULL(val) \
  130. do { \
  131. if (val == nullptr) { \
  132. GELOGE(ge::FAILED, "param[%s] must not be null.", #val); \
  133. return; \
  134. } \
  135. } while (false)
  136. // Check if the parameter is null. If yes, return false and record the error log
  137. #define GE_RT_FALSE_CHECK_NOTNULL(val) \
  138. do { \
  139. if (val == nullptr) { \
  140. GELOGE(ge::FAILED, "param[%s] must not be null.", #val); \
  141. return false; \
  142. } \
  143. } while (false)
  144. // Check if the parameter is out of bounds
  145. #define GE_CHECK_SIZE(size) \
  146. do { \
  147. if (size == 0) { \
  148. GELOGE(ge::FAILED, "param[%s] is out of range", #size); \
  149. return ge::PARAM_INVALID; \
  150. } \
  151. } while (false)
  152. // Check if the value on the left is greater than or equal to the value on the right
  153. #define GE_CHECK_GE(lhs, rhs) \
  154. do { \
  155. if (lhs < rhs) { \
  156. GELOGE(ge::FAILED, "param[%s] is less than[%s]", #lhs, #rhs); \
  157. return ge::PARAM_INVALID; \
  158. } \
  159. } while (false)
  160. // Check if the value on the left is less than or equal to the value on the right
  161. #define GE_CHECK_LE(lhs, rhs) \
  162. do { \
  163. if (lhs > rhs) { \
  164. GELOGE(ge::FAILED, "param[%s] is greater than[%s]", #lhs, #rhs); \
  165. return ge::PARAM_INVALID; \
  166. } \
  167. } while (false)
  168. #define GE_DELETE_NEW_SINGLE(var) \
  169. do { \
  170. if (var != nullptr) { \
  171. delete var; \
  172. var = nullptr; \
  173. } \
  174. } while (false)
  175. #define GE_DELETE_NEW_ARRAY(var) \
  176. do { \
  177. if (var != nullptr) { \
  178. delete[] var; \
  179. var = nullptr; \
  180. } \
  181. } while (false)
  182. #define GE_FREE_RT_LOG(addr) \
  183. do { \
  184. if (addr != nullptr) { \
  185. const rtError_t error = rtFree(addr); \
  186. if (error != RT_ERROR_NONE) { \
  187. GELOGE(RT_FAILED, "Call rtFree failed, error: %#x", error); \
  188. } \
  189. addr = nullptr; \
  190. } \
  191. } while (false)
  192. /**
  193. * @ingroup domi_common
  194. * @brief version of om.proto file
  195. */
  196. static constexpr int32_t OM_PROTO_VERSION = 2;
  197. /**
  198. * Finding an Integer Ceiling Value Without Precision Loss
  199. */
  200. #define CEIL(N, n) (((N) + (n)-1) / (n))
  201. namespace ge {
  202. using google::protobuf::Message;
  203. ///
  204. /// @ingroup domi_common
  205. /// @brief Reads the proto structure from an array.
  206. /// @param [in] data proto data to be read
  207. /// @param [in] size proto data size
  208. /// @param [out] proto Memory for storing the proto file
  209. /// @return true success
  210. /// @return false fail
  211. ///
  212. GE_FUNC_VISIBILITY bool ReadProtoFromArray(const void *data, int size, Message *proto);
  213. ///
  214. /// @ingroup domi_proto
  215. /// @brief Reads the proto file in the text format.
  216. /// @param [in] file path of proto file
  217. /// @param [out] message Memory for storing the proto file
  218. /// @return true success
  219. /// @return false fail
  220. ///
  221. GE_FUNC_VISIBILITY bool ReadProtoFromText(const char *file, google::protobuf::Message *message);
  222. ///
  223. /// @ingroup: domi_common
  224. /// @brief: get length of file
  225. /// @param [in] input_file: path of file
  226. /// @return long: File length. If the file length fails to be obtained, the value -1 is returned.
  227. ///
  228. GE_FUNC_VISIBILITY extern long GetFileLength(const std::string &input_file);
  229. ///
  230. /// @ingroup domi_common
  231. /// @brief Reads all data from a binary file.
  232. /// @param [in] file_name path of file
  233. /// @param [out] buffer Output memory address, which needs to be released by the caller.
  234. /// @param [out] length Output memory size
  235. /// @return false fail
  236. /// @return true success
  237. ///
  238. GE_FUNC_VISIBILITY bool ReadBytesFromBinaryFile(const char *file_name, char **buffer, int &length);
  239. GE_FUNC_VISIBILITY bool ReadBytesFromBinaryFile(const char *file_name, std::vector<char> &buffer);
  240. ///
  241. /// @ingroup domi_common
  242. /// @brief Recursively Creating a Directory
  243. /// @param [in] directory_path Path, which can be a multi-level directory.
  244. /// @return 0 success
  245. /// @return -1 fail
  246. ///
  247. GE_FUNC_VISIBILITY extern int CreateDirectory(const std::string &directory_path);
  248. ///
  249. /// @ingroup domi_common
  250. /// @brief Obtains the current time string.
  251. /// @return Time character string in the format : %Y%m%d%H%M%S, eg: 20171011083555
  252. ///
  253. GE_FUNC_VISIBILITY std::string CurrentTimeInStr();
  254. ///
  255. /// @ingroup domi_common
  256. /// @brief onverts Vector of a number to a string.
  257. /// @param [in] v Vector of a number
  258. /// @return string
  259. ///
  260. template <typename T>
  261. GE_FUNC_VISIBILITY std::string ToString(std::vector<T> &v) {
  262. std::stringstream ss;
  263. ss << "[";
  264. for (T x : v) {
  265. ss << x;
  266. ss << ", ";
  267. }
  268. // Delete the two extra characters at the end of the line.
  269. std::string str = ss.str().substr(0u, ss.str().length() - 2u);
  270. str += "]";
  271. return str;
  272. }
  273. ///
  274. /// @ingroup domi_common
  275. /// @brief Converts RepeatedField to String.
  276. /// @param [in] rpd_field RepeatedField
  277. /// @return string
  278. ///
  279. template <typename T>
  280. GE_FUNC_VISIBILITY std::string ToString(const google::protobuf::RepeatedField<T> &rpd_field) {
  281. std::stringstream ss;
  282. ss << "[";
  283. for (T x : rpd_field) {
  284. ss << x;
  285. ss << ", ";
  286. }
  287. // Delete the two extra characters at the end of the line.
  288. std::string str = ss.str().substr(0u, ss.str().length() - 2u);
  289. str += "]";
  290. return str;
  291. }
  292. ///
  293. /// @ingroup domi_common
  294. /// @brief Obtains the absolute time (timestamp) of the current system.
  295. /// @return Timestamp, in microseconds (US)
  296. ///
  297. ///
  298. GE_FUNC_VISIBILITY uint64_t GetCurrentTimestamp();
  299. ///
  300. /// @ingroup domi_common
  301. /// @brief Obtains the absolute time (timestamp) of the current system.
  302. /// @return Timestamp, in seconds (US)
  303. ///
  304. ///
  305. GE_FUNC_VISIBILITY uint32_t GetCurrentSecondTimestap();
  306. ///
  307. /// @ingroup domi_common
  308. /// @brief Check whether the product of two int64 numbers exceeds the int64 range.
  309. /// @param [in] a
  310. /// @param [in] b
  311. /// @return false: true: The result is within the normal int64 range.
  312. ///
  313. GE_FUNC_VISIBILITY bool CheckInt64MulOverflow(int64_t a, int64_t b);
  314. ///
  315. /// @ingroup domi_common
  316. /// @brief Absolute path for obtaining files.
  317. /// @param [in] path of input file
  318. /// @param [out] Absolute path of a file. If the absolute path cannot be obtained, an empty string is returned
  319. ///
  320. GE_FUNC_VISIBILITY std::string RealPath(const char *path);
  321. ///
  322. /// @ingroup domi_common
  323. /// @brief Check whether the specified input file path is valid.
  324. /// 1. The specified path cannot be empty.
  325. /// 2. The path can be converted to an absolute path.
  326. /// 3. The file path exists and is readable.
  327. /// @param [in] file_path path of input file
  328. /// @param [out] result
  329. ///
  330. GE_FUNC_VISIBILITY bool CheckInputPathValid(const std::string &file_path, const std::string &atc_param = "");
  331. ///
  332. /// @ingroup domi_common
  333. /// @brief Checks whether the specified output file path is valid.
  334. /// @param [in] file_path path of output file
  335. /// @param [out] result
  336. ///
  337. GE_FUNC_VISIBILITY bool CheckOutputPathValid(const std::string &file_path, const std::string &atc_param = "");
  338. ///
  339. /// @ingroup domi_common
  340. /// @brief Check whether the file path meets the whitelist verification requirements.
  341. /// @param [in] filePath file path
  342. /// @param [out] result
  343. ///
  344. GE_FUNC_VISIBILITY bool ValidateStr(const std::string &filePath, const std::string &mode);
  345. ///
  346. /// @ingroup domi_common
  347. /// @brief Check path invalid
  348. /// @param [in] path, path to be checked
  349. /// @param [in] length, length of path
  350. /// @return 0 success
  351. /// @return -1 fail
  352. ///
  353. GE_FUNC_VISIBILITY Status CheckPath(const char *path, size_t length);
  354. } // namespace ge
  355. #endif // INC_FRAMEWORK_COMMON_UTIL_H_

图引擎模块(GE)是MindSpore的一个子模块,其代码由C++实现,位于前端模块ME和底层硬件之间,起到承接作用。图引擎模块以ME下发的图作为输入,然后进行一系列的深度图优化操作,最后输出一张可以在底层硬件上高效运行的图。GE针对昇腾AI处理器的硬件结构特点,做了特定的优化工作,以此来充分发挥出昇腾AI处理器的强大算力。在进行模型训练/推理时,GE会被自动调用而用户并不感知。GE主要由GE API和GE Core两部分组成,详细的架构图如下所示