You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.

util.h 15 kB

5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423
  1. /**
  2. * Copyright 2019-2020 Huawei Technologies Co., Ltd
  3. *
  4. * Licensed under the Apache License, Version 2.0 (the "License");
  5. * you may not use this file except in compliance with the License.
  6. * You may obtain a copy of the License at
  7. *
  8. * http://www.apache.org/licenses/LICENSE-2.0
  9. *
  10. * Unless required by applicable law or agreed to in writing, software
  11. * distributed under the License is distributed on an "AS IS" BASIS,
  12. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13. * See the License for the specific language governing permissions and
  14. * limitations under the License.
  15. */
  16. #ifndef INC_FRAMEWORK_COMMON_UTIL_H_
  17. #define INC_FRAMEWORK_COMMON_UTIL_H_
  18. #include <google/protobuf/text_format.h>
  19. #include <limits.h>
  20. #include <math.h>
  21. #include <sstream>
  22. #include <string>
  23. #include <vector>
  24. #include "framework/common/debug/ge_log.h"
  25. #include "framework/common/debug/log.h"
  26. #include "framework/common/scope_guard.h"
  27. #include "framework/common/ge_inner_error_codes.h"
  28. #include "mmpa/mmpa_api.h"
  29. #define CHECK_FALSE_EXEC(expr, exec_expr, ...) \
  30. { \
  31. bool b = (expr); \
  32. if (!b) { \
  33. exec_expr; \
  34. } \
  35. };
  36. // new ge marco
  37. // Encapsulate common resource releases
  38. #define GE_MAKE_GUARD_RTMEM(var) \
  39. GE_MAKE_GUARD(var, [&] { \
  40. if (var) GE_CHK_RT(rtFreeHost(var)); \
  41. });
  42. #define GE_MAKE_GUARD_RTSTREAM(var) \
  43. GE_MAKE_GUARD(var, [&] { \
  44. if (var) GE_CHK_RT(rtStreamDestroy(var)); \
  45. });
  46. #define GE_MAKE_GUARD_RTEVENT(var) \
  47. GE_MAKE_GUARD(var, [&] { \
  48. if (var) GE_CHK_RT(rtEventDestroy(var)); \
  49. });
  50. #define GE_MAKE_GUARD_TENSOR(var) \
  51. GE_MAKE_GUARD(var, [&] { \
  52. if (var) GE_CHK_CCE(ccDestroyTensorDescriptor(&var)); \
  53. });
  54. #define GE_MAKE_GUARD_FILTER_DESC(var) \
  55. GE_MAKE_GUARD(var, [&] { \
  56. if (var) GE_CHK_CCE(ccDestroyFilterDescriptor(&var)); \
  57. });
  58. // For propagating errors when calling a function.
  59. #define GE_RETURN_IF_ERROR(expr) \
  60. do { \
  61. const ::ge::Status _status = (expr); \
  62. if (_status) return _status; \
  63. } while (0)
  64. #define GE_RETURN_WITH_LOG_IF_ERROR(expr, ...) \
  65. do { \
  66. const ::ge::Status _status = (expr); \
  67. if (_status) { \
  68. GE_LOGE(__VA_ARGS__); \
  69. return _status; \
  70. } \
  71. } while (0)
  72. // check whether the parameter is true. If it is, return FAILED and record the error log
  73. #define GE_RETURN_WITH_LOG_IF_TRUE(condition, ...) \
  74. do { \
  75. if (condition) { \
  76. GE_LOGE(__VA_ARGS__); \
  77. return ge::FAILED; \
  78. } \
  79. } while (0)
  80. // Check if the parameter is false. If yes, return FAILED and record the error log
  81. #define GE_RETURN_WITH_LOG_IF_FALSE(condition, ...) \
  82. do { \
  83. bool _condition = (condition); \
  84. if (!_condition) { \
  85. GE_LOGE(__VA_ARGS__); \
  86. return ge::FAILED; \
  87. } \
  88. } while (0)
  89. // Checks whether the parameter is true. If so, returns PARAM_INVALID and records the error log
  90. #define GE_RT_PARAM_INVALID_WITH_LOG_IF_TRUE(condition, ...) \
  91. do { \
  92. if (condition) { \
  93. GE_LOGE(__VA_ARGS__); \
  94. return ge::PARAM_INVALID; \
  95. } \
  96. } while (0)
  97. // Check if the parameter is false. If yes, return PARAM_INVALID and record the error log
  98. #define GE_RT_PARAM_INVALID_WITH_LOG_IF_FALSE(condition, ...) \
  99. do { \
  100. bool _condition = (condition); \
  101. if (!_condition) { \
  102. GE_LOGE(__VA_ARGS__); \
  103. return ge::PARAM_INVALID; \
  104. } \
  105. } while (0)
  106. // Check if the parameter is null. If yes, return PARAM_INVALID and record the error
  107. #define GE_CHECK_NOTNULL(val) \
  108. do { \
  109. if (val == nullptr) { \
  110. GE_LOGE(param[#val] must not be null.); \
  111. return ge::PARAM_INVALID; \
  112. } \
  113. } while (0)
  114. // Check if the parameter is null. If yes, return PARAM_INVALID and record the error
  115. #define GE_CHECK_NOTNULL_JUST_RETURN(val) \
  116. do { \
  117. if (val == nullptr) { \
  118. GE_LOGE(param[#val] must not be null.); \
  119. return; \
  120. } \
  121. } while (0)
  122. // Check whether the parameter is null. If so, execute the exec_expr expression and record the error log
  123. #define GE_CHECK_NOTNULL_EXEC(val, exec_expr) \
  124. do { \
  125. if (val == nullptr) { \
  126. GE_LOGE(param[#val] must not be null.); \
  127. exec_expr; \
  128. } \
  129. } while (0)
  130. // Check whether the parameter is null. If yes, return directly and record the error log
  131. #define GE_RT_VOID_CHECK_NOTNULL(val) \
  132. do { \
  133. if (val == nullptr) { \
  134. GE_LOGE(param[#val] must not be null.); \
  135. return; \
  136. } \
  137. } while (0)
  138. // Check if the parameter is null. If yes, return false and record the error log
  139. #define GE_RT_FALSE_CHECK_NOTNULL(val) \
  140. do { \
  141. if (val == nullptr) { \
  142. GE_LOGE(param[#val] must not be null.); \
  143. return false; \
  144. } \
  145. } while (0)
  146. // Check if the parameter is out of bounds
  147. #define GE_CHECK_SIZE(size) \
  148. do { \
  149. if (size == 0) { \
  150. GE_LOGE(param[#size] is out of range); \
  151. return ge::PARAM_INVALID; \
  152. } \
  153. } while (0)
  154. // Macros that define the size variable
  155. #define GE_DEFINE_BYTE_SIZE(_var_name, _expr, _sizeof) \
  156. uint32_t _var_name; \
  157. do { \
  158. uint32_t _expr_size = (_expr); \
  159. uint32_t _sizeof_size = (_sizeof); \
  160. if (_expr_size > (0xffffffff) / _sizeof_size) { \
  161. GE_LOGE(byte size : #_var_name is out of range); \
  162. return ge::PARAM_INVALID; \
  163. } \
  164. _var_name = _sizeof_size * _expr_size; \
  165. } while (0);
  166. // Check if the container is empty
  167. #define GE_CHECK_VECTOR_NOT_EMPTY(vector) \
  168. do { \
  169. if (vector.empty()) { \
  170. GE_LOGE(param[#vector] is empty !); \
  171. return ge::FAILED; \
  172. } \
  173. } while (0)
  174. #define GE_CHECK_POSITIVE_SIZE_RANGE(size) \
  175. do { \
  176. if (size <= 0) { \
  177. GE_LOGE(param[#size] is not a positive number); \
  178. return ge::PARAM_INVALID; \
  179. } \
  180. } while (0)
  181. // Check if the value on the left is greater than or equal to the value on the right
  182. #define GE_CHECK_GE(lhs, rhs) \
  183. do { \
  184. if (lhs < rhs) { \
  185. GE_LOGE(param[#lhs] is less than[#rhs]); \
  186. return ge::PARAM_INVALID; \
  187. } \
  188. } while (0)
  189. // Check if the value on the left is less than or equal to the value on the right
  190. #define GE_CHECK_LE(lhs, rhs) \
  191. do { \
  192. if (lhs > rhs) { \
  193. GE_LOGE(param[#lhs] is greater than[#rhs]); \
  194. return ge::PARAM_INVALID; \
  195. } \
  196. } while (0)
  197. #define GE_DELETE_NEW_SINGLE(var) \
  198. { \
  199. if (var != nullptr) { \
  200. delete var; \
  201. var = nullptr; \
  202. } \
  203. };
  204. #define GE_DELETE_NEW_ARRAY(var) \
  205. { \
  206. if (var != nullptr) { \
  207. delete[] var; \
  208. var = nullptr; \
  209. } \
  210. };
  211. /**
  212. * @ingroup domi_common
  213. * @brief version of om.proto file
  214. */
  215. static constexpr int32_t OM_PROTO_VERSION = 2;
  216. /**
  217. * Finding an Integer Ceiling Value Without Precision Loss
  218. */
  219. #define CEIL(N, n) (((N) + (n)-1) / (n))
  220. namespace ge {
  221. using google::protobuf::Message;
  222. ///
  223. /// @ingroup domi_common
  224. /// @brief Maximum file path length
  225. ///
  226. const int32_t DOMI_MAX_PATH_LEN = 256;
  227. ///
  228. /// @ingroup domi_common
  229. /// @brief proto file in bianary format
  230. /// @param [in] file path of proto file
  231. /// @param [out] proto memory for storing the proto file
  232. /// @return true success
  233. /// @return false fail
  234. ///
  235. bool ReadProtoFromBinaryFile(const char *file, Message *proto);
  236. ///
  237. /// @ingroup domi_common
  238. /// @brief Reads the proto structure from an array.
  239. /// @param [in] data proto data to be read
  240. /// @param [in] size proto data size
  241. /// @param [out] proto Memory for storing the proto file
  242. /// @return true success
  243. /// @return false fail
  244. ///
  245. bool ReadProtoFromArray(const void *data, int size, Message *proto);
  246. ///
  247. /// @ingroup domi_proto
  248. /// @brief Reads the proto file in the text format.
  249. /// @param [in] file path of proto file
  250. /// @param [out] message Memory for storing the proto file
  251. /// @return true success
  252. /// @return false fail
  253. ///
  254. bool ReadProtoFromText(const char *file, google::protobuf::Message *message);
  255. bool ReadProtoFromMem(const char *data, int size, google::protobuf::Message *message);
  256. ///
  257. /// @ingroup: domi_common
  258. /// @brief: get length of file
  259. /// @param [in] input_file: path of file
  260. /// @return long: File length. If the file length fails to be obtained, the value -1 is returned.
  261. ///
  262. extern long GetFileLength(const std::string &input_file);
  263. ///
  264. /// @ingroup domi_common
  265. /// @brief Reads all data from a binary file.
  266. /// @param [in] file_name path of file
  267. /// @param [out] buffer Output memory address, which needs to be released by the caller.
  268. /// @param [out] length Output memory size
  269. /// @return false fail
  270. /// @return true success
  271. ///
  272. bool ReadBytesFromBinaryFile(const char *file_name, char **buffer, int &length);
  273. bool ReadBytesFromBinaryFile(const char *file_name, std::vector<char> &buffer);
  274. ///
  275. /// @ingroup domi_common
  276. /// @brief Recursively Creating a Directory
  277. /// @param [in] directory_path Path, which can be a multi-level directory.
  278. /// @return 0 success
  279. /// @return -1 fail
  280. ///
  281. extern int CreateDirectory(const std::string &directory_path);
  282. ///
  283. /// @ingroup domi_common
  284. /// @brief Obtains the current time string.
  285. /// @return Time character string in the format : %Y%m%d%H%M%S, eg: 20171011083555
  286. ///
  287. std::string CurrentTimeInStr();
  288. ///
  289. /// @ingroup domi_common
  290. /// @brief onverts Vector of a number to a string.
  291. /// @param [in] v Vector of a number
  292. /// @return string
  293. ///
  294. template <typename T>
  295. std::string ToString(std::vector<T> &v) {
  296. std::stringstream ss;
  297. ss << "[";
  298. for (T x : v) {
  299. ss << x;
  300. ss << ", ";
  301. }
  302. std::string strRet =
  303. ss.str().substr(0, ss.str().length() - 2); // Delete the two extra characters at the end of the line.
  304. strRet += "]";
  305. return strRet;
  306. }
  307. ///
  308. /// @ingroup domi_common
  309. /// @brief Converts RepeatedField to String.
  310. /// @param [in] rpd_field RepeatedField
  311. /// @return string
  312. ///
  313. template <typename T>
  314. std::string ToString(const google::protobuf::RepeatedField<T> &rpd_field) {
  315. std::stringstream ss;
  316. ss << "[";
  317. for (T x : rpd_field) {
  318. ss << x;
  319. ss << ", ";
  320. }
  321. std::string strRet =
  322. ss.str().substr(0, ss.str().length() - 2); // Delete the two extra characters at the end of the line.
  323. strRet += "]";
  324. return strRet;
  325. }
  326. ///
  327. /// @ingroup domi_common
  328. /// @brief Obtains the absolute time (timestamp) of the current system.
  329. /// @return Timestamp, in microseconds (US)
  330. ///
  331. ///
  332. uint64_t GetCurrentTimestap();
  333. ///
  334. /// @ingroup domi_common
  335. /// @brief Check whether the product of two int64 numbers exceeds the int64 range.
  336. /// @param [in] a
  337. /// @param [in] b
  338. /// @return false: true: The result is within the normal int64 range.
  339. ///
  340. bool CheckInt64MulOverflow(int64_t a, int64_t b);
  341. ///
  342. /// @ingroup domi_common
  343. /// @brief Absolute path for obtaining files.
  344. /// @param [in] path of input file
  345. /// @param [out] Absolute path of a file. If the absolute path cannot be obtained, an empty string is returned
  346. ///
  347. std::string RealPath(const char *path);
  348. ///
  349. /// @ingroup domi_common
  350. /// @brief Check whether the specified input file path is valid.
  351. /// 1. The specified path cannot be empty.
  352. /// 2. The path can be converted to an absolute path.
  353. /// 3. The file path exists and is readable.
  354. /// @param [in] file_path path of input file
  355. /// @param [out] result
  356. ///
  357. bool CheckInputPathValid(const std::string &file_path);
  358. ///
  359. /// @ingroup domi_common
  360. /// @brief Checks whether the specified output file path is valid.
  361. /// @param [in] file_path path of output file
  362. /// @param [out] result
  363. ///
  364. bool CheckOutputPathValid(const std::string &file_path);
  365. ///
  366. /// @ingroup domi_common
  367. /// @brief Check whether the file path meets the whitelist verification requirements.
  368. /// @param [in] filePath file path
  369. /// @param [out] result
  370. ///
  371. bool ValidateStr(const std::string &filePath, const std::string &mode);
  372. } // namespace ge
  373. #endif // INC_FRAMEWORK_COMMON_UTIL_H_

图引擎模块(GE)是MindSpore的一个子模块,其代码由C++实现,位于前端模块ME和底层硬件之间,起到承接作用。图引擎模块以ME下发的图作为输入,然后进行一系列的深度图优化操作,最后输出一张可以在底层硬件上高效运行的图。GE针对昇腾AI处理器的硬件结构特点,做了特定的优化工作,以此来充分发挥出昇腾AI处理器的强大算力。在进行模型训练/推理时,GE会被自动调用而用户并不感知。GE主要由GE API和GE Core两部分组成,详细的架构图如下所示