You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.

util.h 13 kB

5 years ago
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378
  1. /**
  2. * Copyright 2019-2020 Huawei Technologies Co., Ltd
  3. *
  4. * Licensed under the Apache License, Version 2.0 (the "License");
  5. * you may not use this file except in compliance with the License.
  6. * You may obtain a copy of the License at
  7. *
  8. * http://www.apache.org/licenses/LICENSE-2.0
  9. *
  10. * Unless required by applicable law or agreed to in writing, software
  11. * distributed under the License is distributed on an "AS IS" BASIS,
  12. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13. * See the License for the specific language governing permissions and
  14. * limitations under the License.
  15. */
  16. #ifndef INC_FRAMEWORK_COMMON_UTIL_H_
  17. #define INC_FRAMEWORK_COMMON_UTIL_H_
  18. #include <google/protobuf/text_format.h>
  19. #include <limits.h>
  20. #include <math.h>
  21. #include <sstream>
  22. #include <string>
  23. #include <vector>
  24. #include "framework/common/debug/ge_log.h"
  25. #include "framework/common/debug/log.h"
  26. #include "framework/common/scope_guard.h"
  27. #include "framework/common/ge_inner_error_codes.h"
  28. #include "mmpa/mmpa_api.h"
  29. #define CHECK_FALSE_EXEC(expr, exec_expr, ...) \
  30. { \
  31. bool b = (expr); \
  32. if (!b) { \
  33. exec_expr; \
  34. } \
  35. };
  36. // new ge marco
  37. // Encapsulate common resource releases
  38. #define GE_MAKE_GUARD_RTMEM(var) \
  39. GE_MAKE_GUARD(var, [&] { \
  40. if (var) GE_CHK_RT(rtFreeHost(var)); \
  41. });
  42. #define GE_MAKE_GUARD_RTSTREAM(var) \
  43. GE_MAKE_GUARD(var, [&] { \
  44. if (var) GE_CHK_RT(rtStreamDestroy(var)); \
  45. });
  46. #define GE_MAKE_GUARD_RTEVENT(var) \
  47. GE_MAKE_GUARD(var, [&] { \
  48. if (var) GE_CHK_RT(rtEventDestroy(var)); \
  49. });
  50. #define GE_MAKE_GUARD_TENSOR(var) \
  51. GE_MAKE_GUARD(var, [&] { \
  52. if (var) GE_CHK_CCE(ccDestroyTensorDescriptor(&var)); \
  53. });
  54. #define GE_MAKE_GUARD_FILTER_DESC(var) \
  55. GE_MAKE_GUARD(var, [&] { \
  56. if (var) GE_CHK_CCE(ccDestroyFilterDescriptor(&var)); \
  57. });
  58. #define GE_RETURN_WITH_LOG_IF_ERROR(expr, ...) \
  59. do { \
  60. const ::ge::Status _status = (expr); \
  61. if (_status) { \
  62. DOMI_LOGE(__VA_ARGS__); \
  63. return _status; \
  64. } \
  65. } while (0)
  66. // Check if the parameter is false. If yes, return FAILED and record the error log
  67. #define GE_RETURN_WITH_LOG_IF_FALSE(condition, ...) \
  68. do { \
  69. bool _condition = (condition); \
  70. if (!_condition) { \
  71. DOMI_LOGE(__VA_ARGS__); \
  72. return ge::FAILED; \
  73. } \
  74. } while (0)
  75. // Check if the parameter is null. If yes, return PARAM_INVALID and record the error
  76. #define GE_CHECK_NOTNULL(val) \
  77. do { \
  78. if (val == nullptr) { \
  79. DOMI_LOGE(param[#val] must not be null.); \
  80. return ge::PARAM_INVALID; \
  81. } \
  82. } while (0)
  83. // Check if the parameter is null. If yes, return PARAM_INVALID and record the error
  84. #define GE_CHECK_NOTNULL_JUST_RETURN(val) \
  85. do { \
  86. if (val == nullptr) { \
  87. DOMI_LOGE(param[#val] must not be null.); \
  88. return; \
  89. } \
  90. } while (0)
  91. // Check whether the parameter is null. If so, execute the exec_expr expression and record the error log
  92. #define GE_CHECK_NOTNULL_EXEC(val, exec_expr) \
  93. do { \
  94. if (val == nullptr) { \
  95. DOMI_LOGE(param[#val] must not be null.); \
  96. exec_expr; \
  97. } \
  98. } while (0)
  99. // Check whether the parameter is null. If yes, return directly and record the error log
  100. #define GE_RT_VOID_CHECK_NOTNULL(val) \
  101. do { \
  102. if (val == nullptr) { \
  103. DOMI_LOGE(param[#val] must not be null.); \
  104. return; \
  105. } \
  106. } while (0)
  107. // Check if the parameter is null. If yes, return false and record the error log
  108. #define GE_RT_FALSE_CHECK_NOTNULL(val) \
  109. do { \
  110. if (val == nullptr) { \
  111. DOMI_LOGE(param[#val] must not be null.); \
  112. return false; \
  113. } \
  114. } while (0)
  115. // Check if the parameter is out of bounds
  116. #define GE_CHECK_SIZE(size) \
  117. do { \
  118. if (size == 0) { \
  119. DOMI_LOGE(param[#size] is out of range); \
  120. return ge::PARAM_INVALID; \
  121. } \
  122. } while (0)
  123. // Macros that define the size variable
  124. #define GE_DEFINE_BYTE_SIZE(_var_name, _expr, _sizeof) \
  125. uint32_t _var_name; \
  126. do { \
  127. uint32_t _expr_size = (_expr); \
  128. uint32_t _sizeof_size = (_sizeof); \
  129. if (_expr_size > (0xffffffff) / _sizeof_size) { \
  130. DOMI_LOGE(byte size : #_var_name is out of range); \
  131. return ge::PARAM_INVALID; \
  132. } \
  133. _var_name = _sizeof_size * _expr_size; \
  134. } while (0);
  135. // Check if the container is empty
  136. #define GE_CHECK_VECTOR_NOT_EMPTY(vector) \
  137. do { \
  138. if (vector.empty()) { \
  139. DOMI_LOGE(param[#vector] is empty !); \
  140. return ge::FAILED; \
  141. } \
  142. } while (0)
  143. // Check if the value on the left is greater than or equal to the value on the right
  144. #define GE_CHECK_GE(lhs, rhs) \
  145. do { \
  146. if (lhs < rhs) { \
  147. DOMI_LOGE(param[#lhs] is less than[#rhs]); \
  148. return ge::PARAM_INVALID; \
  149. } \
  150. } while (0)
  151. // Check if the value on the left is less than or equal to the value on the right
  152. #define GE_CHECK_LE(lhs, rhs) \
  153. do { \
  154. if (lhs > rhs) { \
  155. DOMI_LOGE(param[#lhs] is greater than[#rhs]); \
  156. return ge::PARAM_INVALID; \
  157. } \
  158. } while (0)
  159. #define GE_DELETE_NEW_SINGLE(var) \
  160. { \
  161. if (var != nullptr) { \
  162. delete var; \
  163. var = nullptr; \
  164. } \
  165. };
  166. #define GE_DELETE_NEW_ARRAY(var) \
  167. { \
  168. if (var != nullptr) { \
  169. delete[] var; \
  170. var = nullptr; \
  171. } \
  172. };
  173. ///
  174. /// @ingroup domi_common
  175. /// @brief version of om.proto file
  176. ///
  177. static constexpr int32_t OM_PROTO_VERSION = 2;
  178. // Finding an Integer Ceiling Value Without Precision Loss
  179. #define CEIL(N, n) (((N) + (n)-1) / (n))
  180. namespace ge {
  181. using google::protobuf::Message;
  182. ///
  183. /// @ingroup domi_common
  184. /// @brief Maximum file path length
  185. ///
  186. const int32_t DOMI_MAX_PATH_LEN = 256;
  187. ///
  188. /// @ingroup domi_common
  189. /// @brief proto file in bianary format
  190. /// @param [in] file path of proto file
  191. /// @param [out] proto memory for storing the proto file
  192. /// @return true success
  193. /// @return false fail
  194. ///
  195. bool ReadProtoFromBinaryFile(const char *file, Message *proto);
  196. ///
  197. /// @ingroup domi_common
  198. /// @brief Reads the proto structure from an array.
  199. /// @param [in] data proto data to be read
  200. /// @param [in] size proto data size
  201. /// @param [out] proto Memory for storing the proto file
  202. /// @return true success
  203. /// @return false fail
  204. ///
  205. bool ReadProtoFromArray(const void *data, int size, Message *proto);
  206. ///
  207. /// @ingroup domi_proto
  208. /// @brief Reads the proto file in the text format.
  209. /// @param [in] file path of proto file
  210. /// @param [out] message Memory for storing the proto file
  211. /// @return true success
  212. /// @return false fail
  213. ///
  214. bool ReadProtoFromText(const char *file, google::protobuf::Message *message);
  215. bool ReadProtoFromMem(const char *data, int size, google::protobuf::Message *message);
  216. ///
  217. /// @ingroup: domi_common
  218. /// @brief: get length of file
  219. /// @param [in] input_file: path of file
  220. /// @return long: File length. If the file length fails to be obtained, the value -1 is returned.
  221. ///
  222. extern long GetFileLength(const std::string &input_file);
  223. ///
  224. /// @ingroup domi_common
  225. /// @brief Reads all data from a binary file.
  226. /// @param [in] file_name path of file
  227. /// @param [out] buffer Output memory address, which needs to be released by the caller.
  228. /// @param [out] length Output memory size
  229. /// @return false fail
  230. /// @return true success
  231. ///
  232. bool ReadBytesFromBinaryFile(const char *file_name, char **buffer, int &length);
  233. bool ReadBytesFromBinaryFile(const char *file_name, std::vector<char> &buffer);
  234. ///
  235. /// @ingroup domi_common
  236. /// @brief Recursively Creating a Directory
  237. /// @param [in] directory_path Path, which can be a multi-level directory.
  238. /// @return 0 success
  239. /// @return -1 fail
  240. ///
  241. extern int CreateDirectory(const std::string &directory_path);
  242. ///
  243. /// @ingroup domi_common
  244. /// @brief Obtains the current time string.
  245. /// @return Time character string in the format : %Y%m%d%H%M%S, eg: 20171011083555
  246. ///
  247. std::string CurrentTimeInStr();
  248. ///
  249. /// @ingroup domi_common
  250. /// @brief onverts Vector of a number to a string.
  251. /// @param [in] v Vector of a number
  252. /// @return string
  253. ///
  254. template <typename T>
  255. std::string ToString(std::vector<T> &v) {
  256. std::stringstream ss;
  257. ss << "[";
  258. for (T x : v) {
  259. ss << x;
  260. ss << ", ";
  261. }
  262. std::string strRet =
  263. ss.str().substr(0, ss.str().length() - 2); // Delete the two extra characters at the end of the line.
  264. strRet += "]";
  265. return strRet;
  266. }
  267. ///
  268. /// @ingroup domi_common
  269. /// @brief Converts RepeatedField to String.
  270. /// @param [in] rpd_field RepeatedField
  271. /// @return string
  272. ///
  273. template <typename T>
  274. std::string ToString(const google::protobuf::RepeatedField<T> &rpd_field) {
  275. std::stringstream ss;
  276. ss << "[";
  277. for (T x : rpd_field) {
  278. ss << x;
  279. ss << ", ";
  280. }
  281. std::string strRet =
  282. ss.str().substr(0, ss.str().length() - 2); // Delete the two extra characters at the end of the line.
  283. strRet += "]";
  284. return strRet;
  285. }
  286. ///
  287. /// @ingroup domi_common
  288. /// @brief Obtains the absolute time (timestamp) of the current system.
  289. /// @return Timestamp, in microseconds (US)
  290. ///
  291. ///
  292. uint64_t GetCurrentTimestap();
  293. ///
  294. /// @ingroup domi_common
  295. /// @brief Check whether the product of two int64 numbers exceeds the int64 range.
  296. /// @param [in] a
  297. /// @param [in] b
  298. /// @return false: true: The result is within the normal int64 range.
  299. ///
  300. bool CheckInt64MulOverflow(int64_t a, int64_t b);
  301. ///
  302. /// @ingroup domi_common
  303. /// @brief Absolute path for obtaining files.
  304. /// @param [in] path of input file
  305. /// @param [out] Absolute path of a file. If the absolute path cannot be obtained, an empty string is returned
  306. ///
  307. std::string RealPath(const char *path);
  308. ///
  309. /// @ingroup domi_common
  310. /// @brief Check whether the specified input file path is valid.
  311. /// 1. The specified path cannot be empty.
  312. /// 2. The path can be converted to an absolute path.
  313. /// 3. The file path exists and is readable.
  314. /// @param [in] file_path path of input file
  315. /// @param [out] result
  316. ///
  317. bool CheckInputPathValid(const std::string &file_path);
  318. ///
  319. /// @ingroup domi_common
  320. /// @brief Checks whether the specified output file path is valid.
  321. /// @param [in] file_path path of output file
  322. /// @param [out] result
  323. ///
  324. bool CheckOutputPathValid(const std::string &file_path);
  325. ///
  326. /// @ingroup domi_common
  327. /// @brief Check whether the file path meets the whitelist verification requirements.
  328. /// @param [in] filePath file path
  329. /// @param [out] result
  330. ///
  331. bool ValidateStr(const std::string &filePath, const std::string &mode);
  332. } // namespace ge
  333. #endif // INC_FRAMEWORK_COMMON_UTIL_H_

图引擎模块(GE)是MindSpore的一个子模块,其代码由C++实现,位于前端模块ME和底层硬件之间,起到承接作用。图引擎模块以ME下发的图作为输入,然后进行一系列的深度图优化操作,最后输出一张可以在底层硬件上高效运行的图。GE针对昇腾AI处理器的硬件结构特点,做了特定的优化工作,以此来充分发挥出昇腾AI处理器的强大算力。在进行模型训练/推理时,GE会被自动调用而用户并不感知。GE主要由GE API和GE Core两部分组成,详细的架构图如下所示

Contributors (1)