You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.

util.cc 23 kB

5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
4 years ago
4 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
4 years ago
5 years ago
5 years ago
4 years ago
5 years ago
4 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
4 years ago
5 years ago
5 years ago
5 years ago
4 years ago
5 years ago
4 years ago
5 years ago
5 years ago
4 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
4 years ago
5 years ago
5 years ago
5 years ago
5 years ago
4 years ago
4 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
4 years ago
4 years ago
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551
  1. /**
  2. * Copyright 2019-2020 Huawei Technologies Co., Ltd
  3. *
  4. * Licensed under the Apache License, Version 2.0 (the "License");
  5. * you may not use this file except in compliance with the License.
  6. * You may obtain a copy of the License at
  7. *
  8. * http://www.apache.org/licenses/LICENSE-2.0
  9. *
  10. * Unless required by applicable law or agreed to in writing, software
  11. * distributed under the License is distributed on an "AS IS" BASIS,
  12. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13. * See the License for the specific language governing permissions and
  14. * limitations under the License.
  15. */
  16. #include "framework/common/util.h"
  17. #include <fcntl.h>
  18. #include <sys/stat.h>
  19. #include <regex.h>
  20. #include <unistd.h>
  21. #include <algorithm>
  22. #include <climits>
  23. #include <cstdlib>
  24. #include <ctime>
  25. #include <fstream>
  26. #include "common/util/error_manager/error_manager.h"
  27. #include "external/ge/ge_api_error_codes.h"
  28. #include "framework/common/debug/ge_log.h"
  29. #include "framework/common/fmk_types.h"
  30. #include "framework/common/ge_inner_error_codes.h"
  31. #include "google/protobuf/io/coded_stream.h"
  32. #include "google/protobuf/io/zero_copy_stream_impl.h"
  33. #include "mmpa/mmpa_api.h"
  34. using google::protobuf::io::CodedInputStream;
  35. using google::protobuf::io::FileInputStream;
  36. using google::protobuf::io::ZeroCopyInputStream;
  37. namespace {
  38. /*
  39. * kProtoReadBytesLimit and kWarningThreshold are real arguments of CodedInputStream::SetTotalBytesLimit.
  40. * In order to prevent integer overflow and excessive memory allocation during protobuf processing,
  41. * it is necessary to limit the length of proto message (call SetTotalBytesLimit function).
  42. * In theory, the minimum message length that causes an integer overflow is 512MB, and the default is 64MB.
  43. * If the limit of warning_threshold is exceeded, the exception information will be printed in stderr.
  44. * If such an exception is encountered during operation,
  45. * the proto file can be divided into several small files or the limit value can be increased.
  46. */
  47. const int kProtoReadBytesLimit = INT_MAX; // Max size of 2 GB minus 1 byte.
  48. const int kWarningThreshold = 536870912 * 2; // 536870912 represent 512M
  49. /// The maximum length of the file.
  50. const uint32_t kMaxFileSizeLimit = UINT32_MAX; // 4G for now
  51. const int kMaxBuffSize = 256;
  52. const char *const kPathValidReason = "The path can only contain 'a-z' 'A-Z' '0-9' '-' '.' '_' and chinese character";
  53. constexpr uint32_t kMaxConfigFileByte = 10 * 1024 * 1024;
  54. } // namespace
  55. namespace ge {
  56. static bool ReadProtoFromCodedInputStream(CodedInputStream &coded_stream, Message *proto) {
  57. GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(proto == nullptr, return false, "incorrect parameter. nullptr == proto");
  58. coded_stream.SetTotalBytesLimit(kProtoReadBytesLimit, kWarningThreshold);
  59. return proto->ParseFromCodedStream(&coded_stream);
  60. }
  61. FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY bool ReadProtoFromBinaryFile(const char *file, Message *proto) {
  62. GE_CHK_BOOL_TRUE_EXEC_WITH_LOG((file == nullptr || proto == nullptr), return false,
  63. "Input parameter file or proto is nullptr!");
  64. std::string real_path = RealPath(file);
  65. GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(real_path.empty(), return false, "pb file path '%s' not valid", file);
  66. GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(GetFileLength(real_path) == -1, return false, "file size not valid.");
  67. std::ifstream fs(real_path, std::ifstream::in | std::ifstream::binary);
  68. if (!fs.is_open()) {
  69. ErrorManager::GetInstance().ATCReportErrMessage("E19001", {"file", "errmsg"}, {file, "ifstream is_open failed"});
  70. GELOGE(ge::FAILED, "Open real path[%s] failed.", file);
  71. return false;
  72. }
  73. google::protobuf::io::IstreamInputStream istream(&fs);
  74. google::protobuf::io::CodedInputStream coded_stream(&istream);
  75. bool ret = ReadProtoFromCodedInputStream(coded_stream, proto);
  76. fs.close();
  77. if (!ret) {
  78. ErrorManager::GetInstance().ATCReportErrMessage("E19005", {"file"}, {file});
  79. GELOGE(ge::FAILED, "Parse file[%s] failed.", file);
  80. return ret;
  81. }
  82. return ret;
  83. }
  84. FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY bool ReadProtoFromArray(const void *data, int size, Message *proto) {
  85. GE_CHK_BOOL_TRUE_EXEC_WITH_LOG((proto == nullptr || data == nullptr || size == 0), return false,
  86. "incorrect parameter. proto is nullptr || data is nullptr || size is 0");
  87. google::protobuf::io::CodedInputStream coded_stream(reinterpret_cast<uint8_t *>(const_cast<void *>(data)), size);
  88. return ReadProtoFromCodedInputStream(coded_stream, proto);
  89. }
  90. // Get file length
  91. long GetFileLength(const std::string &input_file) {
  92. GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(input_file.empty(), return -1, "input_file path is null.");
  93. std::string real_path = RealPath(input_file.c_str());
  94. GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(real_path.empty(), return -1, "input_file path '%s' not valid", input_file.c_str());
  95. unsigned long long file_length = 0;
  96. GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(
  97. mmGetFileSize(input_file.c_str(), &file_length) != EN_OK,
  98. ErrorManager::GetInstance().ATCReportErrMessage("E19001", {"file", "errmsg"}, {input_file, strerror(errno)});
  99. return -1, "Open file[%s] failed. %s", input_file.c_str(), strerror(errno));
  100. GE_CHK_BOOL_TRUE_EXEC_WITH_LOG((file_length == 0),
  101. ErrorManager::GetInstance().ATCReportErrMessage("E19015", {"filepath"}, {input_file});
  102. return -1, "File[%s] size is 0, not valid.", input_file.c_str());
  103. GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(file_length > kMaxFileSizeLimit,
  104. ErrorManager::GetInstance().ATCReportErrMessage(
  105. "E19016", {"filepath", "filesize", "maxlen"},
  106. {input_file, std::to_string(file_length), std::to_string(kMaxFileSizeLimit)});
  107. return -1, "File[%s] size %lld is out of limit: %d.", input_file.c_str(), file_length,
  108. kMaxFileSizeLimit);
  109. return static_cast<long>(file_length);
  110. }
  111. /** @ingroup domi_common
  112. * @brief Read all data from binary file
  113. * @param [in] file_name File path
  114. * @param [out] buffer The address of the output memory, which needs to be released by the caller
  115. * @param [out] length Output memory size
  116. * @return false fail
  117. * @return true success
  118. */
  119. FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY bool ReadBytesFromBinaryFile(const char *file_name, char **buffer,
  120. int &length) {
  121. GE_CHK_BOOL_TRUE_EXEC_WITH_LOG((file_name == nullptr), return false, "incorrect parameter. file is nullptr");
  122. GE_CHK_BOOL_TRUE_EXEC_WITH_LOG((buffer == nullptr), return false, "incorrect parameter. buffer is nullptr");
  123. std::string real_path = RealPath(file_name);
  124. GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(real_path.empty(), return false, "file path '%s' not valid", file_name);
  125. std::ifstream file(real_path.c_str(), std::ios::binary | std::ios::ate);
  126. if (!file.is_open()) {
  127. GELOGE(ge::FAILED, "Read file %s failed.", file_name);
  128. return false;
  129. }
  130. length = static_cast<int>(file.tellg());
  131. GE_CHK_BOOL_TRUE_EXEC_WITH_LOG((length <= 0), file.close(); return false, "file length <= 0");
  132. file.seekg(0, std::ios::beg);
  133. *buffer = new (std::nothrow) char[length]();
  134. GE_CHK_BOOL_TRUE_EXEC_RET_STATUS(*buffer == nullptr, false, file.close(), "new an object failed.");
  135. file.read(*buffer, length);
  136. file.close();
  137. return true;
  138. }
  139. FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY bool ReadBytesFromBinaryFile(const char *file_name,
  140. std::vector<char> &buffer) {
  141. GE_CHK_BOOL_TRUE_EXEC_WITH_LOG((file_name == nullptr), return false, "incorrect parameter. file path is null");
  142. std::string real_path = RealPath(file_name);
  143. GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(real_path.empty(), return false, "file path '%s' not valid", file_name);
  144. std::ifstream file(real_path.c_str(), std::ios::binary | std::ios::ate);
  145. if (!file.is_open()) {
  146. GELOGE(ge::FAILED, "Read file %s failed.", file_name);
  147. return false;
  148. }
  149. std::streamsize size = file.tellg();
  150. GE_CHK_BOOL_TRUE_EXEC_WITH_LOG((size <= 0), file.close(); return false, "file length <= 0, not valid.");
  151. GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(size > static_cast<int64_t >(kMaxFileSizeLimit), file.close();
  152. return false, "file size %ld is out of limit: %d.", size, kMaxFileSizeLimit);
  153. file.seekg(0, std::ios::beg); // [no need to check value]
  154. buffer.resize(static_cast<uint64_t>(size)); // [no need to check value]
  155. file.read(&buffer[0], size); // [no need to check value]
  156. file.close();
  157. GELOGI("Read size:%ld", size);
  158. return true;
  159. }
  160. /**
  161. * @ingroup domi_common
  162. * @brief Create directory, support to create multi-level directory
  163. * @param [in] directory_path Path, can be multi-level directory
  164. * @return -1 fail
  165. * @return 0 success
  166. */
  167. FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY int CreateDirectory(const std::string &directory_path) {
  168. GE_CHK_BOOL_EXEC(!directory_path.empty(), return -1, "directory path is empty.");
  169. auto dir_path_len = directory_path.length();
  170. if (dir_path_len >= PATH_MAX) {
  171. ErrorManager::GetInstance().ATCReportErrMessage("E19002", {"filepath", "size"},
  172. {directory_path, std::to_string(PATH_MAX)});
  173. GELOGW("Path[%s] len is too long, it must be less than %d", directory_path.c_str(), PATH_MAX);
  174. return -1;
  175. }
  176. char tmp_dir_path[PATH_MAX] = {0};
  177. for (size_t i = 0; i < dir_path_len; i++) {
  178. tmp_dir_path[i] = directory_path[i];
  179. if ((tmp_dir_path[i] == '\\') || (tmp_dir_path[i] == '/')) {
  180. if (access(tmp_dir_path, F_OK) != 0) {
  181. int32_t ret = mmMkdir(tmp_dir_path, S_IRUSR | S_IWUSR | S_IXUSR); // 700
  182. if (ret != 0) {
  183. if (errno != EEXIST) {
  184. ErrorManager::GetInstance().ATCReportErrMessage("E19006", {"path"}, {directory_path});
  185. GELOGW("Can not create directory %s. Make sure the directory exists and writable.", directory_path.c_str());
  186. return ret;
  187. }
  188. }
  189. }
  190. }
  191. }
  192. int32_t ret = mmMkdir(const_cast<char *>(directory_path.c_str()), S_IRUSR | S_IWUSR | S_IXUSR); // 700
  193. if (ret != 0) {
  194. if (errno != EEXIST) {
  195. ErrorManager::GetInstance().ATCReportErrMessage("E19006", {"path"}, {directory_path});
  196. GELOGW("Can not create directory %s. Make sure the directory exists and writable.", directory_path.c_str());
  197. return ret;
  198. }
  199. }
  200. return 0;
  201. }
  202. FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY std::string CurrentTimeInStr() {
  203. std::time_t now = std::time(nullptr);
  204. std::tm *ptm = std::localtime(&now);
  205. if (ptm == nullptr) {
  206. GELOGE(ge::FAILED, "Localtime failed.");
  207. return "";
  208. }
  209. const int kTimeBufferLen = 32;
  210. char buffer[kTimeBufferLen + 1] = {0};
  211. // format: 20171122042550
  212. std::strftime(buffer, kTimeBufferLen, "%Y%m%d%H%M%S", ptm);
  213. return std::string(buffer);
  214. }
  215. FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY bool ReadProtoFromText(const char *file,
  216. google::protobuf::Message *message) {
  217. GE_CHK_BOOL_TRUE_EXEC_WITH_LOG((file == nullptr || message == nullptr), return false,
  218. "incorrect parameter. nullptr == file || nullptr == message");
  219. std::string real_path = RealPath(file);
  220. GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(real_path.empty(), ErrorManager::GetInstance().ATCReportErrMessage(
  221. "E19000", {"path", "errmsg"}, {file, strerror(errno)});
  222. return false, "Path[%s]'s realpath is empty, errmsg[%s]", file, strerror(errno));
  223. GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(GetFileLength(real_path) == -1, return false, "file size not valid.");
  224. std::ifstream fs(real_path.c_str(), std::ifstream::in);
  225. if (!fs.is_open()) {
  226. ErrorManager::GetInstance().ATCReportErrMessage("E19017", {"realpth", "protofile"}, {real_path, file});
  227. GELOGE(ge::FAILED, "Fail to open proto file real path is '%s' when orginal file path is '%s'.", real_path.c_str(),
  228. file);
  229. return false;
  230. }
  231. google::protobuf::io::IstreamInputStream input(&fs);
  232. bool ret = google::protobuf::TextFormat::Parse(&input, message);
  233. GE_IF_BOOL_EXEC(!ret, ErrorManager::GetInstance().ATCReportErrMessage("E19018", {"protofile"}, {file});
  234. GELOGE(ret,
  235. "Parse file[%s] through [google::protobuf::TextFormat::Parse] failed, "
  236. "please check whether the file is a valid protobuf format file.",
  237. file));
  238. fs.close();
  239. return ret;
  240. }
  241. FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY bool ReadProtoFromMem(const char *data, int size,
  242. google::protobuf::Message *message) {
  243. GE_CHK_BOOL_TRUE_EXEC_WITH_LOG((data == nullptr || message == nullptr), return false,
  244. "incorrect parameter. data is nullptr || message is nullptr");
  245. std::string str(data, static_cast<size_t>(size));
  246. std::istringstream fs(str);
  247. google::protobuf::io::IstreamInputStream input(&fs);
  248. bool ret = google::protobuf::TextFormat::Parse(&input, message);
  249. GE_IF_BOOL_EXEC(
  250. !ret, GELOGE(ret, "Call [google::protobuf::TextFormat::Parse] func ret fail, please check your text file."));
  251. return ret;
  252. }
  253. FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY uint64_t GetCurrentTimestamp() {
  254. struct timeval tv {};
  255. int ret = gettimeofday(&tv, nullptr);
  256. GE_LOGE_IF(ret != 0, "Func gettimeofday may failed: ret=%d", ret);
  257. auto total_use_time = tv.tv_usec + tv.tv_sec * 1000000; // 1000000: seconds to microseconds
  258. return static_cast<uint64_t>(total_use_time);
  259. }
  260. FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY uint32_t GetCurrentSecondTimestap() {
  261. struct timeval tv {};
  262. int ret = gettimeofday(&tv, nullptr);
  263. GE_LOGE_IF(ret != 0, "Func gettimeofday may failed: ret=%d", ret);
  264. auto total_use_time = tv.tv_sec; // seconds
  265. return static_cast<uint32_t>(total_use_time);
  266. }
  267. FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY bool CheckInt64MulOverflow(int64_t a, int64_t b) {
  268. if (a > 0) {
  269. if (b > 0) {
  270. if (a > (INT64_MAX / b)) {
  271. return false;
  272. }
  273. } else {
  274. if (b < (INT64_MIN / a)) {
  275. return false;
  276. }
  277. }
  278. } else {
  279. if (b > 0) {
  280. if (a < (INT64_MIN / b)) {
  281. return false;
  282. }
  283. } else {
  284. if ((a != 0) && (b < (INT64_MAX / a))) {
  285. return false;
  286. }
  287. }
  288. }
  289. return true;
  290. }
  291. FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY std::string RealPath(const char *path) {
  292. GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(path == nullptr, return "", "path pointer is NULL.");
  293. GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(
  294. strlen(path) >= PATH_MAX,
  295. ErrorManager::GetInstance().ATCReportErrMessage("E19002", {"filepath", "size"}, {path, std::to_string(PATH_MAX)});
  296. return "", "Path[%s] len is too long, it must be less than %d", path, PATH_MAX);
  297. // Nullptr is returned when the path does not exist or there is no permission
  298. // Return absolute path when path is accessible
  299. std::string res;
  300. char resolved_path[PATH_MAX] = {0};
  301. if (realpath(path, resolved_path) != nullptr) {
  302. res = resolved_path;
  303. }
  304. return res;
  305. }
  306. FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY bool CheckInputPathValid(const std::string &file_path,
  307. const std::string &atc_param) {
  308. // The specified path is empty
  309. std::map<std::string, std::string> args_map;
  310. if (file_path.empty()) {
  311. ErrorManager::GetInstance().ATCReportErrMessage("E10004", {"parameter"}, {atc_param});
  312. GELOGW("Input parameter %s is empty.", file_path.c_str());
  313. return false;
  314. }
  315. std::string real_path = RealPath(file_path.c_str());
  316. // Unable to get absolute path (does not exist or does not have permission to access)
  317. if (real_path.empty()) {
  318. ErrorManager::GetInstance().ATCReportErrMessage("E19000", {"path", "errmsg"}, {file_path, strerror(errno)});
  319. GELOGW("Path[%s]'s realpath is empty, errmsg[%s]", file_path.c_str(), strerror(errno));
  320. return false;
  321. }
  322. // A regular matching expression to verify the validity of the input file path
  323. // Path section: Support upper and lower case letters, numbers dots(.) chinese and underscores
  324. // File name section: Support upper and lower case letters, numbers, underscores chinese and dots(.)
  325. std::string mode = "^[\u4e00-\u9fa5A-Za-z0-9./_-]+$";
  326. GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(
  327. !ValidateStr(real_path, mode),
  328. ErrorManager::GetInstance().ATCReportErrMessage("E10001", {"parameter", "value", "reason"},
  329. {atc_param, real_path, kPathValidReason});
  330. return false, "Invalid value for %s[%s], %s.", atc_param.c_str(), real_path.c_str(), kPathValidReason);
  331. // The absolute path points to a file that is not readable
  332. if (access(real_path.c_str(), R_OK) != 0) {
  333. ErrorManager::GetInstance().ATCReportErrMessage("E19003", {"file", "errmsg"}, {file_path.c_str(), strerror(errno)});
  334. GELOGW("Read file[%s] failed, errmsg[%s]", file_path.c_str(), strerror(errno));
  335. return false;
  336. }
  337. return true;
  338. }
  339. FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY bool CheckOutputPathValid(const std::string &file_path,
  340. const std::string &atc_param) {
  341. // The specified path is empty
  342. if (file_path.empty()) {
  343. ErrorManager::GetInstance().ATCReportErrMessage("E10004", {"parameter"}, {atc_param});
  344. GELOGW("Input parameter's value is empty.");
  345. return false;
  346. }
  347. GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(
  348. strlen(file_path.c_str()) >= PATH_MAX, ErrorManager::GetInstance().ATCReportErrMessage(
  349. "E19002", {"filepath", "size"}, {file_path, std::to_string(PATH_MAX)});
  350. return "", "Path[%s] len is too long, it must be less than %d", file_path.c_str(), PATH_MAX);
  351. // A regular matching expression to verify the validity of the input file path
  352. // Path section: Support upper and lower case letters, numbers dots(.) chinese and underscores
  353. // File name section: Support upper and lower case letters, numbers, underscores chinese and dots(.)
  354. std::string mode = "^[\u4e00-\u9fa5A-Za-z0-9./_-]+$";
  355. GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(
  356. !ValidateStr(file_path, mode),
  357. ErrorManager::GetInstance().ATCReportErrMessage("E10001", {"parameter", "value", "reason"},
  358. {atc_param, file_path, kPathValidReason});
  359. return false, "Invalid value for %s[%s], %s.", atc_param.c_str(), file_path.c_str(), kPathValidReason);
  360. std::string real_path = RealPath(file_path.c_str());
  361. // Can get absolute path (file exists)
  362. if (!real_path.empty()) {
  363. // File is not readable or writable
  364. if (access(real_path.c_str(), W_OK | F_OK) != 0) {
  365. ErrorManager::GetInstance().ATCReportErrMessage("E19004", {"file", "errmsg"}, {real_path, strerror(errno)});
  366. GELOGW("Write file[%s] failed, errmsg[%s]", real_path.c_str(), strerror(errno));
  367. return false;
  368. }
  369. } else {
  370. // Find the last separator
  371. int path_split_pos = static_cast<int>(file_path.size() - 1);
  372. for (; path_split_pos >= 0; path_split_pos--) {
  373. if (file_path[path_split_pos] == '\\' || file_path[path_split_pos] == '/') {
  374. break;
  375. }
  376. }
  377. if (path_split_pos == 0) {
  378. return true;
  379. }
  380. if (path_split_pos != -1) {
  381. std::string prefix_path = std::string(file_path).substr(0, static_cast<size_t>(path_split_pos));
  382. // Determine whether the specified path is valid by creating the path
  383. if (CreateDirectory(prefix_path) != 0) {
  384. ErrorManager::GetInstance().ATCReportErrMessage("E19006", {"path"}, {file_path});
  385. GELOGW("Can not create directory[%s].", file_path.c_str());
  386. return false;
  387. }
  388. }
  389. }
  390. return true;
  391. }
  392. FMK_FUNC_HOST_VISIBILITY bool ValidateStr(const std::string &str, const std::string &mode) {
  393. char ebuff[kMaxBuffSize];
  394. regex_t reg;
  395. int cflags = REG_EXTENDED | REG_NOSUB;
  396. int ret = regcomp(&reg, mode.c_str(), cflags);
  397. if (ret) {
  398. regerror(ret, &reg, ebuff, kMaxBuffSize);
  399. GELOGW("regcomp failed, reason: %s", ebuff);
  400. regfree(&reg);
  401. return true;
  402. }
  403. ret = regexec(&reg, str.c_str(), 0, NULL, 0);
  404. if (ret) {
  405. regerror(ret, &reg, ebuff, kMaxBuffSize);
  406. GELOGE(ge::PARAM_INVALID, "regexec failed, reason: %s", ebuff);
  407. regfree(&reg);
  408. return false;
  409. }
  410. regfree(&reg);
  411. return true;
  412. }
  413. FMK_FUNC_HOST_VISIBILITY bool IsValidFile(const char *file_path) {
  414. if (file_path == nullptr) {
  415. GELOGE(PARAM_INVALID, "Config path is null.");
  416. return false;
  417. }
  418. if (!CheckInputPathValid(file_path)) {
  419. GELOGE(PARAM_INVALID, "Config path is invalid: %s", file_path);
  420. return false;
  421. }
  422. // Normalize the path
  423. std::string resolved_file_path = RealPath(file_path);
  424. if (resolved_file_path.empty()) {
  425. GELOGE(PARAM_INVALID, "Invalid input file path [%s], make sure that the file path is correct.", file_path);
  426. return false;
  427. }
  428. mmStat_t stat = {0};
  429. int32_t ret = mmStatGet(resolved_file_path.c_str(), &stat);
  430. if (ret != EN_OK) {
  431. GELOGE(PARAM_INVALID, "cannot get config file status, which path is %s, maybe not exist, return %d, errcode %d",
  432. resolved_file_path.c_str(), ret, mmGetErrorCode());
  433. return false;
  434. }
  435. if ((stat.st_mode & S_IFMT) != S_IFREG) {
  436. GELOGE(PARAM_INVALID, "config file is not a common file, which path is %s, mode is %u", resolved_file_path.c_str(),
  437. stat.st_mode);
  438. return false;
  439. }
  440. if (stat.st_size > kMaxConfigFileByte) {
  441. GELOGE(PARAM_INVALID, "config file %s size[%ld] is larger than max config file Bytes[%u]",
  442. resolved_file_path.c_str(), stat.st_size, kMaxConfigFileByte);
  443. return false;
  444. }
  445. return true;
  446. }
  447. FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status CheckPath(const char *path, size_t length) {
  448. if (path == nullptr) {
  449. GELOGE(PARAM_INVALID, "Config path is invalid.");
  450. return PARAM_INVALID;
  451. }
  452. if (strlen(path) != length) {
  453. GELOGE(PARAM_INVALID, "Path is invalid or length of config path is not equal to given length.");
  454. return PARAM_INVALID;
  455. }
  456. if (length == 0 || length > MMPA_MAX_PATH) {
  457. GELOGE(PARAM_INVALID, "Length of config path is invalid.");
  458. return PARAM_INVALID;
  459. }
  460. INT32 is_dir = mmIsDir(path);
  461. if (is_dir != EN_OK) {
  462. GELOGE(PATH_INVALID, "Open directory %s failed, maybe it is not exit or not a dir", path);
  463. return PATH_INVALID;
  464. }
  465. if (mmAccess2(path, M_R_OK) != EN_OK) {
  466. GELOGE(PATH_INVALID, "Read path[%s] failed, errmsg[%s]", path, strerror(errno));
  467. return PATH_INVALID;
  468. }
  469. return SUCCESS;
  470. }
  471. } // namespace ge

图引擎模块(GE)是MindSpore的一个子模块,其代码由C++实现,位于前端模块ME和底层硬件之间,起到承接作用。图引擎模块以ME下发的图作为输入,然后进行一系列的深度图优化操作,最后输出一张可以在底层硬件上高效运行的图。GE针对昇腾AI处理器的硬件结构特点,做了特定的优化工作,以此来充分发挥出昇腾AI处理器的强大算力。在进行模型训练/推理时,GE会被自动调用而用户并不感知。GE主要由GE API和GE Core两部分组成,详细的架构图如下所示