You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.

util.cc 26 kB

5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
4 years ago
5 years ago
5 years ago
5 years ago
5 years ago
4 years ago
5 years ago
5 years ago
4 years ago
4 years ago
4 years ago
5 years ago
4 years ago
5 years ago
4 years ago
5 years ago
5 years ago
4 years ago
4 years ago
5 years ago
4 years ago
4 years ago
5 years ago
4 years ago
5 years ago
4 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
4 years ago
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591
  1. /**
  2. * Copyright 2020 Huawei Technologies Co., Ltd
  3. *
  4. * Licensed under the Apache License, Version 2.0 (the "License");
  5. * you may not use this file except in compliance with the License.
  6. * You may obtain a copy of the License at
  7. *
  8. * http://www.apache.org/licenses/LICENSE-2.0
  9. *
  10. * Unless required by applicable law or agreed to in writing, software
  11. * distributed under the License is distributed on an "AS IS" BASIS,
  12. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13. * See the License for the specific language governing permissions and
  14. * limitations under the License.
  15. */
  16. #include "framework/common/util.h"
  17. #include <sys/stat.h>
  18. #ifdef __GNUC__
  19. #include <regex.h>
  20. #else
  21. #include <regex>
  22. #endif
  23. #include <algorithm>
  24. #include <climits>
  25. #include <cstdlib>
  26. #include <ctime>
  27. #include <fstream>
  28. #include "common/util/error_manager/error_manager.h"
  29. #include "external/ge/ge_api_error_codes.h"
  30. #include "framework/common/debug/ge_log.h"
  31. #include "framework/common/fmk_types.h"
  32. #include "framework/common/ge_inner_error_codes.h"
  33. #include "google/protobuf/io/coded_stream.h"
  34. #include "google/protobuf/io/zero_copy_stream_impl.h"
  35. #include "mmpa/mmpa_api.h"
  36. using google::protobuf::io::CodedInputStream;
  37. using google::protobuf::io::FileInputStream;
  38. using google::protobuf::io::ZeroCopyInputStream;
  39. namespace {
  40. /*
  41. * kProtoReadBytesLimit and kWarningThreshold are real arguments of CodedInputStream::SetTotalBytesLimit.
  42. * In order to prevent integer overflow and excessive memory allocation during protobuf processing,
  43. * it is necessary to limit the length of proto message (call SetTotalBytesLimit function).
  44. * In theory, the minimum message length that causes an integer overflow is 512MB, and the default is 64MB.
  45. * If the limit of warning_threshold is exceeded, the exception information will be printed in stderr.
  46. * If such an exception is encountered during operation,
  47. * the proto file can be divided into several small files or the limit value can be increased.
  48. */
  49. const int kFileSizeOutLimitedOrOpenFailed = -1;
  50. const int kProtoReadBytesLimit = INT_MAX; // Max size of 2 GB minus 1 byte.
  51. const int kWarningThreshold = 1073741824; // 536870912 * 2 536870912 represent 512M
  52. /// The maximum length of the file.
  53. const uint32_t kMaxFileSizeLimit = UINT32_MAX; // 4G for now
  54. const int kMaxBuffSize = 256;
  55. const char *const kPathValidReason = "The path can only contain 'a-z' 'A-Z' '0-9' '-' '.' '_' and chinese character";
  56. constexpr uint32_t kMaxConfigFileByte = 10485760; // 10 * 1024 * 1024
  57. } // namespace
  58. namespace ge {
  59. static bool ReadProtoFromCodedInputStream(CodedInputStream &coded_stream, Message *proto) {
  60. GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(proto == nullptr, return false, "incorrect parameter. nullptr == proto");
  61. coded_stream.SetTotalBytesLimit(kProtoReadBytesLimit, kWarningThreshold);
  62. return proto->ParseFromCodedStream(&coded_stream);
  63. }
  64. FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY bool ReadProtoFromArray(const void *data, int size, Message *proto) {
  65. GE_CHK_BOOL_TRUE_EXEC_WITH_LOG((proto == nullptr || data == nullptr || size == 0), return false,
  66. "incorrect parameter. proto is nullptr || data is nullptr || size is 0");
  67. google::protobuf::io::CodedInputStream coded_stream(reinterpret_cast<uint8_t *>(const_cast<void *>(data)), size);
  68. return ReadProtoFromCodedInputStream(coded_stream, proto);
  69. }
  70. // Get file length
  71. long GetFileLength(const std::string &input_file) {
  72. GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(input_file.empty(), return -1, "input_file path is null.");
  73. std::string real_path = RealPath(input_file.c_str());
  74. GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(real_path.empty(), return -1, "input_file path '%s' not valid", input_file.c_str());
  75. unsigned long long file_length = 0;
  76. GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(
  77. mmGetFileSize(input_file.c_str(), &file_length) != EN_OK,
  78. ErrorManager::GetInstance().ATCReportErrMessage("E19001", {"file", "errmsg"}, {input_file, strerror(errno)});
  79. return kFileSizeOutLimitedOrOpenFailed, "Open file[%s] failed. errmsg:%s", input_file.c_str(), strerror(errno));
  80. GE_CHK_BOOL_TRUE_EXEC_WITH_LOG((file_length == 0),
  81. REPORT_INNER_ERROR("E19999", "file:%s size is 0, not valid", input_file.c_str());
  82. return -1, "File[%s] size is 0, not valid.", input_file.c_str());
  83. GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(
  84. file_length > kMaxFileSizeLimit,
  85. REPORT_INNER_ERROR("E19999", "file:%s size:%lld is out of limit: %d.", input_file.c_str(), file_length,
  86. kMaxFileSizeLimit);
  87. return kFileSizeOutLimitedOrOpenFailed, "File[%s] size %lld is out of limit: %d.", input_file.c_str(), file_length,
  88. kMaxFileSizeLimit);
  89. return static_cast<long>(file_length);
  90. }
  91. /** @ingroup domi_common
  92. * @brief Read all data from binary file
  93. * @param [in] file_name File path
  94. * @param [out] buffer The address of the output memory, which needs to be released by the caller
  95. * @param [out] length Output memory size
  96. * @return false fail
  97. * @return true success
  98. */
  99. FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY bool ReadBytesFromBinaryFile(const char *file_name, char **buffer,
  100. int &length) {
  101. GE_CHK_BOOL_TRUE_EXEC_WITH_LOG((file_name == nullptr), return false, "incorrect parameter. file is nullptr");
  102. GE_CHK_BOOL_TRUE_EXEC_WITH_LOG((buffer == nullptr), return false, "incorrect parameter. buffer is nullptr");
  103. std::string real_path = RealPath(file_name);
  104. GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(real_path.empty(), return false, "file path '%s' not valid", file_name);
  105. std::ifstream file(real_path.c_str(), std::ios::binary | std::ios::ate);
  106. if (!file.is_open()) {
  107. GELOGE(ge::FAILED, "[Read][File]Failed, file %s", file_name);
  108. REPORT_CALL_ERROR("E19999", "Read file %s failed", file_name);
  109. return false;
  110. }
  111. length = static_cast<int>(file.tellg());
  112. GE_CHK_BOOL_TRUE_EXEC_WITH_LOG((length <= 0), file.close(); return false, "file length <= 0");
  113. file.seekg(0, std::ios::beg);
  114. *buffer = new (std::nothrow) char[length]();
  115. GE_CHK_BOOL_TRUE_EXEC_RET_STATUS(*buffer == nullptr, false, file.close(), "new an object failed.");
  116. file.read(*buffer, length);
  117. file.close();
  118. return true;
  119. }
  120. FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY bool ReadBytesFromBinaryFile(const char *file_name,
  121. std::vector<char> &buffer) {
  122. GE_CHK_BOOL_TRUE_EXEC_WITH_LOG((file_name == nullptr), return false, "incorrect parameter. file path is null");
  123. std::string real_path = RealPath(file_name);
  124. GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(real_path.empty(), return false, "file path '%s' not valid", file_name);
  125. std::ifstream file(real_path.c_str(), std::ios::binary | std::ios::ate);
  126. if (!file.is_open()) {
  127. GELOGE(ge::FAILED, "[Read][File]Failed, file %s", file_name);
  128. REPORT_CALL_ERROR("E19999", "Read file %s failed", file_name);
  129. return false;
  130. }
  131. std::streamsize size = file.tellg();
  132. GE_CHK_BOOL_TRUE_EXEC_WITH_LOG((size <= 0), file.close(); return false, "file length <= 0, not valid.");
  133. GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(size > static_cast<int64_t>(kMaxFileSizeLimit), file.close();
  134. return false, "file size %ld is out of limit: %d.", size, kMaxFileSizeLimit);
  135. file.seekg(0, std::ios::beg); // [no need to check value]
  136. buffer.resize(static_cast<uint64_t>(size)); // [no need to check value]
  137. file.read(&buffer[0], size); // [no need to check value]
  138. file.close();
  139. GELOGI("Read size:%ld", size);
  140. return true;
  141. }
  142. /**
  143. * @ingroup domi_common
  144. * @brief Create directory, support to create multi-level directory
  145. * @param [in] directory_path Path, can be multi-level directory
  146. * @return -1 fail
  147. * @return 0 success
  148. */
  149. FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY int CreateDirectory(const std::string &directory_path) {
  150. GE_CHK_BOOL_EXEC(!directory_path.empty(), return -1, "directory path is empty.");
  151. auto dir_path_len = directory_path.length();
  152. if (dir_path_len >= MMPA_MAX_PATH) {
  153. ErrorManager::GetInstance().ATCReportErrMessage("E19002", {"filepath", "size"},
  154. {directory_path, std::to_string(MMPA_MAX_PATH)});
  155. GELOGW("Path[%s] len is too long, it must be less than %d", directory_path.c_str(), MMPA_MAX_PATH);
  156. return -1;
  157. }
  158. char tmp_dir_path[MMPA_MAX_PATH] = {0};
  159. for (size_t i = 0; i < dir_path_len; i++) {
  160. tmp_dir_path[i] = directory_path[i];
  161. if ((tmp_dir_path[i] == '\\') || (tmp_dir_path[i] == '/')) {
  162. if (mmAccess2(tmp_dir_path, M_F_OK) != EN_OK) {
  163. int32_t ret = mmMkdir(tmp_dir_path, M_IRUSR | M_IWUSR | M_IXUSR); // 700
  164. if (ret != 0) {
  165. if (errno != EEXIST) {
  166. REPORT_CALL_ERROR("E19999",
  167. "Can not create directory %s. Make sure the directory exists and writable. errmsg:%s",
  168. directory_path.c_str(), strerror(errno));
  169. GELOGW("Can not create directory %s. Make sure the directory exists and writable. errmsg:%s",
  170. directory_path.c_str(), strerror(errno));
  171. return ret;
  172. }
  173. }
  174. }
  175. }
  176. }
  177. int32_t ret = mmMkdir(const_cast<char *>(directory_path.c_str()), M_IRUSR | M_IWUSR | M_IXUSR); // 700
  178. if (ret != 0) {
  179. if (errno != EEXIST) {
  180. REPORT_CALL_ERROR("E19999",
  181. "Can not create directory %s. Make sure the directory exists and writable. errmsg:%s",
  182. directory_path.c_str(), strerror(errno));
  183. GELOGW("Can not create directory %s. Make sure the directory exists and writable. errmsg:%s",
  184. directory_path.c_str(), strerror(errno));
  185. return ret;
  186. }
  187. }
  188. return 0;
  189. }
  190. FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY std::string CurrentTimeInStr() {
  191. std::time_t now = std::time(nullptr);
  192. std::tm *ptm = std::localtime(&now);
  193. if (ptm == nullptr) {
  194. GELOGE(ge::FAILED, "[Check][Param]Localtime incorrect, errmsg %s", strerror(errno));
  195. REPORT_CALL_ERROR("E19999", "Localtime incorrect, errmsg %s", strerror(errno));
  196. return "";
  197. }
  198. const int kTimeBufferLen = 32;
  199. char buffer[kTimeBufferLen + 1] = {0};
  200. // format: 20171122042550
  201. std::strftime(buffer, kTimeBufferLen, "%Y%m%d%H%M%S", ptm);
  202. return std::string(buffer);
  203. }
  204. FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY bool ReadProtoFromText(const char *file,
  205. google::protobuf::Message *message) {
  206. GE_CHK_BOOL_TRUE_EXEC_WITH_LOG((file == nullptr || message == nullptr), return false,
  207. "incorrect parameter. nullptr == file || nullptr == message");
  208. std::string real_path = RealPath(file);
  209. GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(real_path.empty(), ErrorManager::GetInstance().ATCReportErrMessage(
  210. "E19000", {"path", "errmsg"}, {file, strerror(errno)});
  211. return false, "Path[%s]'s realpath is empty, errmsg[%s]", file, strerror(errno));
  212. GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(GetFileLength(real_path) == -1, return false, "file size not valid.");
  213. std::ifstream fs(real_path.c_str(), std::ifstream::in);
  214. if (!fs.is_open()) {
  215. REPORT_INNER_ERROR("E19999", "open file:%s failed", real_path.c_str());
  216. GELOGE(ge::FAILED, "[Open][ProtoFile]Failed, real path %s, orginal file path %s",
  217. real_path.c_str(), file);
  218. return false;
  219. }
  220. google::protobuf::io::IstreamInputStream input(&fs);
  221. bool ret = google::protobuf::TextFormat::Parse(&input, message);
  222. GE_IF_BOOL_EXEC(!ret, ErrorManager::GetInstance().ATCReportErrMessage("E19018", {"protofile"}, {file});
  223. GELOGE(ret, "[Parse][File]Through [google::protobuf::TextFormat::Parse] failed, "
  224. "file %s", file));
  225. fs.close();
  226. return ret;
  227. }
  228. FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY bool ReadProtoFromMem(const char *data, int size,
  229. google::protobuf::Message *message) {
  230. GE_CHK_BOOL_TRUE_EXEC_WITH_LOG((data == nullptr || message == nullptr), return false,
  231. "incorrect parameter. data is nullptr || message is nullptr");
  232. std::string str(data, static_cast<size_t>(size));
  233. std::istringstream fs(str);
  234. google::protobuf::io::IstreamInputStream input(&fs);
  235. bool ret = google::protobuf::TextFormat::Parse(&input, message);
  236. GE_IF_BOOL_EXEC(
  237. !ret, GELOGE(ret, "Call [google::protobuf::TextFormat::Parse] func ret fail, please check your text file."));
  238. return ret;
  239. }
  240. FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY uint64_t GetCurrentTimestamp() {
  241. mmTimeval tv{};
  242. int ret = mmGetTimeOfDay(&tv, nullptr);
  243. GE_LOGE_IF(ret != EN_OK, "Func gettimeofday may failed, ret:%d, errmsg:%s", ret, strerror(errno));
  244. auto total_use_time = tv.tv_usec + tv.tv_sec * 1000000; // 1000000: seconds to microseconds
  245. return static_cast<uint64_t>(total_use_time);
  246. }
  247. FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY uint32_t GetCurrentSecondTimestap() {
  248. mmTimeval tv{};
  249. int ret = mmGetTimeOfDay(&tv, nullptr);
  250. GE_LOGE_IF(ret != EN_OK, "Func gettimeofday may failed, ret:%d, errmsg:%s", ret, strerror(errno));
  251. auto total_use_time = tv.tv_sec; // seconds
  252. return static_cast<uint32_t>(total_use_time);
  253. }
  254. FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY bool CheckInt64MulOverflow(int64_t a, int64_t b) {
  255. if (a > 0) {
  256. if (b > 0) {
  257. if (a > (INT64_MAX / b)) {
  258. return false;
  259. }
  260. } else {
  261. if (b < (INT64_MIN / a)) {
  262. return false;
  263. }
  264. }
  265. } else {
  266. if (b > 0) {
  267. if (a < (INT64_MIN / b)) {
  268. return false;
  269. }
  270. } else {
  271. if ((a != 0) && (b < (INT64_MAX / a))) {
  272. return false;
  273. }
  274. }
  275. }
  276. return true;
  277. }
  278. FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY std::string RealPath(const char *path) {
  279. GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(path == nullptr, return "", "path pointer is NULL.");
  280. GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(strlen(path) >= MMPA_MAX_PATH,
  281. ErrorManager::GetInstance().ATCReportErrMessage("E19002", {"filepath", "size"},
  282. {path, std::to_string(MMPA_MAX_PATH)});
  283. return "", "Path[%s] len is too long, it must be less than %d", path, MMPA_MAX_PATH);
  284. // Nullptr is returned when the path does not exist or there is no permission
  285. // Return absolute path when path is accessible
  286. std::string res;
  287. char resolved_path[MMPA_MAX_PATH] = {0};
  288. if (mmRealPath(path, resolved_path, MMPA_MAX_PATH) == EN_OK) {
  289. res = resolved_path;
  290. }
  291. return res;
  292. }
  293. void PathValidErrReport(const std::string &file_path, const std::string &atc_param, const std::string &reason) {
  294. if (!atc_param.empty()) {
  295. REPORT_INPUT_ERROR("E10001", std::vector<std::string>({"parameter", "value", "reason"}),
  296. std::vector<std::string>({atc_param, file_path, reason}));
  297. } else {
  298. REPORT_INNER_ERROR("E19999", "Path[%s] invalid, reason:%s", file_path.c_str(), reason.c_str());
  299. }
  300. }
  301. FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY bool CheckInputPathValid(const std::string &file_path,
  302. const std::string &atc_param) {
  303. // The specified path is empty
  304. std::map<std::string, std::string> args_map;
  305. if (file_path.empty()) {
  306. if (!atc_param.empty()) {
  307. REPORT_INPUT_ERROR("E10004", std::vector<std::string>({"parameter"}), std::vector<std::string>({atc_param}));
  308. } else {
  309. REPORT_INNER_ERROR("E19999", "Param file_path is empty, check invalid.");
  310. }
  311. GELOGW("Input parameter %s is empty.", file_path.c_str());
  312. return false;
  313. }
  314. std::string real_path = RealPath(file_path.c_str());
  315. // Unable to get absolute path (does not exist or does not have permission to access)
  316. if (real_path.empty()) {
  317. std::string reason = "realpath error, errmsg:" + std::string(strerror(errno));
  318. PathValidErrReport(file_path, atc_param, reason);
  319. GELOGW("Path[%s]'s realpath is empty, errmsg[%s]", file_path.c_str(), strerror(errno));
  320. return false;
  321. }
  322. // A regular matching expression to verify the validity of the input file path
  323. // Path section: Support upper and lower case letters, numbers dots(.) chinese and underscores
  324. // File name section: Support upper and lower case letters, numbers, underscores chinese and dots(.)
  325. #ifdef __GNUC__
  326. std::string mode = "^[\u4e00-\u9fa5A-Za-z0-9./_-]+$";
  327. #else
  328. std::string mode = "^[a-zA-Z]:([\\\\/][^\\s\\\\/:*?<>\"|][^\\\\/:*?<>\"|]*)*([/\\\\][^\\s\\\\/:*?<>\"|])?$";
  329. #endif
  330. GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(
  331. !ValidateStr(real_path, mode),
  332. PathValidErrReport(file_path, atc_param, kPathValidReason);
  333. return false, "Invalid value for %s[%s], %s.", atc_param.c_str(), real_path.c_str(), kPathValidReason);
  334. // The absolute path points to a file that is not readable
  335. if (mmAccess2(real_path.c_str(), M_R_OK) != EN_OK) {
  336. PathValidErrReport(file_path, atc_param, "cat not access, errmsg:" + std::string(strerror(errno)));
  337. GELOGW("Read file[%s] failed, errmsg[%s]", file_path.c_str(), strerror(errno));
  338. return false;
  339. }
  340. return true;
  341. }
  342. FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY bool CheckOutputPathValid(const std::string &file_path,
  343. const std::string &atc_param) {
  344. // The specified path is empty
  345. if (file_path.empty()) {
  346. if (!atc_param.empty()) {
  347. REPORT_INPUT_ERROR("E10004", std::vector<std::string>({"parameter"}), std::vector<std::string>({atc_param}));
  348. } else {
  349. REPORT_INNER_ERROR("E19999", "Param file_path is empty, check invalid.");
  350. }
  351. ErrorManager::GetInstance().ATCReportErrMessage("E10004", {"parameter"}, {atc_param});
  352. GELOGW("Input parameter's value is empty.");
  353. return false;
  354. }
  355. GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(strlen(file_path.c_str()) >= MMPA_MAX_PATH,
  356. std::string reason = "len is too long, it must be less than " +
  357. std::to_string(MMPA_MAX_PATH);
  358. PathValidErrReport(file_path, atc_param, reason);
  359. return false, "Path[%s] len is too long, it must be less than %d", file_path.c_str(),
  360. MMPA_MAX_PATH);
  361. // A regular matching expression to verify the validity of the input file path
  362. // Path section: Support upper and lower case letters, numbers dots(.) chinese and underscores
  363. // File name section: Support upper and lower case letters, numbers, underscores chinese and dots(.)
  364. #ifdef __GNUC__
  365. std::string mode = "^[\u4e00-\u9fa5A-Za-z0-9./_-]+$";
  366. #else
  367. std::string mode = "^[a-zA-Z]:([\\\\/][^\\s\\\\/:*?<>\"|][^\\\\/:*?<>\"|]*)*([/\\\\][^\\s\\\\/:*?<>\"|])?$";
  368. #endif
  369. GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(
  370. !ValidateStr(file_path, mode),
  371. PathValidErrReport(file_path, atc_param, kPathValidReason);
  372. return false, "Invalid value for %s[%s], %s.", atc_param.c_str(), file_path.c_str(), kPathValidReason);
  373. std::string real_path = RealPath(file_path.c_str());
  374. // Can get absolute path (file exists)
  375. if (!real_path.empty()) {
  376. // File is not readable or writable
  377. if (mmAccess2(real_path.c_str(), M_W_OK | M_F_OK) != EN_OK) {
  378. PathValidErrReport(file_path, atc_param, "cat not access, errmsg:" + std::string(strerror(errno)));
  379. GELOGW("Write file[%s] failed, errmsg[%s]", real_path.c_str(), strerror(errno));
  380. return false;
  381. }
  382. } else {
  383. // Find the last separator
  384. int path_split_pos = static_cast<int>(file_path.size() - 1);
  385. for (; path_split_pos >= 0; path_split_pos--) {
  386. if (file_path[path_split_pos] == '\\' || file_path[path_split_pos] == '/') {
  387. break;
  388. }
  389. }
  390. if (path_split_pos == 0) {
  391. return true;
  392. }
  393. if (path_split_pos != -1) {
  394. std::string prefix_path = std::string(file_path).substr(0, static_cast<size_t>(path_split_pos));
  395. // Determine whether the specified path is valid by creating the path
  396. if (CreateDirectory(prefix_path) != 0) {
  397. PathValidErrReport(file_path, atc_param, "Can not create directory");
  398. GELOGW("Can not create directory[%s].", file_path.c_str());
  399. return false;
  400. }
  401. }
  402. }
  403. return true;
  404. }
  405. FMK_FUNC_HOST_VISIBILITY bool ValidateStr(const std::string &str, const std::string &mode) {
  406. #ifdef __GNUC__
  407. char ebuff[kMaxBuffSize];
  408. regex_t reg;
  409. int cflags = REG_EXTENDED | REG_NOSUB;
  410. int ret = regcomp(&reg, mode.c_str(), cflags);
  411. if (ret) {
  412. regerror(ret, &reg, ebuff, kMaxBuffSize);
  413. GELOGW("regcomp failed, reason: %s", ebuff);
  414. regfree(&reg);
  415. return true;
  416. }
  417. ret = regexec(&reg, str.c_str(), 0, NULL, 0);
  418. if (ret) {
  419. regerror(ret, &reg, ebuff, kMaxBuffSize);
  420. GELOGE(ge::PARAM_INVALID, "[Rgexec][Param]Failed, reason %s", ebuff);
  421. REPORT_CALL_ERROR("E19999", "Rgexec failed, reason %s", ebuff);
  422. regfree(&reg);
  423. return false;
  424. }
  425. regfree(&reg);
  426. return true;
  427. #else
  428. std::wstring wstr(str.begin(), str.end());
  429. std::wstring wmode(mode.begin(), mode.end());
  430. std::wsmatch match;
  431. bool res = false;
  432. try {
  433. std::wregex reg(wmode, std::regex::icase);
  434. // Matching string part
  435. res = regex_match(wstr, match, reg);
  436. res = regex_search(str, std::regex("[`!@#$%^&*()|{}';',<>?]"));
  437. } catch (std::exception &ex) {
  438. GELOGW("The directory %s is invalid, error: %s.", str.c_str(), ex.what());
  439. return false;
  440. }
  441. return !(res) && (str.size() == match.str().size());
  442. #endif
  443. }
  444. FMK_FUNC_HOST_VISIBILITY bool IsValidFile(const char *file_path) {
  445. if (file_path == nullptr) {
  446. GELOGE(PARAM_INVALID, "[Check][Param]Config path is null");
  447. REPORT_INNER_ERROR("E19999", "Config path is null");
  448. return false;
  449. }
  450. if (!CheckInputPathValid(file_path)) {
  451. GELOGE(PARAM_INVALID, "[Check][Param]Config path %s is invalid", file_path);
  452. REPORT_CALL_ERROR("E19999", "Config path %s is invalid", file_path);
  453. return false;
  454. }
  455. // Normalize the path
  456. std::string resolved_file_path = RealPath(file_path);
  457. if (resolved_file_path.empty()) {
  458. GELOGE(PARAM_INVALID, "[Check][Param]Invalid input file path %s, errmsg %s", file_path, strerror(errno));
  459. REPORT_CALL_ERROR("E19999", "Invalid input file path %s, errmsg %s", file_path, strerror(errno));
  460. return false;
  461. }
  462. mmStat_t stat = {0};
  463. int32_t ret = mmStatGet(resolved_file_path.c_str(), &stat);
  464. if (ret != EN_OK) {
  465. GELOGE(PARAM_INVALID, "[Get][FileStatus]Failed, which path %s maybe not exist, "
  466. "return %d, errcode %d", resolved_file_path.c_str(), ret, mmGetErrorCode());
  467. REPORT_CALL_ERROR("E19999", "Get config file status failed, which path %s maybe not exist, "
  468. "return %d, errcode %d", resolved_file_path.c_str(), ret, mmGetErrorCode());
  469. return false;
  470. }
  471. if ((stat.st_mode & S_IFMT) != S_IFREG) {
  472. GELOGE(PARAM_INVALID, "[Check][Param]Config file is not a common file, which path is %s, "
  473. "mode is %u", resolved_file_path.c_str(), stat.st_mode);
  474. REPORT_CALL_ERROR("E19999", "Config file is not a common file, which path is %s, "
  475. "mode is %u", resolved_file_path.c_str(), stat.st_mode);
  476. return false;
  477. }
  478. if (stat.st_size > kMaxConfigFileByte) {
  479. GELOGE(PARAM_INVALID, "[Check][Param]Config file %s size %ld is larger than max config "
  480. "file Bytes %u", resolved_file_path.c_str(), stat.st_size, kMaxConfigFileByte);
  481. REPORT_CALL_ERROR("E19999", "Config file %s size %ld is larger than max config file Bytes %u",
  482. resolved_file_path.c_str(), stat.st_size, kMaxConfigFileByte);
  483. return false;
  484. }
  485. return true;
  486. }
  487. FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status CheckPath(const char *path, size_t length) {
  488. if (path == nullptr) {
  489. GELOGE(PARAM_INVALID, "[Check][Param]Config path is invalid");
  490. REPORT_CALL_ERROR("E19999", "Config path is invalid");
  491. return PARAM_INVALID;
  492. }
  493. if (strlen(path) != length) {
  494. GELOGE(PARAM_INVALID, "[Check][Param]Path %s is invalid or length %zu "
  495. "not equal to given length %zu", path, strlen(path), length);
  496. REPORT_CALL_ERROR("E19999", "Path %s is invalid or length %zu "
  497. "not equal to given length %zu", path, strlen(path), length);
  498. return PARAM_INVALID;
  499. }
  500. if (length == 0 || length > MMPA_MAX_PATH) {
  501. GELOGE(PARAM_INVALID, "[Check][Param]Length of config path %zu is invalid", length);
  502. REPORT_INNER_ERROR("E19999", "Length of config path %zu is invalid", length);
  503. return PARAM_INVALID;
  504. }
  505. INT32 is_dir = mmIsDir(path);
  506. if (is_dir != EN_OK) {
  507. GELOGE(PATH_INVALID, "[Open][Directory]Failed, directory path %s, errmsg %s",
  508. path, strerror(errno));
  509. REPORT_CALL_ERROR("E19999", "Open directory %s failed, errmsg %s", path, strerror(errno));
  510. return PATH_INVALID;
  511. }
  512. if (mmAccess2(path, M_R_OK) != EN_OK) {
  513. GELOGE(PATH_INVALID, "[Read][Path]Failed, path %s, errmsg %s", path, strerror(errno));
  514. REPORT_CALL_ERROR("E19999", "Read path %s failed, errmsg %s", path, strerror(errno));
  515. return PATH_INVALID;
  516. }
  517. return SUCCESS;
  518. }
  519. } // namespace ge

图引擎模块(GE)是MindSpore的一个子模块,其代码由C++实现,位于前端模块ME和底层硬件之间,起到承接作用。图引擎模块以ME下发的图作为输入,然后进行一系列的深度图优化操作,最后输出一张可以在底层硬件上高效运行的图。GE针对昇腾AI处理器的硬件结构特点,做了特定的优化工作,以此来充分发挥出昇腾AI处理器的强大算力。在进行模型训练/推理时,GE会被自动调用而用户并不感知。GE主要由GE API和GE Core两部分组成,详细的架构图如下所示