You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.

memory_dumper.cc 5.9 kB

5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167
  1. /**
  2. * Copyright 2019-2020 Huawei Technologies Co., Ltd
  3. *
  4. * Licensed under the Apache License, Version 2.0 (the "License");
  5. * you may not use this file except in compliance with the License.
  6. * You may obtain a copy of the License at
  7. *
  8. * http://www.apache.org/licenses/LICENSE-2.0
  9. *
  10. * Unless required by applicable law or agreed to in writing, software
  11. * distributed under the License is distributed on an "AS IS" BASIS,
  12. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13. * See the License for the specific language governing permissions and
  14. * limitations under the License.
  15. */
  16. #include "common/debug/memory_dumper.h"
  17. #include <fcntl.h>
  18. #include <unistd.h>
  19. #include <string>
  20. #include "framework/common/debug/log.h"
  21. #include "framework/common/debug/ge_log.h"
  22. #include "framework/common/util.h"
  23. #include "framework/common/ge_inner_error_codes.h"
  24. using std::string;
  25. namespace {
  26. const int kInvalidFd = (-1);
  27. } // namespace
  28. namespace ge {
  29. FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY MemoryDumper::MemoryDumper() : fd_(kInvalidFd) {}
  30. FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY MemoryDumper::~MemoryDumper() { Close(); }
  31. // Dump the data to the file
  32. FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status MemoryDumper::DumpToFile(const char *filename, void *data,
  33. int64_t len) {
  34. #ifdef FMK_SUPPORT_DUMP
  35. GE_CHECK_NOTNULL(filename);
  36. GE_CHECK_NOTNULL(data);
  37. if (len == 0) {
  38. GELOGE(FAILED, "len is 0.");
  39. return PARAM_INVALID;
  40. }
  41. // Open the file
  42. int fd = OpenFile(filename);
  43. if (fd == kInvalidFd) {
  44. GELOGE(FAILED, "Open file failed.");
  45. return FAILED;
  46. }
  47. // Write the data to the file
  48. Status ret = SUCCESS;
  49. int32_t mmpa_ret = mmWrite(fd, data, len);
  50. // mmWrite return -1:Failed to write data to file;return -2:Invalid parameter
  51. if (mmpa_ret == EN_ERROR || mmpa_ret == EN_INVALID_PARAM) {
  52. GELOGE(FAILED, "Write to file failed. errno = %d, %s", mmpa_ret, strerror(errno));
  53. ret = FAILED;
  54. }
  55. // Close the file
  56. if (mmClose(fd) != EN_OK) { // mmClose return 0: success
  57. GELOGE(FAILED, "Close file failed.");
  58. ret = FAILED;
  59. }
  60. return ret;
  61. #else
  62. GELOGW("need to define FMK_SUPPORT_DUMP for dump op input and output.");
  63. return SUCCESS;
  64. #endif
  65. }
  66. // Open file
  67. FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status MemoryDumper::Open(const char *filename) {
  68. GE_CHK_BOOL_RET_STATUS(filename != nullptr, FAILED, "Incorrect parameter. filename is nullptr");
  69. // Try to remove file first for reduce the close time by overwriting way
  70. // (The process of file closing will be about 100~200ms slower per file when written by overwriting way)
  71. // If remove file failed, then try to open it with overwriting way
  72. int ret = remove(filename);
  73. // If remove file failed, print the warning log
  74. if (ret != 0) {
  75. GELOGW("Remove file failed.");
  76. }
  77. fd_ = OpenFile(filename);
  78. if (fd_ == kInvalidFd) {
  79. GELOGE(FAILED, "Open %s failed.", filename);
  80. return FAILED;
  81. }
  82. return SUCCESS;
  83. }
  84. // Dump the data to file
  85. FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status MemoryDumper::Dump(void *data, uint32_t len) const {
  86. GE_CHK_BOOL_RET_STATUS(data != nullptr, FAILED, "Incorrect parameter. data is nullptr");
  87. #ifdef FMK_SUPPORT_DUMP
  88. int32_t mmpa_ret = mmWrite(fd_, data, len);
  89. // mmWrite return -1:failed to write data to file;return -2:invalid parameter
  90. if (mmpa_ret == EN_ERROR || mmpa_ret == EN_INVALID_PARAM) {
  91. GELOGE(FAILED, "Write to file failed. errno = %d, %s", mmpa_ret, strerror(errno));
  92. return FAILED;
  93. }
  94. return SUCCESS;
  95. #else
  96. GELOGW("need to define FMK_SUPPORT_DUMP for dump op input and output.");
  97. return SUCCESS;
  98. #endif
  99. }
  100. // Close file
  101. void MemoryDumper::Close() noexcept {
  102. // Close file
  103. if (fd_ != kInvalidFd && mmClose(fd_) != EN_OK) {
  104. GELOGW("Close file failed.");
  105. }
  106. fd_ = kInvalidFd;
  107. }
  108. // Open file
  109. int MemoryDumper::OpenFile(const char *filename) {
  110. GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(filename == nullptr, return kInvalidFd, "Incorrect parameter. filename is nullptr");
  111. // Find the last separator
  112. int path_split_pos = static_cast<int>(strlen(filename) - 1);
  113. for (; path_split_pos >= 0; path_split_pos--) {
  114. GE_IF_BOOL_EXEC(filename[path_split_pos] == '\\' || filename[path_split_pos] == '/', break;)
  115. }
  116. // Get the absolute path
  117. string real_path;
  118. char tmp_path[PATH_MAX] = {0};
  119. GE_IF_BOOL_EXEC(
  120. -1 != path_split_pos, string prefix_path = std::string(filename).substr(0, path_split_pos);
  121. string last_path = std::string(filename).substr(path_split_pos, strlen(filename) - 1);
  122. GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(prefix_path.length() >= PATH_MAX, return kInvalidFd, "Prefix path is too long!");
  123. GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(realpath(prefix_path.c_str(), tmp_path) == nullptr, return kInvalidFd,
  124. "Dir %s does not exit.", prefix_path.c_str());
  125. real_path = std::string(tmp_path) + last_path;)
  126. GE_IF_BOOL_EXEC(
  127. path_split_pos == -1 || path_split_pos == 0,
  128. GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(strlen(filename) >= PATH_MAX, return kInvalidFd, "Prefix path is too long!");
  129. GE_IF_BOOL_EXEC(realpath(filename, tmp_path) == nullptr,
  130. GELOGI("File %s does not exit, it will be created.", filename));
  131. real_path = std::string(tmp_path);)
  132. // Open file, only the current user can read and write, to avoid malicious application access
  133. // Using the O_EXCL, if the file already exists,return failed to avoid privilege escalation vulnerability.
  134. mode_t mode = S_IRUSR | S_IWUSR;
  135. int32_t fd = mmOpen2(real_path.c_str(), O_RDWR | O_CREAT | O_APPEND, mode);
  136. if (fd == EN_ERROR || fd == EN_INVALID_PARAM) {
  137. GELOGE(kInvalidFd, "open file failed. errno = %d, %s", fd, strerror(errno));
  138. return kInvalidFd;
  139. }
  140. return fd;
  141. }
  142. } // namespace ge

图引擎模块(GE)是MindSpore的一个子模块,其代码由C++实现,位于前端模块ME和底层硬件之间,起到承接作用。图引擎模块以ME下发的图作为输入,然后进行一系列的深度图优化操作,最后输出一张可以在底层硬件上高效运行的图。GE针对昇腾AI处理器的硬件结构特点,做了特定的优化工作,以此来充分发挥出昇腾AI处理器的强大算力。在进行模型训练/推理时,GE会被自动调用而用户并不感知。GE主要由GE API和GE Core两部分组成,详细的架构图如下所示