You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.

ge_utils.cmake 14 kB

5 years ago
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349
  1. include(FetchContent)
  2. set(FETCHCONTENT_QUIET OFF)
  3. function(graphengine_add_submodule_obj des_submodule_objs sub_dir submodule_name_obj)
  4. add_subdirectory(${sub_dir})
  5. if(NOT TARGET ${submodule_name_obj})
  6. message(FATAL_ERROR "Can not find submodule '${submodule_name_obj}'. in ${CMAKE_CURRENT_LIST_FILE}")
  7. endif()
  8. if("$<TARGET_OBJECTS:${submodule_name_obj}>" IN_LIST ${des_submodule_objs})
  9. message(FATAL_ERROR "submodule '${submodule_name_obj}' added more than once. in ${CMAKE_CURRENT_LIST_FILE}")
  10. endif()
  11. set(${des_submodule_objs} ${${des_submodule_objs}} $<TARGET_OBJECTS:${submodule_name_obj}> PARENT_SCOPE)
  12. endfunction()
  13. get_filename_component(_MS_LIB_CACHE ~/.mslib REALPATH)
  14. if (NOT EXISTS ${_MS_LIB_CACHE})
  15. file(MAKE_DIRECTORY ${_MS_LIB_CACHE})
  16. endif ()
  17. # set(FETCHCONTENT_BASE_DIR ${_MS_LIB_CACHE})
  18. # set(CMAKE_PREFIX_PATH ${_MS_LIB_CACHE})
  19. if (DEFINED ENV{MSLIBS_SERVER})
  20. set(LOCAL_LIBS_SERVER $ENV{MSLIBS_SERVER})
  21. message("LOCAL_LIBS_SERVER: ${LOCAL_LIBS_SERVER}")
  22. endif ()
  23. if(LOCAL_LIBS_SERVER)
  24. if (NOT ENV{no_proxy})
  25. set(ENV{no_proxy} "${LOCAL_LIBS_SERVER}")
  26. else()
  27. string(FIND $ENV{no_proxy} ${LOCAL_LIBS_SERVER} IP_POS)
  28. if (${IP_POS} EQUAL -1)
  29. set(ENV{no_proxy} "$ENV{no_proxy},${LOCAL_LIBS_SERVER}")
  30. endif ()
  31. endif ()
  32. endif()
  33. function(__download_pkg pkg_name pkg_url pkg_md5)
  34. if(LOCAL_LIBS_SERVER)
  35. get_filename_component(_URL_FILE_NAME ${pkg_url} NAME)
  36. set(pkg_url "http://${LOCAL_LIBS_SERVER}:8081/libs/${pkg_name}/${_URL_FILE_NAME}" ${pkg_url})
  37. endif()
  38. FetchContent_Declare(
  39. ${pkg_name}
  40. URL ${pkg_url}
  41. URL_HASH MD5=${pkg_md5}
  42. )
  43. FetchContent_GetProperties(${pkg_name})
  44. message("download: ${${pkg_name}_SOURCE_DIR} , ${pkg_name} , ${pkg_url}")
  45. if(NOT ${pkg_name}_POPULATED)
  46. FetchContent_Populate(${pkg_name})
  47. set(${pkg_name}_SOURCE_DIR ${${pkg_name}_SOURCE_DIR} PARENT_SCOPE)
  48. endif()
  49. endfunction()
  50. function(__download_pkg_with_git pkg_name pkg_url pkg_git_commit pkg_md5)
  51. if(LOCAL_LIBS_SERVER)
  52. set(pkg_url "http://${LOCAL_LIBS_SERVER}:8081/libs/${pkg_name}/${pkg_git_commit}")
  53. FetchContent_Declare(
  54. ${pkg_name}
  55. URL ${pkg_url}
  56. URL_HASH MD5=${pkg_md5}
  57. )
  58. else()
  59. FetchContent_Declare(
  60. ${pkg_name}
  61. GIT_REPOSITORY ${pkg_url}
  62. GIT_TAG ${pkg_git_commit})
  63. endif()
  64. FetchContent_GetProperties(${pkg_name})
  65. message("download: ${${pkg_name}_SOURCE_DIR} , ${pkg_name} , ${pkg_url}")
  66. if(NOT ${pkg_name}_POPULATED)
  67. FetchContent_Populate(${pkg_name})
  68. set(${pkg_name}_SOURCE_DIR ${${pkg_name}_SOURCE_DIR} PARENT_SCOPE)
  69. endif()
  70. endfunction()
  71. function(__find_pkg_then_add_target pkg_name pkg_exe)
  72. unset(${pkg_name}_LIBS)
  73. message("_FIND:${${pkg_name}_BASE_DIR}")
  74. if(pkg_exe)
  75. find_program(${pkg_exe}_EXE ${pkg_exe} PATHS ${${pkg_name}_BASE_DIR}/bin NO_DEFAULT_PATH)
  76. if(NOT ${pkg_exe}_EXE)
  77. return()
  78. endif()
  79. add_executable(${pkg_name}::${pkg_exe} IMPORTED GLOBAL)
  80. set_target_properties(${pkg_name}::${pkg_exe} PROPERTIES
  81. IMPORTED_LOCATION ${${pkg_exe}_EXE}
  82. )
  83. message("found ${${pkg_exe}_EXE}")
  84. endif()
  85. foreach(_LIB_NAME ${ARGN})
  86. set(_LIB_SEARCH_NAME ${_LIB_NAME})
  87. set(_LIB_TYPE SHARED)
  88. if (${pkg_name}_USE_STATIC_LIBS)
  89. set(_LIB_SEARCH_NAME "${CMAKE_STATIC_LIBRARY_PREFIX}${_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}")
  90. set(_LIB_TYPE STATIC)
  91. endif ()
  92. set(${_LIB_NAME}_LIB ${_LIB_NAME}_LIB-NOTFOUND)
  93. find_library(${_LIB_NAME}_LIB ${_LIB_SEARCH_NAME} PATHS ${${pkg_name}_BASE_DIR}/lib NO_DEFAULT_PATH)
  94. if(NOT ${_LIB_NAME}_LIB)
  95. return()
  96. endif()
  97. add_library(${pkg_name}::${_LIB_NAME} ${_LIB_TYPE} IMPORTED GLOBAL)
  98. set_target_properties(${pkg_name}::${_LIB_NAME} PROPERTIES
  99. INTERFACE_INCLUDE_DIRECTORIES "${${pkg_name}_BASE_DIR}/include"
  100. IMPORTED_LOCATION ${${_LIB_NAME}_LIB}
  101. )
  102. list(APPEND ${pkg_name}_LIBS ${pkg_name}::${_LIB_NAME})
  103. message("found ${${_LIB_NAME}_LIB}")
  104. STRING( REGEX REPLACE "(.+)/(.+)" "\\1" LIBPATH ${${_LIB_NAME}_LIB})
  105. set(${pkg_name}_LIBPATH ${LIBPATH} CACHE STRING INTERNAL)
  106. endforeach(_LIB_NAME)
  107. set(${pkg_name}_LIBS ${${pkg_name}_LIBS} PARENT_SCOPE)
  108. endfunction()
  109. function(__exec_cmd)
  110. set(options )
  111. set(oneValueArgs WORKING_DIRECTORY)
  112. set(multiValueArgs COMMAND)
  113. cmake_parse_arguments(EXEC "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN} )
  114. execute_process(COMMAND ${EXEC_COMMAND}
  115. WORKING_DIRECTORY ${EXEC_WORKING_DIRECTORY}
  116. RESULT_VARIABLE RESULT)
  117. if(NOT RESULT EQUAL "0")
  118. message(FATAL_ERROR "error! when ${EXEC_COMMAND} in ${EXEC_WORKING_DIRECTORY}")
  119. endif()
  120. endfunction()
  121. function(__check_patches pkg_patches)
  122. # check patches
  123. if (PKG_PATCHES)
  124. file(TOUCH ${_MS_LIB_CACHE}/${pkg_name}_patch.md5)
  125. file(READ ${_MS_LIB_CACHE}/${pkg_name}_patch.md5 ${pkg_name}_PATCHES_MD5)
  126. message("patches md5:${${pkg_name}_PATCHES_MD5}")
  127. set(${pkg_name}_PATCHES_NEW_MD5 )
  128. foreach(_PATCH ${PKG_PATCHES})
  129. file(MD5 ${_PATCH} _PF_MD5)
  130. set(${pkg_name}_PATCHES_NEW_MD5 "${${pkg_name}_PATCHES_NEW_MD5},${_PF_MD5}")
  131. endforeach(_PATCH)
  132. if (NOT ${pkg_name}_PATCHES_MD5 STREQUAL ${pkg_name}_PATCHES_NEW_MD5)
  133. set(${pkg_name}_PATCHES ${PKG_PATCHES})
  134. file(REMOVE_RECURSE "${_MS_LIB_CACHE}/${pkg_name}-subbuild")
  135. file(WRITE ${_MS_LIB_CACHE}/${pkg_name}_patch.md5 ${${pkg_name}_PATCHES_NEW_MD5})
  136. message("patches changed : ${${pkg_name}_PATCHES_NEW_MD5}")
  137. endif ()
  138. endif ()
  139. endfunction()
  140. set(GE_FIND_NO_DEFAULT_PATH NO_CMAKE_PATH NO_CMAKE_ENVIRONMENT_PATH NO_SYSTEM_ENVIRONMENT_PATH
  141. NO_CMAKE_BUILDS_PATH NO_CMAKE_PACKAGE_REGISTRY NO_CMAKE_SYSTEM_PATH
  142. NO_CMAKE_SYSTEM_PACKAGE_REGISTRY)
  143. set(GE_FIND_NO_DEFAULT_PATH ${GE_FIND_NO_DEFAULT_PATH} PARENT_SCOPE)
  144. function(graphengine_add_pkg pkg_name )
  145. set(options )
  146. set(oneValueArgs URL MD5 GIT_REPOSITORY GIT_TAG VER EXE DIR HEAD_ONLY)
  147. set(multiValueArgs CMAKE_OPTION LIBS PRE_CONFIGURE_COMMAND CONFIGURE_COMMAND BUILD_OPTION INSTALL_INCS INSTALL_LIBS PATCHES)
  148. cmake_parse_arguments(PKG "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN} )
  149. set(__FIND_PKG_NAME ${pkg_name})
  150. string(TOLOWER ${pkg_name} pkg_name)
  151. message("pkg name:${__FIND_PKG_NAME},${pkg_name}")
  152. set(${pkg_name}_PATCHES_HASH )
  153. foreach(_PATCH ${PKG_PATCHES})
  154. file(MD5 ${_PATCH} _PF_MD5)
  155. set(${pkg_name}_PATCHES_HASH "${${pkg_name}_PATCHES_HASH},${_PF_MD5}")
  156. endforeach(_PATCH)
  157. # check options
  158. set(${pkg_name}_CONFIG_TXT
  159. "${CMAKE_CXX_COMPILER_VERSION}-${CMAKE_C_COMPILER_VERSION}
  160. ${ARGN} - ${${pkg_name}_USE_STATIC_LIBS}- ${${pkg_name}_PATCHES_HASH}
  161. ${${pkg_name}_CXXFLAGS}--${${pkg_name}_CFLAGS}--${${pkg_name}_LDFLAGS}")
  162. string(REPLACE ";" "-" ${pkg_name}_CONFIG_TXT ${${pkg_name}_CONFIG_TXT})
  163. string(MD5 ${pkg_name}_CONFIG_HASH ${${pkg_name}_CONFIG_TXT})
  164. message("${pkg_name} config hash: ${${pkg_name}_CONFIG_HASH}")
  165. set(${pkg_name}_BASE_DIR ${_MS_LIB_CACHE}/${pkg_name}_${${pkg_name}_CONFIG_HASH})
  166. set(${pkg_name}_DIRPATH ${${pkg_name}_BASE_DIR} CACHE STRING INTERNAL)
  167. if(EXISTS ${${pkg_name}_BASE_DIR}/options.txt AND PKG_HEAD_ONLY)
  168. set(${pkg_name}_INC ${${pkg_name}_BASE_DIR}/${PKG_HEAD_ONLY} PARENT_SCOPE)
  169. add_library(${pkg_name} INTERFACE)
  170. target_include_directories(${pkg_name} INTERFACE ${${pkg_name}_INC})
  171. return()
  172. endif ()
  173. if(NOT PKG_EXE)
  174. set(PKG_EXE 0)
  175. endif()
  176. set(${__FIND_PKG_NAME}_ROOT ${${pkg_name}_BASE_DIR})
  177. set(${__FIND_PKG_NAME}_ROOT ${${pkg_name}_BASE_DIR} PARENT_SCOPE)
  178. if (PKG_LIBS)
  179. __find_pkg_then_add_target(${pkg_name} ${PKG_EXE} ${PKG_LIBS})
  180. if(${pkg_name}_LIBS)
  181. set(${pkg_name}_INC ${${pkg_name}_BASE_DIR}/include PARENT_SCOPE)
  182. message("Found libs: ${${pkg_name}_LIBS}")
  183. return()
  184. endif()
  185. elseif(NOT PKG_HEAD_ONLY)
  186. find_package(${__FIND_PKG_NAME} ${PKG_VER} NO_CMAKE_SYSTEM_PATH NO_SYSTEM_ENVIRONMENT_PATH)
  187. if (${__FIND_PKG_NAME}_FOUND)
  188. set(${pkg_name}_INC ${${pkg_name}_BASE_DIR}/include PARENT_SCOPE)
  189. message("Found pkg: ${__FIND_PKG_NAME}")
  190. return()
  191. endif ()
  192. endif ()
  193. if (NOT PKG_DIR)
  194. if (PKG_GIT_REPOSITORY)
  195. __download_pkg_with_git(${pkg_name} ${PKG_GIT_REPOSITORY} ${PKG_GIT_TAG} ${PKG_MD5})
  196. else()
  197. __download_pkg(${pkg_name} ${PKG_URL} ${PKG_MD5})
  198. endif()
  199. else()
  200. set(${pkg_name}_SOURCE_DIR ${PKG_DIR})
  201. endif ()
  202. file(WRITE ${${pkg_name}_BASE_DIR}/options.txt ${${pkg_name}_CONFIG_TXT})
  203. message("${pkg_name}_SOURCE_DIR : ${${pkg_name}_SOURCE_DIR}")
  204. foreach(_PATCH_FILE ${PKG_PATCHES})
  205. message("patching ${${pkg_name}_SOURCE_DIR} -p1 < ${_PATCH_FILE}")
  206. execute_process(COMMAND patch -p1 INPUT_FILE ${_PATCH_FILE}
  207. WORKING_DIRECTORY ${${pkg_name}_SOURCE_DIR}
  208. RESULT_VARIABLE Result)
  209. if(NOT Result EQUAL "0")
  210. message(FATAL_ERROR "Failed patch: ${_PATCH_FILE}")
  211. endif()
  212. endforeach(_PATCH_FILE)
  213. file(LOCK ${${pkg_name}_BASE_DIR} DIRECTORY GUARD FUNCTION RESULT_VARIABLE ${pkg_name}_LOCK_RET TIMEOUT 600)
  214. if(NOT ${pkg_name}_LOCK_RET EQUAL "0")
  215. message(FATAL_ERROR "error! when try lock ${${pkg_name}_BASE_DIR} : ${${pkg_name}_LOCK_RET}")
  216. endif()
  217. if(${pkg_name}_SOURCE_DIR)
  218. if (PKG_HEAD_ONLY)
  219. file(GLOB ${pkg_name}_SOURCE_SUBDIRS ${${pkg_name}_SOURCE_DIR}/*)
  220. file(COPY ${${pkg_name}_SOURCE_SUBDIRS} DESTINATION ${${pkg_name}_BASE_DIR})
  221. set(${pkg_name}_INC ${${pkg_name}_BASE_DIR}/${PKG_HEAD_ONLY} PARENT_SCOPE)
  222. add_library(${pkg_name} INTERFACE)
  223. target_include_directories(${pkg_name} INTERFACE ${${pkg_name}_INC})
  224. elseif (PKG_CMAKE_OPTION)
  225. # in cmake
  226. file(MAKE_DIRECTORY ${${pkg_name}_SOURCE_DIR}/_build)
  227. if (${pkg_name}_CFLAGS)
  228. set(${pkg_name}_CMAKE_CFLAGS "-DCMAKE_C_FLAGS=${${pkg_name}_CFLAGS}")
  229. endif ()
  230. if (${pkg_name}_CXXFLAGS)
  231. set(${pkg_name}_CMAKE_CXXFLAGS "-DCMAKE_CXX_FLAGS=${${pkg_name}_CXXFLAGS}")
  232. endif ()
  233. if (${pkg_name}_LDFLAGS)
  234. if (${pkg_name}_USE_STATIC_LIBS)
  235. #set(${pkg_name}_CMAKE_LDFLAGS "-DCMAKE_STATIC_LINKER_FLAGS=${${pkg_name}_LDFLAGS}")
  236. else()
  237. set(${pkg_name}_CMAKE_LDFLAGS "-DCMAKE_SHARED_LINKER_FLAGS=${${pkg_name}_LDFLAGS}")
  238. endif ()
  239. endif ()
  240. __exec_cmd(COMMAND ${CMAKE_COMMAND} ${PKG_CMAKE_OPTION} -G ${CMAKE_GENERATOR}
  241. ${${pkg_name}_CMAKE_CFLAGS} ${${pkg_name}_CMAKE_CXXFLAGS} ${${pkg_name}_CMAKE_LDFLAGS}
  242. -DCMAKE_INSTALL_PREFIX=${${pkg_name}_BASE_DIR} ..
  243. WORKING_DIRECTORY ${${pkg_name}_SOURCE_DIR}/_build)
  244. __exec_cmd(COMMAND ${CMAKE_COMMAND} --build . --target install -- -j8
  245. WORKING_DIRECTORY ${${pkg_name}_SOURCE_DIR}/_build)
  246. else()
  247. if (${pkg_name}_CFLAGS)
  248. set(${pkg_name}_MAKE_CFLAGS "CFLAGS=${${pkg_name}_CFLAGS}")
  249. endif ()
  250. if (${pkg_name}_CXXFLAGS)
  251. set(${pkg_name}_MAKE_CXXFLAGS "CXXFLAGS=${${pkg_name}_CXXFLAGS}")
  252. endif ()
  253. if (${pkg_name}_LDFLAGS)
  254. set(${pkg_name}_MAKE_LDFLAGS "LDFLAGS=${${pkg_name}_LDFLAGS}")
  255. endif ()
  256. # in configure && make
  257. if (PKG_PRE_CONFIGURE_COMMAND)
  258. __exec_cmd(COMMAND ${PKG_PRE_CONFIGURE_COMMAND}
  259. WORKING_DIRECTORY ${${pkg_name}_SOURCE_DIR})
  260. endif ()
  261. if (PKG_CONFIGURE_COMMAND)
  262. __exec_cmd(COMMAND ${PKG_CONFIGURE_COMMAND}
  263. ${${pkg_name}_MAKE_CFLAGS} ${${pkg_name}_MAKE_CXXFLAGS} ${${pkg_name}_MAKE_LDFLAGS}
  264. --prefix=${${pkg_name}_BASE_DIR}
  265. WORKING_DIRECTORY ${${pkg_name}_SOURCE_DIR})
  266. endif ()
  267. set(${pkg_name}_BUILD_OPTION ${PKG_BUILD_OPTION})
  268. if (NOT PKG_CONFIGURE_COMMAND)
  269. set(${pkg_name}_BUILD_OPTION ${${pkg_name}_BUILD_OPTION}
  270. ${${pkg_name}_MAKE_CFLAGS} ${${pkg_name}_MAKE_CXXFLAGS} ${${pkg_name}_MAKE_LDFLAGS})
  271. endif ()
  272. # build
  273. __exec_cmd(COMMAND ${CMAKE_MAKE_PROGRAM} ${${pkg_name}_BUILD_OPTION} -j8
  274. WORKING_DIRECTORY ${${pkg_name}_SOURCE_DIR})
  275. if (PKG_INSTALL_INCS OR PKG_INSTALL_LIBS)
  276. file(GLOB ${pkg_name}_INSTALL_INCS ${${pkg_name}_SOURCE_DIR}/${PKG_INSTALL_INCS})
  277. file(GLOB ${pkg_name}_INSTALL_LIBS ${${pkg_name}_SOURCE_DIR}/${PKG_INSTALL_LIBS})
  278. file(COPY ${${pkg_name}_INSTALL_INCS} DESTINATION ${${pkg_name}_BASE_DIR}/include)
  279. file(COPY ${${pkg_name}_INSTALL_LIBS} DESTINATION ${${pkg_name}_BASE_DIR}/lib)
  280. else()
  281. __exec_cmd(COMMAND ${CMAKE_MAKE_PROGRAM} install WORKING_DIRECTORY ${${pkg_name}_SOURCE_DIR})
  282. endif ()
  283. endif ()
  284. endif()
  285. if (PKG_LIBS)
  286. __find_pkg_then_add_target(${pkg_name} ${PKG_EXE} ${PKG_LIBS})
  287. set(${pkg_name}_INC ${${pkg_name}_BASE_DIR}/include PARENT_SCOPE)
  288. if(NOT ${pkg_name}_LIBS)
  289. message(FATAL_ERROR "Can not find pkg: ${pkg_name}")
  290. endif()
  291. else()
  292. find_package(${__FIND_PKG_NAME} ${PKG_VER} QUIET)
  293. if (${__FIND_PKG_NAME}_FOUND)
  294. set(${pkg_name}_INC ${${pkg_name}_BASE_DIR}/include PARENT_SCOPE)
  295. message("Found pkg: ${${__FIND_PKG_NAME}_LIBRARIES}")
  296. return()
  297. endif ()
  298. endif ()
  299. endfunction()

图引擎模块(GE)是MindSpore的一个子模块,其代码由C++实现,位于前端模块ME和底层硬件之间,起到承接作用。图引擎模块以ME下发的图作为输入,然后进行一系列的深度图优化操作,最后输出一张可以在底层硬件上高效运行的图。GE针对昇腾AI处理器的硬件结构特点,做了特定的优化工作,以此来充分发挥出昇腾AI处理器的强大算力。在进行模型训练/推理时,GE会被自动调用而用户并不感知。GE主要由GE API和GE Core两部分组成,详细的架构图如下所示

Contributors (1)