You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.

build.sh 5.4 kB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210
  1. #!/bin/bash
  2. # Copyright 2019-2020 Huawei Technologies Co., Ltd
  3. #
  4. # Licensed under the Apache License, Version 2.0 (the "License");
  5. # you may not use this file except in compliance with the License.
  6. # You may obtain a copy of the License at
  7. #
  8. # http://www.apache.org/licenses/LICENSE-2.0
  9. #
  10. # Unless required by applicable law or agreed to in writing, software
  11. # distributed under the License is distributed on an "AS IS" BASIS,
  12. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13. # See the License for the specific language governing permissions and
  14. # limitations under the License.
  15. # ============================================================================
  16. set -e
  17. BASEPATH=$(cd "$(dirname $0)"; pwd)
  18. OUTPUT_PATH="${BASEPATH}/output"
  19. export BUILD_PATH="${BASEPATH}/build/"
  20. # print usage message
  21. usage()
  22. {
  23. echo "Usage:"
  24. echo "sh build.sh [-j[n]] [-h] [-v] [-s] [-t] [-u] [-c] [-S on|off]"
  25. echo ""
  26. echo "Options:"
  27. echo " -h Print usage"
  28. echo " -u Only compile ut, not execute"
  29. echo " -s Build st"
  30. echo " -j[n] Set the number of threads used for building Metadef, default is 8"
  31. echo " -t Build and execute ut"
  32. echo " -c Build ut with coverage tag"
  33. echo " -v Display build command"
  34. echo " -S Enable enable download cmake compile dependency from gitee , default off"
  35. echo "to be continued ..."
  36. }
  37. # check value of input is 'on' or 'off'
  38. # usage: check_on_off arg_value arg_name
  39. check_on_off()
  40. {
  41. if [[ "X$1" != "Xon" && "X$1" != "Xoff" ]]; then
  42. echo "Invalid value $1 for option -$2"
  43. usage
  44. exit 1
  45. fi
  46. }
  47. # parse and set options
  48. checkopts()
  49. {
  50. VERBOSE=""
  51. THREAD_NUM=8
  52. # ENABLE_METADEF_UT_ONLY_COMPILE="off"
  53. ENABLE_GE_UT="off"
  54. ENABLE_GE_ST="off"
  55. ENABLE_GE_COV="off"
  56. GE_ONLY="on"
  57. ENABLE_GITEE="off"
  58. # Process the options
  59. while getopts 'ustchj:vS:' opt
  60. do
  61. OPTARG=$(echo ${OPTARG} | tr '[A-Z]' '[a-z]')
  62. case "${opt}" in
  63. u)
  64. # ENABLE_GE_UT_ONLY_COMPILE="on"
  65. ENABLE_GE_UT="on"
  66. GE_ONLY="off"
  67. ;;
  68. s)
  69. ENABLE_GE_ST="on"
  70. ;;
  71. t)
  72. ENABLE_GE_UT="on"
  73. GE_ONLY="off"
  74. ;;
  75. c)
  76. ENABLE_GE_COV="on"
  77. GE_ONLY="off"
  78. ;;
  79. h)
  80. usage
  81. exit 0
  82. ;;
  83. j)
  84. THREAD_NUM=$OPTARG
  85. ;;
  86. v)
  87. VERBOSE="VERBOSE=1"
  88. ;;
  89. S)
  90. check_on_off $OPTARG S
  91. ENABLE_GITEE="$OPTARG"
  92. echo "enable download from gitee"
  93. ;;
  94. *)
  95. echo "Undefined option: ${opt}"
  96. usage
  97. exit 1
  98. esac
  99. done
  100. }
  101. checkopts "$@"
  102. mk_dir() {
  103. local create_dir="$1" # the target to make
  104. mkdir -pv "${create_dir}"
  105. echo "created ${create_dir}"
  106. }
  107. # Meatdef build start
  108. echo "---------------- Metadef build start ----------------"
  109. # create build path
  110. build_metadef()
  111. {
  112. echo "create build directory and build Metadef";
  113. mk_dir "${BUILD_PATH}"
  114. cd "${BUILD_PATH}"
  115. CMAKE_ARGS="-DBUILD_PATH=$BUILD_PATH -DGE_ONLY=$GE_ONLY"
  116. if [[ "X$ENABLE_GE_COV" = "Xon" ]]; then
  117. CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_GE_COV=ON"
  118. fi
  119. if [[ "X$ENABLE_GE_UT" = "Xon" ]]; then
  120. CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_GE_UT=ON"
  121. fi
  122. if [[ "X$ENABLE_GE_ST" = "Xon" ]]; then
  123. CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_GE_ST=ON"
  124. fi
  125. if [[ "X$ENABLE_GITEE" = "Xon" ]]; then
  126. CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_GITEE=ON"
  127. fi
  128. CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_OPEN_SRC=True -DCMAKE_INSTALL_PREFIX=${OUTPUT_PATH}"
  129. echo "${CMAKE_ARGS}"
  130. cmake ${CMAKE_ARGS} ..
  131. if [ 0 -ne $? ]
  132. then
  133. echo "execute command: cmake ${CMAKE_ARGS} .. failed."
  134. return 1
  135. fi
  136. make ${VERBOSE} -j${THREAD_NUM} && make install
  137. if [ 0 -ne $? ]
  138. then
  139. echo "execute command: make ${VERBOSE} -j${THREAD_NUM} && make install failed."
  140. return 1
  141. fi
  142. echo "Metadef build success!"
  143. }
  144. g++ -v
  145. mk_dir ${OUTPUT_PATH}
  146. build_metadef || { echo "Metadef build failed."; return; }
  147. echo "---------------- Metadef build finished ----------------"
  148. rm -f ${OUTPUT_PATH}/libgmock*.so
  149. rm -f ${OUTPUT_PATH}/libgtest*.so
  150. rm -f ${OUTPUT_PATH}/lib*_stub.so
  151. chmod -R 750 ${OUTPUT_PATH}
  152. find ${OUTPUT_PATH} -name "*.so*" -print0 | xargs -0 chmod 500
  153. echo "---------------- Metadef output generated ----------------"
  154. # generate output package in tar form, including ut/st libraries/executables
  155. generate_package()
  156. {
  157. cd "${BASEPATH}"
  158. METADEF_LIB_PATH="lib"
  159. ACL_PATH="acllib/lib64"
  160. FWK_PATH="fwkacllib/lib64"
  161. ATC_PATH="atc/lib64"
  162. COMMON_LIB=("libgraph.so" "libregister.so")
  163. rm -rf ${OUTPUT_PATH:?}/${FWK_PATH}/
  164. rm -rf ${OUTPUT_PATH:?}/${ACL_PATH}/
  165. rm -rf ${OUTPUT_PATH:?}/${ATC_PATH}/
  166. mk_dir "${OUTPUT_PATH}/${FWK_PATH}"
  167. mk_dir "${OUTPUT_PATH}/${ATC_PATH}"
  168. mk_dir "${OUTPUT_PATH}/${ACL_PATH}"
  169. find output/ -name metadef_lib.tar -exec rm {} \;
  170. cd "${OUTPUT_PATH}"
  171. for lib in "${COMMON_LIB[@]}";
  172. do
  173. find ${OUTPUT_PATH}/${METADEF_LIB_PATH} -maxdepth 1 -name "$lib" -exec cp -f {} ${OUTPUT_PATH}/${FWK_PATH} \;
  174. find ${OUTPUT_PATH}/${METADEF_LIB_PATH} -maxdepth 1 -name "$lib" -exec cp -f {} ${OUTPUT_PATH}/${ATC_PATH} \;
  175. done
  176. find ${OUTPUT_PATH}/${METADEF_LIB_PATH} -maxdepth 1 -name "libc_sec.so" -exec cp -f {} ${OUTPUT_PATH}/${ATC_PATH} \;
  177. tar -cf metadef_lib.tar fwkacllib atc
  178. }
  179. if [[ "X$ENABLE_GE_UT" = "Xoff" ]]; then
  180. generate_package
  181. fi
  182. echo "---------------- Metadef package archive generated ----------------"

图引擎模块(GE)是MindSpore的一个子模块,其代码由C++实现,位于前端模块ME和底层硬件之间,起到承接作用。图引擎模块以ME下发的图作为输入,然后进行一系列的深度图优化操作,最后输出一张可以在底层硬件上高效运行的图。GE针对昇腾AI处理器的硬件结构特点,做了特定的优化工作,以此来充分发挥出昇腾AI处理器的强大算力。在进行模型训练/推理时,GE会被自动调用而用户并不感知。GE主要由GE API和GE Core两部分组成,详细的架构图如下所示