From ace58cddcfe73a464d97b712a4c282dfada5af57 Mon Sep 17 00:00:00 2001 From: taoxiangdong Date: Sat, 31 Oct 2020 18:02:20 +0800 Subject: [PATCH] update cmake compile dependency --- build.sh | 34 +++++++++++++++++++-- cmake/external_libs/gflags.cmake | 47 ++++++++++++++++++++++++++++ cmake/external_libs/json.cmake | 34 ++++++++++++++++----- cmake/external_libs/onnx.cmake | 33 +++++++++++++++++--- cmake/external_libs/protobuf_shared.cmake | 10 +++++- cmake/external_libs/protobuf_static.cmake | 51 +++++++++++++++++++++++++++++++ cmake/external_libs/protoc.cmake | 2 +- 7 files changed, 195 insertions(+), 16 deletions(-) create mode 100755 cmake/external_libs/gflags.cmake create mode 100755 cmake/external_libs/protobuf_static.cmake diff --git a/build.sh b/build.sh index 6c5642ed..beaf28c0 100644 --- a/build.sh +++ b/build.sh @@ -23,7 +23,7 @@ export BUILD_PATH="${BASEPATH}/build/" usage() { echo "Usage:" - echo "sh build.sh [-j[n]] [-h] [-v] [-s] [-t] [-u] [-c]" + echo "sh build.sh [-j[n]] [-h] [-v] [-s] [-t] [-u] [-c] [-S on|off]" echo "" echo "Options:" echo " -h Print usage" @@ -33,9 +33,21 @@ usage() echo " -t Build and execute ut" echo " -c Build ut with coverage tag" echo " -v Display build command" + echo " -S Enable enable download cmake compile dependency from gitee , default off" echo "to be continued ..." } +# check value of input is 'on' or 'off' +# usage: check_on_off arg_value arg_name +check_on_off() +{ + if [[ "X$1" != "Xon" && "X$1" != "Xoff" ]]; then + echo "Invalid value $1 for option -$2" + usage + exit 1 + fi +} + # parse and set options checkopts() { @@ -46,8 +58,11 @@ checkopts() ENABLE_GE_ST="off" ENABLE_GE_COV="off" GE_ONLY="on" + PLATFORM="inference" + PRODUCT="normal" + ENABLE_GITEE="off" # Process the options - while getopts 'ustchj:v' opt + while getopts 'ustchj:p:g:vS:' opt do OPTARG=$(echo ${OPTARG} | tr '[A-Z]' '[a-z]') case "${opt}" in @@ -77,6 +92,17 @@ checkopts() v) VERBOSE="VERBOSE=1" ;; + p) + PLATFORM=$OPTARG + ;; + g) + PRODUCT=$OPTARG + ;; + S) + check_on_off $OPTARG S + ENABLE_GITEE="$OPTARG" + echo "enable download from gitee" + ;; *) echo "Undefined option: ${opt}" usage @@ -117,6 +143,10 @@ build_graphengine() CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_GE_ST=ON" fi + if [[ "X$ENABLE_GITEE" = "Xon" ]]; then + CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_GITEE=ON" + fi + CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_OPEN_SRC=True -DCMAKE_INSTALL_PREFIX=${OUTPUT_PATH} -DPLATFORM=${PLATFORM} -DPRODUCT=${PRODUCT}" echo "${CMAKE_ARGS}" cmake ${CMAKE_ARGS} .. if [ $? -ne 0 ] diff --git a/cmake/external_libs/gflags.cmake b/cmake/external_libs/gflags.cmake new file mode 100755 index 00000000..0294192e --- /dev/null +++ b/cmake/external_libs/gflags.cmake @@ -0,0 +1,47 @@ +if (HAVE_GFLAGS) + return() +endif() + +include(ExternalProject) +#set(CMAKE_INSTALL_PREFIX ${GE_CODE_DIR}/output) + +if ((${CMAKE_INSTALL_PREFIX} STREQUAL /usr/local) OR + (${CMAKE_INSTALL_PREFIX} STREQUAL "C:/Program Files (x86)/ascend")) + set(CMAKE_INSTALL_PREFIX ${GE_CODE_DIR}/output CACHE STRING "path for install()" FORCE) + message(STATUS "No install prefix selected, default to ${CMAKE_INSTALL_PREFIX}.") +endif() + +if (ENABLE_GITEE) + set(REQ_URL "https://gitee.com/mirrors/gflags/repository/archive/v2.2.2.tar.gz") + set(MD5 "") +else() + set(REQ_URL "https://github.com/gflags/gflags/archive/v2.2.2.tar.gz") + set(MD5 "") +endif () + +ExternalProject_Add(gflags_build + URL ${REQ_URL} + #URL /home/txd/workspace/linux_cmake/pkg/protobuf-3.8.0.tar.gz + #SOURCE_DIR ${GE_CODE_DIR}/../third_party/gflags/src/gflags-2.2.2 + CONFIGURE_COMMAND ${CMAKE_COMMAND} -DCMAKE_CXX_FLAGS="-D_GLIBCXX_USE_CXX11_ABI=0" -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}/gflags + BUILD_COMMAND $(MAKE) + INSTALL_COMMAND $(MAKE) install + EXCLUDE_FROM_ALL TRUE +) + +set(GFLAGS_PKG_DIR ${CMAKE_INSTALL_PREFIX}/gflags) + +add_library(gflags_static STATIC IMPORTED) + +set_target_properties(gflags_static PROPERTIES + IMPORTED_LOCATION ${GFLAGS_PKG_DIR}/lib/libgflags.a +) + +add_library(gflags INTERFACE) +target_include_directories(gflags INTERFACE ${GFLAGS_PKG_DIR}/include) +target_link_libraries(gflags INTERFACE gflags_static) + +add_dependencies(gflags gflags_build) + +#set(HAVE_GFLAGS TRUE CACHE BOOL "gflags build add") +set(HAVE_GFLAGS TRUE) diff --git a/cmake/external_libs/json.cmake b/cmake/external_libs/json.cmake index f2ae5310..5e91c8b7 100644 --- a/cmake/external_libs/json.cmake +++ b/cmake/external_libs/json.cmake @@ -11,10 +11,30 @@ else() set(INCLUDE "./") endif () -graphengine_add_pkg(ge_nlohmann_json - VER 3.6.1 - HEAD_ONLY ${INCLUDE} - URL ${REQ_URL} - MD5 ${MD5}) -include_directories(${ge_nlohmann_json_INC}) -add_library(graphengine::json ALIAS ge_nlohmann_json) \ No newline at end of file +set(JSON_SRC_DIR ${CMAKE_BINARY_DIR}/opensrc/json/include) +if (ENABLE_GITEE) + set(REQ_URL "https://gitee.com/mirrors/JSON-for-Modern-CPP/repository/archive/v3.6.1.zip") + set(MD5 "5bda78ce308e6cfcf614dcf1d5ff27a7") + set(JSON_INCLUDE_DIR "${JSON_SRC_DIR}/include") +else() + set(REQ_URL "https://github.com/nlohmann/json/releases/download/v3.6.1/include.zip") + set(MD5 "0dc903888211db3a0f170304cd9f3a89") + set(JSON_INCLUDE_DIR ${JSON_SRC_DIR}) +endif () +ExternalProject_Add(json_build + URL ${REQ_URL} + #URL /home/txd/workspace/cloud_code/pkg/include.zip + SOURCE_DIR ${JSON_SRC_DIR} + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + EXCLUDE_FROM_ALL TRUE +) + + +add_library(json INTERFACE) +target_include_directories(json INTERFACE ${JSON_INCLUDE_DIR}) +add_dependencies(json json_build) + +#set(HAVE_JSON TRUE CACHE BOOL "json build add") +set(HAVE_JSON TRUE) diff --git a/cmake/external_libs/onnx.cmake b/cmake/external_libs/onnx.cmake index a092f964..f28c41c7 100644 --- a/cmake/external_libs/onnx.cmake +++ b/cmake/external_libs/onnx.cmake @@ -1,3 +1,11 @@ +include(ExternalProject) + +#set(ONNX_SRC_DIR /home/txd/workspace/cloud_code/graphengine/build/graphengine/open_source/onnx) +#set(ONNX_PROTO ${ONNX_SRC_DIR}/onnx/onnx.proto) +set(ONNX_PROTO_DIR ${CMAKE_BINARY_DIR}/onnx) +set(ONNX_PROTO_FILE ${ONNX_PROTO_DIR}/onnx.proto) +file(MAKE_DIRECTORY ${ONNX_PROTO_DIR}) + if (ENABLE_GITEE) set(REQ_URL "https://gitee.com/mirrors/ONNX/repository/archive/v1.6.0.tar.gz") set(MD5 "1bdbcecdd68ea8392630467646776e02") @@ -6,8 +14,23 @@ else() set(MD5 "512f2779d6215d4a36f366b6b9acdf1e") endif () -graphengine_add_pkg(onnx - VER 1.6.0 - HEAD_ONLY ./ - URL ${REQ_URL} - MD5 ${MD5}) +ExternalProject_Add(onnx + URL ${REQ_URL} + #URL /home/txd/workspace/cloud_code/pkg/onnx-1.6.0.tar.gz + #URL_HASH SHA256=3b88c3fe521151651a0403c4d131cb2e0311bd28b753ef692020a432a81ce345 + #SOURCE_DIR ${ONNX_SRC_DIR} + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + #INSTALL_COMMAND "" + INSTALL_COMMAND ${CMAKE_COMMAND} -E copy /onnx/onnx.proto ${ONNX_PROTO_FILE} + #BUILD_ALWAYS TRUE + EXCLUDE_FROM_ALL TRUE +) + +macro(onnx_protobuf_generate comp c_var h_var) + add_custom_command(OUTPUT ${ONNX_PROTO_FILE} + DEPENDS onnx + ) + ge_protobuf_generate(${comp} ${c_var} ${h_var} ${ONNX_PROTO_FILE}) +endmacro() + diff --git a/cmake/external_libs/protobuf_shared.cmake b/cmake/external_libs/protobuf_shared.cmake index a2b922fa..93c7907c 100644 --- a/cmake/external_libs/protobuf_shared.cmake +++ b/cmake/external_libs/protobuf_shared.cmake @@ -11,10 +11,18 @@ if ((${CMAKE_INSTALL_PREFIX} STREQUAL /usr/local) OR message(STATUS "No install prefix selected, default to ${CMAKE_INSTALL_PREFIX}.") endif() +if (ENABLE_GITEE) + set(REQ_URL "https://gitee.com/mirrors/protobuf_source/repository/archive/v3.8.0.tar.gz") + set(MD5 "eba86ae9f07ba5cfbaf8af3bc4e84236") +else() + set(REQ_URL "https://github.com/protocolbuffers/protobuf/archive/v3.8.0.tar.gz") + set(MD5 "3d9e32700639618a4d2d342c99d4507a") +endif () + set(protobuf_CXXFLAGS "-Wno-maybe-uninitialized -Wno-unused-parameter -fPIC -fstack-protector-all -D_FORTIFY_SOURCE=2 -D_GLIBCXX_USE_CXX11_ABI=0 -O2") set(protobuf_LDFLAGS "-Wl,-z,relro,-z,now,-z,noexecstack") ExternalProject_Add(protobuf_build - URL https://github.com/protocolbuffers/protobuf/archive/v3.8.0.tar.gz + URL ${REQ_URL} #URL /home/txd/workspace/linux_cmake/pkg/protobuf-3.8.0.tar.gz #SOURCE_DIR ${METADEF_DIR}/../../third_party/protobuf/src/protobuf-3.8.0 #DOWNLOAD_COMMAND ${CMAKE_COMMAND} -E copy_directory ${METADEF_DIR}/../../third_party/protobuf/src/protobuf-3.8.0 diff --git a/cmake/external_libs/protobuf_static.cmake b/cmake/external_libs/protobuf_static.cmake new file mode 100755 index 00000000..57f4fd05 --- /dev/null +++ b/cmake/external_libs/protobuf_static.cmake @@ -0,0 +1,51 @@ +include(ExternalProject) +include(GNUInstallDirs) +#set(CMAKE_INSTALL_PREFIX ${GE_CODE_DIR}/output) + +if ((${CMAKE_INSTALL_PREFIX} STREQUAL /usr/local) OR + (${CMAKE_INSTALL_PREFIX} STREQUAL "C:/Program Files (x86)/ascend")) + set(CMAKE_INSTALL_PREFIX ${GE_CODE_DIR}/output CACHE STRING "path for install()" FORCE) + message(STATUS "No install prefix selected, default to ${CMAKE_INSTALL_PREFIX}.") +endif() + +if (ENABLE_GITEE) + set(REQ_URL "https://gitee.com/mirrors/protobuf_source/repository/archive/v3.8.0.tar.gz") + set(MD5 "eba86ae9f07ba5cfbaf8af3bc4e84236") +else() + set(REQ_URL "https://github.com/protocolbuffers/protobuf/archive/v3.8.0.tar.gz") + set(MD5 "3d9e32700639618a4d2d342c99d4507a") +endif () + +set(protobuf_CXXFLAGS "-Wno-maybe-uninitialized -Wno-unused-parameter -fPIC -fstack-protector-all -D_FORTIFY_SOURCE=2 -D_GLIBCXX_USE_CXX11_ABI=0 -O2") +set(protobuf_LDFLAGS "-Wl,-z,relro,-z,now,-z,noexecstack") +set(PROTOBUF_STATIC_PKG_DIR ${CMAKE_INSTALL_PREFIX}/protobuf_static) +ExternalProject_Add(protobuf_static_build + URL ${REQ_URL} + #URL /home/txd/workspace/linux_cmake/pkg/protobuf-3.8.0.tar.gz + #SOURCE_DIR ${GE_CODE_DIR}/../third_party/protobuf/src/protobuf-3.8.0 + CONFIGURE_COMMAND ${CMAKE_COMMAND} + -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} + -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} + -DCMAKE_INSTALL_LIBDIR=${CMAKE_INSTALL_LIBDIR} + -DCMAKE_LINKER=${CMAKE_LINKER} + -DCMAKE_AR=${CMAKE_AR} + -DCMAKE_RANLIB=${CMAKE_RANLIB} + -Dprotobuf_WITH_ZLIB=OFF + -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_CXX_FLAGS=${protobuf_CXXFLAGS} -DCMAKE_CXX_LDFLAGS=${protobuf_LDFLAGS} -DCMAKE_INSTALL_PREFIX=${PROTOBUF_STATIC_PKG_DIR} /cmake + BUILD_COMMAND $(MAKE) + INSTALL_COMMAND $(MAKE) install + EXCLUDE_FROM_ALL TRUE +) +include(GNUInstallDirs) + +add_library(protobuf_static_lib STATIC IMPORTED) + +set_target_properties(protobuf_static_lib PROPERTIES + IMPORTED_LOCATION ${PROTOBUF_STATIC_PKG_DIR}/${CMAKE_INSTALL_LIBDIR}/libprotobuf.a +) + +add_library(protobuf_static INTERFACE) +target_include_directories(protobuf_static INTERFACE ${PROTOBUF_STATIC_PKG_DIR}/include) +target_link_libraries(protobuf_static INTERFACE protobuf_static_lib) + +add_dependencies(protobuf_static protobuf_static_build) diff --git a/cmake/external_libs/protoc.cmake b/cmake/external_libs/protoc.cmake index ed63afed..d1d329f2 100644 --- a/cmake/external_libs/protoc.cmake +++ b/cmake/external_libs/protoc.cmake @@ -8,7 +8,7 @@ include(GNUInstallDirs) if ((${CMAKE_INSTALL_PREFIX} STREQUAL /usr/local) OR (${CMAKE_INSTALL_PREFIX} STREQUAL "C:/Program Files (x86)/ascend")) - set(CMAKE_INSTALL_PREFIX ${GE_SOURCE_DIR}/output CACHE STRING "path for install()" FORCE) + set(CMAKE_INSTALL_PREFIX ${GE_CODE_DIR}/output CACHE STRING "path for install()" FORCE) message(STATUS "No install prefix selected, default to ${CMAKE_INSTALL_PREFIX}.") endif()