diff --git a/CMakeLists.txt b/CMakeLists.txt index 4ece7996..ebf15c21 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -95,6 +95,7 @@ option(MGE_WITH_ROCM "Enable ROCM support" OFF) option(MGE_WITH_LARGE_ARCHIVE "Enable big archive link support" OFF) option(MGE_BUILD_WITH_ASAN "Enable build with ASAN, need compiler support" OFF) option(MGE_WITH_CUSTOM_OP "Build with Custom op" OFF) +option(MGE_SYNC_THIRD_PARTY "help sync third_party submodule" OFF) if(MSVC OR WIN32) # FIXME: static link Windows vc runtime with some version from Visual Studio have some @@ -631,12 +632,6 @@ if(NOT MGE_ENABLE_EXCEPTIONS) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-exceptions") endif() -if(MGE_WITH_TEST) - include(cmake/gtest.cmake) -endif() - -include(cmake/gflags.cmake) - if(MGE_BUILD_IMPERATIVE_RT) set(CMAKE_CXX_STANDARD 17) endif() @@ -653,6 +648,17 @@ if(MGE_INFERENCE_ONLY) set(MGE_BUILD_IMPERATIVE_RT OFF) endif() +# please do any include(cmake/* after do this execute_process +if(MGE_SYNC_THIRD_PARTY) + include(cmake/third_party_sync.cmake) +endif() + +if(MGE_WITH_TEST) + include(cmake/gtest.cmake) +endif() + +include(cmake/gflags.cmake) + if(MGE_WITH_JIT_MLIR OR MGE_BUILD_IMPERATIVE_RT) include(cmake/llvm-project.cmake) endif() diff --git a/cmake/third_party_sync.cmake b/cmake/third_party_sync.cmake new file mode 100644 index 00000000..f4d35750 --- /dev/null +++ b/cmake/third_party_sync.cmake @@ -0,0 +1,46 @@ +set(SYNC_THIRD_PARTY_CMD "bash") +list(APPEND SYNC_THIRD_PARTY_CMD "${CMAKE_CURRENT_SOURCE_DIR}/third_party/prepare.sh") + +if(NOT MGE_WITH_JIT_MLIR AND NOT MGE_BUILD_IMPERATIVE_RT) + list(APPEND SYNC_THIRD_PARTY_CMD "-a") +endif() + +if(NOT MGE_WITH_TEST) + list(APPEND SYNC_THIRD_PARTY_CMD "-b") +endif() + +if((NOT MGE_WITH_MKLDNN) OR (NOT ${MGE_ARCH} STREQUAL "x86_64")) + list(APPEND SYNC_THIRD_PARTY_CMD "-c") +endif() + +if(NOT MGE_WITH_HALIDE) + list(APPEND SYNC_THIRD_PARTY_CMD "-d") +endif() + +if(NOT MGE_WITH_DISTRIBUTED) + list(APPEND SYNC_THIRD_PARTY_CMD "-e") + list(APPEND SYNC_THIRD_PARTY_CMD "-i") +endif() + +if(NOT MGE_WITH_CUDA) + list(APPEND SYNC_THIRD_PARTY_CMD "-f") +endif() + +if(NOT MGE_BUILD_IMPERATIVE_RT) + list(APPEND SYNC_THIRD_PARTY_CMD "-g") +endif() + +if(NOT ${MGE_BLAS} STREQUAL "OpenBLAS") + list(APPEND SYNC_THIRD_PARTY_CMD "-j") +endif() + +message("sync third_party with command: ${SYNC_THIRD_PARTY_CMD}") +execute_process( + COMMAND ${SYNC_THIRD_PARTY_CMD} + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} + RESULT_VARIABLE sync_third_party_cmd_ret) +if(NOT sync_third_party_cmd_ret EQUAL 0) + message( + FATAL_ERROR "Error run sync third_party please run ${SYNC_THIRD_PARTY_CMD} manually" + ) +endif() diff --git a/scripts/cmake-build/BUILD_README.md b/scripts/cmake-build/BUILD_README.md index 5aade7cb..ab581349 100755 --- a/scripts/cmake-build/BUILD_README.md +++ b/scripts/cmake-build/BUILD_README.md @@ -36,6 +36,8 @@ if you are use github MegEngine and build for Windows XP, please 2b: cp lib file to third_party/mkl/x86_32/lib/ ``` +About `third_party/prepare.sh`, also support to be managed by `CMake`, just config `EXTRA_CMAKE_ARGS="-DMGE_SYNC_THIRD_PARTY=ON` before run `scripts/cmake-build/*.sh` + But some dependencies need to be installed manually: * [CUDA](https://developer.nvidia.com/cuda-toolkit-archive)(>=10.1), [cuDNN](https://developer.nvidia.com/cudnn)(>=7.6) are required when building MegBrain with CUDA support. diff --git a/third_party/prepare.sh b/third_party/prepare.sh index 7ed44278..5d8365bf 100755 --- a/third_party/prepare.sh +++ b/third_party/prepare.sh @@ -15,34 +15,135 @@ if [ "$(printf '%s\n' "$requiredGitVersion" "$currentGitVersion" | ${SORT} -V | echo "Please update your Git version. (foud version $currentGitVersion, required version >= $requiredGitVersion)" exit -1 fi + +SYNC_LLVM_PROJECT=True +SYNC_GTEST_PROJECT=True +SYNC_DNNL_PROJECT=True +SYNC_HALIDE_PROJECT=True +SYNC_PROTOBUF_PROJECT=True +SYNC_CUTLASS_PROJECT=True +SYNC_IMPERATIVE_RT_PROJECT=True +SYNC_DISTRIBUTED_PROJECT=True +SYNC_OPENBLAS_PROJECT=True +function usage() { + echo "$0 args1 args2 .." + echo "available args detail:" + echo "-a : do not sync llvm-project" + echo "-b : do not sync gtest" + echo "-c : do not sync intel-mkl-dnn" + echo "-d : do not sync Halide" + echo "-e : do not sync protobuf" + echo "-f : do not sync cutlass" + echo "-g : do not sync IMPERATIVE_RT project" + echo "-i : do not sync DISTRIBUTED project" + echo "-j : do not sync OpenBLAS project" + echo "-h : show usage" + exit -1 +} + +while getopts "abcdefghij" arg +do + case $arg in + a) + echo "do not sync llvm-project" + SYNC_LLVM_PROJECT=False + ;; + b) + echo "do not sync gtest" + SYNC_GTEST_PROJECT=False + ;; + c) + echo "do not sync intel-mkl-dnn" + SYNC_DNNL_PROJECT=False + ;; + d) + echo "do not sync Halide" + SYNC_HALIDE_PROJECT=False + ;; + e) + echo "do not sync protobuf" + SYNC_PROTOBUF_PROJECT=False + ;; + f) + echo "do not sync cutlass" + SYNC_CUTLASS_PROJECT=False + ;; + g) + echo "do not sync IMPERATIVE_RT project" + SYNC_IMPERATIVE_RT_PROJECT=False + ;; + i) + echo "do not sync DISTRIBUTED project" + SYNC_DISTRIBUTED_PROJECT=False + ;; + j) + echo "do not sync OpenBLAS project" + SYNC_OPENBLAS_PROJECT=False + ;; + h) + echo "show usage" + usage + ;; + ?) + echo "unkonw argument" + usage + ;; + esac +done function git_submodule_update() { git submodule sync git submodule update -f --init midout - git submodule update -f --init intel-mkl-dnn - git submodule update -f --init Halide - git submodule update -f --init protobuf - git submodule update -f --init gtest git submodule update -f --init flatbuffers - git submodule update -f --init cutlass git submodule update -f --init Json - git submodule update -f --init pybind11 - git submodule update -f --init llvm-project - git submodule update -f --init range-v3 - git submodule update -f --init libzmq - git submodule update -f --init cppzmq - git submodule update -f --init OpenBLAS - git submodule update -f --init cpuinfo git submodule update -f --init gflags + git submodule update -f --init cpuinfo git submodule update -f --init cpp_redis git submodule update -f --init tacopie - git submodule update -f --init MegRay - pushd MegRay/third_party >/dev/null + if [ ${SYNC_DNNL_PROJECT} = "True" ];then + git submodule update -f --init intel-mkl-dnn + fi + + if [ ${SYNC_HALIDE_PROJECT} = "True" ];then + git submodule update -f --init Halide + fi + + if [ ${SYNC_PROTOBUF_PROJECT} = "True" ];then + git submodule update -f --init protobuf + fi + + if [ ${SYNC_GTEST_PROJECT} = "True" ];then + git submodule update -f --init gtest + fi + + if [ ${SYNC_CUTLASS_PROJECT} = "True" ];then + git submodule update -f --init cutlass + fi + + if [ ${SYNC_LLVM_PROJECT} = "True" ];then + git submodule update -f --init llvm-project + fi + + if [ ${SYNC_IMPERATIVE_RT_PROJECT} = "True" ];then + git submodule update -f --init pybind11 + git submodule update -f --init range-v3 + fi + + if [ ${SYNC_DISTRIBUTED_PROJECT} = "True" ];then + git submodule update -f --init libzmq + git submodule update -f --init cppzmq + git submodule update -f --init MegRay + pushd MegRay/third_party >/dev/null git submodule sync git submodule update -f --init nccl git submodule update -f --init gdrcopy git submodule update -f --init ucx - popd >/dev/null + popd >/dev/null + fi + + if [ ${SYNC_OPENBLAS_PROJECT} = "True" ];then + git submodule update -f --init OpenBLAS + fi } if [[ -z "${ALREADY_UPDATE_SUBMODULES}" ]]; then