OpenI
/
MegEngine
Not watched
1
0
Fork 0
Code
Releases 31 Wiki Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain
Browse Source

feat(build): change make to ninja 

GitOrigin-RevId: ccff72e39a
release-1.4
Megvii Engine Team 4 years ago
parent
e70c9a3839
commit
2f1e7c5352
14 changed files with 364 additions and 101 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +4
    -0
      CMakeLists.txt
  2. +4
    -4
      scripts/cmake-build/BUILD_README.md
  3. +19
    -8
      scripts/cmake-build/cross_build_android_arm_inference.sh
  4. +20
    -5
      scripts/cmake-build/cross_build_ios_arm_inference.sh
  5. +18
    -8
      scripts/cmake-build/cross_build_linux_arm_inference.sh
  6. +34
    -21
      scripts/cmake-build/host_build.sh
  7. +36
    -5
      scripts/cmake-build/utils/utils.sh
  8. +46
    -12
      scripts/whl/macos/macos_build_whl.sh
  9. +1
    -1
      scripts/whl/macos/macos_whl_env_prepare.sh
  10. +14
    -12
      scripts/whl/manylinux2014/build_wheel_common.sh
  11. +39
    -5
      scripts/whl/manylinux2014/do_build_common.sh
  12. +4
    -1
      scripts/whl/manylinux2014/init_image.sh
  13. +81
    -0
      scripts/whl/utils/utils.sh
  14. +44
    -19
      scripts/whl/windows/windows_build_whl.sh

+ 4
- 0
CMakeLists.txt View File

@@ -1,4 +1,8 @@
cmake_minimum_required(VERSION 3.15.2)
message(STATUS "CMAKE_GENERATOR: ${CMAKE_GENERATOR}" )
if (NOT ${CMAKE_GENERATOR} STREQUAL "Ninja")
message(WARNING "CMAKE_GENERATOR NOT EQUAL Ninja, which we do not recommend")
endif()

include (cmake/FetchMegBrainVersion.cmake)
project(MegEngine LANGUAGES C CXX VERSION ${MGB_VER_STRING})


+ 4
- 4
scripts/cmake-build/BUILD_README.md View File

@@ -19,7 +19,7 @@
* commands:
```
1: installl Visual Studio (need support LLVM/clang-cl), eg 2019. Please install LLVM-10, VS LLVM linker have issue, please replace lld-link.exe, which can be download from https://releases.llvm.org/download.html#10.0.0
2: install extension of VS: Python/Cmake/LLVM
2: install extension of VS: Python/Cmake/LLVM/Ninja
3: now we support cuda10.1+cudnn7.6+TensorRT6.0 on Windows, as Windows can only use DLL in fact with cudnn/TensorRT, so please install the same version;
3a: install cuda10.1 to C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.1
3b: install cudnn7.6 to C:\Program Files\NVIDIA GPU Computing Toolkit\cudnn-10.1-windows10-x64-v7.6.5.32
@@ -33,9 +33,9 @@
### Linux host build
* commands:
```
1: install Cmake, which version >= 3.15.2
1: install Cmake, which version >= 3.15.2, ninja-build
2: install gcc/g++, which version >= 6, (gcc/g++ >= 7, if need build training mode)
3: install build-essential git git-lfs gfortran libgfortran-6-dev autoconf gnupg flex bison gperf curl zlib1g-dev gcc-multilib g++-multilib lib32ncurses5-dev libxml2-utils xsltproc unzip libtool librdmacm-dev rdmacm-utils python3-dev swig python3-numpy texinfo
3: install build-essential git git-lfs gfortran libgfortran-6-dev autoconf gnupg flex bison gperf curl zlib1g-dev gcc-multilib g++-multilib lib32ncurses5-dev libxml2-utils xsltproc unzip libtool librdmacm-dev rdmacm-utils python3-dev python3-numpy texinfo
4: CUDA env(if enable CUDA), version detail refer to README.md
```

@@ -44,7 +44,7 @@
```
1: install Cmake, which version >= 3.15.2
2: install brew: /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install.sh)"
3: brew install python python3 swig coreutils
3: brew install python python3 coreutils ninja
4: install at least xcode command line tool: https://developer.apple.com/xcode/
5: about cuda: we do not support CUDA on MacOS
6: python3 -m pip install numpy (if you want to build with training mode)


+ 19
- 8
scripts/cmake-build/cross_build_android_arm_inference.sh View File

@@ -7,6 +7,9 @@ MGE_ARMV8_2_FEATURE_FP16=OFF
MGE_DISABLE_FLOAT16=OFF
ARCH=arm64-v8a
REMOVE_OLD_BUILD=false
NINJA_VERBOSE=OFF
NINJA_DRY_RUN=OFF

echo "EXTRA_CMAKE_ARGS: ${EXTRA_CMAKE_ARGS}"

function usage() {
@@ -17,13 +20,15 @@ function usage() {
echo "-k : open MGE_DISABLE_FLOAT16 for NEON "
echo "-a : config build arch available: ${ARCHS[@]}"
echo "-r : remove old build dir before make, default off"
echo "-v : ninja with verbose and explain, default off"
echo "-n : ninja with -n dry run (don't run commands but act like they succeeded)"
echo "-h : show usage"
echo "append other cmake config by export EXTRA_CMAKE_ARGS=..."
echo "example: $0 -d"
exit -1
}

while getopts "rkhdfa:" arg
while getopts "nvrkhdfa:" arg
do
case $arg in
d)
@@ -62,6 +67,14 @@ do
echo "config REMOVE_OLD_BUILD=true"
REMOVE_OLD_BUILD=true
;;
v)
echo "config NINJA_VERBOSE=ON"
NINJA_VERBOSE=ON
;;
n)
echo "config NINJA_DRY_RUN=ON"
NINJA_DRY_RUN=ON
;;
?)
echo "unkonw argument"
usage
@@ -77,14 +90,12 @@ echo "ARCH: $ARCH"
echo "----------------------------------------------------"

READLINK=readlink
MAKEFILE_TYPE="Unix"
OS=$(uname -s)

if [ $OS = "Darwin" ];then
READLINK=greadlink
elif [[ $OS =~ "NT" ]]; then
echo "BUILD in NT ..."
MAKEFILE_TYPE="Unix"
fi

SRC_DIR=$($READLINK -f "`dirname $0`/../../")
@@ -105,7 +116,6 @@ function cmake_build() {
echo "build type: $BUILD_TYPE"
echo "build ABI: $BUILD_ABI"
echo "build native level: $BUILD_NATIVE_LEVEL"
echo "BUILD MAKEFILE_TYPE: $MAKEFILE_TYPE"
try_remove_old_build $REMOVE_OLD_BUILD $BUILD_DIR $INSTALL_DIR

echo "create build dir"
@@ -113,7 +123,7 @@ function cmake_build() {
mkdir -p $INSTALL_DIR
cd_real_build_dir $BUILD_DIR
unset IFS
cmake -G "$MAKEFILE_TYPE Makefiles" \
bash -c "cmake -G Ninja \
-DCMAKE_TOOLCHAIN_FILE="$NDK_ROOT/build/cmake/android.toolchain.cmake" \
-DANDROID_NDK="$NDK_ROOT" \
-DCMAKE_BUILD_TYPE=$BUILD_TYPE \
@@ -125,10 +135,11 @@ function cmake_build() {
-DMGE_DISABLE_FLOAT16=$MGE_DISABLE_FLOAT16 \
-DCMAKE_INSTALL_PREFIX=$INSTALL_DIR \
${EXTRA_CMAKE_ARGS} \
$SRC_DIR
$SRC_DIR "

make -j$(nproc) ${Target}
make install/strip
config_ninja_target_cmd ${NINJA_VERBOSE} "OFF" "" ${NINJA_DRY_RUN}
bash -c "${NINJA_CMD}"
${NINJA_BASE} install/strip
}

build_flatc $SRC_DIR $REMOVE_OLD_BUILD


+ 20
- 5
scripts/cmake-build/cross_build_ios_arm_inference.sh View File

@@ -7,6 +7,9 @@ MGE_ARMV8_2_FEATURE_FP16=OFF
MGE_DISABLE_FLOAT16=OFF
ARCH=arm64
REMOVE_OLD_BUILD=false
NINJA_VERBOSE=OFF
NINJA_DRY_RUN=OFF

echo "EXTRA_CMAKE_ARGS: ${EXTRA_CMAKE_ARGS}"

function usage() {
@@ -17,13 +20,15 @@ function usage() {
echo "-k : open MGE_DISABLE_FLOAT16 for NEON "
echo "-a : config build arch available: ${ARCHS[@]}"
echo "-r : remove old build dir before make, default off"
echo "-v : ninja with verbose and explain, default off"
echo "-n : ninja with -n dry run (don't run commands but act like they succeeded)"
echo "-h : show usage"
echo "append other cmake config by export EXTRA_CMAKE_ARGS=..."
echo "example: $0 -d"
exit -1
}

while getopts "rkhdfa:" arg
while getopts "nvrkhdfa:" arg
do
case $arg in
d)
@@ -62,6 +67,14 @@ do
echo "config REMOVE_OLD_BUILD=true"
REMOVE_OLD_BUILD=true
;;
v)
echo "config NINJA_VERBOSE=ON"
NINJA_VERBOSE=ON
;;
n)
echo "config NINJA_DRY_RUN=ON"
NINJA_DRY_RUN=ON
;;
?)
echo "unkonw argument"
usage
@@ -109,7 +122,8 @@ function cmake_build() {
mkdir -p $BUILD_DIR
mkdir -p $INSTALL_DIR
cd_real_build_dir $BUILD_DIR
cmake -DCMAKE_TOOLCHAIN_FILE=$TOOLCHAIN \
bash -c "cmake -G Ninja \
-DCMAKE_TOOLCHAIN_FILE=$TOOLCHAIN \
-DCMAKE_BUILD_TYPE=$BUILD_TYPE \
-DIOS_TOOLCHAIN_ROOT=$TOOLCHAIN \
-DOS_PLATFORM=$OS_PLATFORM \
@@ -121,11 +135,12 @@ function cmake_build() {
-DMGE_ARMV8_2_FEATURE_FP16= $MGE_ARMV8_2_FEATURE_FP16 \
-DMGE_DISABLE_FLOAT16=$MGE_DISABLE_FLOAT16 \
-DCMAKE_INSTALL_PREFIX=$INSTALL_DIR \
-DCMAKE_MAKE_PROGRAM=ninja \
${EXTRA_CMAKE_ARGS} \
$SRC_DIR
$SRC_DIR "

make -j$(nproc)
make install
config_ninja_target_cmd ${NINJA_VERBOSE} "OFF" "install" ${NINJA_DRY_RUN}
bash -c "${NINJA_CMD}"
}

build_flatc $SRC_DIR $REMOVE_OLD_BUILD


+ 18
- 8
scripts/cmake-build/cross_build_linux_arm_inference.sh View File

@@ -8,8 +8,11 @@ MGE_ARMV8_2_FEATURE_FP16=OFF
MGE_DISABLE_FLOAT16=OFF
ARCH=arm64-v8a
REMOVE_OLD_BUILD=false
NINJA_VERBOSE=OFF
NINJA_DRY_RUN=OFF
CMAKE_C_FLAGS="-Wno-psabi"
CMAKE_CXX_FLAGS="-Wno-psabi"

echo "EXTRA_CMAKE_ARGS: ${EXTRA_CMAKE_ARGS}"

function usage() {
@@ -21,13 +24,15 @@ function usage() {
echo "-k : open MGE_DISABLE_FLOAT16 for NEON "
echo "-a : config build arch available: ${ARCHS[@]}"
echo "-r : remove old build dir before make, default off"
echo "-v : ninja with verbose and explain, default off"
echo "-n : ninja with -n dry run (don't run commands but act like they succeeded)"
echo "-h : show usage"
echo "append other cmake config by export EXTRA_CMAKE_ARGS=..."
echo "example: $0 -d"
exit -1
}

while getopts "rkhdcfa:" arg
while getopts "nvrkhdcfa:" arg
do
case $arg in
d)
@@ -70,6 +75,14 @@ do
echo "config REMOVE_OLD_BUILD=true"
REMOVE_OLD_BUILD=true
;;
v)
echo "config NINJA_VERBOSE=ON"
NINJA_VERBOSE=ON
;;
n)
echo "config NINJA_DRY_RUN=ON"
NINJA_DRY_RUN=ON
;;
?)
echo "unkonw argument"
usage
@@ -86,14 +99,12 @@ echo "ARCH: $ARCH"
echo "----------------------------------------------------"

READLINK=readlink
MAKEFILE_TYPE="Unix"
OS=$(uname -s)

if [ $OS = "Darwin" ];then
READLINK=greadlink
elif [[ $OS =~ "NT" ]]; then
echo "BUILD in NT ..."
MAKEFILE_TYPE="Unix"
fi

if [ ! $OS = "Linux" ] && [ $MGE_WITH_CUDA = "ON" ];then
@@ -125,14 +136,13 @@ function cmake_build() {
echo "build type: $BUILD_TYPE"
echo "build toolchain: $TOOLCHAIN"
echo "MGE_WITH_CUDA: $MGE_WITH_CUDA"
echo "BUILD MAKEFILE_TYPE: $MAKEFILE_TYPE"
try_remove_old_build $REMOVE_OLD_BUILD $BUILD_DIR $INSTALL_DIR

echo "create build dir"
mkdir -p $BUILD_DIR
mkdir -p $INSTALL_DIR
cd_real_build_dir $BUILD_DIR
cmake -G "$MAKEFILE_TYPE Makefiles" \
bash -c "cmake -G Ninja \
-DCMAKE_C_FLAGS=$CMAKE_C_FLAGS \
-DCMAKE_CXX_FLAGS=$CMAKE_CXX_FLAGS \
-DCMAKE_TOOLCHAIN_FILE=$TOOLCHAIN \
@@ -143,10 +153,10 @@ function cmake_build() {
-DMGE_DISABLE_FLOAT16=$MGE_DISABLE_FLOAT16 \
-DCMAKE_INSTALL_PREFIX=$INSTALL_DIR \
${EXTRA_CMAKE_ARGS} \
$SRC_DIR
$SRC_DIR "

make -j$(nproc)
make install/strip
config_ninja_target_cmd ${NINJA_VERBOSE} "OFF" "install/strip" ${NINJA_DRY_RUN}
bash -c "${NINJA_CMD}"
}

build_flatc $SRC_DIR $REMOVE_OLD_BUILD


+ 34
- 21
scripts/cmake-build/host_build.sh View File

@@ -9,6 +9,9 @@ function usage() {
echo "-t : Build with training mode, default inference only"
echo "-m : Build with m32 mode(only for windows build), default m64"
echo "-r : remove old build dir before make, default off"
echo "-v : ninja with verbose and explain, default off"
echo "-s : Do not build develop even build with training mode, default on when build with training, always for wheel"
echo "-n : ninja with -n dry run (don't run commands but act like they succeeded)"
echo "-h : show usage"
echo "append other cmake config by export EXTRA_CMAKE_ARGS=..."
echo "example: $0 -d"
@@ -22,9 +25,13 @@ MGE_WINDOWS_BUILD_ARCH=x64
MGE_WINDOWS_BUILD_MARCH=m64
MGE_ARCH=x86_64
REMOVE_OLD_BUILD=false
NINJA_VERBOSE=OFF
BUILD_DEVELOP=ON
NINJA_DRY_RUN=OFF

echo "EXTRA_CMAKE_ARGS: ${EXTRA_CMAKE_ARGS}"

while getopts "rhdctm" arg
while getopts "nsrhdctmv" arg
do
case $arg in
d)
@@ -47,6 +54,18 @@ do
echo "config REMOVE_OLD_BUILD=true"
REMOVE_OLD_BUILD=true
;;
s)
echo "config BUILD_DEVELOP=OFF"
BUILD_DEVELOP=OFF
;;
v)
echo "config NINJA_VERBOSE=ON"
NINJA_VERBOSE=ON
;;
n)
echo "config NINJA_DRY_RUN=ON"
NINJA_DRY_RUN=ON
;;
m)
echo "build for m32(only valid use for windows)"
MGE_WINDOWS_BUILD_ARCH=x86
@@ -81,6 +100,11 @@ fi
SRC_DIR=$($READLINK -f "`dirname $0`/../../")
source $SRC_DIR/scripts/cmake-build/utils/utils.sh

if [ ${MGE_INFERENCE_ONLY} = "ON" ]; then
echo "config BUILD_DEVELOP=OFF when MGE_INFERENCE_ONLY=ON"
BUILD_DEVELOP=OFF
fi

function cmake_build() {
BUILD_DIR=$SRC_DIR/build_dir/host/MGE_WITH_CUDA_$1/MGE_INFERENCE_ONLY_$2/$3/build
INSTALL_DIR=$BUILD_DIR/../install
@@ -98,16 +122,17 @@ function cmake_build() {
mkdir -p $BUILD_DIR
mkdir -p $INSTALL_DIR
cd_real_build_dir $BUILD_DIR
cmake \
# fork a new bash to handle EXTRA_CMAKE_ARGS env with space
bash -c "cmake -G Ninja \
-DCMAKE_BUILD_TYPE=$BUILD_TYPE \
-DMGE_INFERENCE_ONLY=$MGE_INFERENCE_ONLY \
-DMGE_WITH_CUDA=$MGE_WITH_CUDA \
-DCMAKE_INSTALL_PREFIX=$INSTALL_DIR \
${EXTRA_CMAKE_ARGS} \
$SRC_DIR
${SRC_DIR} "

make -j$(nproc)
make install/strip
config_ninja_target_cmd ${NINJA_VERBOSE} ${BUILD_DEVELOP} "install/strip" ${NINJA_DRY_RUN}
bash -c "${NINJA_CMD}"
}

function windows_env_err() {
@@ -199,18 +224,6 @@ function prepare_env_for_windows_build() {
export PATH=/c/Users/${USER}/swigwin-4.0.2::$PATH
}

WINDOWS_BUILD_TARGET="Ninja all"
if [[ -z ${MAKE_DEVELOP} ]]
then
MAKE_DEVELOP="false"
fi
function config_windows_build_target() {
if [ ${MAKE_DEVELOP} = "true" ]; then
echo "build all and develop for pytest test"
WINDOWS_BUILD_TARGET="Ninja all && Ninja develop"
fi
}

function cmake_build_windows() {
# windows do not support long path, so we cache the BUILD_DIR ASAP
prepare_env_for_windows_build
@@ -243,9 +256,10 @@ function cmake_build_windows() {
-DCMAKE_C_COMPILER=clang-cl.exe \
-DCMAKE_CXX_COMPILER=clang-cl.exe \
-DCMAKE_MAKE_PROGRAM=ninja.exe \
${EXTRA_CMAKE_ARGS} \
../../.. && \
${WINDOWS_BUILD_TARGET}"
${EXTRA_CMAKE_ARGS} ../../.. "

config_ninja_target_cmd ${NINJA_VERBOSE} ${BUILD_DEVELOP} "" ${NINJA_DRY_RUN}
cmd.exe /c " vcvarsall.bat $MGE_WINDOWS_BUILD_ARCH && ${NINJA_CMD} "
}

if [[ $OS =~ "NT" ]]; then
@@ -254,7 +268,6 @@ if [[ $OS =~ "NT" ]]; then
echo "pls remove -t or remove -m"
exit -1
fi
config_windows_build_target
cmake_build_windows $MGE_WITH_CUDA $MGE_INFERENCE_ONLY $BUILD_TYPE
else
cmake_build $MGE_WITH_CUDA $MGE_INFERENCE_ONLY $BUILD_TYPE


+ 36
- 5
scripts/cmake-build/utils/utils.sh View File

@@ -1,12 +1,13 @@
#!/usr/bin/env bash
set -e

MAKEFILE_TYPE="Unix"
OS=$(uname -s)
NINJA_CMD=""
NINJA_BASE="ninja"

if [[ $OS =~ "NT" ]]; then
echo "BUILD in NT ..."
MAKEFILE_TYPE="Unix"
NINJA_BASE="Ninja"
fi

READLINK=readlink
@@ -38,7 +39,7 @@ function build_flatc() {
mkdir -p $INSTALL_DIR

cd_real_build_dir $BUILD_DIR
cmake -G "$MAKEFILE_TYPE Makefiles" \
cmake -G Ninja \
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_INSTALL_PREFIX=$INSTALL_DIR \
-DFLATBUFFERS_BUILD_TESTS=OFF \
@@ -47,8 +48,8 @@ function build_flatc() {
-DFLATBUFFERS_LIBCXX_WITH_CLANG=OFF \
$SRC_DIR/third_party/flatbuffers

make -j$(nproc)
make install/strip
${NINJA_BASE} all
${NINJA_BASE} install/strip
}

function try_remove_old_build() {
@@ -65,3 +66,33 @@ function try_remove_old_build() {
echo "strip remove old build"
fi
}

function config_ninja_target_cmd() {
NINJA_CMD="${NINJA_BASE} all"
if [ $# -eq 4 ]; then
_NINJA_VERBOSE=$1
_BUILD_DEVELOP=$2
_INSTALL_ALL_TARGET=$3
_NINJA_DRY_RUN=$4
else
echo "err call config_ninja_target_cmd"
exit -1
fi

if [ ${_NINJA_DRY_RUN} = "ON" ]; then
NINJA_CMD="${NINJA_CMD} -d explain -n"
else
if [ ${_NINJA_VERBOSE} = "ON" ]; then
NINJA_CMD="${NINJA_CMD} -d explain -v"
fi
if [ ${_BUILD_DEVELOP} = "ON" ]; then
echo "add develop target"
NINJA_CMD="${NINJA_CMD} && ${NINJA_BASE} develop"
fi
if [ -n "${_INSTALL_ALL_TARGET}" ]; then
NINJA_CMD="${NINJA_CMD} && ${NINJA_BASE} ${_INSTALL_ALL_TARGET}"
fi
fi

echo "build ${NINJA_BASE} target command: ${NINJA_CMD}"
}

+ 46
- 12
scripts/whl/macos/macos_build_whl.sh View File

@@ -34,6 +34,8 @@ function append_path_env_and_check() {
append_path_env_and_check

SRC_DIR=$($READLINK -f "`dirname $0`/../../../")
source ${SRC_DIR}/scripts/whl/utils/utils.sh

ALL_PYTHON=${ALL_PYTHON}
FULL_PYTHON_VER="3.5.9 3.6.10 3.7.7 3.8.3"
if [[ -z ${ALL_PYTHON} ]]
@@ -91,9 +93,23 @@ function depend_real_copy() {
cp "${MEGENGINE_LIB}" ${REAL_DST}
}

BUILD_DIR=${SRC_DIR}/build_dir/host/MGE_WITH_CUDA_OFF/MGE_INFERENCE_ONLY_OFF/Release/build/

# here we just treat dnn/src/common/conv_bias.cpp should not in the increment build file list
INCREMENT_KEY_WORDS="conv_bias.cpp.o is dirty"
IS_IN_FIRST_LOOP=TRUE

function do_build() {
for ver in ${ALL_PYTHON}
do
# we want run a full clean build at the first loop
if [ ${IS_IN_FIRST_LOOP} = "TRUE" ]; then
# TODO: may all cmake issue can be resolved after rm CMakeCache?
# if YES, remove this to use old cache and speed up CI
echo "warning: remove old build_dir for the first loop"
rm -rf ${BUILD_DIR}
fi

#config
config_python_env ${ver}

@@ -108,26 +124,43 @@ function do_build() {
fi
echo "PYTHON_LIBRARY: ${PYTHON_LIBRARY}"
echo "PYTHON_INCLUDE_DIR: ${PYTHON_INCLUDE_DIR}"
#append cmake args for config python
export EXTRA_CMAKE_ARGS="-DCMAKE_PREFIX_PATH=${PYTHON_DIR} -DPYTHON_LIBRARY=${PYTHON_LIBRARY} -DPYTHON_INCLUDE_DIR=${PYTHON_INCLUDE_DIR} "
#config build type to RelWithDebInfo to enable MGB_ENABLE_DEBUG_UTIL etc
export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DCMAKE_BUILD_TYPE=RelWithDebInfo "
export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DCMAKE_BUILD_TYPE=RelWithDebInfo"
#append cmake args for config python
export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DPYTHON_EXECUTABLE=${PYTHON_DIR}/bin/python3"
export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DPYTHON_LIBRARY=${PYTHON_LIBRARY}"
export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DPYTHON_INCLUDE_DIR=${PYTHON_INCLUDE_DIR}"

#we use std::visit in src, so set osx version minimum to 10.14, but 10.14 have objdump
#issue, so we now config to 10.15, whl name to 10.14
#TODO: can set to 10.12 after remove use std::visit
export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DCMAKE_OSX_DEPLOYMENT_TARGET=10.15 "

#call build and install
#FIXME: cmake do not triger update python config, after
#change PYTHON_LIBRARY and PYTHON_INCLUDE_DIR, so add
#-r to remove build cache after a new ver build, which
#will be more slow build than without -r
echo "build whl with legacy python rt"
${SRC_DIR}/scripts/cmake-build/host_build.sh -t -r
if [ -d "${BUILD_DIR}" ]; then
# insure rm have args
touch ${BUILD_DIR}/empty.so
touch ${BUILD_DIR}/CMakeCache.txt
find ${BUILD_DIR} -name "*.so" | xargs rm
# as we now use increment build mode when switch python
# But I do not known any more issue at CMakeLists.txt or not
# so Force remove CMakeCache.txt
find ${BUILD_DIR} -name CMakeCache.txt | xargs rm
fi

HOST_BUILD_ARGS="-t -s"

# call ninja dry run and check increment is invalid or not
if [ ${IS_IN_FIRST_LOOP} = "FALSE" ]; then
ninja_dry_run_and_check_increment "${SRC_DIR}/scripts/cmake-build/host_build.sh" "${HOST_BUILD_ARGS}" "${INCREMENT_KEY_WORDS}"
fi

# call real build
echo "host_build.sh HOST_BUILD_ARGS: ${HOST_BUILD_ARGS}"
${SRC_DIR}/scripts/cmake-build/host_build.sh ${HOST_BUILD_ARGS}

#call setup.py
BUILD_DIR=${SRC_DIR}/build_dir/host/MGE_WITH_CUDA_OFF/MGE_INFERENCE_ONLY_OFF/Release/build/
# check python api call setup.py
cd ${BUILD_DIR}
check_build_ninja_python_api ${ver}

rm -rf staging
mkdir -p staging
@@ -180,6 +213,7 @@ function do_build() {
echo "macos whl package location: ${MACOS_WHL_HOME}"
ls ${MACOS_WHL_HOME}
echo "##############################################################################################"
IS_IN_FIRST_LOOP=FALSE
done
}



+ 1
- 1
scripts/whl/macos/macos_whl_env_prepare.sh View File

@@ -20,7 +20,7 @@ function try_install_brew() {
}

function install_brew_package() {
BREW_PACKAGE="openssl readline sqlite3 xz gdbm zlib pyenv wget swig coreutils llvm git-lfs"
BREW_PACKAGE="openssl readline sqlite3 xz gdbm zlib pyenv wget swig coreutils llvm git-lfs ninja"
for pak in ${BREW_PACKAGE}
do
echo "###### do command: brew install ${pak}"


+ 14
- 12
scripts/whl/manylinux2014/build_wheel_common.sh View File

@@ -50,12 +50,13 @@ elif [ $SDK_NAME == "cu111" ];then
${CUDNN_LIB_DIR}/libcudnn_ops_train.so.8:\
${CUDNN_LIB_DIR}/libcudnn.so.8"
EXTRA_CMAKE_FLAG=" -DMGE_WITH_CUDNN_SHARED=ON -DMGE_WITH_CUBLAS_SHARED=ON \
-gencode arch=compute_61,code=sm_61 \
arch=compute_70,code=sm_70 \
arch=compute_75,code=sm_75 \
arch=compute_80,code=sm_80 \
arch=compute_86,code=sm_86 \
arch=compute_86,code=compute_86"
-DMGE_CUDA_GENCODE=\"-gencode arch=compute_61,code=sm_61 \
-gencode arch=compute_70,code=sm_70 \
-gencode arch=compute_75,code=sm_75 \
-gencode arch=compute_80,code=sm_80 \
-gencode arch=compute_86,code=sm_86 \
-gencode arch=compute_86,code=compute_86\" "

REQUIR_CUDA_VERSION="11010"
REQUIR_CUDNN_VERSION="8.0.4"
REQUIR_TENSORRT_VERSION="7.2.2.3"
@@ -73,12 +74,13 @@ elif [ $SDK_NAME == "cu112" ];then
${CUDNN_LIB_DIR}/libcudnn_ops_train.so.8:\
${CUDNN_LIB_DIR}/libcudnn.so.8"
EXTRA_CMAKE_FLAG=" -DMGE_WITH_CUDNN_SHARED=ON -DMGE_WITH_CUBLAS_SHARED=ON \
-gencode arch=compute_61,code=sm_61 \
arch=compute_70,code=sm_70 \
arch=compute_75,code=sm_75 \
arch=compute_80,code=sm_80 \
arch=compute_86,code=sm_86 \
arch=compute_86,code=compute_86"
-DMGE_CUDA_GENCODE=\"-gencode arch=compute_61,code=sm_61 \
-gencode arch=compute_70,code=sm_70 \
-gencode arch=compute_75,code=sm_75 \
-gencode arch=compute_80,code=sm_80 \
-gencode arch=compute_86,code=sm_86 \
-gencode arch=compute_86,code=compute_86\" "

REQUIR_CUDA_VERSION="11020"
REQUIR_CUDNN_VERSION="8.0.4"
REQUIR_TENSORRT_VERSION="7.2.2.3"


+ 39
- 5
scripts/whl/manylinux2014/do_build_common.sh View File

@@ -75,33 +75,66 @@ then
fi

SRC_DIR=$(readlink -f "`dirname $0`/../../../")
source ${SRC_DIR}/scripts/whl/utils/utils.sh

BUILD_DIR=${SRC_DIR}/build_dir/host/MGE_WITH_CUDA_OFF/MGE_INFERENCE_ONLY_OFF/Release/build/
if [ ${BUILD_WHL_CPU_ONLY} = "OFF" ]; then
BUILD_DIR=${SRC_DIR}/build_dir/host/MGE_WITH_CUDA_ON/MGE_INFERENCE_ONLY_OFF/Release/build/
fi

# here we just treat cu file should not in the increment build file list
INCREMENT_KEY_WORDS=".cu.o is dirty"
IS_IN_FIRST_LOOP=TRUE

for ver in ${ALL_PYTHON}
do
# we want run a full clean build at the first loop
if [ ${IS_IN_FIRST_LOOP} = "TRUE" ]; then
# TODO: may all cmake issue can be resolved after rm CMakeCache?
# if YES, remove this to use old cache and speed up CI
echo "warning: remove old build_dir for the first loop"
rm -rf ${BUILD_DIR}
fi

python_ver=${ver:0:2}
MAJOR=${python_ver:0:1}
MINOR=${ver:1}
PYTHON_DIR=/opt/python/cp${python_ver}-cp${ver}/
export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} ${EXTRA_CMAKE_FLAG}"
export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DCMAKE_BUILD_TYPE=RelWithDebInfo"
export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DCMAKE_PREFIX_PATH=${PYTHON_DIR}"
export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DPYTHON_EXECUTABLE=${PYTHON_DIR}/bin/python3"
export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DPYTHON_LIBRARY=${PYTHON_DIR}lib/"
export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DPYTHON_INCLUDE_DIR=${PYTHON_DIR}include/python${MAJOR}.${MINOR}"
export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DMGE_WITH_ATLAS=ON"

# TODO: after change to Ninja, only the fisrt loop need add -r to save build time
if [ -d "${BUILD_DIR}" ]; then
# insure rm have args
touch ${BUILD_DIR}/empty.so
touch ${BUILD_DIR}/CMakeCache.txt
find ${BUILD_DIR} -name "*.so" | xargs rm
# as we now use increment build mode when switch python
# But I do not known any more issue at CMakeLists.txt or not
# so Force remove CMakeCache.txt
find ${BUILD_DIR} -name CMakeCache.txt | xargs rm
fi

HOST_BUILD_ARGS="-t -s"
if [ ${BUILD_WHL_CPU_ONLY} = "OFF" ]; then
${SRC_DIR}/scripts/cmake-build/host_build.sh -c -t -r
else
${SRC_DIR}/scripts/cmake-build/host_build.sh -t -r
HOST_BUILD_ARGS="${HOST_BUILD_ARGS} -c"
fi

# call ninja dry run and check increment is invalid or not
if [ ${IS_IN_FIRST_LOOP} = "FALSE" ]; then
ninja_dry_run_and_check_increment "${SRC_DIR}/scripts/cmake-build/host_build.sh" "${HOST_BUILD_ARGS}" "${INCREMENT_KEY_WORDS}"
fi

# call real build
echo "host_build.sh HOST_BUILD_ARGS: ${HOST_BUILD_ARGS}"
${SRC_DIR}/scripts/cmake-build/host_build.sh ${HOST_BUILD_ARGS}

# check python api call setup.py
cd ${BUILD_DIR}
check_build_ninja_python_api ${ver}
rm -rf staging
mkdir -p staging
cp -a imperative/python/{megengine,setup.py,requires.txt,requires-style.txt,requires-test.txt} staging/
@@ -128,4 +161,5 @@ do
# compat for root-less docker env to remove output at host side
chmod -R 777 .
echo "python $ver done"
IS_IN_FIRST_LOOP=FALSE
done

+ 4
- 1
scripts/whl/manylinux2014/init_image.sh View File

@@ -7,8 +7,11 @@ LLVM_URL='https://github.com/llvm-mirror/llvm/archive/release_60.tar.gz'
CLANG_URL='https://github.com/llvm-mirror/clang/archive/release_60.tar.gz'


yum install -y pcre-devel devtoolset-9-libatomic-devel.x86_64
yum install -y pcre-devel devtoolset-9-libatomic-devel.x86_64 ninja-build
yum install -y devtoolset-8 devtoolset-8-libatomic-devel.x86_64
# install a default python3 for cmake PYTHON3_EXECUTABLE_WITHOUT_VERSION
yum install -y python3
python3 -m pip install numpy

for ver in 35m 36m 37m 38
do


+ 81
- 0
scripts/whl/utils/utils.sh View File

@@ -0,0 +1,81 @@
#!/usr/bin/env bash
set -e

OS=$(uname -s)

function ninja_dry_run_and_check_increment() {
if [ $# -eq 3 ]; then
_BUILD_SHELL=$1
_BUILD_FLAGS="$2 -n"
_INCREMENT_KEY_WORDS=$3
else
echo "err call ninja_dry_run_and_check_increment"
exit -1
fi

${_BUILD_SHELL} ${_BUILD_FLAGS} 2>&1 | tee dry_run.log

DIRTY_LOG=`cat dry_run.log`
if [[ "${DIRTY_LOG}" =~ ${_INCREMENT_KEY_WORDS} ]]; then
echo "+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
echo "python3 switch increment build failed, some MR make a wrong CMakeLists.txt depends"
echo "or build env can not find default python3 in PATH env"
echo "please refs for PYTHON3_EXECUTABLE_WITHOUT_VERSION define at SRC_ROOT/CMakeLists.txt"
echo "+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
exit -1
fi
}

PYTHON_API_INCLUDES=""

function check_build_ninja_python_api() {
INCLUDE_KEYWORD=""
IS_MINOR_HIT=FALSE
if [ $# -eq 1 ]; then
ver=$1
echo "org args: ${ver}"
if [[ $OS =~ "NT" ]]; then
INCLUDE_KEYWORD="${ver}\\\\include"
PYTHON_API_INCLUDES="3.5.4\\\\include 3.6.8\\\\include 3.7.7\\\\include 3.8.3\\\\include"
elif [[ $OS =~ "Linux" ]]; then
INCLUDE_KEYWORD="include/python3.${ver:1:1}"
PYTHON_API_INCLUDES="include/python3.5 include/python3.6 include/python3.7 include/python3.8"
elif [[ $OS =~ "Darwin" ]]; then
INCLUDE_KEYWORD="include/python3.${ver:2:1}"
PYTHON_API_INCLUDES="include/python3.5 include/python3.6 include/python3.7 include/python3.8"
else
echo "unknown OS: ${OS}"
exit -1
fi
else
echo "err call check_build_ninja_python_api"
exit -1
fi
echo "try check python INCLUDE_KEYWORD: ${INCLUDE_KEYWORD} is invalid in ninja.build or not"

NINJA_BUILD=`cat build.ninja`
for PYTHON_API_INCLUDE in ${PYTHON_API_INCLUDES}
do
echo "check PYTHON_API_INCLUDE vs INCLUDE_KEYWORD : (${PYTHON_API_INCLUDE} : ${INCLUDE_KEYWORD})"
if [ ${PYTHON_API_INCLUDE} = ${INCLUDE_KEYWORD} ]; then
if [[ "${NINJA_BUILD}" =~ ${PYTHON_API_INCLUDE} ]]; then
echo "hit INCLUDE_KEYWORD: ${INCLUDE_KEYWORD} in build.ninja"
IS_MINOR_HIT="TRUE"
else
echo "Err happened can not find INCLUDE_KEYWORD: ${INCLUDE_KEYWORD} in build.ninja"
exit -1
fi
else
if [[ "${NINJA_BUILD}" =~ ${PYTHON_API_INCLUDE} ]]; then
echo "Err happened: find PYTHON_API_INCLUDE: ${PYTHON_API_INCLUDE} in build.ninja"
echo "But now INCLUDE_KEYWORD: ${INCLUDE_KEYWORD}"
exit -1
fi
fi
done

if [ ${IS_MINOR_HIT} = "FALSE" ]; then
echo "Err happened, can not hit any MINOR api in ninja.build"
exit -1
fi
}

+ 44
- 19
scripts/whl/windows/windows_build_whl.sh View File

@@ -23,6 +23,8 @@ function append_path_env_and_check() {
append_path_env_and_check

SRC_DIR=$(READLINK -f "`dirname $0`/../../../")
source ${SRC_DIR}/scripts/whl/utils/utils.sh

ALL_PYTHON=${ALL_PYTHON}
FULL_PYTHON_VER="3.5.4 3.6.8 3.7.7 3.8.3"
if [[ -z ${ALL_PYTHON} ]]
@@ -43,12 +45,12 @@ mkdir -p ${WINDOWS_WHL_HOME}
function config_python_env() {
PYTHON_DIR=/c/Users/${USER}/mge_whl_python_env/$1
PYTHON_BIN=${PYTHON_DIR}
if [ ! -f "$PYTHON_BIN/python3.exe" ]; then
if [ ! -f "${PYTHON_BIN}/python3.exe" ]; then
echo "ERR: can not find $PYTHON_BIN , Invalid python package"
echo "now support list: ${FULL_PYTHON_VER}"
err_env
else
echo "put python3 to env..."
echo "put ${PYTHON_BIN}/python3.exe to env..."
export PATH=${PYTHON_BIN}:$PATH
which python3
fi
@@ -104,11 +106,24 @@ function copy_more_dll() {
fi
}

BUILD_DIR=${SRC_DIR}/build_dir/host/build/

# here we just treat cu file should not in the increment build file list
INCREMENT_KEY_WORDS=".cu.obj is dirty"
IS_IN_FIRST_LOOP=TRUE

function do_build() {
for ver in ${ALL_PYTHON}
do
#config
# we want run a full clean build at the first loop
if [ ${IS_IN_FIRST_LOOP} = "TRUE" ]; then
# TODO: may all cmake issue can be resolved after rm CMakeCache?
# if YES, remove this to use old cache and speed up CI
echo "warning: remove old build_dir for the first loop"
rm -rf ${BUILD_DIR}
fi

#config python3
config_python_env ${ver}

#check env
@@ -122,35 +137,44 @@ function do_build() {
fi
echo "PYTHON_LIBRARY: ${PYTHON_LIBRARY}"
echo "PYTHON_INCLUDE_DIR: ${PYTHON_INCLUDE_DIR}"
#append cmake args for config python
#FIXME: ninja handle err with cmake 3.17 when assgin PYTHON_LIBRARY
#But after put python3.exe to HEAD of PATH by config_python_env, cmake can also handle the
#right PYTHON_LIBRARY and PYTHON_INCLUDE_DIR, at the same time, clang-cl need swig target
#force LINK a real PYTHON_LIBRARY file, after test we do not find the symbols conflict with python
#export EXTRA_CMAKE_ARGS="-DPYTHON_LIBRARY=${PYTHON_LIBRARY} -DPYTHON_INCLUDE_DIR=${PYTHON_INCLUDE_DIR} "
#config build type to RelWithDebInfo to enable MGB_ENABLE_DEBUG_UTIL etc
export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DCMAKE_BUILD_TYPE=RelWithDebInfo "

#call build and install
#FIXME: cmake do not triger update python config, after
#change PYTHON_LIBRARY and PYTHON_INCLUDE_DIR, so add
#-r to remove build cache after a new ver build, which
#will be more slow build than without -r
BUILD_ARGS=" -t -r"
HOST_BUILD_ARGS=" -t -s"

if [ ${BUILD_WHL_CPU_ONLY} = "OFF" ]; then
echo "build windows whl with cuda"
BUILD_ARGS="${BUILD_ARGS} -c "
HOST_BUILD_ARGS="${HOST_BUILD_ARGS} -c "
else
echo "build windows whl with cpu only"
fi

echo "host_build.sh BUILD_ARGS: ${BUILD_ARGS}"
${SRC_DIR}/scripts/cmake-build/host_build.sh ${BUILD_ARGS}
if [ -d "${BUILD_DIR}" ]; then
# insure rm have args
touch ${BUILD_DIR}/empty.pyd
touch ${BUILD_DIR}/CMakeCache.txt
/usr/bin/find ${BUILD_DIR} -name "*.pyd" | xargs rm
# ninja/cmake on windows will handle error if just export
# PYTHON_LIBRARY/PYTHON_INCLUDE_DIR/PYTHON_EXECUTABLE
# But after put python3.exe to HEAD of PATH by config_python_env
# and force remove CMakeCache.txt, ninja/cmake will auto update
# PYTHON_LIBRARY/PYTHON_INCLUDE_DIR/PYTHON_EXECUTABLE
/usr/bin/find ${BUILD_DIR} -name CMakeCache.txt | xargs rm
fi
echo "host_build.sh HOST_BUILD_ARGS: ${HOST_BUILD_ARGS}"

BUILD_DIR=${SRC_DIR}/build_dir/host/build/
cd ${BUILD_DIR}
# call ninja dry run and check increment is invalid or not
if [ ${IS_IN_FIRST_LOOP} = "FALSE" ]; then
ninja_dry_run_and_check_increment "${SRC_DIR}/scripts/cmake-build/host_build.sh" "${HOST_BUILD_ARGS}" "${INCREMENT_KEY_WORDS}"
fi

#call real build
${SRC_DIR}/scripts/cmake-build/host_build.sh ${HOST_BUILD_ARGS}

# check python api call setup.py
cd ${BUILD_DIR}
check_build_ninja_python_api ${ver}
rm -rf staging
mkdir -p staging
cp -a imperative/python/{megengine,setup.py,requires.txt,requires-style.txt,requires-test.txt} staging/
@@ -178,6 +202,7 @@ function do_build() {
echo "windows whl package location: ${WINDOWS_WHL_HOME}"
ls ${WINDOWS_WHL_HOME}
echo "##############################################################################################"
IS_IN_FIRST_LOOP=FALSE
done
}



Write Preview
Loading…
Cancel
Save