feat(build): change make to ninja

GitOrigin-RevId: ccff72e39a

CMakeLists.txt

@@ -1,4 +1,8 @@
 cmake_minimum_required(VERSION 3.15.2)
+message(STATUS "CMAKE_GENERATOR: ${CMAKE_GENERATOR}" )
+if (NOT ${CMAKE_GENERATOR} STREQUAL "Ninja")
+    message(WARNING "CMAKE_GENERATOR NOT EQUAL Ninja, which we do not recommend")
+endif()
 
 include (cmake/FetchMegBrainVersion.cmake)
 project(MegEngine LANGUAGES C CXX VERSION ${MGB_VER_STRING})

scripts/cmake-build/BUILD_README.md

@@ -19,7 +19,7 @@
 * commands:
 ```
 1: installl Visual Studio (need support LLVM/clang-cl), eg 2019. Please install LLVM-10, VS LLVM linker have issue, please replace lld-link.exe, which can be download from https://releases.llvm.org/download.html#10.0.0
-2: install extension of VS: Python/Cmake/LLVM
+2: install extension of VS: Python/Cmake/LLVM/Ninja
 3: now we support cuda10.1+cudnn7.6+TensorRT6.0 on Windows, as Windows can only use DLL in fact with cudnn/TensorRT, so please install the same version;
     3a: install cuda10.1 to C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.1
     3b: install cudnn7.6 to C:\Program Files\NVIDIA GPU Computing Toolkit\cudnn-10.1-windows10-x64-v7.6.5.32
@@ -33,9 +33,9 @@
 ### Linux host build
 * commands:
 ```
-1: install Cmake, which version >= 3.15.2
+1: install Cmake, which version >= 3.15.2, ninja-build
 2: install gcc/g++, which version >= 6, (gcc/g++ >= 7, if need build training mode)
-3: install build-essential git git-lfs gfortran libgfortran-6-dev autoconf gnupg flex bison gperf curl zlib1g-dev gcc-multilib g++-multilib lib32ncurses5-dev libxml2-utils xsltproc unzip libtool librdmacm-dev rdmacm-utils python3-dev swig python3-numpy texinfo
+3: install build-essential git git-lfs gfortran libgfortran-6-dev autoconf gnupg flex bison gperf curl zlib1g-dev gcc-multilib g++-multilib lib32ncurses5-dev libxml2-utils xsltproc unzip libtool librdmacm-dev rdmacm-utils python3-dev python3-numpy texinfo
 4: CUDA env(if enable CUDA), version detail refer to README.md
 ```
 
@@ -44,7 +44,7 @@
 ```
 1: install Cmake, which version >= 3.15.2
 2: install brew: /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install.sh)"
-3: brew install python python3 swig coreutils
+3: brew install python python3 coreutils ninja
 4: install at least xcode command line tool: https://developer.apple.com/xcode/
 5: about cuda: we do not support CUDA on MacOS
 6: python3 -m pip install numpy (if you want to build with training mode)

scripts/cmake-build/cross_build_android_arm_inference.sh

@@ -7,6 +7,9 @@ MGE_ARMV8_2_FEATURE_FP16=OFF
 MGE_DISABLE_FLOAT16=OFF
 ARCH=arm64-v8a
 REMOVE_OLD_BUILD=false
+NINJA_VERBOSE=OFF
+NINJA_DRY_RUN=OFF
+
 echo "EXTRA_CMAKE_ARGS: ${EXTRA_CMAKE_ARGS}"
 
 function usage() {
@@ -17,13 +20,15 @@ function usage() {
     echo "-k : open MGE_DISABLE_FLOAT16 for NEON "
     echo "-a : config build arch available: ${ARCHS[@]}"
     echo "-r : remove old build dir before make, default off"
+    echo "-v : ninja with verbose and explain, default off"
+    echo "-n : ninja with -n dry run (don't run commands but act like they succeeded)"
     echo "-h : show usage"
     echo "append other cmake config by export EXTRA_CMAKE_ARGS=..."
     echo "example: $0 -d"
     exit -1
 }
 
-while getopts "rkhdfa:" arg
+while getopts "nvrkhdfa:" arg
 do
     case $arg in
         d)
@@ -62,6 +67,14 @@ do
             echo "config REMOVE_OLD_BUILD=true"
             REMOVE_OLD_BUILD=true
             ;;
+        v)
+            echo "config NINJA_VERBOSE=ON"
+            NINJA_VERBOSE=ON
+            ;;
+        n)
+            echo "config NINJA_DRY_RUN=ON"
+            NINJA_DRY_RUN=ON
+            ;;
         ?)
             echo "unkonw argument"
             usage
@@ -77,14 +90,12 @@ echo "ARCH: $ARCH"
 echo "----------------------------------------------------"
 
 READLINK=readlink
-MAKEFILE_TYPE="Unix"
 OS=$(uname -s)
 
 if [ $OS = "Darwin" ];then
     READLINK=greadlink
 elif [[ $OS =~ "NT" ]]; then
     echo "BUILD in NT ..."
-    MAKEFILE_TYPE="Unix"
 fi
 
 SRC_DIR=$($READLINK -f "`dirname $0`/../../")
@@ -105,7 +116,6 @@ function cmake_build() {
     echo "build type: $BUILD_TYPE"
     echo "build ABI: $BUILD_ABI"
     echo "build native level: $BUILD_NATIVE_LEVEL"
-    echo "BUILD MAKEFILE_TYPE: $MAKEFILE_TYPE"
     try_remove_old_build $REMOVE_OLD_BUILD $BUILD_DIR $INSTALL_DIR
 
     echo "create build dir"
@@ -113,7 +123,7 @@ function cmake_build() {
     mkdir -p $INSTALL_DIR
     cd_real_build_dir $BUILD_DIR
     unset IFS
-    cmake -G "$MAKEFILE_TYPE Makefiles" \
+    bash -c "cmake -G Ninja \
         -DCMAKE_TOOLCHAIN_FILE="$NDK_ROOT/build/cmake/android.toolchain.cmake" \
         -DANDROID_NDK="$NDK_ROOT" \
         -DCMAKE_BUILD_TYPE=$BUILD_TYPE \
@@ -125,10 +135,11 @@ function cmake_build() {
         -DMGE_DISABLE_FLOAT16=$MGE_DISABLE_FLOAT16 \
         -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR \
         ${EXTRA_CMAKE_ARGS} \
-        $SRC_DIR
+        $SRC_DIR "
 
-    make -j$(nproc) ${Target}
-    make install/strip
+    config_ninja_target_cmd ${NINJA_VERBOSE} "OFF" "" ${NINJA_DRY_RUN}
+    bash -c "${NINJA_CMD}"
+    ${NINJA_BASE} install/strip
 }
 
 build_flatc $SRC_DIR $REMOVE_OLD_BUILD

scripts/cmake-build/cross_build_ios_arm_inference.sh

@@ -7,6 +7,9 @@ MGE_ARMV8_2_FEATURE_FP16=OFF
 MGE_DISABLE_FLOAT16=OFF
 ARCH=arm64
 REMOVE_OLD_BUILD=false
+NINJA_VERBOSE=OFF
+NINJA_DRY_RUN=OFF
+
 echo "EXTRA_CMAKE_ARGS: ${EXTRA_CMAKE_ARGS}"
 
 function usage() {
@@ -17,13 +20,15 @@ function usage() {
     echo "-k : open MGE_DISABLE_FLOAT16 for NEON "
     echo "-a : config build arch available: ${ARCHS[@]}"
     echo "-r : remove old build dir before make, default off"
+    echo "-v : ninja with verbose and explain, default off"
+    echo "-n : ninja with -n dry run (don't run commands but act like they succeeded)"
     echo "-h : show usage"
     echo "append other cmake config by export EXTRA_CMAKE_ARGS=..."
     echo "example: $0 -d"
     exit -1
 }
 
-while getopts "rkhdfa:" arg
+while getopts "nvrkhdfa:" arg
 do
     case $arg in
         d)
@@ -62,6 +67,14 @@ do
             echo "config REMOVE_OLD_BUILD=true"
             REMOVE_OLD_BUILD=true
             ;;
+        v)
+            echo "config NINJA_VERBOSE=ON"
+            NINJA_VERBOSE=ON
+            ;;
+        n)
+            echo "config NINJA_DRY_RUN=ON"
+            NINJA_DRY_RUN=ON
+            ;;
         ?)
             echo "unkonw argument"
             usage
@@ -109,7 +122,8 @@ function cmake_build() {
     mkdir -p $BUILD_DIR
     mkdir -p $INSTALL_DIR
     cd_real_build_dir $BUILD_DIR
-    cmake -DCMAKE_TOOLCHAIN_FILE=$TOOLCHAIN \
+    bash -c "cmake -G Ninja \
+        -DCMAKE_TOOLCHAIN_FILE=$TOOLCHAIN \
         -DCMAKE_BUILD_TYPE=$BUILD_TYPE \
         -DIOS_TOOLCHAIN_ROOT=$TOOLCHAIN \
         -DOS_PLATFORM=$OS_PLATFORM \
@@ -121,11 +135,12 @@ function cmake_build() {
         -DMGE_ARMV8_2_FEATURE_FP16= $MGE_ARMV8_2_FEATURE_FP16 \
         -DMGE_DISABLE_FLOAT16=$MGE_DISABLE_FLOAT16 \
         -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR \
+        -DCMAKE_MAKE_PROGRAM=ninja \
         ${EXTRA_CMAKE_ARGS} \
-        $SRC_DIR
+        $SRC_DIR "
 
-    make -j$(nproc)
-    make install
+    config_ninja_target_cmd ${NINJA_VERBOSE} "OFF" "install" ${NINJA_DRY_RUN}
+    bash -c "${NINJA_CMD}"
 }
 
 build_flatc $SRC_DIR $REMOVE_OLD_BUILD

scripts/cmake-build/cross_build_linux_arm_inference.sh

@@ -8,8 +8,11 @@ MGE_ARMV8_2_FEATURE_FP16=OFF
 MGE_DISABLE_FLOAT16=OFF
 ARCH=arm64-v8a
 REMOVE_OLD_BUILD=false
+NINJA_VERBOSE=OFF
+NINJA_DRY_RUN=OFF
 CMAKE_C_FLAGS="-Wno-psabi"
 CMAKE_CXX_FLAGS="-Wno-psabi"
+
 echo "EXTRA_CMAKE_ARGS: ${EXTRA_CMAKE_ARGS}"
 
 function usage() {
@@ -21,13 +24,15 @@ function usage() {
     echo "-k : open MGE_DISABLE_FLOAT16 for NEON "
     echo "-a : config build arch available: ${ARCHS[@]}"
     echo "-r : remove old build dir before make, default off"
+    echo "-v : ninja with verbose and explain, default off"
+    echo "-n : ninja with -n dry run (don't run commands but act like they succeeded)"
     echo "-h : show usage"
     echo "append other cmake config by export EXTRA_CMAKE_ARGS=..."
     echo "example: $0 -d"
     exit -1
 }
 
-while getopts "rkhdcfa:" arg
+while getopts "nvrkhdcfa:" arg
 do
     case $arg in
         d)
@@ -70,6 +75,14 @@ do
             echo "config REMOVE_OLD_BUILD=true"
             REMOVE_OLD_BUILD=true
             ;;
+        v)
+            echo "config NINJA_VERBOSE=ON"
+            NINJA_VERBOSE=ON
+            ;;
+        n)
+            echo "config NINJA_DRY_RUN=ON"
+            NINJA_DRY_RUN=ON
+            ;;
         ?)
             echo "unkonw argument"
             usage
@@ -86,14 +99,12 @@ echo "ARCH: $ARCH"
 echo "----------------------------------------------------"
 
 READLINK=readlink
-MAKEFILE_TYPE="Unix"
 OS=$(uname -s)
 
 if [ $OS = "Darwin" ];then
     READLINK=greadlink
 elif [[ $OS =~ "NT" ]]; then
     echo "BUILD in NT ..."
-    MAKEFILE_TYPE="Unix"
 fi
 
 if [ ! $OS = "Linux" ] && [ $MGE_WITH_CUDA = "ON" ];then
@@ -125,14 +136,13 @@ function cmake_build() {
     echo "build type: $BUILD_TYPE"
     echo "build toolchain: $TOOLCHAIN"
     echo "MGE_WITH_CUDA: $MGE_WITH_CUDA"
-    echo "BUILD MAKEFILE_TYPE: $MAKEFILE_TYPE"
     try_remove_old_build $REMOVE_OLD_BUILD $BUILD_DIR $INSTALL_DIR
 
     echo "create build dir"
     mkdir -p $BUILD_DIR
     mkdir -p $INSTALL_DIR
     cd_real_build_dir $BUILD_DIR
-    cmake -G "$MAKEFILE_TYPE Makefiles" \
+    bash -c "cmake -G Ninja \
         -DCMAKE_C_FLAGS=$CMAKE_C_FLAGS \
         -DCMAKE_CXX_FLAGS=$CMAKE_CXX_FLAGS \
         -DCMAKE_TOOLCHAIN_FILE=$TOOLCHAIN \
@@ -143,10 +153,10 @@ function cmake_build() {
         -DMGE_DISABLE_FLOAT16=$MGE_DISABLE_FLOAT16 \
         -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR \
         ${EXTRA_CMAKE_ARGS} \
-        $SRC_DIR
+        $SRC_DIR "
 
-    make -j$(nproc)
-    make install/strip
+    config_ninja_target_cmd ${NINJA_VERBOSE} "OFF" "install/strip" ${NINJA_DRY_RUN}
+    bash -c "${NINJA_CMD}"
 }
 
 build_flatc $SRC_DIR $REMOVE_OLD_BUILD

scripts/cmake-build/host_build.sh

@@ -9,6 +9,9 @@ function usage() {
     echo "-t : Build with training mode, default inference only"
     echo "-m : Build with m32 mode(only for windows build), default m64"
     echo "-r : remove old build dir before make, default off"
+    echo "-v : ninja with verbose and explain, default off"
+    echo "-s : Do not build develop even build with training mode, default on when build with training, always for wheel"
+    echo "-n : ninja with -n dry run (don't run commands but act like they succeeded)"
     echo "-h : show usage"
     echo "append other cmake config by export EXTRA_CMAKE_ARGS=..."
     echo "example: $0 -d"
@@ -22,9 +25,13 @@ MGE_WINDOWS_BUILD_ARCH=x64
 MGE_WINDOWS_BUILD_MARCH=m64
 MGE_ARCH=x86_64
 REMOVE_OLD_BUILD=false
+NINJA_VERBOSE=OFF
+BUILD_DEVELOP=ON
+NINJA_DRY_RUN=OFF
+
 echo "EXTRA_CMAKE_ARGS: ${EXTRA_CMAKE_ARGS}"
 
-while getopts "rhdctm" arg
+while getopts "nsrhdctmv" arg
 do
     case $arg in
         d)
@@ -47,6 +54,18 @@ do
             echo "config REMOVE_OLD_BUILD=true"
             REMOVE_OLD_BUILD=true
             ;;
+        s)
+            echo "config  BUILD_DEVELOP=OFF"
+            BUILD_DEVELOP=OFF
+            ;;
+        v)
+            echo "config NINJA_VERBOSE=ON"
+            NINJA_VERBOSE=ON
+            ;;
+        n)
+            echo "config NINJA_DRY_RUN=ON"
+            NINJA_DRY_RUN=ON
+            ;;
         m)
             echo "build for m32(only valid use for windows)"
             MGE_WINDOWS_BUILD_ARCH=x86
@@ -81,6 +100,11 @@ fi
 SRC_DIR=$($READLINK -f "`dirname $0`/../../")
 source $SRC_DIR/scripts/cmake-build/utils/utils.sh
 
+if [ ${MGE_INFERENCE_ONLY} = "ON" ]; then
+    echo "config BUILD_DEVELOP=OFF when MGE_INFERENCE_ONLY=ON"
+    BUILD_DEVELOP=OFF
+fi
+
 function cmake_build() {
     BUILD_DIR=$SRC_DIR/build_dir/host/MGE_WITH_CUDA_$1/MGE_INFERENCE_ONLY_$2/$3/build
     INSTALL_DIR=$BUILD_DIR/../install
@@ -98,16 +122,17 @@ function cmake_build() {
     mkdir -p $BUILD_DIR
     mkdir -p $INSTALL_DIR
     cd_real_build_dir $BUILD_DIR
-    cmake \
+    # fork a new bash to handle EXTRA_CMAKE_ARGS env with space
+    bash -c "cmake -G Ninja \
         -DCMAKE_BUILD_TYPE=$BUILD_TYPE \
         -DMGE_INFERENCE_ONLY=$MGE_INFERENCE_ONLY \
         -DMGE_WITH_CUDA=$MGE_WITH_CUDA \
         -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR \
         ${EXTRA_CMAKE_ARGS} \
-        $SRC_DIR
+        ${SRC_DIR} "
 
-    make -j$(nproc)
-    make install/strip
+    config_ninja_target_cmd ${NINJA_VERBOSE} ${BUILD_DEVELOP} "install/strip" ${NINJA_DRY_RUN}
+    bash -c "${NINJA_CMD}"
 }
 
 function windows_env_err() {
@@ -199,18 +224,6 @@ function prepare_env_for_windows_build() {
     export PATH=/c/Users/${USER}/swigwin-4.0.2::$PATH
 }
 
-WINDOWS_BUILD_TARGET="Ninja all"
-if [[ -z ${MAKE_DEVELOP} ]]
-then
-    MAKE_DEVELOP="false"
-fi
-function config_windows_build_target() {
-    if [ ${MAKE_DEVELOP} = "true" ]; then
-        echo "build all and develop for pytest test"
-        WINDOWS_BUILD_TARGET="Ninja all && Ninja develop"
-    fi
-}
-
 function cmake_build_windows() {
     # windows do not support long path, so we cache the BUILD_DIR ASAP
     prepare_env_for_windows_build
@@ -243,9 +256,10 @@ function cmake_build_windows() {
         -DCMAKE_C_COMPILER=clang-cl.exe \
         -DCMAKE_CXX_COMPILER=clang-cl.exe \
         -DCMAKE_MAKE_PROGRAM=ninja.exe \
-        ${EXTRA_CMAKE_ARGS} \
-        ../../.. && \
-        ${WINDOWS_BUILD_TARGET}"
+        ${EXTRA_CMAKE_ARGS} ../../.. "
+
+    config_ninja_target_cmd ${NINJA_VERBOSE} ${BUILD_DEVELOP} "" ${NINJA_DRY_RUN}
+    cmd.exe /c " vcvarsall.bat $MGE_WINDOWS_BUILD_ARCH && ${NINJA_CMD} "
 }
 
 if [[ $OS =~ "NT" ]]; then
@@ -254,7 +268,6 @@ if [[ $OS =~ "NT" ]]; then
         echo "pls remove -t or remove -m"
         exit -1
     fi
-    config_windows_build_target
     cmake_build_windows $MGE_WITH_CUDA $MGE_INFERENCE_ONLY $BUILD_TYPE
 else
     cmake_build $MGE_WITH_CUDA $MGE_INFERENCE_ONLY $BUILD_TYPE

scripts/cmake-build/utils/utils.sh

@@ -1,12 +1,13 @@
 #!/usr/bin/env bash
 set -e
 
-MAKEFILE_TYPE="Unix"
 OS=$(uname -s)
+NINJA_CMD=""
+NINJA_BASE="ninja"
 
 if [[ $OS =~ "NT" ]]; then
     echo "BUILD in NT ..."
-    MAKEFILE_TYPE="Unix"
+    NINJA_BASE="Ninja"
 fi
 
 READLINK=readlink
@@ -38,7 +39,7 @@ function build_flatc() {
     mkdir -p $INSTALL_DIR
 
     cd_real_build_dir $BUILD_DIR
-    cmake -G "$MAKEFILE_TYPE Makefiles" \
+    cmake -G Ninja \
         -DCMAKE_BUILD_TYPE=Release \
         -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR \
         -DFLATBUFFERS_BUILD_TESTS=OFF \
@@ -47,8 +48,8 @@ function build_flatc() {
         -DFLATBUFFERS_LIBCXX_WITH_CLANG=OFF \
         $SRC_DIR/third_party/flatbuffers
 
-    make -j$(nproc)
-    make install/strip
+    ${NINJA_BASE} all
+    ${NINJA_BASE} install/strip
 }
 
 function try_remove_old_build() {
@@ -65,3 +66,33 @@ function try_remove_old_build() {
         echo "strip remove old build"
     fi
 }
+
+function config_ninja_target_cmd() {
+    NINJA_CMD="${NINJA_BASE} all"
+    if [ $# -eq 4 ]; then
+        _NINJA_VERBOSE=$1
+        _BUILD_DEVELOP=$2
+        _INSTALL_ALL_TARGET=$3
+        _NINJA_DRY_RUN=$4
+    else
+        echo "err call config_ninja_target_cmd"
+        exit -1
+    fi
+
+    if [ ${_NINJA_DRY_RUN} = "ON" ]; then
+        NINJA_CMD="${NINJA_CMD} -d explain -n"
+    else
+        if [ ${_NINJA_VERBOSE} = "ON" ]; then
+            NINJA_CMD="${NINJA_CMD} -d explain -v"
+        fi
+        if [ ${_BUILD_DEVELOP} = "ON" ]; then
+            echo "add develop target"
+            NINJA_CMD="${NINJA_CMD} && ${NINJA_BASE} develop"
+        fi
+        if [ -n "${_INSTALL_ALL_TARGET}" ]; then
+            NINJA_CMD="${NINJA_CMD} && ${NINJA_BASE} ${_INSTALL_ALL_TARGET}"
+        fi
+    fi
+
+    echo "build ${NINJA_BASE} target command: ${NINJA_CMD}"
+}

scripts/whl/macos/macos_build_whl.sh

@@ -34,6 +34,8 @@ function append_path_env_and_check() {
 append_path_env_and_check
 
 SRC_DIR=$($READLINK -f "`dirname $0`/../../../")
+source ${SRC_DIR}/scripts/whl/utils/utils.sh
+
 ALL_PYTHON=${ALL_PYTHON}
 FULL_PYTHON_VER="3.5.9 3.6.10 3.7.7 3.8.3"
 if [[ -z ${ALL_PYTHON} ]]
@@ -91,9 +93,23 @@ function depend_real_copy() {
     cp "${MEGENGINE_LIB}" ${REAL_DST}
 }
 
+BUILD_DIR=${SRC_DIR}/build_dir/host/MGE_WITH_CUDA_OFF/MGE_INFERENCE_ONLY_OFF/Release/build/
+
+# here we just treat dnn/src/common/conv_bias.cpp should not in the increment build file list
+INCREMENT_KEY_WORDS="conv_bias.cpp.o is dirty"
+IS_IN_FIRST_LOOP=TRUE
+
 function do_build() {
     for ver in ${ALL_PYTHON}
     do
+        # we want run a full clean build at the first loop
+        if [ ${IS_IN_FIRST_LOOP} = "TRUE" ]; then
+            # TODO: may all cmake issue can be resolved after rm CMakeCache?
+            # if YES, remove this to use old cache and speed up CI
+            echo "warning: remove old build_dir for the first loop"
+            rm -rf ${BUILD_DIR}
+        fi
+
         #config
         config_python_env ${ver}
 
@@ -108,26 +124,43 @@ function do_build() {
         fi
         echo "PYTHON_LIBRARY: ${PYTHON_LIBRARY}"
         echo "PYTHON_INCLUDE_DIR: ${PYTHON_INCLUDE_DIR}"
-        #append cmake args for config python
-        export EXTRA_CMAKE_ARGS="-DCMAKE_PREFIX_PATH=${PYTHON_DIR} -DPYTHON_LIBRARY=${PYTHON_LIBRARY} -DPYTHON_INCLUDE_DIR=${PYTHON_INCLUDE_DIR} "
         #config build type to RelWithDebInfo to enable MGB_ENABLE_DEBUG_UTIL etc
-        export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DCMAKE_BUILD_TYPE=RelWithDebInfo "
+        export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DCMAKE_BUILD_TYPE=RelWithDebInfo"
+        #append cmake args for config python
+        export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DPYTHON_EXECUTABLE=${PYTHON_DIR}/bin/python3"
+        export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DPYTHON_LIBRARY=${PYTHON_LIBRARY}"
+        export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DPYTHON_INCLUDE_DIR=${PYTHON_INCLUDE_DIR}"
+
         #we use std::visit in src, so set osx version minimum to 10.14, but 10.14 have objdump
         #issue, so we now config to 10.15, whl name to 10.14
         #TODO: can set to 10.12 after remove use std::visit
         export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DCMAKE_OSX_DEPLOYMENT_TARGET=10.15 "
 
-        #call build and install
-        #FIXME: cmake do not triger update python config, after
-        #change PYTHON_LIBRARY and PYTHON_INCLUDE_DIR, so add
-        #-r to remove build cache after a new ver build, which
-        #will be more slow build than without -r
-        echo "build whl with legacy python rt"
-        ${SRC_DIR}/scripts/cmake-build/host_build.sh -t -r
+        if [ -d "${BUILD_DIR}" ]; then
+            # insure rm have args
+            touch ${BUILD_DIR}/empty.so
+            touch ${BUILD_DIR}/CMakeCache.txt
+            find ${BUILD_DIR} -name "*.so" | xargs rm
+            # as we now use increment build mode when switch python
+            # But I do not known any more issue at CMakeLists.txt or not
+            # so Force remove CMakeCache.txt
+            find ${BUILD_DIR} -name CMakeCache.txt | xargs rm
+        fi
+
+        HOST_BUILD_ARGS="-t -s"
+
+        # call ninja dry run and check increment is invalid or not
+        if [ ${IS_IN_FIRST_LOOP} = "FALSE" ]; then
+            ninja_dry_run_and_check_increment "${SRC_DIR}/scripts/cmake-build/host_build.sh" "${HOST_BUILD_ARGS}" "${INCREMENT_KEY_WORDS}"
+        fi
+
+        # call real build
+        echo "host_build.sh HOST_BUILD_ARGS: ${HOST_BUILD_ARGS}"
+        ${SRC_DIR}/scripts/cmake-build/host_build.sh ${HOST_BUILD_ARGS}
 
-        #call setup.py
-        BUILD_DIR=${SRC_DIR}/build_dir/host/MGE_WITH_CUDA_OFF/MGE_INFERENCE_ONLY_OFF/Release/build/
+        # check python api call setup.py
         cd ${BUILD_DIR}
+        check_build_ninja_python_api ${ver}
 
         rm -rf staging
         mkdir -p staging
@@ -180,6 +213,7 @@ function do_build() {
         echo "macos whl package location: ${MACOS_WHL_HOME}"
         ls ${MACOS_WHL_HOME}
         echo "##############################################################################################"
+        IS_IN_FIRST_LOOP=FALSE
     done
 }
 

scripts/whl/macos/macos_whl_env_prepare.sh

@@ -20,7 +20,7 @@ function try_install_brew() {
 }
 
 function install_brew_package() {
-    BREW_PACKAGE="openssl readline sqlite3 xz gdbm zlib pyenv wget swig coreutils llvm git-lfs"
+    BREW_PACKAGE="openssl readline sqlite3 xz gdbm zlib pyenv wget swig coreutils llvm git-lfs ninja"
     for pak in ${BREW_PACKAGE}
     do
         echo "###### do command: brew install ${pak}"

scripts/whl/manylinux2014/build_wheel_common.sh

@@ -50,12 +50,13 @@ elif [ $SDK_NAME == "cu111" ];then
         ${CUDNN_LIB_DIR}/libcudnn_ops_train.so.8:\
         ${CUDNN_LIB_DIR}/libcudnn.so.8"
     EXTRA_CMAKE_FLAG=" -DMGE_WITH_CUDNN_SHARED=ON -DMGE_WITH_CUBLAS_SHARED=ON \
-        -gencode arch=compute_61,code=sm_61 \
-        arch=compute_70,code=sm_70 \
-        arch=compute_75,code=sm_75 \
-        arch=compute_80,code=sm_80 \
-        arch=compute_86,code=sm_86 \
-        arch=compute_86,code=compute_86" 
+        -DMGE_CUDA_GENCODE=\"-gencode arch=compute_61,code=sm_61 \
+        -gencode arch=compute_70,code=sm_70 \
+        -gencode arch=compute_75,code=sm_75 \
+        -gencode arch=compute_80,code=sm_80 \
+        -gencode arch=compute_86,code=sm_86 \
+        -gencode arch=compute_86,code=compute_86\" "
+
     REQUIR_CUDA_VERSION="11010" 
     REQUIR_CUDNN_VERSION="8.0.4" 
     REQUIR_TENSORRT_VERSION="7.2.2.3" 
@@ -73,12 +74,13 @@ elif [ $SDK_NAME == "cu112" ];then
         ${CUDNN_LIB_DIR}/libcudnn_ops_train.so.8:\
         ${CUDNN_LIB_DIR}/libcudnn.so.8"
     EXTRA_CMAKE_FLAG=" -DMGE_WITH_CUDNN_SHARED=ON -DMGE_WITH_CUBLAS_SHARED=ON \
-        -gencode arch=compute_61,code=sm_61 \
-        arch=compute_70,code=sm_70 \
-        arch=compute_75,code=sm_75 \
-        arch=compute_80,code=sm_80 \
-        arch=compute_86,code=sm_86 \
-        arch=compute_86,code=compute_86"  
+        -DMGE_CUDA_GENCODE=\"-gencode arch=compute_61,code=sm_61 \
+        -gencode arch=compute_70,code=sm_70 \
+        -gencode arch=compute_75,code=sm_75 \
+        -gencode arch=compute_80,code=sm_80 \
+        -gencode arch=compute_86,code=sm_86 \
+        -gencode arch=compute_86,code=compute_86\" "
+
     REQUIR_CUDA_VERSION="11020" 
     REQUIR_CUDNN_VERSION="8.0.4" 
     REQUIR_TENSORRT_VERSION="7.2.2.3" 

scripts/whl/manylinux2014/do_build_common.sh

@@ -75,33 +75,66 @@ then
 fi
 
 SRC_DIR=$(readlink -f "`dirname $0`/../../../")
+source ${SRC_DIR}/scripts/whl/utils/utils.sh
+
 BUILD_DIR=${SRC_DIR}/build_dir/host/MGE_WITH_CUDA_OFF/MGE_INFERENCE_ONLY_OFF/Release/build/
 if [ ${BUILD_WHL_CPU_ONLY} = "OFF" ]; then
     BUILD_DIR=${SRC_DIR}/build_dir/host/MGE_WITH_CUDA_ON/MGE_INFERENCE_ONLY_OFF/Release/build/
 fi
 
+# here we just treat cu file should not in the increment build file list
+INCREMENT_KEY_WORDS=".cu.o is dirty"
+IS_IN_FIRST_LOOP=TRUE
+
 for ver in ${ALL_PYTHON}
 do
+    # we want run a full clean build at the first loop
+    if [ ${IS_IN_FIRST_LOOP} = "TRUE" ]; then
+        # TODO: may all cmake issue can be resolved after rm CMakeCache?
+        # if YES, remove this to use old cache and speed up CI
+        echo "warning: remove old build_dir for the first loop"
+        rm -rf ${BUILD_DIR}
+    fi
+
     python_ver=${ver:0:2}
     MAJOR=${python_ver:0:1}
     MINOR=${ver:1}
     PYTHON_DIR=/opt/python/cp${python_ver}-cp${ver}/
     export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} ${EXTRA_CMAKE_FLAG}"
     export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DCMAKE_BUILD_TYPE=RelWithDebInfo"
-    export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DCMAKE_PREFIX_PATH=${PYTHON_DIR}"
     export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DPYTHON_EXECUTABLE=${PYTHON_DIR}/bin/python3"
     export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DPYTHON_LIBRARY=${PYTHON_DIR}lib/"
     export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DPYTHON_INCLUDE_DIR=${PYTHON_DIR}include/python${MAJOR}.${MINOR}"
     export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DMGE_WITH_ATLAS=ON"
 
-    # TODO: after change to Ninja, only the fisrt loop need add -r to save build time
+    if [ -d "${BUILD_DIR}" ]; then
+        # insure rm have args
+        touch ${BUILD_DIR}/empty.so
+        touch ${BUILD_DIR}/CMakeCache.txt
+        find ${BUILD_DIR} -name "*.so" | xargs rm
+        # as we now use increment build mode when switch python
+        # But I do not known any more issue at CMakeLists.txt or not
+        # so Force remove CMakeCache.txt
+        find ${BUILD_DIR} -name CMakeCache.txt | xargs rm
+    fi
+
+    HOST_BUILD_ARGS="-t -s"
     if [ ${BUILD_WHL_CPU_ONLY} = "OFF" ]; then
-        ${SRC_DIR}/scripts/cmake-build/host_build.sh -c -t -r
-    else
-        ${SRC_DIR}/scripts/cmake-build/host_build.sh -t -r
+        HOST_BUILD_ARGS="${HOST_BUILD_ARGS} -c"
+    fi
+
+    # call ninja dry run and check increment is invalid or not
+    if [ ${IS_IN_FIRST_LOOP} = "FALSE" ]; then
+        ninja_dry_run_and_check_increment "${SRC_DIR}/scripts/cmake-build/host_build.sh" "${HOST_BUILD_ARGS}" "${INCREMENT_KEY_WORDS}"
     fi
 
+    # call real build
+    echo "host_build.sh HOST_BUILD_ARGS: ${HOST_BUILD_ARGS}"
+    ${SRC_DIR}/scripts/cmake-build/host_build.sh ${HOST_BUILD_ARGS}
+
+    # check python api call setup.py
     cd ${BUILD_DIR}
+    check_build_ninja_python_api ${ver}
     rm -rf staging
     mkdir -p staging
     cp -a imperative/python/{megengine,setup.py,requires.txt,requires-style.txt,requires-test.txt} staging/
@@ -128,4 +161,5 @@ do
     # compat for root-less docker env to remove output at host side
     chmod -R 777 .
     echo "python $ver done"
+    IS_IN_FIRST_LOOP=FALSE
 done

scripts/whl/manylinux2014/init_image.sh

@@ -7,8 +7,11 @@ LLVM_URL='https://github.com/llvm-mirror/llvm/archive/release_60.tar.gz'
 CLANG_URL='https://github.com/llvm-mirror/clang/archive/release_60.tar.gz'
 
 
-yum install -y pcre-devel devtoolset-9-libatomic-devel.x86_64
+yum install -y pcre-devel devtoolset-9-libatomic-devel.x86_64 ninja-build
 yum install -y devtoolset-8 devtoolset-8-libatomic-devel.x86_64
+# install a default python3 for cmake PYTHON3_EXECUTABLE_WITHOUT_VERSION
+yum install -y python3
+python3 -m pip install numpy
 
 for ver in 35m 36m 37m 38 
 do

scripts/whl/utils/utils.sh

@@ -0,0 +1,81 @@
+#!/usr/bin/env bash
+set -e
+
+OS=$(uname -s)
+
+function ninja_dry_run_and_check_increment() {
+    if [ $# -eq 3 ]; then
+        _BUILD_SHELL=$1
+        _BUILD_FLAGS="$2 -n"
+        _INCREMENT_KEY_WORDS=$3
+    else
+        echo "err call ninja_dry_run_and_check_increment"
+        exit -1
+    fi
+
+    ${_BUILD_SHELL} ${_BUILD_FLAGS} 2>&1 | tee dry_run.log
+
+    DIRTY_LOG=`cat dry_run.log`
+    if [[ "${DIRTY_LOG}" =~ ${_INCREMENT_KEY_WORDS} ]]; then
+        echo "+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
+        echo "python3 switch increment build failed, some MR make a wrong CMakeLists.txt depends"
+        echo "or build env can not find default python3 in PATH env"
+        echo "please refs for PYTHON3_EXECUTABLE_WITHOUT_VERSION define at SRC_ROOT/CMakeLists.txt"
+        echo "+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
+        exit -1
+    fi
+}
+
+PYTHON_API_INCLUDES=""
+
+function check_build_ninja_python_api() {
+    INCLUDE_KEYWORD=""
+    IS_MINOR_HIT=FALSE
+    if [ $# -eq 1 ]; then
+        ver=$1
+        echo "org args: ${ver}"
+        if [[ $OS =~ "NT" ]]; then
+            INCLUDE_KEYWORD="${ver}\\\\include"
+            PYTHON_API_INCLUDES="3.5.4\\\\include 3.6.8\\\\include 3.7.7\\\\include 3.8.3\\\\include"
+        elif [[ $OS =~ "Linux" ]]; then
+            INCLUDE_KEYWORD="include/python3.${ver:1:1}"
+            PYTHON_API_INCLUDES="include/python3.5 include/python3.6 include/python3.7 include/python3.8"
+        elif [[ $OS =~ "Darwin" ]]; then
+            INCLUDE_KEYWORD="include/python3.${ver:2:1}"
+            PYTHON_API_INCLUDES="include/python3.5 include/python3.6 include/python3.7 include/python3.8"
+        else
+            echo "unknown OS: ${OS}"
+            exit -1
+        fi
+    else
+        echo "err call check_build_ninja_python_api"
+        exit -1
+    fi
+    echo "try check python INCLUDE_KEYWORD: ${INCLUDE_KEYWORD} is invalid in ninja.build or not"
+
+    NINJA_BUILD=`cat build.ninja`
+    for PYTHON_API_INCLUDE in ${PYTHON_API_INCLUDES}
+    do
+        echo "check PYTHON_API_INCLUDE vs INCLUDE_KEYWORD : (${PYTHON_API_INCLUDE} : ${INCLUDE_KEYWORD})"
+        if [ ${PYTHON_API_INCLUDE} = ${INCLUDE_KEYWORD} ]; then
+            if [[ "${NINJA_BUILD}" =~ ${PYTHON_API_INCLUDE} ]]; then
+                echo "hit INCLUDE_KEYWORD: ${INCLUDE_KEYWORD} in build.ninja"
+                IS_MINOR_HIT="TRUE"
+            else
+                echo "Err happened can not find INCLUDE_KEYWORD: ${INCLUDE_KEYWORD} in build.ninja"
+                exit -1
+            fi
+        else
+            if [[ "${NINJA_BUILD}" =~ ${PYTHON_API_INCLUDE} ]]; then
+                echo "Err happened: find PYTHON_API_INCLUDE: ${PYTHON_API_INCLUDE} in build.ninja"
+                echo "But now INCLUDE_KEYWORD: ${INCLUDE_KEYWORD}"
+                exit -1
+            fi
+        fi
+    done
+
+    if [ ${IS_MINOR_HIT} = "FALSE" ]; then
+        echo "Err happened, can not hit any MINOR api in ninja.build"
+        exit -1
+    fi
+}

scripts/whl/windows/windows_build_whl.sh

@@ -23,6 +23,8 @@ function append_path_env_and_check() {
 append_path_env_and_check
 
 SRC_DIR=$(READLINK -f "`dirname $0`/../../../")
+source ${SRC_DIR}/scripts/whl/utils/utils.sh
+
 ALL_PYTHON=${ALL_PYTHON}
 FULL_PYTHON_VER="3.5.4 3.6.8 3.7.7 3.8.3"
 if [[ -z ${ALL_PYTHON} ]]
@@ -43,12 +45,12 @@ mkdir -p ${WINDOWS_WHL_HOME}
 function config_python_env() {
     PYTHON_DIR=/c/Users/${USER}/mge_whl_python_env/$1
     PYTHON_BIN=${PYTHON_DIR}
-    if [ ! -f "$PYTHON_BIN/python3.exe" ]; then
+    if [ ! -f "${PYTHON_BIN}/python3.exe" ]; then
         echo "ERR: can not find $PYTHON_BIN , Invalid python package"
         echo "now support list: ${FULL_PYTHON_VER}"
         err_env
     else
-        echo "put python3 to env..."
+        echo "put ${PYTHON_BIN}/python3.exe to env..."
         export PATH=${PYTHON_BIN}:$PATH
         which python3
     fi
@@ -104,11 +106,24 @@ function copy_more_dll() {
     fi
 }
 
+BUILD_DIR=${SRC_DIR}/build_dir/host/build/
+
+# here we just treat cu file should not in the increment build file list
+INCREMENT_KEY_WORDS=".cu.obj is dirty"
+IS_IN_FIRST_LOOP=TRUE
 
 function do_build() {
     for ver in ${ALL_PYTHON}
     do
-        #config
+        # we want run a full clean build at the first loop
+        if [ ${IS_IN_FIRST_LOOP} = "TRUE" ]; then
+            # TODO: may all cmake issue can be resolved after rm CMakeCache?
+            # if YES, remove this to use old cache and speed up CI
+            echo "warning: remove old build_dir for the first loop"
+            rm -rf ${BUILD_DIR}
+        fi
+
+        #config python3
         config_python_env ${ver}
 
         #check env
@@ -122,35 +137,44 @@ function do_build() {
         fi
         echo "PYTHON_LIBRARY: ${PYTHON_LIBRARY}"
         echo "PYTHON_INCLUDE_DIR: ${PYTHON_INCLUDE_DIR}"
-        #append cmake args for config python
-        #FIXME: ninja handle err with cmake 3.17 when assgin PYTHON_LIBRARY
-        #But after put python3.exe to HEAD of PATH by config_python_env, cmake can also handle the
-        #right PYTHON_LIBRARY and PYTHON_INCLUDE_DIR, at the same time, clang-cl need swig target
-        #force LINK a real PYTHON_LIBRARY file, after test we do not find the symbols conflict with python
-        #export EXTRA_CMAKE_ARGS="-DPYTHON_LIBRARY=${PYTHON_LIBRARY} -DPYTHON_INCLUDE_DIR=${PYTHON_INCLUDE_DIR} "
         #config build type to RelWithDebInfo to enable MGB_ENABLE_DEBUG_UTIL etc
         export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DCMAKE_BUILD_TYPE=RelWithDebInfo "
 
         #call build and install
-        #FIXME: cmake do not triger update python config, after
-        #change PYTHON_LIBRARY and PYTHON_INCLUDE_DIR, so add
-        #-r to remove build cache after a new ver build, which
-        #will be more slow build than without -r
-        BUILD_ARGS=" -t -r"
+        HOST_BUILD_ARGS=" -t -s"
 
         if [ ${BUILD_WHL_CPU_ONLY} = "OFF" ]; then
             echo "build windows whl with cuda"
-            BUILD_ARGS="${BUILD_ARGS} -c "
+            HOST_BUILD_ARGS="${HOST_BUILD_ARGS} -c "
         else
             echo "build windows whl with cpu only"
         fi
 
-        echo "host_build.sh BUILD_ARGS: ${BUILD_ARGS}"
-        ${SRC_DIR}/scripts/cmake-build/host_build.sh ${BUILD_ARGS}
+        if [ -d "${BUILD_DIR}" ]; then
+            # insure rm have args
+            touch ${BUILD_DIR}/empty.pyd
+            touch ${BUILD_DIR}/CMakeCache.txt
+            /usr/bin/find ${BUILD_DIR} -name "*.pyd" | xargs rm
+            # ninja/cmake on windows will handle error if just export
+            # PYTHON_LIBRARY/PYTHON_INCLUDE_DIR/PYTHON_EXECUTABLE
+            # But after put python3.exe to HEAD of PATH by config_python_env
+            # and force remove CMakeCache.txt, ninja/cmake will auto update
+            # PYTHON_LIBRARY/PYTHON_INCLUDE_DIR/PYTHON_EXECUTABLE
+            /usr/bin/find ${BUILD_DIR} -name CMakeCache.txt | xargs rm
+        fi
+        echo "host_build.sh HOST_BUILD_ARGS: ${HOST_BUILD_ARGS}"
 
-        BUILD_DIR=${SRC_DIR}/build_dir/host/build/
-        cd ${BUILD_DIR}
+        # call ninja dry run and check increment is invalid or not
+        if [ ${IS_IN_FIRST_LOOP} = "FALSE" ]; then
+            ninja_dry_run_and_check_increment "${SRC_DIR}/scripts/cmake-build/host_build.sh" "${HOST_BUILD_ARGS}" "${INCREMENT_KEY_WORDS}"
+        fi
 
+        #call real build
+        ${SRC_DIR}/scripts/cmake-build/host_build.sh ${HOST_BUILD_ARGS}
+
+        # check python api call setup.py
+        cd ${BUILD_DIR}
+        check_build_ninja_python_api ${ver}
         rm -rf staging
         mkdir -p staging
         cp -a imperative/python/{megengine,setup.py,requires.txt,requires-style.txt,requires-test.txt} staging/
@@ -178,6 +202,7 @@ function do_build() {
         echo "windows whl package location: ${WINDOWS_WHL_HOME}"
         ls ${WINDOWS_WHL_HOME}
         echo "##############################################################################################"
+        IS_IN_FIRST_LOOP=FALSE
     done
 }