From c6069d02d72fe0f6a62865b5fd1796d0c7900804 Mon Sep 17 00:00:00 2001 From: Megvii Engine Team Date: Mon, 22 Mar 2021 18:53:48 +0800 Subject: [PATCH] chore(scripts): remove manylinux2010 build enviroment --- scripts/whl/manylinux2010/.dockerignore | 3 - scripts/whl/manylinux2010/.gitignore | 1 - scripts/whl/manylinux2010/Dockerfile | 15 ---- scripts/whl/manylinux2010/build_image.sh | 5 -- scripts/whl/manylinux2010/build_wheel.sh | 103 -------------------------- scripts/whl/manylinux2010/do_build.sh | 117 ------------------------------ scripts/whl/manylinux2010/init_image.sh | 100 ------------------------- scripts/whl/manylinux2010/setup_mirror.sh | 65 ----------------- 8 files changed, 409 deletions(-) delete mode 100644 scripts/whl/manylinux2010/.dockerignore delete mode 100644 scripts/whl/manylinux2010/.gitignore delete mode 100644 scripts/whl/manylinux2010/Dockerfile delete mode 100755 scripts/whl/manylinux2010/build_image.sh delete mode 100755 scripts/whl/manylinux2010/build_wheel.sh delete mode 100755 scripts/whl/manylinux2010/do_build.sh delete mode 100755 scripts/whl/manylinux2010/init_image.sh delete mode 100755 scripts/whl/manylinux2010/setup_mirror.sh diff --git a/scripts/whl/manylinux2010/.dockerignore b/scripts/whl/manylinux2010/.dockerignore deleted file mode 100644 index b8342df1..00000000 --- a/scripts/whl/manylinux2010/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -/output/ -/build_image.sh -/build_wheel.sh diff --git a/scripts/whl/manylinux2010/.gitignore b/scripts/whl/manylinux2010/.gitignore deleted file mode 100644 index 16be8f21..00000000 --- a/scripts/whl/manylinux2010/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/output/ diff --git a/scripts/whl/manylinux2010/Dockerfile b/scripts/whl/manylinux2010/Dockerfile deleted file mode 100644 index 0a434fdb..00000000 --- a/scripts/whl/manylinux2010/Dockerfile +++ /dev/null @@ -1,15 +0,0 @@ -FROM quay.io/pypa/manylinux2010_x86_64:2020-01-31-046f791 - -ENV UID=1024 \ - PATH=${PATH}:/usr/local/cuda/bin \ - LIBRARY_PATH=${LIBRARY_PATH}:/usr/local/cuda/lib64:/usr/local/cuda/lib64/stubs:/opt/cudnn/lib64:/opt/tensorrt/lib \ - LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/cuda/lib64:/usr/local/cuda/lib64/stubs:/opt/cudnn/lib64:/opt/tensorrt/lib \ - CPATH=${CPATH}:/usr/local/cuda/include:/opt/cudnn/include:/opt/tensorrt/include - -ARG platform -COPY setup_mirror.sh . -RUN ./setup_mirror.sh "$platform" - -ADD init_image.sh /tmp -RUN /tmp/init_image.sh && rm -f /tmp/init_image.sh - diff --git a/scripts/whl/manylinux2010/build_image.sh b/scripts/whl/manylinux2010/build_image.sh deleted file mode 100755 index fd686fd4..00000000 --- a/scripts/whl/manylinux2010/build_image.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -e - -cd $(dirname $0) - -docker build -t env_manylinux2010:latest . diff --git a/scripts/whl/manylinux2010/build_wheel.sh b/scripts/whl/manylinux2010/build_wheel.sh deleted file mode 100755 index b9d16150..00000000 --- a/scripts/whl/manylinux2010/build_wheel.sh +++ /dev/null @@ -1,103 +0,0 @@ -#!/bin/bash -e - -CWD=$(dirname $0) -BASEDIR=$(readlink -f ${CWD}/../../..) -OUTPUTDIR=$(readlink -f ${CWD}/output) -USERID=$(id -u) -TMPFS_ARGS="--tmpfs /tmp:exec" - -BUILD_WHL_CPU_ONLY=${BUILD_WHL_CPU_ONLY} -if [[ -z ${BUILD_WHL_CPU_ONLY} ]] -then - BUILD_WHL_CPU_ONLY="OFF" -fi - -echo ${BASEDIR} -pushd ${BASEDIR}/third_party >/dev/null - ./prepare.sh -popd >/dev/null - -cd ${CWD} -mkdir -p ${OUTPUTDIR} - -if [ ${BUILD_WHL_CPU_ONLY} = "OFF" ]; then - if [[ -z ${CUDA_ROOT_DIR} ]]; then - echo "Environment variable CUDA_ROOT_DIR not set." - exit -1 - fi - if [[ -z ${CUDNN_ROOT_DIR} ]]; then - echo "Environment variable CUDNN_ROOT_DIR not set." - exit -1 - fi - if [[ -z ${TENSORRT_ROOT_DIR} ]]; then - echo "Environment variable TENSORRT_ROOT_DIR not set." - exit -1 - fi - - ## YOU SHOULD MODIFY CUDA VERSION AS BELOW WHEN UPGRADE - REQUIR_CUDA_VERSION="10010" - REQUIR_CUDNN_VERSION="7.6.3" - REQUIR_TENSORRT_VERSION="6.0.1.5" - - CUDA_ROOT_DIR_=${CUDA_ROOT_DIR%*/} - CUDNN_ROOT_DIR_=${CUDNN_ROOT_DIR%*/} - TENSORRT_ROOT_DIR_=${TENSORRT_ROOT_DIR%*/} - - CUDA_VERSION_PATH=${CUDA_ROOT_DIR_}/include/cuda.h - CUDNN_VERSION_PATH=${CUDNN_ROOT_DIR_}/include/cudnn.h - TENSORRT_VERSION_PATH=${TENSORRT_ROOT_DIR_}/include/NvInferVersion.h - - if [ ! -e $CUDA_VERSION_PATH ] ; then - echo file $CUDA_VERSION_PATH is not exist - echo please check the Environment must use CUDA-10.1 NO.$REQUIR_CUDA_VERSION - exit -1 - fi - if [ ! -e $CUDNN_VERSION_PATH ] ; then - echo file $CUDNN_VERSION_PATH is not exist - echo please check the Environment must use CUDNN-V$REQUIR_CUDNN_VERSION - exit -1 - fi - if [ ! -e $TENSORRT_VERSION_PATH ] ; then - echo file $TENSORRT_VERSION_PATH is not exist - echo please check the Environment must use TensorRT-$REQUIR_TENSORRT_VERSION - exit -1 - fi - - CUDA_VERSION_CONTEXT=$(head -85 ${CUDA_VERSION_PATH}) - CUDNN_VERSION_CONTEXT=$(head -62 ${CUDNN_VERSION_PATH}) - TENSORRT_VERSION_CONTEXT=$(tail -12 ${TENSORRT_VERSION_PATH}) - - CUDA_API_VERSION=$(echo $CUDA_VERSION_CONTEXT | grep -Eo "define __CUDA_API_VERSION * +([0-9]+)") - CUDA_VERSION=${CUDA_API_VERSION:0-5} - echo CUDA_VERSION:$CUDA_VERSION - - CUDNN_VERSION_MAJOR=$(echo $CUDNN_VERSION_CONTEXT | grep -Eo "define CUDNN_MAJOR * +([0-9]+)") - CUDNN_VERSION_MINOR=$(echo $CUDNN_VERSION_CONTEXT | grep -Eo "define CUDNN_MINOR * +([0-9]+)") - CUDNN_VERSION_PATCH=$(echo $CUDNN_VERSION_CONTEXT | grep -Eo "define CUDNN_PATCHLEVEL * +([0-9]+)") - CUDNN_VERSION=${CUDNN_VERSION_MAJOR:0-1}.${CUDNN_VERSION_MINOR:0-1}.${CUDNN_VERSION_PATCH:0-1} - echo CUDNN_VERSION:$CUDNN_VERSION - - TENSORRT_VERSION_MAJOR=$(echo $TENSORRT_VERSION_CONTEXT | grep -Eo "NV_TENSORRT_MAJOR * +([0-9]+)") - TENSORRT_VERSION_MINOR=$(echo $TENSORRT_VERSION_CONTEXT | grep -Eo "NV_TENSORRT_MINOR * +([0-9]+)") - TENSORRT_VERSION_PATCH=$(echo $TENSORRT_VERSION_CONTEXT | grep -Eo "NV_TENSORRT_PATCH * +([0-9]+)") - TENSORRT_VERSION_BUILD=$(echo $TENSORRT_VERSION_CONTEXT | grep -Eo "NV_TENSORRT_BUILD * +([0-9]+)") - TENSORRT_VERSION=${TENSORRT_VERSION_MAJOR:0-1}.${TENSORRT_VERSION_MINOR:0-1}.${TENSORRT_VERSION_PATCH:0-1}.${TENSORRT_VERSION_BUILD:0-1} - echo TENSORRT_VERSION:$TENSORRT_VERSION - - if [ $CUDA_VERSION != $REQUIR_CUDA_VERSION ] ; then - echo please check the Environment must use CUDA-10.1 NO.$REQUIR_CUDA_VERSION - exit -1 - fi - - if [ $CUDNN_VERSION != $REQUIR_CUDNN_VERSION ] ; then - echo please check the Environment must use CUDNN-V$REQUIR_CUDNN_VERSION - exit -1 - fi - - if [ $TENSORRT_VERSION != $REQUIR_TENSORRT_VERSION ] ; then - echo please check the Environment must use TENSORRT-$REQUIR_TENSORRT_VERSION - exit -1 - fi -fi - -docker run -it --rm $TMPFS_ARGS -e UID=${USERID} -e LOCAL_VERSION=${LOCAL_VERSION} -e BUILD_WHL_CPU_ONLY=${BUILD_WHL_CPU_ONLY} -e ALL_PYTHON="${ALL_PYTHON}" -v ${CUDA_ROOT_DIR}:/usr/local/cuda -v ${CUDNN_ROOT_DIR}:/opt/cudnn -v ${TENSORRT_ROOT_DIR}:/opt/tensorrt -v ${BASEDIR}:/home/code -v ${OUTPUTDIR}:/home/output:rw env_manylinux2010:latest /home/code/scripts/whl/manylinux2010/do_build.sh diff --git a/scripts/whl/manylinux2010/do_build.sh b/scripts/whl/manylinux2010/do_build.sh deleted file mode 100755 index 1a8e158b..00000000 --- a/scripts/whl/manylinux2010/do_build.sh +++ /dev/null @@ -1,117 +0,0 @@ -#!/bin/bash -e -ALL_PYTHON=${ALL_PYTHON} -if [[ -z ${ALL_PYTHON} ]] -then - ALL_PYTHON="35m 36m 37m 38" -fi - -BUILD_WHL_CPU_ONLY=${BUILD_WHL_CPU_ONLY} -if [[ -z ${BUILD_WHL_CPU_ONLY} ]] -then - BUILD_WHL_CPU_ONLY="OFF" -fi - -SRC_DIR=$(readlink -f "`dirname $0`/../../../") -BUILD_DIR=${SRC_DIR}/build_dir/host/MGE_WITH_CUDA_OFF/MGE_INFERENCE_ONLY_OFF/Release/build/ -if [ ${BUILD_WHL_CPU_ONLY} = "OFF" ]; then - BUILD_DIR=${SRC_DIR}/build_dir/host/MGE_WITH_CUDA_ON/MGE_INFERENCE_ONLY_OFF/Release/build/ -fi -NEW_LIB_PATH=core/lib - -function handle_strip() { - echo "now handle strip $1" - objcopy --only-keep-debug $1 $1.dbg - strip -s $1 - objcopy --add-gnu-debuglink=$1.dbg $1 - rm $1.dbg -} - -function patch_elf_depend_lib() { - echo "handle common depend lib" - LIBS_DIR=${BUILD_DIR}/staging/megengine/core/lib - mkdir -p ${LIBS_DIR} - cp /usr/lib64/libatomic.so.1 ${LIBS_DIR} - - patchelf --remove-rpath ${BUILD_DIR}/staging/megengine/core/_imperative_rt.so - patchelf --force-rpath --set-rpath '$ORIGIN/lib' ${BUILD_DIR}/staging/megengine/core/_imperative_rt.so - cp ${BUILD_DIR}/src/libmegengine_export.so ${LIBS_DIR} - patchelf --remove-rpath ${LIBS_DIR}/libmegengine_export.so - patchelf --force-rpath --set-rpath '$ORIGIN/.' ${LIBS_DIR}/libmegengine_export.so - - - if [ ${BUILD_WHL_CPU_ONLY} = "OFF" ]; then - echo "handle cuda lib" - CUDA_VER=10.1 - cp ${BUILD_DIR}/dnn/cuda-stub/libcuda.so ${LIBS_DIR} - cp /usr/local/cuda/lib64/libnvrtc.so.${CUDA_VER} ${LIBS_DIR} - cp /usr/local/cuda/lib64/libnvToolsExt.so.1 ${LIBS_DIR} - fi -} - -for ver in ${ALL_PYTHON} -do - USE_AUDITWHEEL="ON" - python_ver=${ver:0:2} - MAJOR=${python_ver:0:1} - MINOR=${ver:1} - PYTHON_DIR=/opt/python/cp${python_ver}-cp${ver}/ - export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DCMAKE_BUILD_TYPE=RelWithDebInfo" - export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DCMAKE_PREFIX_PATH=${PYTHON_DIR}" - export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DPYTHON_EXECUTABLE=${PYTHON_DIR}/bin/python3" - export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DPYTHON_LIBRARY=${PYTHON_DIR}lib/" - export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DPYTHON_INCLUDE_DIR=${PYTHON_DIR}include/python${MAJOR}.${MINOR}" - export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DMGE_WITH_ATLAS=ON" - - if [ ${BUILD_WHL_CPU_ONLY} = "OFF" ]; then - ${SRC_DIR}/scripts/cmake-build/host_build.sh -c -t -r - else - ${SRC_DIR}/scripts/cmake-build/host_build.sh -t -r - fi - - cd ${BUILD_DIR} - rm -rf staging - mkdir -p staging - cp -a imperative/python/{megengine,setup.py,requires.txt,requires-style.txt,requires-test.txt} staging/ - - - if [ ${BUILD_WHL_CPU_ONLY} = "OFF" ]; then - cd ${BUILD_DIR}/dnn/cuda-stub/ - strip -s libcuda.so - ln -sf libcuda.so libcuda.so.1 - fi - - handle_strip ${BUILD_DIR}/src/libmegengine_export.so - - cd ${BUILD_DIR}/staging/megengine/core - handle_strip _imperative_rt.so - - mkdir -p lib/ucx - - if [ ${BUILD_WHL_CPU_ONLY} = "OFF" ]; then - cp -L /usr/local/cuda/lib*/libnvrtc-builtins.so lib - fi - - - if [ ${USE_AUDITWHEEL} = "OFF" ]; then - patch_elf_depend_lib - fi - - cd ${BUILD_DIR}/staging/ - ${PYTHON_DIR}/bin/python setup.py bdist_wheel - cd /home/output - if [ ${USE_AUDITWHEEL} = "ON" ]; then - LD_LIBRARY_PATH=${BUILD_DIR}/dnn/cuda-stub:$LD_LIBRARY_PATH auditwheel repair -L ${NEW_LIB_PATH} ${BUILD_DIR}/staging/dist/Meg*.whl - else - mkdir -p ${SRC_DIR}/scripts/whl/manylinux2010/output/wheelhouse - cd ${BUILD_DIR}/staging/dist/ - org_whl_name=`ls Meg*${ver}*.whl` - compat_whl_name=`echo ${org_whl_name} | sed 's/linux/manylinux2010/'` - echo "org whl name: ${org_whl_name}" - echo "comapt whl name: ${compat_whl_name}" - mv ${org_whl_name} ${SRC_DIR}/scripts/whl/manylinux2010/output/wheelhouse/${compat_whl_name} - cd /home/output - fi - chown -R ${UID}.${UID} . - # compat for root-less docker env to remove output at host side - chmod -R 777 . -done diff --git a/scripts/whl/manylinux2010/init_image.sh b/scripts/whl/manylinux2010/init_image.sh deleted file mode 100755 index 0d44f5f8..00000000 --- a/scripts/whl/manylinux2010/init_image.sh +++ /dev/null @@ -1,100 +0,0 @@ -#!/bin/bash -e - -GET_PIP_URL='https://bootstrap.pypa.io/get-pip.py' -SWIG_URL='https://downloads.sourceforge.net/project/swig/swig/swig-3.0.12/swig-3.0.12.tar.gz?use_mirror=autoselect' -LLVM_URL='https://github.com/llvm-mirror/llvm/archive/release_60.tar.gz' -CLANG_URL='https://github.com/llvm-mirror/clang/archive/release_60.tar.gz' - -yum erase -y cmake cmake28 -yum install -y python34-pip pcre-devel devtoolset-8-libatomic-devel.x86_64 - -pip3 install --no-cache-dir --only-binary :all: -U pip==19.1 -pip3 install --no-cache-dir --only-binary :all: cmake==3.16.3 - -for ver in 35m 36m 37m 38 -do - python_ver=${ver:0:2} - curl ${GET_PIP_URL} | /opt/python/cp${python_ver}-cp${ver}/bin/python - \ - --no-cache-dir --only-binary :all: - /opt/python/cp${python_ver}-cp${ver}/bin/pip install \ - --no-cache-dir --only-binary :all: numpy==1.18.1 setuptools==46.1.3 -done - -pushd /home >/dev/null - echo "Install swig" - curl -sSL ${SWIG_URL} | tar xz - pushd swig-3.0.12 >/dev/null - mkdir build - pushd build >/dev/null - ../configure - make -j$(nproc) - make install - popd >/dev/null - popd >/dev/null - rm -rf swig-3.0.12 - - echo "Install llvm" - curl -sSL ${LLVM_URL} | tar xz - pushd llvm-release_60 >/dev/null - mkdir build - pushd build >/dev/null - cmake .. -DCMAKE_PREFIX_PATH=/opt/python/cp36-cp36m/ \ - -DCMAKE_BUILD_TYPE=Release - make -j$(nproc) - make install - popd >/dev/null - popd >/dev/null - rm -rf llvm-release_60 - - echo "Install clang" - curl -sSL ${CLANG_URL} | tar xz - pushd clang-release_60 >/dev/null - mkdir build - pushd build >/dev/null - cmake .. -DCMAKE_PREFIX_PATH=/opt/python/cp36-cp36m/ \ - -DCMAKE_BUILD_TYPE=Release - make -j$(nproc) - make install - popd >/dev/null - popd >/dev/null - rm -rf clang-release_60 -popd >/dev/null - -pushd /tmp >/dev/null - curl -sSL https://github.com/NixOS/patchelf/archive/0.10.tar.gz | tar xz - pushd /tmp/patchelf-0.10 >/dev/null - patch -p1 <<'EOF' -diff --git a/src/patchelf.cc b/src/patchelf.cc -index 0b4965a..7aae7a4 100644 ---- a/src/patchelf.cc -+++ b/src/patchelf.cc -@@ -1074,13 +1074,6 @@ void ElfFile::modifySoname(sonameMode op, const std::string & - return; - } - -- /* Zero out the previous SONAME */ -- unsigned int sonameSize = 0; -- if (soname) { -- sonameSize = strlen(soname); -- memset(soname, 'X', sonameSize); -- } -- - debug("new SONAME is '%s'\n", newSoname.c_str()); - - /* Grow the .dynstr section to make room for the new SONAME. */ -@@ -1264,7 +1257,6 @@ void ElfFile::modifyRPath(RPathOp op, - unsigned int rpathSize = 0; - if (rpath) { - rpathSize = strlen(rpath); -- memset(rpath, 'X', rpathSize); - } - - debug("new rpath is '%s'\n", newRPath.c_str()); - -EOF - ./bootstrap.sh && ./configure && make install-strip - popd - rm -rf /tmp/patchelf-0.10 -popd - -yum clean all diff --git a/scripts/whl/manylinux2010/setup_mirror.sh b/scripts/whl/manylinux2010/setup_mirror.sh deleted file mode 100755 index f3c9f832..00000000 --- a/scripts/whl/manylinux2010/setup_mirror.sh +++ /dev/null @@ -1,65 +0,0 @@ -#!/bin/bash - -set -e - -function set_tuna_yum_mirror() { - cp /etc/yum.repos.d/CentOS-Base.repo /etc/yum.repos.d/CentOS-Base.repo.bak - local repo=/etc/yum.repos.d/CentOS-Base.repo - local plugin=/etc/yum/pluginconf.d/fastestmirror.conf - sed -i "s/mirrorlist=/#mirrorlist=/g" $repo - sed -i "s/#baseurl/baseurl/g" $repo - sed -i "s/mirror.centos.org/mirrors.tuna.tsinghua.edu.cn/g" $repo - sed -i "s/http/https/g" $repo - sed -i "s/enabled=1/enabled=0/g" $plugin - yum clean all - # Build on brainpp unable to pull epel reo metadata so disable this - # https://unix.stackexchange.com/questions/148144/unable-to-pull-epel-repository-metadata - yum --disablerepo="epel" update nss - yum makecache -} - -function set_epel() { - mv /etc/yum.repos.d/epel.repo /etc/yum.repos.d/epel.repo.backup - mv /etc/yum.repos.d/epel-testing.repo /etc/yum.repos.d/epel-testing.repo.backup - curl -o /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-6.repo -} - -function set_yum_mirror() { - mv /etc/yum.repos.d/CentOS-Base.repo /etc/yum.repos.d/CentOS-Base.repo.backup - curl -o /etc/yum.repos.d/CentOS-Base.repo https://mirrors.aliyun.com/repo/Centos-6.repo - yum makecache -} - -function set_pip_mirror() { -cat > /etc/pip.conf <