OpenI
/
MegEngine
Not watched
1
0
Fork 0
Code
Releases 31 Wiki Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain
Browse Source

feat(android/whl): support android whl 

GitOrigin-RevId: 05df16b494
tags/v1.8.0
Megvii Engine Team 3 years ago
parent
f509b1be9b
commit
3ddc32d3e3
15 changed files with 699 additions and 8 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +4
    -0
      imperative/CMakeLists.txt
  2. +11
    -0
      imperative/python/megengine/__init__.py
  3. +8
    -0
      imperative/python/megengine/data/dataloader.py
  4. +25
    -5
      scripts/whl/BUILD_PYTHON_WHL_README.md
  5. +164
    -0
      scripts/whl/android/android_build_whl.sh
  6. +65
    -0
      scripts/whl/android/android_opencv_python.sh
  7. +155
    -0
      scripts/whl/android/android_whl_env_prepare.sh
  8. +75
    -0
      scripts/whl/android/cv_patch/host_android.patch
  9. +14
    -0
      scripts/whl/android/patchs/Lib-subprocess.py.patch
  10. +13
    -0
      scripts/whl/android/patchs/Lib-tmpfile.py.patch
  11. +11
    -0
      scripts/whl/android/patchs/distutils-command-build.py.patch
  12. +36
    -0
      scripts/whl/android/patchs/no-setuid-servers.patch
  13. +59
    -0
      scripts/whl/android/up_3_9_patch/fix-paths.patch
  14. +43
    -0
      scripts/whl/android/utils.sh
  15. +16
    -3
      scripts/whl/utils/utils.sh

+ 4
- 0
imperative/CMakeLists.txt View File

@@ -47,6 +47,10 @@ else()
${MODULE_NAME} PRIVATE megengine_shared -Wl,--version-script=${MGE_VERSION_SCRIPT})
endif()

if(ANDROID)
target_link_libraries(${MODULE_NAME} PRIVATE ${PYTHON_LIBRARIES})
endif()

add_subdirectory(${PROJECT_SOURCE_DIR}/third_party/range-v3
${PROJECT_BINARY_DIR}/third_party/range-v3)
target_link_libraries(${MODULE_NAME} PRIVATE range-v3)


+ 11
- 0
imperative/python/megengine/__init__.py View File

@@ -12,6 +12,17 @@ import os
import platform
import sys

if os.getenv("TERMUX_VERSION"):
try:
import cv2
except Exception as exc:
print("Run MegEngine python interface at Android/Termux env")
print("!!!You need build opencv-python manually!!!, by run sh:")
print(
"https://github.com/MegEngine/MegEngine/blob/master/scripts/whl/android/android_opencv_python.sh"
)
raise exc

if sys.platform == "win32":
lib_path = os.path.join(os.path.dirname(__file__), "core/lib")
dll_paths = list(filter(os.path.exists, [lib_path,]))


+ 8
- 0
imperative/python/megengine/data/dataloader.py View File

@@ -10,6 +10,7 @@ import collections
import gc
import math
import multiprocessing
import os
import platform
import queue
import random
@@ -150,6 +151,13 @@ class DataLoader:
"pyarrow.plasma does not support ParallelDataLoader on windows, changing num_workers to be zero"
)
self.num_workers = 0
if os.getenv("TERMUX_VERSION"):
# FIXME: termux install pyarrow will build error now
# remove this logic after pyarrow fix this issue
print(
"pyarrow do not support on termux env now, changing num_workers to be zero"
)
self.num_workers = 0
if isinstance(self.dataset, StreamDataset):
if not self.num_workers:
return _SerialStreamDataLoaderIter(self, self.preload)


+ 25
- 5
scripts/whl/BUILD_PYTHON_WHL_README.md View File

@@ -2,6 +2,7 @@
* Windows build (cpu and gpu)
* Linux build (cpu and gpu)
* MacOS build (cpu only)
* Android(termux) build (cpu only)

# Build env prepare
## Linux
@@ -30,6 +31,15 @@
./scripts/whl/macos/macos_whl_env_prepare.sh
```

## Android
* install [termux](https://termux.com/) apk on Android Device
* at least 8G DDR
* at least Android 7
* init wheel build-dependent env by command:
```bash
./scripts/whl/android/android_whl_env_prepare.sh
```

## Windows
* refer to [BUILD_README.md](../cmake-build/BUILD_README.md) Windows section to init base build environment

@@ -43,12 +53,12 @@ commands:
./scripts/whl/manylinux2014/build_wheel_common.sh -sdk cu101
```

* And you can find all of the outputs in `output` directory.If you just want to build for a specific Python verison, you can use `ALL_PYTHON` environment variable. eg:
* And you can find all of the outputs in `output` directory.If you just want to build for a specific Python verison, you can use `ALL_PYTHON` environment variable. such as:
```bash
ALL_PYTHON="36m" ./scripts/whl/manylinux2014/build_wheel_common.sh -sdk cu101
```

* If you just want to build with cpu only version, you can set `-sdk` environment 'cpu'. eg:
* If you just want to build with cpu only version, you can set `-sdk` environment 'cpu'. such as:
```bash
ALL_PYTHON="36m" ./scripts/whl/manylinux2014/build_wheel_common.sh -sdk cpu
```
@@ -58,7 +68,7 @@ ALL_PYTHON="36m" ./scripts/whl/manylinux2014/build_wheel_common.sh -sdk cpu
```bash
./scripts/whl/macos/macos_build_whl.sh
```
* If you just want to build for a specific Python verison, you can use `ALL_PYTHON` environment variable. eg:
* If you just want to build for a specific Python verison, you can use `ALL_PYTHON` environment variable. such as:
```bash
ALL_PYTHON="3.7.7" ./scripts/whl/macos/macos_build_whl.sh
```
@@ -69,12 +79,22 @@ ALL_PYTHON="3.7.7" ./scripts/whl/macos/macos_build_whl.sh
./scripts/whl/windows/windows_build_whl.sh
```

* If you just want to build for a specific Python verison, you can use `ALL_PYTHON` environment variable. eg:
* If you just want to build for a specific Python verison, you can use `ALL_PYTHON` environment variable. such as:
```bash
ALL_PYTHON="3.8.3" ./scripts/whl/windows/windows_build_whl.sh
```

* If you just want to build with cpu only version, you can set `BUILD_WHL_CPU_ONLY` environment 'ON'. eg:
* If you just want to build with cpu only version, you can set `BUILD_WHL_CPU_ONLY` environment 'ON'. such as:
```
BUILD_WHL_CPU_ONLY="ON" ALL_PYTHON="3.8.3" ./scripts/whl/windows/windows_build_whl.sh
```

## Build for Android
* commands:
```bash
scripts/whl/android/android_build_whl.sh
```
* If you just want to build for a specific Python verison, you can use `ALL_PYTHON` environment variable. such as:
```bash
ALL_PYTHON="3.10.1" ./scripts/whl/android/android_build_whl.sh
```

+ 164
- 0
scripts/whl/android/android_build_whl.sh View File

@@ -0,0 +1,164 @@
#!/bin/bash -e

SRC_DIR=$(readlink -f "`dirname $0`/../../../")
cd ${SRC_DIR}
source scripts/whl/android/utils.sh

ANDROID_WHL_HOME=${SRC_DIR}/scripts/whl/android/ANDROID_WHL_HOME
if [ -e "${ANDROID_WHL_HOME}" ]; then
echo "remove old android whl file"
rm -rf ${ANDROID_WHL_HOME}
fi
mkdir -p ${ANDROID_WHL_HOME}

BUILD_DIR=${SRC_DIR}/build_dir/host/MGE_WITH_CUDA_OFF/MGE_INFERENCE_ONLY_OFF/Release/build/

# We only handle the case where dnn/src/common/conv_bias.cpp is not in the list of incremental build files.
INCREMENT_KEY_WORDS="conv_bias.cpp.o is dirty"
IS_IN_FIRST_LOOP=TRUE

ORG_EXTRA_CMAKE_FLAG=${EXTRA_CMAKE_FLAG}

function handle_strip() {
echo "now handle strip $1"
objcopy --only-keep-debug $1 $1.dbg
strip -s $1
objcopy --add-gnu-debuglink=$1.dbg $1
rm $1.dbg
}

function patch_elf_depend_lib_mgb_mge() {
echo "handle common depend lib for mgb or mge"
LIBS_DIR=${BUILD_DIR}/staging/megengine/core/lib
mkdir -p ${LIBS_DIR}

patchelf --remove-rpath ${BUILD_DIR}/staging/megengine/core/_imperative_rt.so
patchelf --set-rpath '$ORIGIN/lib' ${BUILD_DIR}/staging/megengine/core/_imperative_rt.so
handle_strip ${BUILD_DIR}/staging/megengine/core/_imperative_rt.so

cp ${BUILD_DIR}/src/libmegengine_shared.so ${LIBS_DIR}
patchelf --remove-rpath ${LIBS_DIR}/libmegengine_shared.so
patchelf --set-rpath '$ORIGIN/.' ${LIBS_DIR}/libmegengine_shared.so
# FXIME: third_party LLVM need c++_static > 5.1
# but now clang(13) at termux env do not satisfy it
# may use -static-libstdc++ at CMakeLists.txt after
# upgrade third_party LLVM
cp /data/data/com.termux/files/usr/lib/libc++_shared.so ${LIBS_DIR}
handle_strip ${LIBS_DIR}/libmegengine_shared.so
}

function patch_elf_depend_lib_megenginelite() {
echo "handle common depend lib for megenginelite"
LIBS_DIR=${BUILD_DIR}/staging/megenginelite/libs
mkdir -p ${LIBS_DIR}

cp ${BUILD_DIR}/lite/liblite_shared_whl.so ${LIBS_DIR}/
patchelf --remove-rpath ${LIBS_DIR}/liblite_shared_whl.so
patchelf --set-rpath '$ORIGIN/../../megengine/core/lib' ${LIBS_DIR}/liblite_shared_whl.so
handle_strip ${LIBS_DIR}/liblite_shared_whl.so
}
function do_build() {
mge_python_env_root="${HOME}/mge_python_env"
for ver in ${ALL_PYTHON}
do
python_install_dir=${mge_python_env_root}/${ver}/install
# we want to run a full clean build in the first loop
if [ ${IS_IN_FIRST_LOOP} = "TRUE" ]; then
# TODO: can all cmake issues be resolved after removing CMakeCache?
# if YES, remove this logic to use old cache and speed up CI
echo "warning: remove old build_dir for the first loop"
rm -rf ${BUILD_DIR}
fi

# insert python3_install_dir into head of PATH to enable CMake find it
if [ -e ${python_install_dir}/bin/python3 ];then
echo "will use ${python_install_dir}/bin/python3 to build mge wheel"
export PATH=${python_install_dir}/bin:$PATH
else
echo "ERROR: can not find python3 in: ${python_install_dir}/bin"
echo "please run: %{SRC_DIR}/scripts/whl/android/android_whl_env_prepare.sh to prepare env"
exit -1
fi

export EXTRA_CMAKE_ARGS="${ORG_EXTRA_CMAKE_FLAG} -DCMAKE_BUILD_TYPE=RelWithDebInfo"
export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DMGE_WITH_CUSTOM_OP=ON"

if [ -d "${BUILD_DIR}" ]; then
# insure rm have args
touch ${BUILD_DIR}/empty.so
touch ${BUILD_DIR}/CMakeCache.txt
find ${BUILD_DIR} -name "*.so" | xargs rm
# Force remove CMakeCache.txt to avoid error owing to unknown issue in CMakeLists.txt
# which comes from using increment build mode when switching python version
find ${BUILD_DIR} -name CMakeCache.txt | xargs rm
fi

HOST_BUILD_ARGS="-t -s"

# call ninja dry run and check increment is invalid or not
if [ ${IS_IN_FIRST_LOOP} = "FALSE" ]; then
ninja_dry_run_and_check_increment "${SRC_DIR}/scripts/cmake-build/host_build.sh" "${HOST_BUILD_ARGS}" "${INCREMENT_KEY_WORDS}"
fi

# call real build
echo "host_build.sh HOST_BUILD_ARGS: ${HOST_BUILD_ARGS}"
bash ${SRC_DIR}/scripts/cmake-build/host_build.sh ${HOST_BUILD_ARGS}

# check python api call setup.py
cd ${BUILD_DIR}
check_build_ninja_python_api ${ver}
rm -rf staging
mkdir -p staging
cp -a imperative/python/{megengine,setup.py,requires.txt,requires-style.txt,requires-test.txt} staging/
cp -a ${SRC_DIR}/src/custom/include staging/megengine/core/include/

patch_elf_depend_lib_mgb_mge

# handle megenginelite
cd ${BUILD_DIR}
mkdir -p staging/megenginelite
cp ${SRC_DIR}/lite/pylite/megenginelite/* staging/megenginelite/
patch_elf_depend_lib_megenginelite

cd ${BUILD_DIR}/staging
python3 setup.py bdist_wheel
cd ${BUILD_DIR}/staging/dist/
cp ${BUILD_DIR}/staging/dist/Meg*.whl ${ANDROID_WHL_HOME}

cd ${SRC_DIR}
echo ""
echo "##############################################################################################"
echo "android whl package location: ${ANDROID_WHL_HOME}"
ls ${ANDROID_WHL_HOME}
echo "##############################################################################################"
IS_IN_FIRST_LOOP=FALSE
done
}

function third_party_prepare() {
echo "init third_party..."
bash ${SRC_DIR}/third_party/prepare.sh
# fix flatbuffers build at pure LLVM env(not cross link gcc)
# TODO: pr to flatbuffers to fix this issue
sed -i 's/lc++abi/lc/g' ${SRC_DIR}/third_party/flatbuffers/CMakeLists.txt
}

function remove_requires() {
# do not worry about this, we will provide 'scripts/whl/android/android_opencv_python.sh'
# to build opencv-python from opencv src!! This function may be removed after termux fixes
# this issue
cd ${SRC_DIR}
git checkout imperative/python/requires.txt
sed -i '/opencv-python/d' imperative/python/requires.txt
# FIXME: termux install pyarrow will build error now
# remove this logic after pyarrow fix this issue
# see imperative/python/megengine/data/dataloader.py
# for detail, now will use _SerialStreamDataLoaderIter
sed -i '/pyarrow/d' imperative/python/requires.txt
cd -
}
######################
check_termux_env
third_party_prepare
remove_requires
do_build

+ 65
- 0
scripts/whl/android/android_opencv_python.sh View File

@@ -0,0 +1,65 @@
#!/bin/bash -e

# This script is a workaround of installing opencv-python in termux.

SRC_DIR=$(readlink -f "`dirname $0`/../../../")
cd ${SRC_DIR}
source scripts/whl/android/utils.sh

function install_apt_package() {
APT_PACKAGE="build-essential cmake libjpeg-turbo libpng python clang"
echo "try to install: ${APT_PACKAGE}"
apt install ${APT_PACKAGE}
}

function build_opencv_python() {
python3 -m pip install numpy
mge_python_env_root="${HOME}/mge_python_env"
mkdir -p ${mge_python_env_root}
cd ${mge_python_env_root}

opencv_repo_dir=${mge_python_env_root}/opencv
if [ -d ${opencv_repo_dir}/.git ];then
echo "already find opencv repo"
cd ${opencv_repo_dir}
git reset --hard
git clean -xdf
git fetch
else
cd ${mge_python_env_root}
rm -rf ${opencv_repo_dir}
git clone https://github.com/opencv/opencv.git
fi
# Build and test latest version by default. You can modify OPENCV_VER to build and test another version!!
python3_site=`python3 -c 'import site; print(site.getsitepackages()[0])'`
OPENCV_VER="3.4.15"
git checkout ${OPENCV_VER}
if [ -e ${python3_site}/cv2/__init__.py ];then
echo "python3 already build cv2, skip build it, if you want to rebuild, you can do: rm -rf ${python3_site}/cv2"
else
cd ${opencv_repo_dir}
git checkout ${OPENCV_VE}
git apply ${SRC_DIR}/scripts/whl/android/cv_patch/*.patch
mkdir -p build
cd build
echo "will install to ${python3_site}"
PYTHON3_EXECUTABLE=`command -v python3`
LDFLAGS=" -llog -lpython3" cmake -DCMAKE_BUILD_TYPE=RELEASE -DBUILD_opencv_python3=on \
-DBUILD_opencv_python2=off -DWITH_QT=OFF -DWITH_GTK=OFF -DBUILD_ANDROID_PROJECTS=OFF \
-DBUILD_ANDROID_EXAMPLES=OFF -DBUILD_FAT_JAVA_LIB=OFF -DBUILD_ANDROID_SERVICE=OFF \
-DHAVE_opencv_python3=ON -D__INSTALL_PATH_PYTHON3=${python3_site} \
-DPYTHON3_EXECUTABLE=${PYTHON3_EXECUTABLE} \
-DOPENCV_PYTHON_INSTALL_PATH=${python3_site} -DCMAKE_INSTALL_PREFIX=${python3_site} .. \
&& make -j$(nproc) && make install

# check if build successfully
cd ~
python3 -c 'import cv2;print(cv2.__version__)'
fi
}

############install env now###########
echo "run at root dir: ${SRC_DIR}"
check_termux_env
install_apt_package
build_opencv_python

+ 155
- 0
scripts/whl/android/android_whl_env_prepare.sh View File

@@ -0,0 +1,155 @@
#!/bin/bash -e

# Installing package by pkg in termux is unwise because pkg will upgrade
# package to the latest version which is undesired sometimes. We will
# use apt as default package tool. If your env is already broken,e.g.
# clang does not work， you can execute following commands to fix it:
# pkg update
# pkg upgrade

SRC_DIR=$(readlink -f "`dirname $0`/../../../")
cd ${SRC_DIR}
source scripts/whl/android/utils.sh

function install_apt_package() {
APT_PACKAGE="proot git wget clang cmake libandroid-spawn binutils build-essential ninja texinfo patchelf python"
echo "try install: ${APT_PACKAGE}"
apt install ${APT_PACKAGE}
echo "check termux status by running: clang --version"
log=`clang --version || true`
if [[ "${log}" =~ "clang version" ]]; then
echo "valid env after installing apt package"
else
echo "Failed to run clang command, please check termux env!!! You can run: pkg update && pkg upgrade to try to solve it"
echo "raw log: ${log}"
exit -1
fi

}

function patch_termux_env() {
# do not try to modify other project build files to adapt to only-llvm environment
echo "many projects can not build without gcc libs, so we create a fake gcc libs linked to librt"
RT_LIB_TARGET="${PREFIX}/lib/librt.so"
if [ -e ${RT_LIB_TARGET} ];then
echo "find librt.so and link it to libgcc.so"
GCC_LIB_TARGET="${PREFIX}/lib/libgcc.so"
if [ -e ${GCC_LIB_TARGET} ];then
echo "already find: ${GCC_LIB_TARGET} skip it"
else
create_libgcc_cmd="ln -s ${RT_LIB_TARGET} ${GCC_LIB_TARGET}"
echo "run cmd: $create_libgcc_cmd"
${create_libgcc_cmd}
fi
else
echo "broken termux env, can not find librt.so"
exit -1
fi
}

function build_python() {
# Up to now many tools (build multi python3) are not supported in termux so that we have to build them from source.
# This function will be changed when some tools are supported in termux, e.g. pyenv.
mge_python_env_root="${HOME}/mge_python_env"
mkdir -p ${mge_python_env_root}
cd ${mge_python_env_root}

if [ -e ${PREFIX}/local/lib/libffi.a ];then
echo "always find libffi, skip build it"
else
echo "build libffi for python module"
rm -rf libffi
git clone https://github.com/libffi/libffi.git
cd libffi
termux-chroot "./autogen.sh && ./configure && make -j$(nproc) && make install"
# remove dynamic lib to force python to use static lib
rm ${PREFIX}/local/lib/libffi.so*
fi

if [ -e ${PREFIX}/local/lib/libz.a ];then
echo "always find libzlib, skip build it"
else
echo "build zlib for python module"
rm -rf zlib
git clone https://github.com/madler/zlib.git
cd zlib
termux-chroot "CFLAGS=\"-O3 -fPIC\" ./configure && make -j$(nproc) && make install"
# remove dynamic lib to force python to use static lib
rm ${PREFIX}/local/lib/libz.so*
fi

cpython_repo_dir=${mge_python_env_root}/cpython
if [ -d ${cpython_repo_dir}/.git ];then
echo "already find cpython repo"
cd ${cpython_repo_dir}
git reset --hard
git clean -xdf
git fetch
else
cd ${mge_python_env_root}
rm -rf ${cpython_repo_dir}
git clone https://github.com/python/cpython.git
fi
for ver in ${ALL_PYTHON}
do
install_dir=${mge_python_env_root}/${ver}/install
if [ -e ${install_dir}/bin/python3 ];then
echo "always find python3, skip build it"
else
mkdir -p ${install_dir}
echo "try build python: ${ver} to ${install_dir}"
cd ${cpython_repo_dir}
git reset --hard
git clean -xdf
git checkout v${ver}
index=`awk -v a="${ver}" -v b="." 'BEGIN{print index(a,b)}'`
sub_str=${ver:${index}}
index_s=`awk -v a="${sub_str}" -v b="." 'BEGIN{print index(a,b)}'`
((finally_index = ${index} + ${index_s}))
finally_verson=${ver:0:${finally_index}-1}
MINOR_VER=${ver:${index}:${finally_index}-${index}-1}
finally_verson="python${finally_verson}"
echo "finally_verson is: ${finally_verson}"
# apply patchs
git apply ${SRC_DIR}/scripts/whl/android/patchs/*.patch
if [[ ${MINOR_VER} -gt 8 ]]
then
echo "apply more patchs"
git apply ${SRC_DIR}/scripts/whl/android/up_3_9_patch/*.patch
fi

termux-chroot "FLAGS=\"-D__ANDROID_API__=24 -Wno-unused-value -Wno-empty-body -Qunused-arguments -Wno-error\" \
./configure CFLAGS=\"${FLAGS}\" CPPFLAGS=\"${FLAGS}\" CC=clang CXX=clang++ --enable-shared --prefix=${install_dir} \
&& sed -i 's/-Werror=implicit-function-declaration//g' Makefile && \
sed -i 's/\$(LN) -f \$(INSTSONAME)/cp \$(INSTSONAME)/g' Makefile && make -j$(nproc) && make install"

# after building successfully, patchelf to make python work out of termux-chroot env
cd ${install_dir}/bin
# Some python versions won't link to python3 automatically, so we create link manually here'
rm -rf python3
if [[ ${MINOR_VER} -gt 7 ]]
then
echo "python3 remove suffix m after 3.8"
patchelf --add-rpath ${install_dir}/lib ${finally_verson}
cp ${finally_verson} python3
else
echo "python3 with suffix m before 3.8, add it"
patchelf --add-rpath ${install_dir}/lib "${finally_verson}m"
cp "${finally_verson}m" ${finally_verson}
cp ${finally_verson} python3
fi
echo "finally try run python3"
./python3 --version
./python3 -m pip install --upgrade pip
./python3 -m pip install numpy wheel

fi
done
}

############install env now###########
echo "run at root dir: ${SRC_DIR}"
check_termux_env
install_apt_package
patch_termux_env
build_python

+ 75
- 0
scripts/whl/android/cv_patch/host_android.patch View File

@@ -0,0 +1,75 @@
diff --git a/CMakeLists.txt b/CMakeLists.txt
index f6a2da5310..10354312c9 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -643,7 +643,7 @@ if(UNIX)
if(NOT APPLE)
CHECK_INCLUDE_FILE(pthread.h HAVE_PTHREAD)
if(ANDROID)
- set(OPENCV_LINKER_LIBS ${OPENCV_LINKER_LIBS} dl m log)
+ set(OPENCV_LINKER_LIBS ${OPENCV_LINKER_LIBS} dl m log z)
elseif(CMAKE_SYSTEM_NAME MATCHES "FreeBSD|NetBSD|DragonFly|OpenBSD|Haiku")
set(OPENCV_LINKER_LIBS ${OPENCV_LINKER_LIBS} m pthread)
elseif(EMSCRIPTEN)
diff --git a/cmake/OpenCVDetectPython.cmake b/cmake/OpenCVDetectPython.cmake
index 4ff02a77d3..db1305448f 100644
--- a/cmake/OpenCVDetectPython.cmake
+++ b/cmake/OpenCVDetectPython.cmake
@@ -123,7 +123,7 @@ if(NOT ${found})
if(_found)
set(_version_major_minor "${_version_major}.${_version_minor}")
- if(NOT ANDROID AND NOT APPLE_FRAMEWORK)
+ if(TRUE)
ocv_check_environment_variables(${library_env} ${include_dir_env})
if(NOT ${${library_env}} STREQUAL "")
set(PYTHON_LIBRARY "${${library_env}}")
@@ -175,7 +175,7 @@ if(NOT ${found})
endif()
endif()
- if(NOT ANDROID AND NOT IOS)
+ if(TRUE)
if(CMAKE_HOST_UNIX)
execute_process(COMMAND ${_executable} -c "from distutils.sysconfig import *; print(get_python_lib())"
RESULT_VARIABLE _cvpy_process
@@ -240,7 +240,7 @@ if(NOT ${found})
OUTPUT_STRIP_TRAILING_WHITESPACE)
endif()
endif()
- endif(NOT ANDROID AND NOT IOS)
+ endif()
endif()
# Export return values
@@ -285,6 +285,17 @@ find_python("${OPENCV_PYTHON3_VERSION}" "${MIN_VER_PYTHON3}" PYTHON3_LIBRARY PYT
PYTHON3_INCLUDE_DIR PYTHON3_INCLUDE_DIR2 PYTHON3_PACKAGES_PATH
PYTHON3_NUMPY_INCLUDE_DIRS PYTHON3_NUMPY_VERSION)
+message("DEBUG PYTHON3_LIBRARIES: ${PYTHON3_LIBRARIES}")
+message("DEBUG PYTHON3_INCLUDE_DIR: ${PYTHON3_INCLUDE_DIR}")
+string(COMPARE EQUAL "${PYTHON3_LIBRARIES}" "" result)
+if(result)
+ message(FATAL_ERROR "can not find PYTHON3_LIBRARIES")
+endif()
+
+string(COMPARE EQUAL "${PYTHON3_INCLUDE_DIR}" "" result)
+if(result)
+ message(FATAL_ERROR "can not find PYTHON3_INCLUDE_DIR")
+endif()
if(PYTHON_DEFAULT_EXECUTABLE)
set(PYTHON_DEFAULT_AVAILABLE "TRUE")
diff --git a/modules/python/CMakeLists.txt b/modules/python/CMakeLists.txt
index a51acf386e..5605a54a32 100644
--- a/modules/python/CMakeLists.txt
+++ b/modules/python/CMakeLists.txt
@@ -3,7 +3,7 @@
# ----------------------------------------------------------------------------
if(DEFINED OPENCV_INITIAL_PASS) # OpenCV build
-if(ANDROID OR APPLE_FRAMEWORK OR WINRT)
+ if(False)
ocv_module_disable_(python2)
ocv_module_disable_(python3)
return()

+ 14
- 0
scripts/whl/android/patchs/Lib-subprocess.py.patch View File

@@ -0,0 +1,14 @@
diff -u -r ../Python-3.7.1/Lib/subprocess.py ./Lib/subprocess.py
--- ../Python-3.7.1/Lib/subprocess.py 2018-10-20 06:04:19.000000000 +0000
+++ ./Lib/subprocess.py 2018-10-20 20:17:50.157206343 +0000
@@ -1389,9 +1389,7 @@
args = list(args)
if shell:
- # On Android the default shell is at '/system/bin/sh'.
- unix_shell = ('/system/bin/sh' if
- hasattr(sys, 'getandroidapilevel') else '/bin/sh')
+ unix_shell = ('@TERMUX_PREFIX@/bin/sh')
args = [unix_shell, "-c"] + args
if executable:
args[0] = executable

+ 13
- 0
scripts/whl/android/patchs/Lib-tmpfile.py.patch View File

@@ -0,0 +1,13 @@
diff --git a/Lib/tempfile.py b/Lib/tempfile.py
index 531cbf32f1..dae57e22bd 100644
--- a/Lib/tempfile.py
+++ b/Lib/tempfile.py
@@ -170,7 +170,7 @@ def _candidate_tempdir_list():
_os.path.expandvars(r'%SYSTEMROOT%\Temp'),
r'c:\temp', r'c:\tmp', r'\temp', r'\tmp' ])
else:
- dirlist.extend([ '/tmp', '/var/tmp', '/usr/tmp' ])
+ dirlist.extend([ '@TERMUX_PREFIX@/tmp' ])
# As a last resort, the current directory.
try:

+ 11
- 0
scripts/whl/android/patchs/distutils-command-build.py.patch View File

@@ -0,0 +1,11 @@
--- ./Lib/distutils/command/build.py.orig 2019-11-19 23:19:08.878782120 +0000
+++ ./Lib/distutils/command/build.py 2019-11-19 23:19:18.410915201 +0000
@@ -118,7 +118,7 @@
if self.executable is None and sys.executable:
self.executable = os.path.normpath(sys.executable)
-
+ self.parallel = 1
if isinstance(self.parallel, str):
try:
self.parallel = int(self.parallel)

+ 36
- 0
scripts/whl/android/patchs/no-setuid-servers.patch View File

@@ -0,0 +1,36 @@
diff -uNr Python-3.9.2/Lib/http/server.py Python-3.9.2.mod/Lib/http/server.py
--- Python-3.9.2/Lib/http/server.py 2021-02-19 14:31:44.000000000 +0200
+++ Python-3.9.2.mod/Lib/http/server.py 2021-03-20 16:08:34.173543081 +0200
@@ -1165,10 +1165,6 @@
return
# Child
try:
- try:
- os.setuid(nobody)
- except OSError:
- pass
os.dup2(self.rfile.fileno(), 0)
os.dup2(self.wfile.fileno(), 1)
os.execve(scriptfile, args, env)
diff -uNr Python-3.9.2/Lib/smtpd.py Python-3.9.2.mod/Lib/smtpd.py
--- Python-3.9.2/Lib/smtpd.py 2021-02-19 14:31:44.000000000 +0200
+++ Python-3.9.2.mod/Lib/smtpd.py 2021-03-20 16:11:48.785629393 +0200
@@ -9,7 +9,8 @@
-n
This program generally tries to setuid `nobody', unless this flag is
set. The setuid call will fail if this program is not run as root (in
- which case, use this flag).
+ which case, use this flag). Ignored in Termux as no setuid done on this
+ platform.
--version
-V
@@ -863,7 +864,7 @@
class Options:
- setuid = True
+ setuid = False
classname = 'PureProxy'
size_limit = None
enable_SMTPUTF8 = False

+ 59
- 0
scripts/whl/android/up_3_9_patch/fix-paths.patch View File

@@ -0,0 +1,59 @@
diff -uNr Python-3.6.2/Lib/aifc.py Python-3.6.2.mod/Lib/aifc.py
--- Python-3.6.2/Lib/aifc.py 2017-07-08 06:33:27.000000000 +0300
+++ Python-3.6.2.mod/Lib/aifc.py 2017-09-15 15:09:08.092797061 +0300
@@ -920,7 +920,7 @@
if __name__ == '__main__':
import sys
if not sys.argv[1:]:
- sys.argv.append('/usr/demos/data/audio/bach.aiff')
+ sys.argv.append('@TERMUX_PREFIX@/demos/data/audio/bach.aiff')
fn = sys.argv[1]
with open(fn, 'r') as f:
print("Reading", fn)
diff -uNr Python-3.6.2/Lib/mailcap.py Python-3.6.2.mod/Lib/mailcap.py
--- Python-3.6.2/Lib/mailcap.py 2017-07-08 06:33:27.000000000 +0300
+++ Python-3.6.2.mod/Lib/mailcap.py 2017-09-15 15:08:41.312797081 +0300
@@ -55,7 +55,8 @@
# Don't bother with getpwuid()
home = '.' # Last resort
mailcaps = [home + '/.mailcap', '/etc/mailcap',
- '/usr/etc/mailcap', '/usr/local/etc/mailcap']
+ '/usr/etc/mailcap', '/usr/local/etc/mailcap',
+ '@TERMUX_PREFIX@/etc/mailcap']
return mailcaps
diff -uNr Python-3.6.2/Lib/mimetypes.py Python-3.6.2.mod/Lib/mimetypes.py
--- Python-3.6.2/Lib/mimetypes.py 2017-07-08 06:33:27.000000000 +0300
+++ Python-3.6.2.mod/Lib/mimetypes.py 2017-09-15 15:08:05.522797106 +0300
@@ -49,6 +49,7 @@
"/usr/local/lib/netscape/mime.types",
"/usr/local/etc/httpd/conf/mime.types", # Apache 1.2
"/usr/local/etc/mime.types", # Apache 1.3
+ "@TERMUX_PREFIX@/etc/mime.types", # Termux
]
inited = False
diff -uNr Python-3.6.2/Lib/posixpath.py Python-3.6.2.mod/Lib/posixpath.py
--- Python-3.6.2/Lib/posixpath.py 2017-07-08 06:33:27.000000000 +0300
+++ Python-3.6.2.mod/Lib/posixpath.py 2017-09-15 15:07:20.872797138 +0300
@@ -32,7 +32,7 @@
extsep = '.'
sep = '/'
pathsep = ':'
-defpath = '/bin:/usr/bin'
+defpath = '@TERMUX_PREFIX@/bin'
altsep = None
devnull = '/dev/null'
diff -uNr Python-3.9.0/Lib/uuid.py Python-3.9.0.mod/Lib/uuid.py
--- Python-3.9.0/Lib/uuid.py 2020-10-05 20:37:58.000000000 +0530
+++ Python-3.9.0.mod/Lib/uuid.py 2020-10-08 18:25:45.565373486 +0530
@@ -361,7 +361,6 @@

try:
path_dirs = os.environ.get('PATH', os.defpath).split(os.pathsep)
- path_dirs.extend(['/sbin', '/usr/sbin'])
executable = shutil.which(command, path=os.pathsep.join(path_dirs))
if executable is None:
return None

+ 43
- 0
scripts/whl/android/utils.sh View File

@@ -0,0 +1,43 @@
#!/bin/bash -e

source ${SRC_DIR}/scripts/whl/utils/utils.sh

ALL_PYTHON=${ALL_PYTHON}
# FIXME: now imperative py code do not support 3.10
# but megenginelite and megbrain support it, so we
# config with 3.10.1 now
FULL_PYTHON_VER="3.8.3 3.9.9 3.10.1"
if [[ -z ${ALL_PYTHON} ]]
then
ALL_PYTHON=${FULL_PYTHON_VER}
else
check_python_version_is_valid "${ALL_PYTHON}" "${FULL_PYTHON_VER}"
fi

# FIXME python3.10+ self build have some issue, need config env
# _PYTHON_SYSCONFIGDATA_NAME remove this env after find the build issue
# do not care about this, apt install python3.10 do not have this issue
export _PYTHON_SYSCONFIGDATA_NAME="_sysconfigdata__linux_aarch64-linux-android"

function check_termux_env() {
echo "check is in termux env or not"
info=`command -v termux-info || true`
if [[ "${info}" =~ "com.termux" ]]; then
echo "find termux-info at: ${info}"
echo "check env now"
ENVS="PREFIX HOME"
for check_env in ${ENVS}
do
echo "try check env: ${check_env}"
if [[ "${!check_env}" =~ "termux" ]]; then
echo "env ${check_env} is: ${!check_env}"
else
echo "invalid ${check_env} env, may broken termux env"
exit -1
fi
done
else
echo "invalid env, only support build android whl at termux env, please refs to: scripts/whl/BUILD_PYTHON_WHL_README.md to init env"
exit -1
fi
}

+ 16
- 3
scripts/whl/utils/utils.sh View File

@@ -23,7 +23,7 @@ function ninja_dry_run_and_check_increment() {
exit -1
fi

${_BUILD_SHELL} ${_BUILD_FLAGS} 2>&1 | tee dry_run.log
bash ${_BUILD_SHELL} ${_BUILD_FLAGS} 2>&1 | tee dry_run.log

DIRTY_LOG=`cat dry_run.log`
if [[ "${DIRTY_LOG}" =~ ${_INCREMENT_KEY_WORDS} ]]; then
@@ -79,10 +79,23 @@ function check_build_ninja_python_api() {
PYTHON_API_INCLUDES="3.5.4\\\\include 3.6.8\\\\include 3.7.7\\\\include 3.8.3\\\\include"
elif [[ $OS =~ "Linux" ]]; then
INCLUDE_KEYWORD="include/python3.${ver:1:1}"
PYTHON_API_INCLUDES="include/python3.5 include/python3.6 include/python3.7 include/python3.8"
info=`command -v termux-info || true`
if [[ "${info}" =~ "com.termux" ]]; then
echo "find termux-info at: ${info}"
is_punctuation=${ver:4:1}
INCLUDE_KEYWORD="include/python3.${ver:2:1}"
if [ ${is_punctuation} = "." ]; then
INCLUDE_KEYWORD="include/python3.${ver:2:2}"
fi
fi
PYTHON_API_INCLUDES="include/python3.5 include/python3.6 include/python3.7 include/python3.8 include/python3.9 include/python3.10"
elif [[ $OS =~ "Darwin" ]]; then
is_punctuation=${ver:4:1}
INCLUDE_KEYWORD="include/python3.${ver:2:1}"
PYTHON_API_INCLUDES="include/python3.5 include/python3.6 include/python3.7 include/python3.8"
if [ ${is_punctuation} = "." ]; then
INCLUDE_KEYWORD="include/python3.${ver:2:2}"
fi
PYTHON_API_INCLUDES="include/python3.5 include/python3.6 include/python3.7 include/python3.8 include/python3.9 include/python3.10"
else
echo "unknown OS: ${OS}"
exit -1


Write Preview
Loading…
Cancel
Save