feat(osx/python_whl): refine whl doc and macos whl env

GitOrigin-RevId: 1db6f845f6

ci/docker_env/manylinux2010/brainpp_build_image.sh

@@ -1,9 +0,0 @@
-#!/bin/bash
-
-cd $(dirname $0)
-
-docker build -t env_manylinux2010:latest . \
-    --build-arg platform="brainpp" \
-    --build-arg http_proxy="http://proxy.i.brainpp.cn:3128" \
-    --build-arg https_proxy="http://proxy.i.brainpp.cn:3128" \
-    --build-arg no_proxy="brainpp.cn,.brainpp.ml,.megvii-inc.com,.megvii-op.org,127.0.0.1,localhost"

scripts/cmake-build/macos_build_whl.sh

@@ -1,125 +0,0 @@
-#!/bin/bash -e
-
-######################################################################################################################
-#                                             macos build whl env prepare                                            #
-# 1: install xcodebuild for host-build                                                                               #
-# 2: install brew: /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install.sh)" #
-# 3: build/install cmake3-14.4                                                                                       #
-#    wget https://cmake.org/files/v3.14/cmake-3.14.4.tar.gz                                                          #
-#    tar -xzvf cmake-3.14.4.tar.gz;cd cmake-3.14.4;                                                                  #
-#    ./configure; make -j32; sudo make install                                                                       #
-#                                                                                                                    #
-# 4: brew install wget python swig coreutils llvm                                                                    #
-#    echo 'export PATH="/usr/local/opt/llvm/bin:$PATH"' >> ~/.zshrc  (if u use other shell, chang this)              #
-#                                                                                                                    #
-# 5: brew install python@3.8 (sub version may different)                                                             #
-#    /usr/local/Cellar/python@3.8/3.8.3_2/bin/pip3 install numpy                                                     #
-#                                                                                                                    #
-# 6: brew install python@3.7 (sub version may different)                                                             #
-#    /usr/local/Cellar/python@3.7/3.7.8_1/bin/pip3 install numpy                                                     #
-#                                                                                                                    #
-# 7: make link for easy use python3.x (not build necessary)                                                          #
-#    ln -s /usr/local/Cellar/python@3.7/3.7.8_1/bin/pip3.7 /usr/local/bin/pip3.7                                     #
-#    ln -s /usr/local/Cellar/python@3.7/3.7.8_1/bin/python3.7 /usr/local/bin/python3.7                               #
-#    ln -s /usr/local/Cellar/python@3.8/3.8.3_2/bin/pip3.8 /usr/local/bin/pip3.8                                     #
-#    ln -s /usr/local/Cellar/python@3.8/3.8.3_2/bin/python3.8 /usr/local/bin/python3.8                               #
-######################################################################################################################
-
-READLINK=readlink
-OS=$(uname -s)
-
-if [ $OS = "Darwin" ];then
-    READLINK=greadlink
-else
-    echo "ERR: only run at macos env"
-    exit -1
-fi
-
-SRC_DIR=$($READLINK -f "`dirname $0`/../../")
-ALL_PYTHON=${ALL_PYTHON}
-if [[ -z ${ALL_PYTHON} ]]
-then
-    #FIXME: on catalina brew only official support 3.7 and 3.8
-    ALL_PYTHON="37 38"
-fi
-
-PYTHON_DIR=
-PYTHON_LIBRARY=
-PYTHON_INCLUDE_DIR=
-function config_python_env() {
-    if [[ "$1" -eq "38" ]]; then
-        PYTHON_DIR=/usr/local/Cellar/python@3.8/3.8.3_2/Frameworks/Python.framework/Versions/3.8/
-        PYTHON_LIBRARY=${PYTHON_DIR}lib/libpython3.8.dylib
-        PYTHON_INCLUDE_DIR=${PYTHON_DIR}include/python3.8
-    elif [[ "$1" -eq "37" ]]; then
-        PYTHON_DIR=/usr/local/Cellar/python@3.7/3.7.8_1/Frameworks/Python.framework/Versions/3.7/
-        PYTHON_LIBRARY=${PYTHON_DIR}lib/libpython3.7.dylib
-        PYTHON_INCLUDE_DIR=${PYTHON_DIR}include/python3.7m
-    else
-        echo "ERR: DO NOT SUPPORT PYTHON VERSION"
-        exit -1
-    fi
-}
-
-MACOS_WHL_HOME=${SRC_DIR}/scripts/cmake-build/macos_whl_home
-if [ -e "${MACOS_WHL_HOME}" ]; then
-    echo "remove old macos whl file"
-    rm -rf ${MACOS_WHL_HOME}
-fi
-mkdir -p ${MACOS_WHL_HOME}
-
-for ver in ${ALL_PYTHON}
-do
-    #config
-    config_python_env ${ver}
-
-    #check env
-    if [ ! -f "$PYTHON_LIBRARY" ]; then
-        echo "ERR: can not find $PYTHON_LIBRARY , Invalid python package"
-        exit -1
-    fi
-    if [ ! -d "$PYTHON_INCLUDE_DIR" ]; then
-        echo "ERR: can not find $PYTHON_INCLUDE_DIR , Invalid python package"
-        exit -1
-    fi
-    echo "PYTHON_LIBRARY: ${PYTHON_LIBRARY}"
-    echo "PYTHON_INCLUDE_DIR: ${PYTHON_INCLUDE_DIR}"
-
-    #append cmake args for config python
-    export EXTRA_CMAKE_ARGS="-DCMAKE_PREFIX_PATH=${PYTHON_DIR} -DPYTHON_LIBRARY=${PYTHON_LIBRARY} -DPYTHON_INCLUDE_DIR=${PYTHON_INCLUDE_DIR} "
-    #config build type to RelWithDebInfo to enable MGB_ENABLE_DEBUG_UTIL etc
-    export EXTRA_CMAKE_ARGS=${EXTRA_CMAKE_ARGS}" -DCMAKE_BUILD_TYPE=RelWithDebInfo "
-
-    #call build and install
-    #FIXME: cmake do not triger update python config, after
-    #change PYTHON_LIBRARY and PYTHON_INCLUDE_DIR, so add
-    #-r to remove build cache after a new ver build, which
-    #will be more slow build than without -r
-    ${SRC_DIR}/scripts/cmake-build/host_build.sh -t -r
-
-    #call setup.py
-    BUILD_DIR=${SRC_DIR}/build_dir/host/MGE_WITH_CUDA_OFF/MGE_INFERENCE_ONLY_OFF/Release/build/
-    cd ${BUILD_DIR}
-
-    if [ -d "staging" ]; then
-        echo "remove old build cache file"
-        rm -rf staging
-    fi
-    mkdir -p staging
-
-
-    cp -a python_module/{megengine,setup.py,requires.txt,requires-style.txt,requires-test.txt} staging/
-    cd ${BUILD_DIR}/staging/megengine/_internal
-    #FIXME: set lib suffix to dylib may be better, BUT we find after distutils.file_util.copy_file
-    #will change to .so at macos even we set suffix to dylib, at the same time, macos also support .so
-    llvm-strip -s _mgb.so
-    cd ${BUILD_DIR}/staging
-    ${PYTHON_DIR}/bin/python3 setup.py bdist_wheel
-    cp ${BUILD_DIR}/staging/dist/Meg*.whl ${MACOS_WHL_HOME}/
-
-    echo ""
-    echo "##############################################################################################"
-    echo "macos whl package location: ${MACOS_WHL_HOME}"
-    ls ${MACOS_WHL_HOME}
-    echo "##############################################################################################"
-done

scripts/whl/BUILD_PYTHON_WHL_README.md

@@ -0,0 +1,44 @@
+# python whl package build support status
+*  windows build (not ok)
+*  linux build   (ok, cpu or gpu)
+*  macos build   (ok,cpu only)
+# build env prepare
+## linux
+
+    ```
+    1: enable rootless docker env, refs: https://docs.docker.com/engine/security/rootless/
+    2: cd ./scripts/whl/linux/manylinux2010
+    3: ./build_image.sh cpu
+    4: ./build_image.sh cuda
+
+    ```
+
+## macos
+    ```
+    ./scripts/whl/macos/macos_whl_env_prepare.sh
+    ```
+# how to build
+## build for linux
+    MegBrain delivers `wheel` package with `manylinux2010` tag defined in [PEP-571](https://www.python.org/dev/peps/pep-0571/).
+
+    ```
+    ./build_wheel.sh cpu
+    
+    CUDA_ROOT_DIR=/path/to/cuda \
+    CUDNN_ROOT_DIR=/path/to/cudnn \
+    TENSORRT_ROOT_DIR=/path/to/tensorrt \
+    ./build_wheel.sh cuda
+    ```
+
+    And you can find all of the outputs in `output` directory.
+    
+    If you just want to build for a specific Python verison, you can use `ALL_PYTHON` environment variable. eg:
+
+    ```
+    ALL_PYTHON=35m ./build_wheel.sh cpu
+    
+    ```
+## build for macos
+    ```
+    ./scripts/whl/macos/macos_build_whl.sh
+    ```

scripts/whl/macos/macos_build_whl.sh

@@ -0,0 +1,145 @@
+#!/bin/bash -e
+
+READLINK=readlink
+OS=$(uname -s)
+
+if [ $OS = "Darwin" ];then
+    READLINK=greadlink
+else
+    echo "ERR: only run at macos env"
+    exit -1
+fi
+
+function err_env() {
+    echo "check_env failed: pls refs ${SRC_DIR}/scripts/whl/BUILD_PYTHON_WHL_README.md"
+    echo "try call ./scripts/whl/macos/macos_whl_env_prepare.sh to init env"
+    exit -1
+}
+
+function append_path_env_and_check() {
+    export PATH=/usr/local/opt/findutils/libexec/gnubin:$PATH
+    export PATH=/usr/local/opt/binutils/bin:$PATH
+    export PATH=/usr/local/opt/llvm/bin:$PATH
+    export PATH=/Users/${USER}/megengine_use_cmake/install/bin:$PATH
+    if [ ! -f "/usr/local/opt/llvm/bin/llvm-strip" ]; then
+        err_env
+    fi
+
+    which cmake
+    if [ ! -f "/Users/${USER}/megengine_use_cmake/install/bin/cmake" ]; then
+        err_env
+    fi
+}
+
+append_path_env_and_check
+
+SRC_DIR=$($READLINK -f "`dirname $0`/../../../")
+ALL_PYTHON=${ALL_PYTHON}
+FULL_PYTHON_VER="3.5.9 3.6.10 3.7.7 3.8.3"
+if [[ -z ${ALL_PYTHON} ]]
+then
+    ALL_PYTHON=${FULL_PYTHON_VER}
+fi
+
+PYTHON_DIR=
+PYTHON_LIBRARY=
+PYTHON_INCLUDE_DIR=
+MACOS_WHL_HOME=${SRC_DIR}/scripts/whl/macos/macos_whl_home
+if [ -e "${MACOS_WHL_HOME}" ]; then
+    echo "remove old macos whl file"
+    rm -rf ${MACOS_WHL_HOME}
+fi
+mkdir -p ${MACOS_WHL_HOME}
+
+function config_python_env() {
+    PYTHON_DIR=/Users/${USER}/.pyenv/versions/$1/
+    PYTHON_BIN=/Users/${USER}/.pyenv/versions/$1/bin
+    if [ ! -f "$PYTHON_BIN/python3" ]; then
+        echo "ERR: can not find $PYTHON_BIN , Invalid python package"
+        echo "now support list: ${FULL_PYTHON_VER}"
+        err_env
+    else
+        echo "put python3 to env..."
+        export PATH=${PYTHON_BIN}:$PATH
+        which python3
+    fi
+    echo ${ver}
+
+    if [ "$1" = "3.5.9" ]; then
+        PYTHON_LIBRARY=${PYTHON_DIR}lib/libpython3.5m.dylib
+        PYTHON_INCLUDE_DIR=${PYTHON_DIR}include/python3.5m
+    elif [ "$1" = "3.6.10" ]; then
+        PYTHON_LIBRARY=${PYTHON_DIR}lib/libpython3.6m.dylib
+        PYTHON_INCLUDE_DIR=${PYTHON_DIR}include/python3.6m
+    elif [ "$1" = "3.7.7" ]; then
+        PYTHON_LIBRARY=${PYTHON_DIR}lib/libpython3.7m.dylib
+        PYTHON_INCLUDE_DIR=${PYTHON_DIR}include/python3.7m
+    elif [ "$1" = "3.8.3" ]; then
+        PYTHON_LIBRARY=${PYTHON_DIR}lib/libpython3.8.dylib
+        PYTHON_INCLUDE_DIR=${PYTHON_DIR}include/python3.8
+    else
+        echo "ERR: DO NOT SUPPORT PYTHON VERSION"
+        echo "now support list: ${FULL_PYTHON_VER}"
+        exit -1
+    fi
+}
+
+function do_build() {
+    for ver in ${ALL_PYTHON}
+    do
+        #config
+        config_python_env ${ver}
+
+        #check env
+        if [ ! -f "$PYTHON_LIBRARY" ]; then
+            echo "ERR: can not find $PYTHON_LIBRARY , Invalid python package"
+            err_env
+        fi
+        if [ ! -d "$PYTHON_INCLUDE_DIR" ]; then
+            echo "ERR: can not find $PYTHON_INCLUDE_DIR , Invalid python package"
+            err_env
+        fi
+        echo "PYTHON_LIBRARY: ${PYTHON_LIBRARY}"
+        echo "PYTHON_INCLUDE_DIR: ${PYTHON_INCLUDE_DIR}"
+        #append cmake args for config python
+        export EXTRA_CMAKE_ARGS="-DCMAKE_PREFIX_PATH=${PYTHON_DIR} -DPYTHON_LIBRARY=${PYTHON_LIBRARY} -DPYTHON_INCLUDE_DIR=${PYTHON_INCLUDE_DIR} "
+        #config build type to RelWithDebInfo to enable MGB_ENABLE_DEBUG_UTIL etc
+        export EXTRA_CMAKE_ARGS=${EXTRA_CMAKE_ARGS}" -DCMAKE_BUILD_TYPE=RelWithDebInfo "
+
+        #call build and install
+        #FIXME: cmake do not triger update python config, after
+        #change PYTHON_LIBRARY and PYTHON_INCLUDE_DIR, so add
+        #-r to remove build cache after a new ver build, which
+        #will be more slow build than without -r
+        ${SRC_DIR}/scripts/cmake-build/host_build.sh -t -r
+
+        #call setup.py
+        BUILD_DIR=${SRC_DIR}/build_dir/host/MGE_WITH_CUDA_OFF/MGE_INFERENCE_ONLY_OFF/Release/build/
+        cd ${BUILD_DIR}
+
+        if [ -d "staging" ]; then
+            echo "remove old build cache file"
+            rm -rf staging
+        fi
+        mkdir -p staging
+
+
+        cp -a python_module/{megengine,setup.py,requires.txt,requires-style.txt,requires-test.txt} staging/
+        cd ${BUILD_DIR}/staging/megengine/_internal
+        #FIXME: set lib suffix to dylib may be better, BUT we find after distutils.file_util.copy_file
+        #will change to .so at macos even we set suffix to dylib, at the same time, macos also support .so
+        llvm-strip -s _mgb.so
+        cd ${BUILD_DIR}/staging
+        ${PYTHON_DIR}/bin/python3 setup.py bdist_wheel
+        cp ${BUILD_DIR}/staging/dist/Meg*.whl ${MACOS_WHL_HOME}/
+
+        echo ""
+        echo "##############################################################################################"
+        echo "macos whl package location: ${MACOS_WHL_HOME}"
+        ls ${MACOS_WHL_HOME}
+        echo "##############################################################################################"
+    done
+}
+
+######################
+do_build

scripts/whl/macos/macos_whl_env_prepare.sh

@@ -0,0 +1,102 @@
+#!/bin/bash -e
+
+READLINK=readlink
+OS=$(uname -s)
+
+if [ $OS = "Darwin" ];then
+    READLINK=greadlink
+else
+    echo "ERR: only run at macos env"
+    exit -1
+fi
+
+SRC_DIR=$($READLINK -f "`dirname $0`/../../../")
+
+echo ${SRC_DIR}
+ALL_PYTHON="3.5.9 3.6.10 3.7.7 3.8.3"
+
+function try_install_brew() {
+    which brew
+    if [ $? -eq 0 ]; then
+        echo "find install brew, use it"
+    else
+        echo "DO NOT FIND brew, now try install, may ask root password, please input manually!!"
+        /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install.sh)"
+        #try double check 
+        which brew
+        if [ $? -eq 0 ]; then
+            echo "find install brew, use it"
+        else
+            echo "ERR: INSTALL brew failed!!, please install manually!!"
+            exit -1
+        fi
+    fi
+}
+
+function install_brew_package() {
+    BREW_PACKAGE="openssl readline sqlite3 xz gdbm zlib pyenv wget swig coreutils llvm"
+    for pak in ${BREW_PACKAGE}
+    do
+        echo "###### do command: brew install ${pak}"
+        brew install ${pak}
+    done
+}
+
+function install_python_package() {
+    for pak in ${ALL_PYTHON}
+    do
+        echo "###### do command: env PYTHON_CONFIGURE_OPTS=\"--enable-shared\" pyenv install ${pak}"
+        if [ -e /Users/$USER/.pyenv/versions/${pak} ];then
+            echo "FOUND install /Users/$USER/.pyenv/versions/${pak} strip it..."
+        else
+            env PYTHON_CONFIGURE_OPTS="--enable-shared" pyenv install ${pak}
+        fi
+        echo "###### do command: /Users/${USER}/.pyenv/versions/${pak}/bin/python3 -m pip install numpy wheel"
+        /Users/${USER}/.pyenv/versions/${pak}/bin/python3 -m pip install numpy wheel
+        echo "###### do command: /Users/${USER}/.pyenv/versions/${pak}/bin/python3 -m pip install -r ${SRC_DIR}/python_module/requires-test.txt"
+        /Users/${USER}/.pyenv/versions/${pak}/bin/python3 -m pip install -r ${SRC_DIR}/python_module/requires-test.txt
+    done
+}
+
+function install_cmake() {
+    CMAKE_INSTALL_DIR="/Users/${USER}/megengine_use_cmake"
+    if [ -f /Users/${USER}/megengine_use_cmake/install/bin/cmake ];then
+        echo "find old build cmake, strip..."
+    else
+        if [ ! -d /Users/${USER}/megengine_use_cmake ];then
+            echo "create dir for megengine_use_cmake"
+            mkdir -p ${CMAKE_INSTALL_DIR}
+        fi
+
+        rm -rf ${CMAKE_INSTALL_DIR}/src/cmake-3.14.4.tar.gz
+        mkdir ${CMAKE_INSTALL_DIR}/src
+        cd ${CMAKE_INSTALL_DIR}/src
+        wget https://cmake.org/files/v3.14/cmake-3.14.4.tar.gz
+        tar -xvf cmake-3.14.4.tar.gz
+        cd cmake-3.14.4
+        mkdir build
+        cd build
+        ../configure --prefix=${CMAKE_INSTALL_DIR}/install
+        make -j$(nproc)
+        make install
+    fi
+}
+
+function append_path_env_message() {
+    echo "+++++++++++++++++++++++++++++++++++++++++++++++++++++"
+    echo "install env build megengine macos env successfully"
+    echo "pls append path env at build script, if u use customization whl build script"
+    echo "append detail:"
+    echo "/Users/${USER}/megengine_use_cmake/install/bin/"
+    echo "/usr/local/opt/findutils/libexec/gnubin"
+    echo "/usr/local/opt/binutils/bin"
+    echo "/usr/local/opt/llvm/bin"
+    echo "+++++++++++++++++++++++++++++++++++++++++++++++++++++"
+}
+
+############install env now###########
+try_install_brew
+install_brew_package
+install_python_package
+install_cmake
+append_path_env_message

ci/docker_env/manylinux2010/build_wheel.sh → scripts/whl/manylinux2010/build_wheel.sh

@@ -6,6 +6,7 @@ OUTPUTDIR=$(readlink -f ${CWD}/output)
 USERID=$(id -u)
 TMPFS_ARGS="--tmpfs /tmp:exec"
 
+echo ${BASEDIR}
 pushd ${BASEDIR}/third_party >/dev/null
     ./prepare.sh
 popd >/dev/null
@@ -91,6 +92,6 @@ if [ $TENSORRT_VERSION != $REQUIR_TENSORRT_VERSION ] ; then
     exit -1
 fi
 
-docker run -it --rm $TMPFS_ARGS -e UID=${USERID} -e LOCAL_VERSION=${LOCAL_VERSION} -e ALL_PYTHON=${ALL_PYTHON} -v ${CUDA_ROOT_DIR}:/usr/local/cuda -v ${CUDNN_ROOT_DIR}:/opt/cudnn -v ${TENSORRT_ROOT_DIR}:/opt/tensorrt -v ${BASEDIR}:/home/code -v ${OUTPUTDIR}:/home/output:rw env_manylinux2010:latest /home/code/ci/docker_env/manylinux2010/do_build.sh
+docker run -it --rm $TMPFS_ARGS -e UID=${USERID} -e LOCAL_VERSION=${LOCAL_VERSION} -e ALL_PYTHON=${ALL_PYTHON} -v ${CUDA_ROOT_DIR}:/usr/local/cuda -v ${CUDNN_ROOT_DIR}:/opt/cudnn -v ${TENSORRT_ROOT_DIR}:/opt/tensorrt -v ${BASEDIR}:/home/code -v ${OUTPUTDIR}:/home/output:rw env_manylinux2010:latest /home/code/scripts/whl/manylinux2010/do_build.sh