feat(osx/python_whl): make macos HAPPY for build megbrain python package

GitOrigin-RevId: 7ffd9b9541

python_module/src/cpp/megbrain_config.cpp

@@ -274,7 +274,8 @@ void _config::load_opr_library(const char* self_path, const char* lib_path) {
     }
 }
 
-std::vector<std::pair<uint64_t, std::string>> _config::dump_registered_oprs() {
+std::vector<std::pair<unsigned long int, std::string>>
+_config::dump_registered_oprs() {
 #if MGB_ENABLE_DEBUG_UTIL
     return serialization::OprRegistry::dump_registries();
 #else

python_module/src/cpp/megbrain_config.h

@@ -62,7 +62,7 @@ class _config {
         static void load_opr_library(
                 const char* self_path, const char* lib_path);
 
-        static std::vector<std::pair<uint64_t, std::string>>
+        static std::vector<std::pair<unsigned long int, std::string>>
             dump_registered_oprs();
 
         static int create_mm_server(const std::string& server_addr, int port);

scripts/cmake-build/cross_build_android_arm_inference.sh

@@ -7,6 +7,8 @@ MGE_ARMV8_2_FEATURE_FP16=OFF
 MGE_ARMV8_2_FEATURE_DOTPROD=OFF
 MGE_DISABLE_FLOAT16=OFF
 ARCH=arm64-v8a
+REMOVE_OLD_BUILD=false
+echo "EXTRA_CMAKE_ARGS: ${EXTRA_CMAKE_ARGS}"
 
 function usage() {
     echo "$0 args1 args2 .."
@@ -16,12 +18,14 @@ function usage() {
     echo "-p : enable MGE_ARMV8_2_FEATURE_DOTPROD for ARM64, need toolchain and hardware support"
     echo "-k : open MGE_DISABLE_FLOAT16 for NEON "
     echo "-a : config build arch available: ${ARCHS[@]}"
+    echo "-r : remove old build dir before make, default off"
     echo "-h : show usage"
+    echo "append other cmake config by export EXTRA_CMAKE_ARGS=..."
     echo "example: $0 -d"
     exit -1
 }
 
-while getopts "khdfpa:" arg
+while getopts "rkhdfpa:" arg
 do
     case $arg in
         d)
@@ -60,6 +64,10 @@ do
             echo "show usage"
             usage
             ;;
+        r)
+            echo "config REMOVE_OLD_BUILD=true"
+            REMOVE_OLD_BUILD=true
+            ;;
         ?)
             echo "unkonw argument"
             usage
@@ -87,7 +95,7 @@ elif [[ $OS =~ "NT" ]]; then
 fi
 
 SRC_DIR=$($READLINK -f "`dirname $0`/../../")
-source $SRC_DIR/scripts/cmake-build/utils/host_build_flatc.sh
+source $SRC_DIR/scripts/cmake-build/utils/utils.sh
 
 if [ -z $NDK_ROOT ];then
     echo "can not find NDK_ROOT env, pls export you NDK root dir to NDK_ROOT"
@@ -105,14 +113,7 @@ function cmake_build() {
     echo "build ABI: $BUILD_ABI"
     echo "build native level: $BUILD_NATIVE_LEVEL"
     echo "BUILD MAKEFILE_TYPE: $MAKEFILE_TYPE"
-    if [ -e $BUILD_DIR ];then
-        echo "clean old dir: $BUILD_DIR"
-        rm -rf $BUILD_DIR
-    fi
-    if [ -e $INSTALL_DIR ];then
-        echo "clean old dir: $INSTALL_DIR"
-        rm -rf $INSTALL_DIR
-    fi
+    try_remove_old_build $REMOVE_OLD_BUILD $BUILD_DIR $INSTALL_DIR
 
     echo "create build dir"
     mkdir -p $BUILD_DIR
@@ -130,6 +131,7 @@ function cmake_build() {
         -DMGE_ARMV8_2_FEATURE_DOTPROD=$MGE_ARMV8_2_FEATURE_DOTPROD \
         -DMGE_DISABLE_FLOAT16=$MGE_DISABLE_FLOAT16 \
         -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR \
+        ${EXTRA_CMAKE_ARGS} \
         $SRC_DIR
 
     make -j$(nproc)

scripts/cmake-build/cross_build_ios_arm_inference.sh

@@ -7,6 +7,8 @@ MGE_ARMV8_2_FEATURE_FP16=OFF
 MGE_ARMV8_2_FEATURE_DOTPROD=OFF
 MGE_DISABLE_FLOAT16=OFF
 ARCH=arm64
+REMOVE_OLD_BUILD=false
+echo "EXTRA_CMAKE_ARGS: ${EXTRA_CMAKE_ARGS}"
 
 function usage() {
     echo "$0 args1 args2 .."
@@ -16,12 +18,14 @@ function usage() {
     echo "-p : enable MGE_ARMV8_2_FEATURE_DOTPROD for ARM64, need toolchain and hardware support"
     echo "-k : open MGE_DISABLE_FLOAT16 for NEON "
     echo "-a : config build arch available: ${ARCHS[@]}"
+    echo "-r : remove old build dir before make, default off"
     echo "-h : show usage"
+    echo "append other cmake config by export EXTRA_CMAKE_ARGS=..."
     echo "example: $0 -d"
     exit -1
 }
 
-while getopts "khdfpa:" arg
+while getopts "rkhdfpa:" arg
 do
     case $arg in
         d)
@@ -60,6 +64,10 @@ do
             echo "show usage"
             usage
             ;;
+        r)
+            echo "config REMOVE_OLD_BUILD=true"
+            REMOVE_OLD_BUILD=true
+            ;;
         ?)
             echo "unkonw argument"
             usage
@@ -86,7 +94,7 @@ else
 fi
 
 SRC_DIR=$($READLINK -f "`dirname $0`/../../")
-source $SRC_DIR/scripts/cmake-build/utils/host_build_flatc.sh
+source $SRC_DIR/scripts/cmake-build/utils/utils.sh
 
 function cmake_build() {
     BUILD_DIR=$SRC_DIR/build_dir/apple/$3/$4/$1/$BUILD_TYPE/build
@@ -102,14 +110,7 @@ function cmake_build() {
     echo "build OS_PLATFORM: $OS_PLATFORM"
     echo "build XCODE_IOS_PLATFORM: $XCODE_IOS_PLATFORM"
     echo "build IOS_ARCH: $IOS_ARCH"
-    if [ -e $BUILD_DIR ];then
-        echo "clean old dir: $BUILD_DIR"
-        rm -rf $BUILD_DIR
-    fi
-    if [ -e $INSTALL_DIR ];then
-        echo "clean old dir: $INSTALL_DIR"
-        rm -rf $INSTALL_DIR
-    fi
+    try_remove_old_build $REMOVE_OLD_BUILD $BUILD_DIR $INSTALL_DIR
 
     echo "create build dir"
     mkdir -p $BUILD_DIR
@@ -128,6 +129,7 @@ function cmake_build() {
         -DMGE_ARMV8_2_FEATURE_DOTPROD=$MGE_ARMV8_2_FEATURE_DOTPROD \
         -DMGE_DISABLE_FLOAT16=$MGE_DISABLE_FLOAT16 \
         -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR \
+        ${EXTRA_CMAKE_ARGS} \
         $SRC_DIR
 
     make -j$(nproc)

scripts/cmake-build/cross_build_linux_arm_inference.sh

@@ -7,6 +7,8 @@ MGE_ARMV8_2_FEATURE_FP16=OFF
 MGE_ARMV8_2_FEATURE_DOTPROD=OFF
 MGE_DISABLE_FLOAT16=OFF
 ARCH=arm64-v8a
+REMOVE_OLD_BUILD=false
+echo "EXTRA_CMAKE_ARGS: ${EXTRA_CMAKE_ARGS}"
 
 function usage() {
     echo "$0 args1 args2 .."
@@ -16,12 +18,14 @@ function usage() {
     echo "-p : enable MGE_ARMV8_2_FEATURE_DOTPROD for ARM64, need toolchain and hardware support"
     echo "-k : open MGE_DISABLE_FLOAT16 for NEON "
     echo "-a : config build arch available: ${ARCHS[@]}"
+    echo "-r : remove old build dir before make, default off"
     echo "-h : show usage"
+    echo "append other cmake config by export EXTRA_CMAKE_ARGS=..."
     echo "example: $0 -d"
     exit -1
 }
 
-while getopts "khdfpa:" arg
+while getopts "rkhdfpa:" arg
 do
     case $arg in
         d)
@@ -60,6 +64,10 @@ do
             echo "show usage"
             usage
             ;;
+        r)
+            echo "config REMOVE_OLD_BUILD=true"
+            REMOVE_OLD_BUILD=true
+            ;;
         ?)
             echo "unkonw argument"
             usage
@@ -87,7 +95,7 @@ elif [[ $OS =~ "NT" ]]; then
 fi
 
 SRC_DIR=$($READLINK -f "`dirname $0`/../../")
-source $SRC_DIR/scripts/cmake-build/utils/host_build_flatc.sh
+source $SRC_DIR/scripts/cmake-build/utils/utils.sh
 
 function cmake_build() {
     BUILD_DIR=$SRC_DIR/build_dir/gnu-linux/$1/$BUILD_TYPE/build
@@ -98,14 +106,7 @@ function cmake_build() {
     echo "build type: $BUILD_TYPE"
     echo "build toolchain: $TOOLCHAIN"
     echo "BUILD MAKEFILE_TYPE: $MAKEFILE_TYPE"
-    if [ -e $BUILD_DIR ];then
-        echo "clean old dir: $BUILD_DIR"
-        rm -rf $BUILD_DIR
-    fi
-    if [ -e $INSTALL_DIR ];then
-        echo "clean old dir: $INSTALL_DIR"
-        rm -rf $INSTALL_DIR
-    fi
+    try_remove_old_build $REMOVE_OLD_BUILD $BUILD_DIR $INSTALL_DIR
 
     echo "create build dir"
     mkdir -p $BUILD_DIR
@@ -120,6 +121,7 @@ function cmake_build() {
         -DMGE_ARMV8_2_FEATURE_DOTPROD=$MGE_ARMV8_2_FEATURE_DOTPROD \
         -DMGE_DISABLE_FLOAT16=$MGE_DISABLE_FLOAT16 \
         -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR \
+        ${EXTRA_CMAKE_ARGS} \
         $SRC_DIR
 
     make -j$(nproc)

scripts/cmake-build/host_build.sh

@@ -8,7 +8,9 @@ function usage() {
     echo "-c : Build with CUDA, default without CUDA"
     echo "-t : Build with training mode, default inference only"
     echo "-m : Build with m32 mode(only for windows build), default m64"
+    echo "-r : remove old build dir before make, default off"
     echo "-h : show usage"
+    echo "append other cmake config by export EXTRA_CMAKE_ARGS=..."
     echo "example: $0 -d"
     exit -1
 }
@@ -19,8 +21,10 @@ MGE_INFERENCE_ONLY=ON
 MGE_WINDOWS_BUILD_ARCH=x64
 MGE_WINDOWS_BUILD_MARCH=m64
 MGE_ARCH=x86_64
+REMOVE_OLD_BUILD=false
+echo "EXTRA_CMAKE_ARGS: ${EXTRA_CMAKE_ARGS}"
 
-while getopts "hdctm" arg
+while getopts "rhdctm" arg
 do
     case $arg in
         d)
@@ -39,6 +43,10 @@ do
             echo "show usage"
             usage
             ;;
+        r)
+            echo "config REMOVE_OLD_BUILD=true"
+            REMOVE_OLD_BUILD=true
+            ;;
         m)
             echo "build for m32(only use for windows)"
             MGE_WINDOWS_BUILD_ARCH=x86
@@ -71,6 +79,7 @@ elif [[ $OS =~ "NT" ]]; then
 fi
 
 SRC_DIR=$($READLINK -f "`dirname $0`/../../")
+source $SRC_DIR/scripts/cmake-build/utils/utils.sh
 
 function cmake_build() {
     BUILD_DIR=$SRC_DIR/build_dir/host/MGE_WITH_CUDA_$1/MGE_INFERENCE_ONLY_$2/$3/build
@@ -83,14 +92,7 @@ function cmake_build() {
     echo "build type: $BUILD_TYPE"
     echo "MGE_WITH_CUDA: $MGE_WITH_CUDA"
     echo "MGE_INFERENCE_ONLY: $MGE_INFERENCE_ONLY"
-    if [ -e $BUILD_DIR ];then
-        echo "clean old dir: $BUILD_DIR"
-        rm -rf $BUILD_DIR
-    fi
-    if [ -e $INSTALL_DIR ];then
-        echo "clean old dir: $INSTALL_DIR"
-        rm -rf $INSTALL_DIR
-    fi
+    try_remove_old_build $REMOVE_OLD_BUILD $BUILD_DIR $INSTALL_DIR
 
     echo "create build dir"
     mkdir -p $BUILD_DIR
@@ -101,6 +103,7 @@ function cmake_build() {
         -DMGE_INFERENCE_ONLY=$MGE_INFERENCE_ONLY \
         -DMGE_WITH_CUDA=$MGE_WITH_CUDA \
         -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR \
+        ${EXTRA_CMAKE_ARGS} \
         $SRC_DIR
 
     make -j$(nproc)
@@ -156,7 +159,7 @@ function prepare_env_for_windows_build() {
         echo "Ninja valid ..."
     else
         echo "Ninja Invalid: ..."
-        windows_env_er
+        windows_env_err
     fi
 
     export PATH=$VS_PATH/VC/Auxiliary/Build:$PATH
@@ -176,14 +179,7 @@ function cmake_build_windows() {
     echo "build type: $BUILD_TYPE"
     echo "MGE_WITH_CUDA: $MGE_WITH_CUDA"
     echo "MGE_INFERENCE_ONLY: $MGE_INFERENCE_ONLY"
-    if [ -e $BUILD_DIR ];then
-        echo "clean old dir: $BUILD_DIR"
-        rm -rf $BUILD_DIR
-    fi
-    if [ -e $INSTALL_DIR ];then
-        echo "clean old dir: $INSTALL_DIR"
-        rm -rf $INSTALL_DIR
-    fi
+    try_remove_old_build $REMOVE_OLD_BUILD $BUILD_DIR $INSTALL_DIR
 
     echo "create build dir"
     mkdir -p $BUILD_DIR
@@ -202,6 +198,7 @@ function cmake_build_windows() {
         -DCMAKE_C_COMPILER=clang-cl.exe \
         -DCMAKE_CXX_COMPILER=clang-cl.exe \
         -DCMAKE_MAKE_PROGRAM=ninja.exe \
+        ${EXTRA_CMAKE_ARGS} \
         ../../.. && \
         echo \"start Ninja build log to build.log, may take serval min...\" && \
         Ninja load_and_run > build.log"

scripts/cmake-build/macos_build_whl.sh

@@ -0,0 +1,125 @@
+#!/bin/bash -e
+
+######################################################################################################################
+#                                             macos build whl env prepare                                            #
+# 1: install xcodebuild for host-build                                                                               #
+# 2: install brew: /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install.sh)" #
+# 3: build/install cmake3-14.4                                                                                       #
+#    wget https://cmake.org/files/v3.14/cmake-3.14.4.tar.gz                                                          #
+#    tar -xzvf cmake-3.14.4.tar.gz;cd cmake-3.14.4;                                                                  #
+#    ./configure; make -j32; sudo make install                                                                       #
+#                                                                                                                    #
+# 4: brew install wget python swig coreutils llvm                                                                    #
+#    echo 'export PATH="/usr/local/opt/llvm/bin:$PATH"' >> ~/.zshrc  (if u use other shell, chang this)              #
+#                                                                                                                    #
+# 5: brew install python@3.8 (sub version may different)                                                             #
+#    /usr/local/Cellar/python@3.8/3.8.3_2/bin/pip3 install numpy                                                     #
+#                                                                                                                    #
+# 6: brew install python@3.7 (sub version may different)                                                             #
+#    /usr/local/Cellar/python@3.7/3.7.8_1/bin/pip3 install numpy                                                     #
+#                                                                                                                    #
+# 7: make link for easy use python3.x (not build necessary)                                                          #
+#    ln -s /usr/local/Cellar/python@3.7/3.7.8_1/bin/pip3.7 /usr/local/bin/pip3.7                                     #
+#    ln -s /usr/local/Cellar/python@3.7/3.7.8_1/bin/python3.7 /usr/local/bin/python3.7                               #
+#    ln -s /usr/local/Cellar/python@3.8/3.8.3_2/bin/pip3.8 /usr/local/bin/pip3.8                                     #
+#    ln -s /usr/local/Cellar/python@3.8/3.8.3_2/bin/python3.8 /usr/local/bin/python3.8                               #
+######################################################################################################################
+
+READLINK=readlink
+OS=$(uname -s)
+
+if [ $OS = "Darwin" ];then
+    READLINK=greadlink
+else
+    echo "ERR: only run at macos env"
+    exit -1
+fi
+
+SRC_DIR=$($READLINK -f "`dirname $0`/../../")
+ALL_PYTHON=${ALL_PYTHON}
+if [[ -z ${ALL_PYTHON} ]]
+then
+    #FIXME: on catalina brew only official support 3.7 and 3.8
+    ALL_PYTHON="37 38"
+fi
+
+PYTHON_DIR=
+PYTHON_LIBRARY=
+PYTHON_INCLUDE_DIR=
+function config_python_env() {
+    if [[ "$1" -eq "38" ]]; then
+        PYTHON_DIR=/usr/local/Cellar/python@3.8/3.8.3_2/Frameworks/Python.framework/Versions/3.8/
+        PYTHON_LIBRARY=${PYTHON_DIR}lib/libpython3.8.dylib
+        PYTHON_INCLUDE_DIR=${PYTHON_DIR}include/python3.8
+    elif [[ "$1" -eq "37" ]]; then
+        PYTHON_DIR=/usr/local/Cellar/python@3.7/3.7.8_1/Frameworks/Python.framework/Versions/3.7/
+        PYTHON_LIBRARY=${PYTHON_DIR}lib/libpython3.7.dylib
+        PYTHON_INCLUDE_DIR=${PYTHON_DIR}include/python3.7m
+    else
+        echo "ERR: DO NOT SUPPORT PYTHON VERSION"
+        exit -1
+    fi
+}
+
+MACOS_WHL_HOME=${SRC_DIR}/scripts/cmake-build/macos_whl_home
+if [ -e "${MACOS_WHL_HOME}" ]; then
+    echo "remove old macos whl file"
+    rm -rf ${MACOS_WHL_HOME}
+fi
+mkdir -p ${MACOS_WHL_HOME}
+
+for ver in ${ALL_PYTHON}
+do
+    #config
+    config_python_env ${ver}
+
+    #check env
+    if [ ! -f "$PYTHON_LIBRARY" ]; then
+        echo "ERR: can not find $PYTHON_LIBRARY , Invalid python package"
+        exit -1
+    fi
+    if [ ! -d "$PYTHON_INCLUDE_DIR" ]; then
+        echo "ERR: can not find $PYTHON_INCLUDE_DIR , Invalid python package"
+        exit -1
+    fi
+    echo "PYTHON_LIBRARY: ${PYTHON_LIBRARY}"
+    echo "PYTHON_INCLUDE_DIR: ${PYTHON_INCLUDE_DIR}"
+
+    #append cmake args for config python
+    export EXTRA_CMAKE_ARGS="-DCMAKE_PREFIX_PATH=${PYTHON_DIR} -DPYTHON_LIBRARY=${PYTHON_LIBRARY} -DPYTHON_INCLUDE_DIR=${PYTHON_INCLUDE_DIR} "
+    #config build type to RelWithDebInfo to enable MGB_ENABLE_DEBUG_UTIL etc
+    export EXTRA_CMAKE_ARGS=${EXTRA_CMAKE_ARGS}" -DCMAKE_BUILD_TYPE=RelWithDebInfo "
+
+    #call build and install
+    #FIXME: cmake do not triger update python config, after
+    #change PYTHON_LIBRARY and PYTHON_INCLUDE_DIR, so add
+    #-r to remove build cache after a new ver build, which
+    #will be more slow build than without -r
+    ${SRC_DIR}/scripts/cmake-build/host_build.sh -t -r
+
+    #call setup.py
+    BUILD_DIR=${SRC_DIR}/build_dir/host/MGE_WITH_CUDA_OFF/MGE_INFERENCE_ONLY_OFF/Release/build/
+    cd ${BUILD_DIR}
+
+    if [ -d "staging" ]; then
+        echo "remove old build cache file"
+        rm -rf staging
+    fi
+    mkdir -p staging
+
+
+    cp -a python_module/{megengine,setup.py,requires.txt,requires-style.txt,requires-test.txt} staging/
+    cd ${BUILD_DIR}/staging/megengine/_internal
+    #FIXME: set lib suffix to dylib may be better, BUT we find after distutils.file_util.copy_file
+    #will change to .so at macos even we set suffix to dylib, at the same time, macos also support .so
+    llvm-strip -s _mgb.so
+    cd ${BUILD_DIR}/staging
+    ${PYTHON_DIR}/bin/python3 setup.py bdist_wheel
+    cp ${BUILD_DIR}/staging/dist/Meg*.whl ${MACOS_WHL_HOME}/
+
+    echo ""
+    echo "##############################################################################################"
+    echo "macos whl package location: ${MACOS_WHL_HOME}"
+    ls ${MACOS_WHL_HOME}
+    echo "##############################################################################################"
+done

scripts/cmake-build/utils/host_build_flatc.sh → scripts/cmake-build/utils/utils.sh

@@ -37,3 +37,23 @@ function build_flatc() {
     make -j$(nproc)
     make install/strip
 }
+
+function try_remove_old_build() {
+    REMOVE_OLD_BUILD=$1
+    echo $REMOVE_OLD_BUILD
+    BUILD_DIR=$2
+    INSTALL_DIR=$3
+
+    if [ $REMOVE_OLD_BUILD = "true" ]; then
+        if [ -e $BUILD_DIR ];then
+            echo "clean old dir: $BUILD_DIR"
+            rm -rf $BUILD_DIR
+        fi
+        if [ -e $INSTALL_DIR ];then
+            echo "clean old dir: $INSTALL_DIR"
+            rm -rf $INSTALL_DIR
+        fi
+    else
+        echo "strip remove old build"
+    fi
+}

sdk/c-opr-loaders/mace/README.md

@@ -4,7 +4,7 @@
 
 ```bash
 cd $MEGENGINE_HOME
-./scripts/cmake-build/cross_build_android_arm_inference.sh -a arm64-v8a
+./scripts/cmake-build/cross_build_android_arm_inference.sh -a arm64-v8a -r
 ```
 
 After successfully built, load_and_run should be in `$MEGENGINE_HOME/build_dir/android/arm64-v8a/Release/install/bin`

src/serialization/impl/opr_registry.cpp

@@ -159,9 +159,10 @@ void OprRegistry::add_using_dynamic_loader(
 }
 
 #if MGB_ENABLE_DEBUG_UTIL
-std::vector<std::pair<uint64_t, std::string>> OprRegistry::dump_registries() {
-    auto &&id2reg = static_data().id2reg;
-    std::vector<std::pair<uint64_t, std::string>> result;
+std::vector<std::pair<unsigned long int, std::string>>
+OprRegistry::dump_registries() {
+    auto&& id2reg = static_data().id2reg;
+    std::vector<std::pair<unsigned long int, std::string>> result;
     for (auto iter = id2reg.begin(); iter != id2reg.end(); ++iter) {
         if (iter->second.name.size() == 0)
             result.push_back({iter->first, "<special>"});

src/serialization/include/megbrain/serialization/opr_registry.h

@@ -76,7 +76,8 @@ namespace serialization {
 
 #if MGB_ENABLE_DEBUG_UTIL
         //! dump registered oprs
-        static std::vector<std::pair<uint64_t, std::string>> dump_registries();
+        static std::vector<std::pair<unsigned long int, std::string>>
+        dump_registries();
 #endif
     };