!9555 [MD]fix the minddata lite build, keep the third package compiled in the same way

From: @xulei2020
Reviewed-by: 
Signed-off-by:

build.sh

@@ -596,95 +596,6 @@ build_opencl() {
     fi
 }
 
-build_opencv() {
-    # check what platform we are building opencv on
-    cd ${BASEPATH}
-    if [[ "${LITE_PLATFORM}" == "x86_64" ]]; then
-        OPENCV_BIN="${BASEPATH}"/third_party/opencv/build/lib/libopencv_core.so.4.2.0
-    elif [[ "${LITE_PLATFORM}" == "arm32" ]]; then
-        OPENCV_BIN="${BASEPATH}"/third_party/opencv/build/lib/armeabi-v7a/libopencv_core.so
-    else
-        OPENCV_BIN="${BASEPATH}"/third_party/opencv/build/lib/arm64-v8a/libopencv_core.so
-
-    fi
-    if [[ ! -f "${OPENCV_BIN}" ]]; then
-        if [[ "${MSLIBS_SERVER}" ]]; then
-	    cd "${BASEPATH}"/third_party/
-	    rm -rf 4.2.0.tar.gz ./opencv
-	    wget http://${MSLIBS_SERVER}:8081/libs/opencv/4.2.0.tar.gz
-            tar -zxvf ./4.2.0.tar.gz
-	    mv ./opencv-4.2.0 ./opencv
-            rm -rf 4.2.0.tar.gz
-	else
-            git submodule update --init --recursive third_party/opencv
-        fi
-        cd ${BASEPATH}/third_party/opencv
-        rm -rf build && mkdir -p build && cd build && cmake ${CMAKE_MINDDATA_ARGS} -DBUILD_SHARED_LIBS=ON -DBUILD_ANDROID_PROJECTS=OFF \
-          -DBUILD_LIST=core,imgcodecs,imgproc -DBUILD_ZLIB=ON .. && make -j$THREAD_NUM
-    fi
-}
-
-build_jpeg_turbo() {
-    if [ -d  "${BASEPATH}"/third_party/libjpeg-turbo/lib ];then
-        rm -rf "${BASEPATH}"/third_party/libjpeg-turbo/lib
-    fi
-    cd ${BASEPATH}
-    if [[ "${LITE_PLATFORM}" == "x86_64" ]]; then
-        JPEG_TURBO="${BASEPATH}"/third_party/libjpeg-turbo/lib/libjpeg.so.62.3.0
-    else
-        JPEG_TURBO="${BASEPATH}"/third_party/libjpeg-turbo/lib/libjpeg.so
-    fi
-
-    if [[ ! -f "${JPEG_TURBO}" ]]; then
-        if [[ "${MSLIBS_SERVER}" ]]; then
-            cd "${BASEPATH}"/third_party/
-	    rm -rf 2.0.4.tar.gz ./libjpeg-turbo
-	    wget http://${MSLIBS_SERVER}:8081/libs/jpeg_turbo/2.0.4.tar.gz
-	    tar -zxvf ./2.0.4.tar.gz
-	    mv ./libjpeg-turbo-2.0.4 ./libjpeg-turbo
-	    rm -rf ./2.0.4.tar.gz
-        else
-            git submodule update --init --recursive third_party/libjpeg-turbo
-        fi
-
-        cd ${BASEPATH}/third_party/libjpeg-turbo
-        rm -rf build && mkdir -p build && cd build && cmake ${CMAKE_MINDDATA_ARGS} -DCMAKE_BUILD_TYPE=Release \
-          -DCMAKE_INSTALL_PREFIX="${BASEPATH}/third_party/libjpeg-turbo" .. && make -j$THREAD_NUM && make install
-    fi
-}
-
-build_eigen() {
-    cd ${BASEPATH}
-    if [[ "${MSLIBS_SERVER}" ]]; then
-        cd "${BASEPATH}"/third_party/
-	rm -rf ./eigen-3.*.tar.gz ./eigen
-        wget http://${MSLIBS_SERVER}:8081/libs/eigen3/eigen-3.3.7.tar.gz
-        tar -zxvf ./eigen-3.3.7.tar.gz
-        mv ./eigen-3.3.7 ./eigen
-	rm -rf ./eigen-3.*.tar.gz
-    else
-        git submodule update --init --recursive third_party/eigen
-
-    fi
-}
-
-build_minddata_lite_deps()
-{
-  echo "start build minddata lite project"
-  if [[ "${LITE_PLATFORM}" == "arm64" ]]; then
-      CMAKE_MINDDATA_ARGS="-DCMAKE_TOOLCHAIN_FILE=${ANDROID_NDK}/build/cmake/android.toolchain.cmake -DANDROID_NATIVE_API_LEVEL=19    \
-            -DANDROID_NDK=${ANDROID_NDK} -DANDROID_ABI=arm64-v8a -DANDROID_TOOLCHAIN_NAME=aarch64-linux-android-clang                 \
-            -DANDROID_STL=c++_shared -DCMAKE_BUILD_TYPE=${BUILD_TYPE}"
-  elif [[ "${LITE_PLATFORM}" == "arm32" ]]; then
-      CMAKE_MINDDATA_ARGS="-DCMAKE_TOOLCHAIN_FILE=${ANDROID_NDK}/build/cmake/android.toolchain.cmake -DANDROID_NATIVE_API_LEVEL=19    \
-            -DANDROID_NDK=${ANDROID_NDK} -DANDROID_ABI=armeabi-v7a -DANDROID_TOOLCHAIN_NAME=clang                                     \
-            -DANDROID_STL=c++_shared -DCMAKE_BUILD_TYPE=${BUILD_TYPE}"
-  else
-      CMAKE_MINDDATA_ARGS="-DCMAKE_BUILD_TYPE=${BUILD_TYPE}"
-  fi
-  build_eigen
-  build_jpeg_turbo
-}
 
 get_version() {
     VERSION_MAJOR=$(grep "const int ms_version_major =" ${BASEPATH}/mindspore/lite/include/version.h | tr -dc "[0-9]")
@@ -705,10 +616,6 @@ build_lite()
         build_gtest
     fi
 
-    if [[ "${COMPILE_MINDDATA_LITE}" == "lite" || "${COMPILE_MINDDATA_LITE}" == "full" ||  "${COMPILE_MINDDATA_LITE}" == "wrapper" ]]; then
-        build_minddata_lite_deps
-    fi
-
     cd "${BASEPATH}/mindspore/lite"
     if [[ "${INC_BUILD}" == "off" ]]; then
         rm -rf build

cmake/external_libs/jpeg_turbo.cmake

@@ -1,4 +1,3 @@
-set(jpeg_turbo_USE_STATIC_LIBS ON)
 
 if (ENABLE_GITEE)
     set(REQ_URL "https://gitee.com/mirrors/libjpeg-turbo/repository/archive/2.0.4.tar.gz")
@@ -15,13 +14,40 @@ else()
 endif()
 
 set(jpeg_turbo_LDFLAGS "-Wl,-z,relro,-z,now,-z,noexecstack")
+
+
+set(jpeg_turbo_USE_STATIC_LIBS ON)
+set(JPEG_TURBO_PATCHE ${CMAKE_SOURCE_DIR}/third_party/patch/jpeg_turbo/jpeg_turbo.patch001)
+set(CMAKE_OPTION -DCMAKE_BUILD_TYPE=Release -DCMAKE_SKIP_RPATH=TRUE -DWITH_SIMD=ON)
+if (BUILD_LITE)
+    set(jpeg_turbo_USE_STATIC_LIBS OFF)
+    set(JPEG_TURBO_PATCHE ${TOP_DIR}/third_party/patch/jpeg_turbo/jpeg_turbo.patch001)
+    if (PLATFORM_ARM64)
+        set(CMAKE_OPTION  -DCMAKE_TOOLCHAIN_FILE=$ENV{ANDROID_NDK}/build/cmake/android.toolchain.cmake
+                          -DANDROID_NATIVE_API_LEVEL=19
+                          -DANDROID_NDK=$ENV{ANDROID_NDK}
+                          -DANDROID_ABI=arm64-v8a
+                          -DANDROID_TOOLCHAIN_NAME=aarch64-linux-android-clang
+                          -DANDROID_STL=c++_shared -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE})
+    endif()
+    if (PLATFORM_ARM32)
+        set(CMAKE_OPTION  -DCMAKE_TOOLCHAIN_FILE=$ENV{ANDROID_NDK}/build/cmake/android.toolchain.cmake
+                          -DANDROID_NATIVE_API_LEVEL=19
+                          -DANDROID_NDK=$ENV{ANDROID_NDK}
+                          -DANDROID_ABI=armeabi-v7a
+                          -DANDROID_TOOLCHAIN_NAME=aarch64-linux-android-clang
+                          -DANDROID_STL=c++_shared -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE})
+    endif()
+
+endif ()
+
 mindspore_add_pkg(jpeg_turbo
         VER 2.0.4
         LIBS jpeg turbojpeg
         URL ${REQ_URL}
         MD5 ${MD5}
-        CMAKE_OPTION -DCMAKE_BUILD_TYPE=Release -DCMAKE_SKIP_RPATH=TRUE -DWITH_SIMD=ON
-        PATCHES ${CMAKE_SOURCE_DIR}/third_party/patch/jpeg_turbo/jpeg_turbo.patch001
+        CMAKE_OPTION ${CMAKE_OPTION}
+        PATCHES ${JPEG_TURBO_PATCHE}
         )
 include_directories(${jpeg_turbo_INC})
 add_library(mindspore::jpeg_turbo ALIAS jpeg_turbo::jpeg)

cmake/package_lite.cmake

@@ -22,18 +22,16 @@ set(FLATBF_DIR_RUN_X86 ${MAIN_DIR}-${RUN_X86_COMPONENT_NAME}/third_party/flatbuf
 
 if (BUILD_MINDDATA STREQUAL "full" OR BUILD_MINDDATA STREQUAL "wrapper")
     install(DIRECTORY ${TOP_DIR}/mindspore/ccsrc/minddata/dataset/include/ DESTINATION ${MIND_DATA_INC_DIR} COMPONENT ${COMPONENT_NAME} FILES_MATCHING PATTERN "*.h" PATTERN "vision.h" EXCLUDE)
+    file(GLOB JPEGTURBO_LIB_LIST ${jpeg_turbo_LIBPATH}/*.so)
     if (PLATFORM_ARM64)
         install(FILES ${TOP_DIR}/mindspore/lite/build/minddata/libminddata-lite.so DESTINATION ${MIND_DATA_LIB_DIR} COMPONENT ${COMPONENT_NAME})
-        install(FILES ${TOP_DIR}/third_party/libjpeg-turbo/lib/libjpeg.so DESTINATION ${TURBO_DIR}/lib COMPONENT ${COMPONENT_NAME})
-        install(FILES ${TOP_DIR}/third_party/libjpeg-turbo/lib/libturbojpeg.so DESTINATION ${TURBO_DIR}/lib COMPONENT ${COMPONENT_NAME})
+        install(FILES ${JPEGTURBO_LIB_LIST} DESTINATION ${TURBO_DIR}/lib COMPONENT ${COMPONENT_NAME})
     elseif (PLATFORM_ARM32)
         install(FILES ${TOP_DIR}/mindspore/lite/build/minddata/libminddata-lite.so DESTINATION ${MIND_DATA_LIB_DIR} COMPONENT ${COMPONENT_NAME})
-        install(FILES ${TOP_DIR}/third_party/libjpeg-turbo/lib/libjpeg.so DESTINATION ${TURBO_DIR}/lib COMPONENT ${COMPONENT_NAME})
-        install(FILES ${TOP_DIR}/third_party/libjpeg-turbo/lib/libturbojpeg.so DESTINATION ${TURBO_DIR}/lib COMPONENT ${COMPONENT_NAME})
+        install(FILES ${JPEGTURBO_LIB_LIST} DESTINATION ${TURBO_DIR}/lib COMPONENT ${COMPONENT_NAME})
     else ()
         install(FILES ${TOP_DIR}/mindspore/lite/build/minddata/libminddata-lite.so DESTINATION ${MIND_DATA_LIB_DIR_RUN_X86} COMPONENT ${RUN_X86_COMPONENT_NAME})
-        install(FILES ${TOP_DIR}/third_party/libjpeg-turbo/lib/libjpeg.so.62.3.0 DESTINATION ${TURBO_DIR_RUN_X86}/lib RENAME libjpeg.so.62 COMPONENT ${RUN_X86_COMPONENT_NAME})
-        install(FILES ${TOP_DIR}/third_party/libjpeg-turbo/lib/libturbojpeg.so.0.2.0 DESTINATION ${TURBO_DIR_RUN_X86}/lib RENAME libturbojpeg.so.0 COMPONENT ${RUN_X86_COMPONENT_NAME})
+        install(FILES ${JPEGTURBO_LIB_LIST} DESTINATION ${TURBO_DIR_RUN_X86}/lib COMPONENT ${RUN_X86_COMPONENT_NAME})
     endif ()
 endif ()
 

cmake/utils.cmake

@@ -134,9 +134,6 @@ function(__find_pkg_then_add_target pkg_name pkg_exe lib_path)
         endif ()
         set(${_LIB_NAME}_LIB ${_LIB_NAME}_LIB-NOTFOUND)
         find_library(${_LIB_NAME}_LIB ${_LIB_SEARCH_NAME} PATHS ${${pkg_name}_BASE_DIR}/${lib_path} NO_DEFAULT_PATH)
-        if (NOT ${_LIB_NAME}_LIB AND BUILD_LITE AND PLATFORM_ARM)
-            set(${_LIB_NAME}_LIB "${${pkg_name}_BASE_DIR}/${lib_path}/lib${_LIB_SEARCH_NAME}.so")
-        endif(NOT ${_LIB_NAME}_LIB AND BUILD_LITE AND PLATFORM_ARM)
         if(NOT ${_LIB_NAME}_LIB)
             return()
         endif()

mindspore/lite/CMakeLists.txt

@@ -33,8 +33,14 @@ set(BUILD_LITE "on")
 set(PLATFORM_ARM "off")
 if (PLATFORM_ARM64 OR PLATFORM_ARM32)
     set(PLATFORM_ARM "on")
+    #set for cross-compiling toolchain
+    set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY BOTH)
+    set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE BOTH)
+    set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE BOTH)
 endif()
 
+
+
 if (SUPPORT_GPU)
     set(PROCESS_UNIT gpu)
 else ()
@@ -93,6 +99,10 @@ include(${TOP_DIR}/cmake/dependency_utils.cmake)
 include(${TOP_DIR}/cmake/dependency_securec.cmake)
 include(${TOP_DIR}/cmake/external_libs/flatbuffers.cmake)
 
+if (ENABLE_CONVERTER OR BUILD_MINDDATA STREQUAL "full")
+include(${TOP_DIR}/cmake/external_libs/json.cmake)
+endif()
+
 file(GLOB FBS_FILES ${CMAKE_CURRENT_SOURCE_DIR}/schema/*.fbs)
 ms_build_flatbuffers_lite(FBS_FILES ${CMAKE_CURRENT_SOURCE_DIR}/schema/ fbs_src ${CMAKE_BINARY_DIR}/schema "")
 ms_build_flatbuffers_lite(FBS_FILES ${CMAKE_CURRENT_SOURCE_DIR}/schema/ fbs_inner_src ${CMAKE_BINARY_DIR}/schema/inner "inner")
@@ -147,7 +157,6 @@ if (ENABLE_CONVERTER)
         MESSAGE(FATAL_ERROR "Cannot build converter in arm platform")
     endif()
     include_directories(${PYTHON_INCLUDE_DIRS})
-    include(${TOP_DIR}/cmake/external_libs/json.cmake)
     include(${TOP_DIR}/cmake/external_libs/eigen.cmake)
     include(${TOP_DIR}/cmake/external_libs/protobuf.cmake)
     add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/tools/converter)
@@ -183,25 +192,6 @@ if (NOT PLATFORM_ARM32 AND NOT PLATFORM_ARM64)
 endif ()
 
 if (BUILD_MINDDATA STREQUAL "lite" OR BUILD_MINDDATA STREQUAL "full" OR BUILD_MINDDATA STREQUAL "wrapper")
-    # add sentencepiece dependency
-    # include(${TOP_DIR}/cmake/external_libs/sentencepiece.cmake)
-    # json
-    if (NOT ENABLE_CONVERTER)
-        include(${TOP_DIR}/cmake/external_libs/json.cmake)
-    endif ()
-    # eigen
-    include_directories(${TOP_DIR}/third_party/eigen/)
-    # jpeg-turbo
-    add_library(jpeg-turbo SHARED IMPORTED)
-    set_target_properties(jpeg-turbo PROPERTIES
-        IMPORTED_LOCATION ${TOP_DIR}/third_party/libjpeg-turbo/lib/libturbojpeg.so
-    )
-    add_library(jpeg SHARED IMPORTED)
-    set_target_properties(jpeg PROPERTIES
-        IMPORTED_LOCATION ${TOP_DIR}/third_party/libjpeg-turbo/lib/libjpeg.so
-    )
-    include_directories(${TOP_DIR}/third_party/libjpeg-turbo/include)
-
     add_compile_definitions(ENABLE_ANDROID)
     add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/minddata)
 endif ()

mindspore/lite/java/java/app/src/main/native/CMakeLists.txt

@@ -11,6 +11,11 @@ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DMS_VERSION_MAJOR=${MS_VERSION_MAJOR} -DMS_
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DMS_VERSION_MAJOR=${MS_VERSION_MAJOR} -DMS_VERSION_MINOR=${MS_VERSION_MINOR} -DMS_VERSION_REVISION=${MS_VERSION_REVISION}")
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17")
 
+#set for cross-compiling toolchain
+set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY BOTH)
+set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE BOTH)
+set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE BOTH)
+
 if (ENABLE_VERBOSE)
     set(CMAKE_VERBOSE_MAKEFILE on)
 endif ()

mindspore/lite/minddata/CMakeLists.txt

@@ -1,9 +1,18 @@
+find_package(Patch)
+if (NOT Patch_FOUND)
+  message(FATAL_ERROR "Patch not found, please set environment variable MS_PATCH_PATH to path where Patch is located, "
+  "usually found in GIT_PATH/usr/bin on Windows")
+endif ()
+
+include(${TOP_DIR}/cmake/external_libs/jpeg_turbo.cmake)
+
 set(MINDDATA_DIR ${CCSRC_DIR}/minddata/dataset)
 set(CMAKE_CXX_STANDARD 17)
 
 set(CMAKE_CXX_FLAGS_DEBUG "$ENV{CXXFLAGS} -O0 -g2 -ggdb -fno-inline-functions -fno-omit-frame-pointer -D_LIBCPP_INLINE_VISIBILITY='' -D_LIBCPP_DISABLE_EXTERN_TEMPLATE=1 -DHALF_ENABLE_CPP11_USER_LITERALS=0 -D_FORTIFY_SOURCE=2 -Wno-cpp")
 set(CMAKE_CXX_FLAGS_DEBUG "$ENV{CXXFLAGS} -Werror -Wno-return-std-move -Wno-unused-private-field -Wno-unused-lambda-capture -Wno-sign-compare -Wno-overloaded-virtual -Wno-unneeded-internal-declaration -Wno-unused-variable -Wno-pessimizing-move -Wno-inconsistent-missing-override")
 
+
 set(CMAKE_CXX_FLAGS "$ENV{CXXFLAGS} -I/usr/local/include -std=c++17 -Wall -fPIC")
 
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OPTION_CXX_FLAGS}")
@@ -281,8 +290,8 @@ if (BUILD_MINDDATA STREQUAL "full")
     find_package(Threads REQUIRED)
     target_link_libraries(minddata-lite
             securec
-            jpeg-turbo
-            jpeg
+            mindspore::jpeg_turbo
+            mindspore::turbojpeg
             mindspore::json
             Threads::Threads
             )
@@ -349,8 +358,8 @@ elseif (BUILD_MINDDATA STREQUAL "wrapper")
     find_package(Threads REQUIRED)
     target_link_libraries(minddata-lite
             securec
-            jpeg-turbo
-            jpeg
+            mindspore::jpeg_turbo
+            mindspore::turbojpeg
             mindspore::json
             Threads::Threads
             )
@@ -428,7 +437,7 @@ elseif (BUILD_MINDDATA STREQUAL "lite")
     add_library(minddata-lite SHARED
         ${MINDDATA_CORE_SRC_FILES}
         ${MINDDATA_KERNELS_SRC_FILES}
-	${MINDDATA_KERNELS_IMAGE_LITE_CV_FILES}
+	    ${MINDDATA_KERNELS_IMAGE_LITE_CV_FILES}
         ${MINDDATA_KERNELS_IMAGE_SRC_FILES}
         ${MINDDATA_KERNELS_DATA_SRC_FILES}
         ${MINDDATA_DIR}/util/status.cc

third_party/eigen

@@ -1 +0,0 @@
-Subproject commit daf9bbeca26e98da2eed0058835cbb04e0a30ad8

third_party/libjpeg-turbo

@@ -1 +0,0 @@
-Subproject commit b443c541b9a6fdcac214f9f003de0aa13e480ac1

third_party/opencv

@@ -1 +0,0 @@
-Subproject commit bda89a6469aa79ecd8713967916bd754bff1d931