| @@ -12,6 +12,7 @@ else() | |||||
| endif() | endif() | ||||
| set(CMAKE_CXX_FLAGS_DEBUG "$ENV{CXXFLAGS} -O0 -g2 -ggdb -fno-inline-functions -fno-omit-frame-pointer -Wl,--allow-shlib-undefined -D_LIBCPP_INLINE_VISIBILITY='' -D'_LIBCPP_EXTERN_TEMPLATE(...)=' -DHALF_ENABLE_CPP11_USER_LITERALS=0 -D_FORTIFY_SOURCE=2 -Wno-cpp") | set(CMAKE_CXX_FLAGS_DEBUG "$ENV{CXXFLAGS} -O0 -g2 -ggdb -fno-inline-functions -fno-omit-frame-pointer -Wl,--allow-shlib-undefined -D_LIBCPP_INLINE_VISIBILITY='' -D'_LIBCPP_EXTERN_TEMPLATE(...)=' -DHALF_ENABLE_CPP11_USER_LITERALS=0 -D_FORTIFY_SOURCE=2 -Wno-cpp") | ||||
| set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -I/usr/local/include -std=c++17 -Werror -Wall -Wno-deprecated-declarations -fPIC") | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -I/usr/local/include -std=c++17 -Werror -Wall -Wno-deprecated-declarations -fPIC") | ||||
| set(CMAKE_EXPORT_COMPILE_COMMANDS ON) | set(CMAKE_EXPORT_COMPILE_COMMANDS ON) | ||||
| @@ -29,6 +30,18 @@ find_package(Python3 3.7 COMPONENTS Interpreter Development) | |||||
| if(Python3_FOUND) | if(Python3_FOUND) | ||||
| set(PYTHON_INCLUDE_DIRS "${Python3_INCLUDE_DIRS}") | set(PYTHON_INCLUDE_DIRS "${Python3_INCLUDE_DIRS}") | ||||
| set(PYTHON_LIBRARIES "${Python3_LIBRARIES}") | set(PYTHON_LIBRARIES "${Python3_LIBRARIES}") | ||||
| if (WIN32) | |||||
| if (Python3_DIR) | |||||
| message("Python3_DIR set already: " ${Python3_DIR}) | |||||
| else() | |||||
| string(LENGTH ${PYTHON_LIBRARIES} PYTHON_LIBRARIES_LEN) | |||||
| string(LENGTH "libpythonxx.a" Python3_NAME_LEN) | |||||
| math(EXPR Python3_DIR_LEN ${PYTHON_LIBRARIES_LEN}-${Python3_NAME_LEN}) | |||||
| string(SUBSTRING ${Python3_LIBRARIES} 0 ${Python3_DIR_LEN} Python3_DIR) | |||||
| message("Python3_DIR: " ${Python3_DIR}) | |||||
| endif() | |||||
| link_directories(${Python3_DIR}) | |||||
| endif() | |||||
| else() | else() | ||||
| find_python_package(py_inc py_lib) | find_python_package(py_inc py_lib) | ||||
| set(PYTHON_INCLUDE_DIRS "${py_inc}") | set(PYTHON_INCLUDE_DIRS "${py_inc}") | ||||
| @@ -0,0 +1,36 @@ | |||||
| @echo off | |||||
| @title mindspore_build | |||||
| SET BASEPATH=%CD% | |||||
| IF NOT EXIST %BASEPATH%/build ( | |||||
| md "build" | |||||
| ) | |||||
| cd %BASEPATH%/build | |||||
| SET BUILD_PATH=%CD% | |||||
| IF NOT EXIST %BUILD_PATH%/mindspore ( | |||||
| md "mindspore" | |||||
| ) | |||||
| cd %CD%/mindspore | |||||
| cmake -DCMAKE_BUILD_TYPE=Release -DENABLE_CPU=ON -DENABLE_MINDDATA=ON -DUSE_GLOG=ON -G "CodeBlocks - MinGW Makefiles" ../.. | |||||
| IF NOT %errorlevel% == 0 ( | |||||
| goto run_fail | |||||
| ) | |||||
| cmake --build . --target all -- -j6 | |||||
| IF NOT %errorlevel% == 0 ( | |||||
| goto run_fail | |||||
| ) | |||||
| cd %BASEPATH% | |||||
| goto run_eof | |||||
| :run_fail | |||||
| cd %BASEPATH% | |||||
| echo "build fail." | |||||
| :run_eof | |||||
| @@ -1,5 +1,8 @@ | |||||
| set(flatbuffers_CXXFLAGS "-D_FORTIFY_SOURCE=2 -O2") | set(flatbuffers_CXXFLAGS "-D_FORTIFY_SOURCE=2 -O2") | ||||
| set(flatbuffers_CFLAGS "-D_FORTIFY_SOURCE=2 -O2") | set(flatbuffers_CFLAGS "-D_FORTIFY_SOURCE=2 -O2") | ||||
| if (WIN32) | |||||
| set(flatbuffers_USE_STATIC_LIBS ON) | |||||
| endif() | |||||
| mindspore_add_pkg(flatbuffers | mindspore_add_pkg(flatbuffers | ||||
| VER 1.11.0 | VER 1.11.0 | ||||
| LIBS flatbuffers | LIBS flatbuffers | ||||
| @@ -1,11 +1,22 @@ | |||||
| set(onednn_CXXFLAGS "-D_FORTIFY_SOURCE=2 -O2") | set(onednn_CXXFLAGS "-D_FORTIFY_SOURCE=2 -O2") | ||||
| set(onednn_CFLAGS "-D_FORTIFY_SOURCE=2 -O2") | set(onednn_CFLAGS "-D_FORTIFY_SOURCE=2 -O2") | ||||
| mindspore_add_pkg(onednn | |||||
| if (CMAKE_SYSTEM_NAME MATCHES "Windows") | |||||
| mindspore_add_pkg(onednn | |||||
| VER 1.1.1 | |||||
| LIBS dnnl mkldnn | |||||
| HEAD_ONLY ./ | |||||
| RELEASE on | |||||
| URL https://github.com/oneapi-src/oneDNN/releases/download/v1.1.1/dnnl_win_1.1.1_cpu_vcomp.zip | |||||
| MD5 ecaab9ed549643067699c80e5cea1c23) | |||||
| else() | |||||
| mindspore_add_pkg(onednn | |||||
| VER 1.1.2 | VER 1.1.2 | ||||
| LIBS dnnl mkldnn | LIBS dnnl mkldnn | ||||
| URL https://github.com/oneapi-src/oneDNN/archive/v1.1.2.tar.gz | URL https://github.com/oneapi-src/oneDNN/archive/v1.1.2.tar.gz | ||||
| MD5 ab40d52230f3ad1d7a6f06ce0f6bc17a | MD5 ab40d52230f3ad1d7a6f06ce0f6bc17a | ||||
| CMAKE_OPTION -DDNNL_ARCH_OPT_FLAGS='' -DDNNL_CPU_RUNTIME='SEQ' -DDNNL_BUILD_EXAMPLES=OFF -DDNNL_BUILD_TESTS=OFF) | CMAKE_OPTION -DDNNL_ARCH_OPT_FLAGS='' -DDNNL_CPU_RUNTIME='SEQ' -DDNNL_BUILD_EXAMPLES=OFF -DDNNL_BUILD_TESTS=OFF) | ||||
| endif() | |||||
| include_directories(${onednn_INC}) | include_directories(${onednn_INC}) | ||||
| add_library(mindspore::dnnl ALIAS onednn::dnnl) | add_library(mindspore::dnnl ALIAS onednn::dnnl) | ||||
| add_library(mindspore::mkldnn ALIAS onednn::mkldnn) | add_library(mindspore::mkldnn ALIAS onednn::mkldnn) | ||||
| @@ -2,35 +2,72 @@ if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") | |||||
| set(opencv_CXXFLAGS "-fstack-protector-all -Wno-uninitialized -Wno-unused-parameter -D_FORTIFY_SOURCE=2 -O2") | set(opencv_CXXFLAGS "-fstack-protector-all -Wno-uninitialized -Wno-unused-parameter -D_FORTIFY_SOURCE=2 -O2") | ||||
| set(opencv_CFLAGS "-fstack-protector-all -Wno-uninitialized -Wno-unused-parameter -D_FORTIFY_SOURCE=2 -O2") | set(opencv_CFLAGS "-fstack-protector-all -Wno-uninitialized -Wno-unused-parameter -D_FORTIFY_SOURCE=2 -O2") | ||||
| set(opencv_LDFLAGS "-Wl") | set(opencv_LDFLAGS "-Wl") | ||||
| elseif (${CMAKE_SYSTEM_NAME} MATCHES "Windows") | |||||
| set(opencv_CXXFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -D_FORTIFY_SOURCE=2 -O2") | |||||
| set(opencv_CFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -D_FORTIFY_SOURCE=2 -O2") | |||||
| else() | else() | ||||
| set(opencv_CXXFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -D_FORTIFY_SOURCE=2 -O2") | set(opencv_CXXFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -D_FORTIFY_SOURCE=2 -O2") | ||||
| set(opencv_CFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -D_FORTIFY_SOURCE=2 -O2") | set(opencv_CFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -D_FORTIFY_SOURCE=2 -O2") | ||||
| set(opencv_LDFLAGS "-Wl,-z,relro,-z,now,-z,noexecstack") | set(opencv_LDFLAGS "-Wl,-z,relro,-z,now,-z,noexecstack") | ||||
| endif() | endif() | ||||
| mindspore_add_pkg(opencv | |||||
| VER 4.2.0 | |||||
| LIBS opencv_core opencv_imgcodecs opencv_imgproc | |||||
| URL https://github.com/opencv/opencv/archive/4.2.0.tar.gz | |||||
| MD5 e8cb208ce2723481408b604b480183b6 | |||||
| CMAKE_OPTION -DCMAKE_BUILD_TYPE=Release -DWITH_PROTOBUF=OFF -DWITH_WEBP=OFF -DWITH_IPP=OFF -DWITH_ADE=OFF | |||||
| -DBUILD_ZLIB=ON | |||||
| -DBUILD_JPEG=ON | |||||
| -DBUILD_PNG=ON | |||||
| -DBUILD_OPENEXR=ON | |||||
| -DBUILD_TESTS=OFF | |||||
| -DBUILD_PERF_TESTS=OFF | |||||
| -DBUILD_opencv_apps=OFF | |||||
| -DCMAKE_SKIP_RPATH=TRUE | |||||
| -DBUILD_opencv_python3=OFF | |||||
| -DWITH_FFMPEG=OFF | |||||
| -DWITH_TIFF=ON | |||||
| -DBUILD_TIFF=OFF | |||||
| -DWITH_JASPER=OFF | |||||
| -DBUILD_JASPER=OFF | |||||
| -DTIFF_INCLUDE_DIR=${tiff_INC} | |||||
| -DTIFF_LIBRARY=${tiff_LIB}) | |||||
| include_directories(${opencv_INC}/opencv4) | |||||
| add_library(mindspore::opencv_core ALIAS opencv::opencv_core) | |||||
| add_library(mindspore::opencv_imgcodecs ALIAS opencv::opencv_imgcodecs) | |||||
| add_library(mindspore::opencv_imgproc ALIAS opencv::opencv_imgproc) | |||||
| if (WIN32) | |||||
| mindspore_add_pkg(opencv | |||||
| VER 4.2.0 | |||||
| LIBS libopencv_core420.dll.a libopencv_imgcodecs420.dll.a libopencv_imgproc420.dll.a | |||||
| LIB_PATH x64/mingw/lib | |||||
| URL https://github.com/opencv/opencv/archive/4.2.0.tar.gz | |||||
| MD5 e8cb208ce2723481408b604b480183b6 | |||||
| CMAKE_OPTION -DCMAKE_BUILD_TYPE=Release -DWITH_PROTOBUF=OFF -DWITH_WEBP=OFF -DWITH_IPP=OFF -DWITH_ADE=OFF | |||||
| -DBUILD_ZLIB=ON | |||||
| -DBUILD_JPEG=ON | |||||
| -DBUILD_PNG=ON | |||||
| -DBUILD_OPENEXR=ON | |||||
| -DBUILD_TESTS=OFF | |||||
| -DBUILD_PERF_TESTS=OFF | |||||
| -DBUILD_opencv_apps=OFF | |||||
| -DCMAKE_SKIP_RPATH=TRUE | |||||
| -DBUILD_opencv_python3=OFF | |||||
| -DWITH_FFMPEG=OFF | |||||
| -DWITH_TIFF=ON | |||||
| -DBUILD_TIFF=OFF | |||||
| -DWITH_JASPER=OFF | |||||
| -DBUILD_JASPER=OFF | |||||
| -DTIFF_INCLUDE_DIR=${tiff_INC} | |||||
| -DTIFF_LIBRARY=${tiff_LIB}) | |||||
| else() | |||||
| mindspore_add_pkg(opencv | |||||
| VER 4.2.0 | |||||
| LIBS opencv_core opencv_imgcodecs opencv_imgproc | |||||
| URL https://github.com/opencv/opencv/archive/4.2.0.tar.gz | |||||
| MD5 e8cb208ce2723481408b604b480183b6 | |||||
| CMAKE_OPTION -DCMAKE_BUILD_TYPE=Release -DWITH_PROTOBUF=OFF -DWITH_WEBP=OFF -DWITH_IPP=OFF -DWITH_ADE=OFF | |||||
| -DBUILD_ZLIB=ON | |||||
| -DBUILD_JPEG=ON | |||||
| -DBUILD_PNG=ON | |||||
| -DBUILD_OPENEXR=ON | |||||
| -DBUILD_TESTS=OFF | |||||
| -DBUILD_PERF_TESTS=OFF | |||||
| -DBUILD_opencv_apps=OFF | |||||
| -DCMAKE_SKIP_RPATH=TRUE | |||||
| -DBUILD_opencv_python3=OFF | |||||
| -DWITH_FFMPEG=OFF | |||||
| -DWITH_TIFF=ON | |||||
| -DBUILD_TIFF=OFF | |||||
| -DWITH_JASPER=OFF | |||||
| -DBUILD_JASPER=OFF | |||||
| -DTIFF_INCLUDE_DIR=${tiff_INC} | |||||
| -DTIFF_LIBRARY=${tiff_LIB}) | |||||
| endif() | |||||
| if (WIN32) | |||||
| include_directories(${opencv_INC}) | |||||
| add_library(mindspore::opencv_core ALIAS opencv::libopencv_core420.dll.a) | |||||
| add_library(mindspore::opencv_imgcodecs ALIAS opencv::libopencv_imgcodecs420.dll.a) | |||||
| add_library(mindspore::opencv_imgproc ALIAS opencv::libopencv_imgproc420.dll.a) | |||||
| else() | |||||
| include_directories(${opencv_INC}/opencv4) | |||||
| add_library(mindspore::opencv_core ALIAS opencv::opencv_core) | |||||
| add_library(mindspore::opencv_imgcodecs ALIAS opencv::opencv_imgcodecs) | |||||
| add_library(mindspore::opencv_imgproc ALIAS opencv::opencv_imgproc) | |||||
| endif() | |||||
| @@ -77,22 +77,36 @@ function(ms_protobuf_generate_py c_var h_var py_var) | |||||
| list(APPEND ${c_var} "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}.pb.cc") | list(APPEND ${c_var} "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}.pb.cc") | ||||
| list(APPEND ${h_var} "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}.pb.h") | list(APPEND ${h_var} "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}.pb.h") | ||||
| list(APPEND ${py_var} "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}_pb2.py") | list(APPEND ${py_var} "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}_pb2.py") | ||||
| add_custom_command( | |||||
| OUTPUT "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}.pb.cc" | |||||
| "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}.pb.h" | |||||
| "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}_pb2.py" | |||||
| WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} | |||||
| COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_BINARY_DIR}/${rel_path}" | |||||
| COMMAND protobuf::protoc -I${file_dir} --cpp_out=${CMAKE_BINARY_DIR}/${rel_path} ${abs_file} | |||||
| COMMAND protobuf::protoc -I${file_dir} --python_out=${CMAKE_BINARY_DIR}/${rel_path} ${abs_file} | |||||
| COMMAND protobuf::protoc -I${file_dir} --python_out=${CMAKE_BINARY_DIR}/${rel_path} ${abs_file} | |||||
| COMMAND perl -pi -e "s/import (.+_pb2.*)/from . import \\1/" "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}_pb2.py" | |||||
| COMMAND cp "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}_pb2.py" "${PROJECT_SOURCE_DIR}/mindspore/train/" | |||||
| DEPENDS protobuf::protoc ${abs_file} | |||||
| COMMENT "Running C++ protocol buffer compiler on ${file}" VERBATIM ) | |||||
| if (WIN32) | |||||
| add_custom_command( | |||||
| OUTPUT "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}.pb.cc" | |||||
| "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}.pb.h" | |||||
| "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}_pb2.py" | |||||
| WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} | |||||
| COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_BINARY_DIR}/${rel_path}" | |||||
| COMMAND protobuf::protoc -I${file_dir} --cpp_out=${CMAKE_BINARY_DIR}/${rel_path} ${abs_file} | |||||
| COMMAND protobuf::protoc -I${file_dir} --python_out=${CMAKE_BINARY_DIR}/${rel_path} ${abs_file} | |||||
| COMMAND protobuf::protoc -I${file_dir} --python_out=${CMAKE_BINARY_DIR}/${rel_path} ${abs_file} | |||||
| COMMAND perl -pi.bak -e "s/import (.+_pb2.*)/from . import \\1/" "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}_pb2.py" | |||||
| COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}_pb2.py" "${PROJECT_SOURCE_DIR}/mindspore/train/" | |||||
| DEPENDS protobuf::protoc ${abs_file} | |||||
| COMMENT "Running C++ protocol buffer compiler on ${file}" VERBATIM ) | |||||
| else() | |||||
| add_custom_command( | |||||
| OUTPUT "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}.pb.cc" | |||||
| "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}.pb.h" | |||||
| "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}_pb2.py" | |||||
| WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} | |||||
| COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_BINARY_DIR}/${rel_path}" | |||||
| COMMAND protobuf::protoc -I${file_dir} --cpp_out=${CMAKE_BINARY_DIR}/${rel_path} ${abs_file} | |||||
| COMMAND protobuf::protoc -I${file_dir} --python_out=${CMAKE_BINARY_DIR}/${rel_path} ${abs_file} | |||||
| COMMAND protobuf::protoc -I${file_dir} --python_out=${CMAKE_BINARY_DIR}/${rel_path} ${abs_file} | |||||
| COMMAND perl -pi -e "s/import (.+_pb2.*)/from . import \\1/" "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}_pb2.py" | |||||
| COMMAND cp "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}_pb2.py" "${PROJECT_SOURCE_DIR}/mindspore/train/" | |||||
| DEPENDS protobuf::protoc ${abs_file} | |||||
| COMMENT "Running C++ protocol buffer compiler on ${file}" VERBATIM ) | |||||
| endif() | |||||
| endforeach() | endforeach() | ||||
| set_source_files_properties(${${c_var}} ${${h_var}} ${${py_var}} PROPERTIES GENERATED TRUE) | set_source_files_properties(${${c_var}} ${${h_var}} ${${py_var}} PROPERTIES GENERATED TRUE) | ||||
| set(${c_var} ${${c_var}} PARENT_SCOPE) | set(${c_var} ${${c_var}} PARENT_SCOPE) | ||||
| set(${h_var} ${${h_var}} PARENT_SCOPE) | set(${h_var} ${${h_var}} PARENT_SCOPE) | ||||
| @@ -1,19 +1,36 @@ | |||||
| if (WIN32) | |||||
| mindspore_add_pkg(sqlite-head | |||||
| VER 3.31.1 | |||||
| HEAD_ONLY ./ | |||||
| URL https://sqlite.org/2020/sqlite-amalgamation-3310100.zip | |||||
| MD5 2b7bfcdd97dc281903a9aee966213fe4) | |||||
| include_directories(${sqlite-head_INC}) | |||||
| mindspore_add_pkg(sqlite | |||||
| VER 3.31.1 | |||||
| LIBS sqlite3 | |||||
| LIB_PATH ./ | |||||
| HEAD_ONLY ./ | |||||
| RELEASE ON | |||||
| URL https://sqlite.org/2020/sqlite-dll-win64-x64-3310100.zip | |||||
| MD5 662c9d2b05467d590ba5c0443e7fd6bd) | |||||
| set(sqlite_USE_STATIC_LIBS ON) | |||||
| set(sqlite_CXXFLAGS) | |||||
| if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") | |||||
| set(sqlite_CFLAGS "-fstack-protector-all -Wno-uninitialized -Wno-unused-parameter -fPIC -D_FORTIFY_SOURCE=2 -O2") | |||||
| else() | |||||
| set(sqlite_CFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -fPIC -D_FORTIFY_SOURCE=2 -O2") | |||||
| endif() | |||||
| set(sqlite_LDFLAGS "-Wl,-z,relro,-z,now,-z,noexecstack") | |||||
| mindspore_add_pkg(sqlite | |||||
| else () | |||||
| set(sqlite_USE_STATIC_LIBS ON) | |||||
| set(sqlite_CXXFLAGS) | |||||
| if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") | |||||
| set(sqlite_CFLAGS "-fstack-protector-all -Wno-uninitialized -Wno-unused-parameter -fPIC -D_FORTIFY_SOURCE=2 -O2") | |||||
| else() | |||||
| set(sqlite_CFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -fPIC -D_FORTIFY_SOURCE=2 -O2") | |||||
| endif() | |||||
| set(sqlite_LDFLAGS "-Wl,-z,relro,-z,now,-z,noexecstack") | |||||
| mindspore_add_pkg(sqlite | |||||
| VER 3.31.1 | VER 3.31.1 | ||||
| LIBS sqlite3 | LIBS sqlite3 | ||||
| URL https://github.com/sqlite/sqlite/archive/version-3.31.1.tar.gz | URL https://github.com/sqlite/sqlite/archive/version-3.31.1.tar.gz | ||||
| MD5 5f4e7b4016c15f4fb5855615279819da | MD5 5f4e7b4016c15f4fb5855615279819da | ||||
| PATCHES ${CMAKE_SOURCE_DIR}/third_party/patch/sqlite/sqlite.patch001 | PATCHES ${CMAKE_SOURCE_DIR}/third_party/patch/sqlite/sqlite.patch001 | ||||
| CONFIGURE_COMMAND ./configure --enable-shared=no --disable-tcl --disable-editline --enable-json1) | CONFIGURE_COMMAND ./configure --enable-shared=no --disable-tcl --disable-editline --enable-json1) | ||||
| include_directories(${sqlite_INC}) | |||||
| add_library(mindspore::sqlite ALIAS sqlite::sqlite3) | |||||
| include_directories(${sqlite_INC}) | |||||
| endif () | |||||
| add_library(mindspore::sqlite ALIAS sqlite::sqlite3) | |||||
| @@ -1,6 +1,10 @@ | |||||
| set(SECURE_CXX_FLAGS "") | set(SECURE_CXX_FLAGS "") | ||||
| if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") | if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") | ||||
| set(SECURE_CXX_FLAGS "-fstack-protector-all -Wl,-z,relro,-z,now,-z,noexecstack") | |||||
| if (WIN32) | |||||
| set(SECURE_CXX_FLAGS "-fstack-protector-all") | |||||
| else() | |||||
| set(SECURE_CXX_FLAGS "-fstack-protector-all -Wl,-z,relro,-z,now,-z,noexecstack") | |||||
| endif() | |||||
| endif() | endif() | ||||
| set(_ms_tmp_CMAKE_CXX_FLAGS_F ${CMAKE_CXX_FLAGS}) | set(_ms_tmp_CMAKE_CXX_FLAGS_F ${CMAKE_CXX_FLAGS}) | ||||
| set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility=hidden") | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility=hidden") | ||||
| @@ -19,7 +19,11 @@ option(ENABLE_MPI "enable mpi" OFF) | |||||
| option(ENABLE_AKG "enable akg" OFF) | option(ENABLE_AKG "enable akg" OFF) | ||||
| if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") | if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") | ||||
| set(OPTION_CXX_FLAGS "${OPTION_CXX_FLAGS} -fstack-protector-all -Wl,-z,relro,-z,now,-z,noexecstack") | |||||
| if (WIN32) | |||||
| set(OPTION_CXX_FLAGS "${OPTION_CXX_FLAGS} -fstack-protector-all") | |||||
| else() | |||||
| set(OPTION_CXX_FLAGS "${OPTION_CXX_FLAGS} -fstack-protector-all -Wl,-z,relro,-z,now,-z,noexecstack") | |||||
| endif() | |||||
| endif() | endif() | ||||
| if (CMAKE_SYSTEM_NAME MATCHES "Darwin") | if (CMAKE_SYSTEM_NAME MATCHES "Darwin") | ||||
| @@ -106,4 +110,4 @@ endif() | |||||
| if(ENABLE_DUMP_E2E) | if(ENABLE_DUMP_E2E) | ||||
| add_compile_definitions(ENABLE_DUMP_E2E) | add_compile_definitions(ENABLE_DUMP_E2E) | ||||
| endif() | |||||
| endif() | |||||
| @@ -103,7 +103,7 @@ function(__download_pkg_with_git pkg_name pkg_url pkg_git_commit pkg_md5) | |||||
| endfunction() | endfunction() | ||||
| function(__find_pkg_then_add_target pkg_name pkg_exe) | |||||
| function(__find_pkg_then_add_target pkg_name pkg_exe lib_path) | |||||
| unset(${pkg_name}_LIBS) | unset(${pkg_name}_LIBS) | ||||
| @@ -129,15 +129,24 @@ function(__find_pkg_then_add_target pkg_name pkg_exe) | |||||
| set(_LIB_TYPE STATIC) | set(_LIB_TYPE STATIC) | ||||
| endif () | endif () | ||||
| set(${_LIB_NAME}_LIB ${_LIB_NAME}_LIB-NOTFOUND) | set(${_LIB_NAME}_LIB ${_LIB_NAME}_LIB-NOTFOUND) | ||||
| find_library(${_LIB_NAME}_LIB ${_LIB_SEARCH_NAME} PATHS ${${pkg_name}_BASE_DIR}/lib NO_DEFAULT_PATH) | |||||
| find_library(${_LIB_NAME}_LIB ${_LIB_SEARCH_NAME} PATHS ${${pkg_name}_BASE_DIR}/${lib_path} NO_DEFAULT_PATH) | |||||
| if(NOT ${_LIB_NAME}_LIB) | if(NOT ${_LIB_NAME}_LIB) | ||||
| return() | return() | ||||
| endif() | endif() | ||||
| add_library(${pkg_name}::${_LIB_NAME} ${_LIB_TYPE} IMPORTED GLOBAL) | add_library(${pkg_name}::${_LIB_NAME} ${_LIB_TYPE} IMPORTED GLOBAL) | ||||
| set_target_properties(${pkg_name}::${_LIB_NAME} PROPERTIES | |||||
| INTERFACE_INCLUDE_DIRECTORIES "${${pkg_name}_BASE_DIR}/include" | |||||
| IMPORTED_LOCATION ${${_LIB_NAME}_LIB} | |||||
| ) | |||||
| if (WIN32 AND ${_LIB_TYPE} STREQUAL "SHARED") | |||||
| set_target_properties(${pkg_name}::${_LIB_NAME} PROPERTIES IMPORTED_IMPLIB_RELEASE ${${_LIB_NAME}_LIB}) | |||||
| else() | |||||
| set_target_properties(${pkg_name}::${_LIB_NAME} PROPERTIES IMPORTED_LOCATION ${${_LIB_NAME}_LIB}) | |||||
| endif() | |||||
| if (EXISTS ${${pkg_name}_BASE_DIR}/include) | |||||
| set_target_properties(${pkg_name}::${_LIB_NAME} PROPERTIES | |||||
| INTERFACE_INCLUDE_DIRECTORIES "${${pkg_name}_BASE_DIR}/include") | |||||
| endif () | |||||
| list(APPEND ${pkg_name}_LIBS ${pkg_name}::${_LIB_NAME}) | list(APPEND ${pkg_name}_LIBS ${pkg_name}::${_LIB_NAME}) | ||||
| message("found ${${_LIB_NAME}_LIB}") | message("found ${${_LIB_NAME}_LIB}") | ||||
| STRING( REGEX REPLACE "(.+)/(.+)" "\\1" LIBPATH ${${_LIB_NAME}_LIB}) | STRING( REGEX REPLACE "(.+)/(.+)" "\\1" LIBPATH ${${_LIB_NAME}_LIB}) | ||||
| @@ -192,12 +201,18 @@ set(MS_FIND_NO_DEFAULT_PATH ${MS_FIND_NO_DEFAULT_PATH} PARENT_SCOPE) | |||||
| function(mindspore_add_pkg pkg_name ) | function(mindspore_add_pkg pkg_name ) | ||||
| set(options ) | set(options ) | ||||
| set(oneValueArgs URL MD5 GIT_REPOSITORY GIT_TAG VER EXE DIR HEAD_ONLY CMAKE_PATH) | |||||
| set(oneValueArgs URL MD5 GIT_REPOSITORY GIT_TAG VER EXE DIR HEAD_ONLY CMAKE_PATH RELEASE LIB_PATH) | |||||
| set(multiValueArgs CMAKE_OPTION LIBS PRE_CONFIGURE_COMMAND CONFIGURE_COMMAND BUILD_OPTION INSTALL_INCS INSTALL_LIBS PATCHES) | set(multiValueArgs CMAKE_OPTION LIBS PRE_CONFIGURE_COMMAND CONFIGURE_COMMAND BUILD_OPTION INSTALL_INCS INSTALL_LIBS PATCHES) | ||||
| cmake_parse_arguments(PKG "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN} ) | cmake_parse_arguments(PKG "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN} ) | ||||
| if (NOT PKG_CMAKE_PATH) | |||||
| set(PKG_CMAKE_PATH .) | |||||
| if (NOT PKG_LIB_PATH) | |||||
| set(PKG_LIB_PATH lib) | |||||
| endif () | endif () | ||||
| if(NOT PKG_EXE) | |||||
| set(PKG_EXE 0) | |||||
| endif() | |||||
| set(__FIND_PKG_NAME ${pkg_name}) | set(__FIND_PKG_NAME ${pkg_name}) | ||||
| string(TOLOWER ${pkg_name} pkg_name) | string(TOLOWER ${pkg_name} pkg_name) | ||||
| message("pkg name:${__FIND_PKG_NAME},${pkg_name}") | message("pkg name:${__FIND_PKG_NAME},${pkg_name}") | ||||
| @@ -225,18 +240,17 @@ function(mindspore_add_pkg pkg_name ) | |||||
| set(${pkg_name}_INC ${${pkg_name}_BASE_DIR}/${PKG_HEAD_ONLY} PARENT_SCOPE) | set(${pkg_name}_INC ${${pkg_name}_BASE_DIR}/${PKG_HEAD_ONLY} PARENT_SCOPE) | ||||
| add_library(${pkg_name} INTERFACE) | add_library(${pkg_name} INTERFACE) | ||||
| target_include_directories(${pkg_name} INTERFACE ${${pkg_name}_INC}) | target_include_directories(${pkg_name} INTERFACE ${${pkg_name}_INC}) | ||||
| if (${PKG_RELEASE}) | |||||
| __find_pkg_then_add_target(${pkg_name} ${PKG_EXE} ${PKG_LIB_PATH} ${PKG_LIBS}) | |||||
| endif () | |||||
| return() | return() | ||||
| endif () | endif () | ||||
| if(NOT PKG_EXE) | |||||
| set(PKG_EXE 0) | |||||
| endif() | |||||
| set(${__FIND_PKG_NAME}_ROOT ${${pkg_name}_BASE_DIR}) | set(${__FIND_PKG_NAME}_ROOT ${${pkg_name}_BASE_DIR}) | ||||
| set(${__FIND_PKG_NAME}_ROOT ${${pkg_name}_BASE_DIR} PARENT_SCOPE) | set(${__FIND_PKG_NAME}_ROOT ${${pkg_name}_BASE_DIR} PARENT_SCOPE) | ||||
| if (PKG_LIBS) | if (PKG_LIBS) | ||||
| __find_pkg_then_add_target(${pkg_name} ${PKG_EXE} ${PKG_LIBS}) | |||||
| __find_pkg_then_add_target(${pkg_name} ${PKG_EXE} ${PKG_LIB_PATH} ${PKG_LIBS}) | |||||
| if(${pkg_name}_LIBS) | if(${pkg_name}_LIBS) | ||||
| set(${pkg_name}_INC ${${pkg_name}_BASE_DIR}/include PARENT_SCOPE) | set(${pkg_name}_INC ${${pkg_name}_BASE_DIR}/include PARENT_SCOPE) | ||||
| message("Found libs: ${${pkg_name}_LIBS}") | message("Found libs: ${${pkg_name}_LIBS}") | ||||
| @@ -283,8 +297,10 @@ function(mindspore_add_pkg pkg_name ) | |||||
| file(GLOB ${pkg_name}_SOURCE_SUBDIRS ${${pkg_name}_SOURCE_DIR}/*) | file(GLOB ${pkg_name}_SOURCE_SUBDIRS ${${pkg_name}_SOURCE_DIR}/*) | ||||
| file(COPY ${${pkg_name}_SOURCE_SUBDIRS} DESTINATION ${${pkg_name}_BASE_DIR}) | file(COPY ${${pkg_name}_SOURCE_SUBDIRS} DESTINATION ${${pkg_name}_BASE_DIR}) | ||||
| set(${pkg_name}_INC ${${pkg_name}_BASE_DIR}/${PKG_HEAD_ONLY} PARENT_SCOPE) | set(${pkg_name}_INC ${${pkg_name}_BASE_DIR}/${PKG_HEAD_ONLY} PARENT_SCOPE) | ||||
| add_library(${pkg_name} INTERFACE) | |||||
| target_include_directories(${pkg_name} INTERFACE ${${pkg_name}_INC}) | |||||
| if (NOT PKG_RELEASE) | |||||
| add_library(${pkg_name} INTERFACE) | |||||
| target_include_directories(${pkg_name} INTERFACE ${${pkg_name}_INC}) | |||||
| endif () | |||||
| elseif (PKG_CMAKE_OPTION) | elseif (PKG_CMAKE_OPTION) | ||||
| # in cmake | # in cmake | ||||
| @@ -355,7 +371,7 @@ function(mindspore_add_pkg pkg_name ) | |||||
| endif() | endif() | ||||
| if (PKG_LIBS) | if (PKG_LIBS) | ||||
| __find_pkg_then_add_target(${pkg_name} ${PKG_EXE} ${PKG_LIBS}) | |||||
| __find_pkg_then_add_target(${pkg_name} ${PKG_EXE} ${PKG_LIB_PATH} ${PKG_LIBS}) | |||||
| set(${pkg_name}_INC ${${pkg_name}_BASE_DIR}/include PARENT_SCOPE) | set(${pkg_name}_INC ${${pkg_name}_BASE_DIR}/include PARENT_SCOPE) | ||||
| if(NOT ${pkg_name}_LIBS) | if(NOT ${pkg_name}_LIBS) | ||||
| message(FATAL_ERROR "Can not find pkg: ${pkg_name}") | message(FATAL_ERROR "Can not find pkg: ${pkg_name}") | ||||
| @@ -5,6 +5,10 @@ if(ENABLE_CPU) | |||||
| file(GLOB_RECURSE CPU_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} | file(GLOB_RECURSE CPU_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} | ||||
| "device/cpu/*.cc" | "device/cpu/*.cc" | ||||
| ) | ) | ||||
| if (CMAKE_SYSTEM_NAME MATCHES "Windows") | |||||
| set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-attributes -DHAVE_SNPRINTF") | |||||
| add_compile_definitions(BUILDING_DLL) | |||||
| endif() | |||||
| endif() | endif() | ||||
| if(ENABLE_GPU) | if(ENABLE_GPU) | ||||
| @@ -150,7 +154,15 @@ file(GLOB_RECURSE MINDSPORE_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} | |||||
| "kernel/kash/*.cc" | "kernel/kash/*.cc" | ||||
| "device/gpu/distribution/collective_init.cc" | "device/gpu/distribution/collective_init.cc" | ||||
| ) | ) | ||||
| if (ENABLE_CPU) | |||||
| list(REMOVE_ITEM MINDSPORE_SRC_LIST "device/gpu/distribution/collective_init.cc") | |||||
| if (WIN32) | |||||
| list(REMOVE_ITEM MINDSPORE_SRC_LIST "kernel/kernel_query.cc") | |||||
| endif() | |||||
| endif() | |||||
| if (NOT ENABLE_GPU) | |||||
| list(APPEND MINDSPORE_SRC_LIST "device/gpu/distribution/collective_fake_init.cc") | |||||
| endif() | |||||
| file(GLOB_RECURSE MEM_REUSE_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} | file(GLOB_RECURSE MEM_REUSE_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} | ||||
| "pre_activate/mem_reuse/*.cc" | "pre_activate/mem_reuse/*.cc" | ||||
| ) | ) | ||||
| @@ -248,6 +260,7 @@ file(GLOB_RECURSE MS_GVAR_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} | |||||
| add_library(mindspore_gvar SHARED ${MS_GVAR_SRC_LIST}) | add_library(mindspore_gvar SHARED ${MS_GVAR_SRC_LIST}) | ||||
| add_library(mindspore STATIC ${MINDSPORE_SRC_LIST}) | add_library(mindspore STATIC ${MINDSPORE_SRC_LIST}) | ||||
| add_dependencies(mindspore GENERATED_OUTPUT_DIR) | |||||
| if(ENABLE_D) | if(ENABLE_D) | ||||
| list(APPEND MINDSPORE_PROTO_LIST ${MINDSPORE_PROTO_AICPU_LIST}) | list(APPEND MINDSPORE_PROTO_LIST ${MINDSPORE_PROTO_AICPU_LIST}) | ||||
| @@ -315,19 +328,33 @@ if(ENABLE_D) | |||||
| endif() | endif() | ||||
| target_link_libraries(mindspore securec) | target_link_libraries(mindspore securec) | ||||
| target_link_libraries(mindspore dl) | |||||
| if (NOT WIN32) | |||||
| target_link_libraries(mindspore dl) | |||||
| endif() | |||||
| target_link_libraries(mindspore mindspore::flatbuffers) | target_link_libraries(mindspore mindspore::flatbuffers) | ||||
| # link protobuf | # link protobuf | ||||
| if (ENABLE_D) | if (ENABLE_D) | ||||
| target_link_libraries(mindspore mindspore::protobuf) | target_link_libraries(mindspore mindspore::protobuf) | ||||
| endif() | endif() | ||||
| if (WIN32) | |||||
| target_link_libraries(mindspore ${PYTHON_LIBRARIES} mindspore_gvar) | |||||
| endif() | |||||
| # set c_expression building | # set c_expression building | ||||
| if (WIN32) | |||||
| set(PYTHON_MODULE_SOURCE ${MS_GVAR_SRC_LIST} | |||||
| pipeline/init.cc | |||||
| kernel/oplib/oplib.cc | |||||
| ${MINDSPORE_SRC_LIST} ${MS_STEPS_SRC_LIST} ${MS_CCE_SRC_LIST} ${MS_AICPU_SRC_LIST} ${MS_TASKINFO_LIST} ${MS_RT_SRC_LIST} | |||||
| ${GPU_NCCL_LIST} ${MS_HCCL_SRC_LIST} ${MS_PREDICT_SRC_LIST} ${CPU_SRC_LIST} ${MEM_REUSE_SRC_LIST} ${GPU_KERNEL_SRC_LIST}) | |||||
| else() | |||||
| set(PYTHON_MODULE_SOURCE | set(PYTHON_MODULE_SOURCE | ||||
| pipeline/init.cc | pipeline/init.cc | ||||
| kernel/oplib/oplib.cc | kernel/oplib/oplib.cc | ||||
| ${MS_STEPS_SRC_LIST} ${MS_CCE_SRC_LIST} ${MS_AICPU_SRC_LIST} ${MS_TASKINFO_LIST} ${MS_RT_SRC_LIST} | |||||
| ${GPU_NCCL_LIST} ${MS_HCCL_SRC_LIST} ${MS_PREDICT_SRC_LIST} ${CPU_SRC_LIST} ${MEM_REUSE_SRC_LIST} ${GPU_KERNEL_SRC_LIST}) | |||||
| ${MS_STEPS_SRC_LIST} ${MS_CCE_SRC_LIST} ${MS_AICPU_SRC_LIST} ${MS_TASKINFO_LIST} ${MS_RT_SRC_LIST} | |||||
| ${GPU_NCCL_LIST} ${MS_HCCL_SRC_LIST} ${MS_PREDICT_SRC_LIST} ${CPU_SRC_LIST} ${MEM_REUSE_SRC_LIST} ${GPU_KERNEL_SRC_LIST}) | |||||
| endif() | |||||
| set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE) | set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE) | ||||
| pybind11_add_module(_c_expression ${PYTHON_MODULE_SOURCE}) | pybind11_add_module(_c_expression ${PYTHON_MODULE_SOURCE}) | ||||
| @@ -339,6 +366,8 @@ if (CMAKE_SYSTEM_NAME MATCHES "Linux") | |||||
| elseif (CMAKE_SYSTEM_NAME MATCHES "Darwin") | elseif (CMAKE_SYSTEM_NAME MATCHES "Darwin") | ||||
| set_target_properties(_c_expression PROPERTIES MACOSX_RPATH ON) | set_target_properties(_c_expression PROPERTIES MACOSX_RPATH ON) | ||||
| set(ORIGIN_PATH @loader_path) | set(ORIGIN_PATH @loader_path) | ||||
| elseif (CMAKE_SYSTEM_NAME MATCHES "Windows") | |||||
| set(ORIGIN_PATH $ORIGIN) | |||||
| else () | else () | ||||
| MESSAGE(FATAL_ERROR "other platform: ${CMAKE_SYSTEM_NAME}") | MESSAGE(FATAL_ERROR "other platform: ${CMAKE_SYSTEM_NAME}") | ||||
| endif () | endif () | ||||
| @@ -346,11 +375,22 @@ endif () | |||||
| set(ORIGIN_PATH ${ORIGIN_PATH}/lib) | set(ORIGIN_PATH ${ORIGIN_PATH}/lib) | ||||
| set_target_properties(_c_expression PROPERTIES INSTALL_RPATH ${ORIGIN_PATH}) | set_target_properties(_c_expression PROPERTIES INSTALL_RPATH ${ORIGIN_PATH}) | ||||
| target_link_libraries(_c_expression PRIVATE | |||||
| mindspore::pybind11_module | |||||
| mindspore | |||||
| mindspore_gvar | |||||
| ) | |||||
| if (WIN32) | |||||
| target_link_libraries(_c_expression PRIVATE | |||||
| mindspore::pybind11_module | |||||
| securec | |||||
| proto_input | |||||
| mindspore::flatbuffers | |||||
| mindspore::glog | |||||
| ) | |||||
| else() | |||||
| target_link_libraries(_c_expression PRIVATE | |||||
| mindspore::pybind11_module | |||||
| mindspore | |||||
| mindspore_gvar | |||||
| mindspore::glog | |||||
| ) | |||||
| endif() | |||||
| if(ENABLE_GPU) | if(ENABLE_GPU) | ||||
| execute_process(COMMAND bash ${CMAKE_SOURCE_DIR}/third_party/apply_patches.sh | execute_process(COMMAND bash ${CMAKE_SOURCE_DIR}/third_party/apply_patches.sh | ||||
| @@ -493,7 +533,3 @@ if(ENABLE_MINDDATA) | |||||
| add_subdirectory(mindrecord) | add_subdirectory(mindrecord) | ||||
| add_subdirectory(dataset) | add_subdirectory(dataset) | ||||
| endif() | endif() | ||||
| if (USE_GLOG) | |||||
| target_link_libraries(_c_expression PRIVATE mindspore::glog) | |||||
| endif() | |||||
| @@ -80,14 +80,19 @@ set_target_properties(_c_dataengine PROPERTIES | |||||
| ###################################################################### | ###################################################################### | ||||
| ################# Link with external libraries ######################## | ################# Link with external libraries ######################## | ||||
| target_link_libraries(_c_dataengine PRIVATE mindspore mindspore_gvar) | |||||
| target_link_libraries(_c_dataengine PRIVATE mindspore::pybind11_module -ldl mindspore::protobuf ${SECUREC_LIBRARY}) | |||||
| if (WIN32) | |||||
| target_link_libraries(_c_dataengine PRIVATE mindspore) | |||||
| target_link_libraries(_c_dataengine PRIVATE mindspore::pybind11_module ${PYTHON_LIBRARIES} mindspore::protobuf ${SECUREC_LIBRARY}) | |||||
| else() | |||||
| target_link_libraries(_c_dataengine PRIVATE mindspore mindspore_gvar) | |||||
| target_link_libraries(_c_dataengine PRIVATE mindspore::pybind11_module -ldl mindspore::protobuf ${SECUREC_LIBRARY}) | |||||
| endif() | |||||
| target_link_libraries(_c_dataengine PUBLIC mindspore::jpeg_turbo mindspore::opencv_core mindspore::opencv_imgcodecs | target_link_libraries(_c_dataengine PUBLIC mindspore::jpeg_turbo mindspore::opencv_core mindspore::opencv_imgcodecs | ||||
| mindspore::opencv_imgproc) | mindspore::opencv_imgproc) | ||||
| if (ENABLE_GPUQUE) | if (ENABLE_GPUQUE) | ||||
| target_link_libraries(_c_dataengine PRIVATE gpu_queue | target_link_libraries(_c_dataengine PRIVATE gpu_queue | ||||
| ${CUDNN_PATH}/lib64/libcudnn.so | ${CUDNN_PATH}/lib64/libcudnn.so | ||||
| ${CUDA_PATH}/lib64/libcudart.so | |||||
| ${CUDA_PATH}/lib64/libcudart.so | |||||
| ${CUDA_PATH}/lib64/stubs/libcuda.so) | ${CUDA_PATH}/lib64/stubs/libcuda.so) | ||||
| endif () | endif () | ||||
| @@ -96,7 +101,12 @@ if (ENABLE_TDTQUE) | |||||
| endif () | endif () | ||||
| add_dependencies(_c_dataengine _c_mindrecord) | add_dependencies(_c_dataengine _c_mindrecord) | ||||
| target_link_libraries(_c_dataengine PRIVATE _c_mindrecord) | |||||
| if (WIN32) | |||||
| set(MINDRECORD_LINK_OBJECT ${CMAKE_BINARY_DIR}/mindspore/ccsrc/mindrecord/CMakeFiles/_c_mindrecord.dir/objects.a) | |||||
| target_link_libraries(_c_dataengine PRIVATE _c_mindrecord ${MINDRECORD_LINK_OBJECT} mindspore::sqlite) | |||||
| else() | |||||
| target_link_libraries(_c_dataengine PRIVATE _c_mindrecord) | |||||
| endif() | |||||
| if (USE_GLOG) | if (USE_GLOG) | ||||
| target_link_libraries(_c_dataengine PRIVATE mindspore::glog) | target_link_libraries(_c_dataengine PRIVATE mindspore::glog) | ||||
| @@ -19,7 +19,9 @@ | |||||
| #include "dataset/kernels/no_op.h" | #include "dataset/kernels/no_op.h" | ||||
| #include "dataset/kernels/data/one_hot_op.h" | #include "dataset/kernels/data/one_hot_op.h" | ||||
| #include "dataset/kernels/image/center_crop_op.h" | #include "dataset/kernels/image/center_crop_op.h" | ||||
| #if !defined(_WIN32) && !defined(_WIN64) | |||||
| #include "dataset/kernels/image/change_mode_op.h" | #include "dataset/kernels/image/change_mode_op.h" | ||||
| #endif | |||||
| #include "dataset/kernels/image/cut_out_op.h" | #include "dataset/kernels/image/cut_out_op.h" | ||||
| #include "dataset/kernels/image/decode_op.h" | #include "dataset/kernels/image/decode_op.h" | ||||
| #include "dataset/kernels/image/distort_bounding_box_crop_op.h" | #include "dataset/kernels/image/distort_bounding_box_crop_op.h" | ||||
| @@ -279,9 +281,11 @@ void bindTensorOps2(py::module *m) { | |||||
| py::arg("fillG") = RandomCropOp::kDefFillG, py::arg("fillB") = RandomCropOp::kDefFillB); | py::arg("fillG") = RandomCropOp::kDefFillG, py::arg("fillB") = RandomCropOp::kDefFillB); | ||||
| (void)py::class_<HwcToChwOp, TensorOp, std::shared_ptr<HwcToChwOp>>(*m, "ChannelSwapOp").def(py::init<>()); | (void)py::class_<HwcToChwOp, TensorOp, std::shared_ptr<HwcToChwOp>>(*m, "ChannelSwapOp").def(py::init<>()); | ||||
| #if !defined(_WIN32) && !defined(_WIN64) | |||||
| (void)py::class_<ChangeModeOp, TensorOp, std::shared_ptr<ChangeModeOp>>( | (void)py::class_<ChangeModeOp, TensorOp, std::shared_ptr<ChangeModeOp>>( | ||||
| *m, "ChangeModeOp", "Tensor operation to change colors from BGR to RGB") | *m, "ChangeModeOp", "Tensor operation to change colors from BGR to RGB") | ||||
| .def(py::init<>()); | .def(py::init<>()); | ||||
| #endif | |||||
| (void)py::class_<OneHotOp, TensorOp, std::shared_ptr<OneHotOp>>( | (void)py::class_<OneHotOp, TensorOp, std::shared_ptr<OneHotOp>>( | ||||
| *m, "OneHotOp", "Tensor operation to apply one hot encoding. Takes number of classes.") | *m, "OneHotOp", "Tensor operation to apply one hot encoding. Takes number of classes.") | ||||
| @@ -22,6 +22,10 @@ | |||||
| #include <vector> | #include <vector> | ||||
| #include "./securec.h" | #include "./securec.h" | ||||
| #include "utils/log_adapter.h" | #include "utils/log_adapter.h" | ||||
| #if defined(_WIN32) || defined(_WIN64) | |||||
| #undef HAVE_STDDEF_H | |||||
| #undef HAVE_STDLIB_H | |||||
| #endif | |||||
| #include "pybind11/numpy.h" | #include "pybind11/numpy.h" | ||||
| #include "pybind11/pybind11.h" | #include "pybind11/pybind11.h" | ||||
| #include "pybind11/stl.h" | #include "pybind11/stl.h" | ||||
| @@ -85,7 +85,11 @@ Status ShuffleOp::SelfReset() { | |||||
| if (!reshuffle_each_epoch_) { | if (!reshuffle_each_epoch_) { | ||||
| rng_ = std::mt19937_64(shuffle_seed_); | rng_ = std::mt19937_64(shuffle_seed_); | ||||
| } else { | } else { | ||||
| #if defined(_WIN32) || defined(_WIN64) | |||||
| std::random_device random_device; | |||||
| #else | |||||
| std::random_device random_device("/dev/urandom"); | std::random_device random_device("/dev/urandom"); | ||||
| #endif | |||||
| std::uniform_int_distribution<int32_t> distribution(0, std::numeric_limits<int32_t>::max()); | std::uniform_int_distribution<int32_t> distribution(0, std::numeric_limits<int32_t>::max()); | ||||
| shuffle_seed_ = distribution(random_device); | shuffle_seed_ = distribution(random_device); | ||||
| rng_ = std::mt19937_64(shuffle_seed_); | rng_ = std::mt19937_64(shuffle_seed_); | ||||
| @@ -1,22 +1,46 @@ | |||||
| add_library(kernels-image OBJECT | |||||
| center_crop_op.cc | |||||
| change_mode_op.cc | |||||
| cut_out_op.cc | |||||
| decode_op.cc | |||||
| distort_bounding_box_crop_op.cc | |||||
| hwc_to_chw_op.cc | |||||
| image_utils.cc | |||||
| normalize_op.cc | |||||
| pad_op.cc | |||||
| random_color_adjust_op.cc | |||||
| random_crop_decode_resize_op.cc | |||||
| random_crop_and_resize_op.cc | |||||
| random_crop_op.cc | |||||
| random_horizontal_flip_op.cc | |||||
| random_resize_op.cc | |||||
| random_rotation_op.cc | |||||
| random_vertical_flip_op.cc | |||||
| rescale_op.cc | |||||
| resize_bilinear_op.cc | |||||
| resize_op.cc | |||||
| ) | |||||
| if (WIN32) | |||||
| add_library(kernels-image OBJECT | |||||
| center_crop_op.cc | |||||
| cut_out_op.cc | |||||
| decode_op.cc | |||||
| distort_bounding_box_crop_op.cc | |||||
| hwc_to_chw_op.cc | |||||
| image_utils.cc | |||||
| normalize_op.cc | |||||
| pad_op.cc | |||||
| random_color_adjust_op.cc | |||||
| random_crop_decode_resize_op.cc | |||||
| random_crop_and_resize_op.cc | |||||
| random_crop_op.cc | |||||
| random_horizontal_flip_op.cc | |||||
| random_resize_op.cc | |||||
| random_rotation_op.cc | |||||
| random_vertical_flip_op.cc | |||||
| rescale_op.cc | |||||
| resize_bilinear_op.cc | |||||
| resize_op.cc | |||||
| ) | |||||
| else() | |||||
| add_library(kernels-image OBJECT | |||||
| center_crop_op.cc | |||||
| change_mode_op.cc | |||||
| cut_out_op.cc | |||||
| decode_op.cc | |||||
| distort_bounding_box_crop_op.cc | |||||
| hwc_to_chw_op.cc | |||||
| image_utils.cc | |||||
| normalize_op.cc | |||||
| pad_op.cc | |||||
| random_color_adjust_op.cc | |||||
| random_crop_decode_resize_op.cc | |||||
| random_crop_and_resize_op.cc | |||||
| random_crop_op.cc | |||||
| random_horizontal_flip_op.cc | |||||
| random_resize_op.cc | |||||
| random_rotation_op.cc | |||||
| random_vertical_flip_op.cc | |||||
| rescale_op.cc | |||||
| resize_bilinear_op.cc | |||||
| resize_op.cc | |||||
| ) | |||||
| endif() | |||||
| @@ -186,7 +186,11 @@ void JpegSetSource(j_decompress_ptr cinfo, const void *data, int64_t datasize) { | |||||
| (*cinfo->mem->alloc_small)(reinterpret_cast<j_common_ptr>(cinfo), JPOOL_PERMANENT, sizeof(struct jpeg_source_mgr))); | (*cinfo->mem->alloc_small)(reinterpret_cast<j_common_ptr>(cinfo), JPOOL_PERMANENT, sizeof(struct jpeg_source_mgr))); | ||||
| cinfo->src->init_source = JpegInitSource; | cinfo->src->init_source = JpegInitSource; | ||||
| cinfo->src->fill_input_buffer = JpegFillInputBuffer; | cinfo->src->fill_input_buffer = JpegFillInputBuffer; | ||||
| #if defined(_WIN32) || defined(_WIN64) | |||||
| cinfo->src->skip_input_data = reinterpret_cast<void (*)(j_decompress_ptr, long)>(JpegSkipInputData); | |||||
| #else | |||||
| cinfo->src->skip_input_data = JpegSkipInputData; | cinfo->src->skip_input_data = JpegSkipInputData; | ||||
| #endif | |||||
| cinfo->src->resync_to_restart = jpeg_resync_to_restart; | cinfo->src->resync_to_restart = jpeg_resync_to_restart; | ||||
| cinfo->src->term_source = JpegTermSource; | cinfo->src->term_source = JpegTermSource; | ||||
| cinfo->src->bytes_in_buffer = datasize; | cinfo->src->bytes_in_buffer = datasize; | ||||
| @@ -22,6 +22,10 @@ | |||||
| #include <random> | #include <random> | ||||
| #include <string> | #include <string> | ||||
| #include <vector> | #include <vector> | ||||
| #if defined(_WIN32) || defined(_WIN64) | |||||
| #undef HAVE_STDDEF_H | |||||
| #undef HAVE_STDLIB_H | |||||
| #endif | |||||
| #include "./jpeglib.h" | #include "./jpeglib.h" | ||||
| #include "./jerror.h" | #include "./jerror.h" | ||||
| #include <opencv2/imgproc/imgproc.hpp> | #include <opencv2/imgproc/imgproc.hpp> | ||||
| @@ -27,7 +27,7 @@ | |||||
| namespace mindspore { | namespace mindspore { | ||||
| namespace dataset { | namespace dataset { | ||||
| #ifdef _WIN32 | #ifdef _WIN32 | ||||
| char Path::_separator = '\\'; | |||||
| char Path::separator_ = '\\'; | |||||
| #else | #else | ||||
| char Path::separator_ = '/'; | char Path::separator_ = '/'; | ||||
| #endif | #endif | ||||
| @@ -129,7 +129,11 @@ bool Path::IsDirectory() { | |||||
| Status Path::CreateDirectory() { | Status Path::CreateDirectory() { | ||||
| if (!Exists()) { | if (!Exists()) { | ||||
| #if defined(_WIN32) || defined(_WIN64) | |||||
| int rc = mkdir(common::SafeCStr(path_)); | |||||
| #else | |||||
| int rc = mkdir(common::SafeCStr(path_), 0700); | int rc = mkdir(common::SafeCStr(path_), 0700); | ||||
| #endif | |||||
| if (rc) { | if (rc) { | ||||
| std::ostringstream oss; | std::ostringstream oss; | ||||
| oss << "Unable to create directory " << path_ << ". Errno = " << errno; | oss << "Unable to create directory " << path_ << ". Errno = " << errno; | ||||
| @@ -32,7 +32,11 @@ namespace dataset { | |||||
| uint32_t GetSeed() { | uint32_t GetSeed() { | ||||
| uint32_t seed = GlobalContext::config_manager()->seed(); | uint32_t seed = GlobalContext::config_manager()->seed(); | ||||
| if (seed == std::mt19937::default_seed) { | if (seed == std::mt19937::default_seed) { | ||||
| #if defined(_WIN32) || defined(_WIN64) | |||||
| std::random_device random_device; | |||||
| #else | |||||
| std::random_device random_device("/dev/urandom"); | std::random_device random_device("/dev/urandom"); | ||||
| #endif | |||||
| std::uniform_int_distribution<uint32_t> distribution(0, std::numeric_limits<uint32_t>::max()); | std::uniform_int_distribution<uint32_t> distribution(0, std::numeric_limits<uint32_t>::max()); | ||||
| seed = distribution(random_device); | seed = distribution(random_device); | ||||
| } | } | ||||
| @@ -16,7 +16,9 @@ | |||||
| #include "dataset/util/services.h" | #include "dataset/util/services.h" | ||||
| #include <limits.h> | #include <limits.h> | ||||
| #if !defined(_WIN32) && !defined(_WIN64) | |||||
| #include <sys/syscall.h> | #include <sys/syscall.h> | ||||
| #endif | |||||
| #include <unistd.h> | #include <unistd.h> | ||||
| #include <random> | #include <random> | ||||
| #include "dataset/util/circular_pool.h" | #include "dataset/util/circular_pool.h" | ||||
| @@ -28,6 +30,7 @@ namespace dataset { | |||||
| std::unique_ptr<Services> Services::instance_ = nullptr; | std::unique_ptr<Services> Services::instance_ = nullptr; | ||||
| std::once_flag Services::init_instance_flag_; | std::once_flag Services::init_instance_flag_; | ||||
| #if !defined(_WIN32) && !defined(_WIN64) | |||||
| std::string Services::GetUserName() { | std::string Services::GetUserName() { | ||||
| char user[LOGIN_NAME_MAX]; | char user[LOGIN_NAME_MAX]; | ||||
| (void)getlogin_r(user, sizeof(user)); | (void)getlogin_r(user, sizeof(user)); | ||||
| @@ -41,10 +44,15 @@ std::string Services::GetHostName() { | |||||
| } | } | ||||
| int Services::GetLWP() { return syscall(SYS_gettid); } | int Services::GetLWP() { return syscall(SYS_gettid); } | ||||
| #endif | |||||
| std::string Services::GetUniqueID() { | std::string Services::GetUniqueID() { | ||||
| const std::string kStr = "abcdefghijklmnopqrstuvwxyz0123456789"; | const std::string kStr = "abcdefghijklmnopqrstuvwxyz0123456789"; | ||||
| #if defined(_WIN32) || defined(_WIN64) | |||||
| std::mt19937 gen{std::random_device{}()}; | |||||
| #else | |||||
| std::mt19937 gen{std::random_device{"/dev/urandom"}()}; | std::mt19937 gen{std::random_device{"/dev/urandom"}()}; | ||||
| #endif | |||||
| std::uniform_int_distribution<> dist(0, kStr.size() - 1); | std::uniform_int_distribution<> dist(0, kStr.size() - 1); | ||||
| char buffer[UNIQUEID_LEN]; | char buffer[UNIQUEID_LEN]; | ||||
| for (int i = 0; i < UNIQUEID_LEN; i++) { | for (int i = 0; i < UNIQUEID_LEN; i++) { | ||||
| @@ -62,11 +62,13 @@ class Services { | |||||
| std::shared_ptr<MemoryPool> GetServiceMemPool() { return pool_; } | std::shared_ptr<MemoryPool> GetServiceMemPool() { return pool_; } | ||||
| #if !defined(_WIN32) && !defined(_WIN64) | |||||
| static std::string GetUserName(); | static std::string GetUserName(); | ||||
| static std::string GetHostName(); | static std::string GetHostName(); | ||||
| static int GetLWP(); | static int GetLWP(); | ||||
| #endif | |||||
| static std::string GetUniqueID(); | static std::string GetUniqueID(); | ||||
| @@ -16,13 +16,16 @@ | |||||
| #include "dataset/util/sig_handler.h" | #include "dataset/util/sig_handler.h" | ||||
| #include <signal.h> | #include <signal.h> | ||||
| #include <sys/types.h> | #include <sys/types.h> | ||||
| #if !defined(_WIN32) && !defined(_WIN64) | |||||
| #include <ucontext.h> | #include <ucontext.h> | ||||
| #endif | |||||
| #include <unistd.h> | #include <unistd.h> | ||||
| #include "dataset/util/task_manager.h" | #include "dataset/util/task_manager.h" | ||||
| namespace mindspore { | namespace mindspore { | ||||
| namespace dataset { | namespace dataset { | ||||
| // Register the custom signal handlers | // Register the custom signal handlers | ||||
| #if !defined(_WIN32) && !defined(_WIN64) | |||||
| void RegisterHandlers() { | void RegisterHandlers() { | ||||
| struct sigaction new_int_action; | struct sigaction new_int_action; | ||||
| @@ -40,5 +43,6 @@ extern void IntHandler(int sig_num, // The signal that was raised | |||||
| // Wake up the watchdog which is designed as async-signal-safe. | // Wake up the watchdog which is designed as async-signal-safe. | ||||
| TaskManager::WakeUpWatchDog(); | TaskManager::WakeUpWatchDog(); | ||||
| } | } | ||||
| #endif | |||||
| } // namespace dataset | } // namespace dataset | ||||
| } // namespace mindspore | } // namespace mindspore | ||||
| @@ -22,12 +22,14 @@ | |||||
| namespace mindspore { | namespace mindspore { | ||||
| namespace dataset { | namespace dataset { | ||||
| // Register the custom signal handlers | // Register the custom signal handlers | ||||
| #if !defined(_WIN32) && !defined(_WIN64) | |||||
| extern void RegisterHandlers(); | extern void RegisterHandlers(); | ||||
| // A signal handler for SIGINT. Drives interrupt to watchdog | // A signal handler for SIGINT. Drives interrupt to watchdog | ||||
| extern void IntHandler(int sig_num, // The signal that was raised | extern void IntHandler(int sig_num, // The signal that was raised | ||||
| siginfo_t *sig_info, // The siginfo structure. | siginfo_t *sig_info, // The siginfo structure. | ||||
| void *context); // context info | void *context); // context info | ||||
| #endif | |||||
| } // namespace dataset | } // namespace dataset | ||||
| } // namespace mindspore | } // namespace mindspore | ||||
| @@ -14,6 +14,9 @@ | |||||
| * limitations under the License. | * limitations under the License. | ||||
| */ | */ | ||||
| #include "debug/anf_ir_dump.h" | #include "debug/anf_ir_dump.h" | ||||
| #if defined(_WIN32) || defined(_WIN64) | |||||
| #include <stdlib.h> | |||||
| #endif | |||||
| #include <fstream> | #include <fstream> | ||||
| #include <map> | #include <map> | ||||
| #include <memory> | #include <memory> | ||||
| @@ -434,9 +437,15 @@ void DumpIR(const std::string &filename, const FuncGraphPtr &graph, bool dump_fu | |||||
| return; | return; | ||||
| } | } | ||||
| char real_path[PATH_MAX] = {0}; | char real_path[PATH_MAX] = {0}; | ||||
| #if defined(_WIN32) || defined(_WIN64) | |||||
| if (_fullpath(real_path, filename.c_str(), PATH_MAX) == nullptr) { | |||||
| MS_LOG(DEBUG) << "dir " << filename << " does not exit."; | |||||
| } | |||||
| #else | |||||
| if (nullptr == realpath(filename.c_str(), real_path)) { | if (nullptr == realpath(filename.c_str(), real_path)) { | ||||
| MS_LOG(DEBUG) << "Dir " << filename << " does not exit."; | MS_LOG(DEBUG) << "Dir " << filename << " does not exit."; | ||||
| } | } | ||||
| #endif | |||||
| OrderedMap<AnfNodePtr, int32_t> para_map; | OrderedMap<AnfNodePtr, int32_t> para_map; | ||||
| std::string path_string = real_path; | std::string path_string = real_path; | ||||
| @@ -48,9 +48,15 @@ std::string GetMsIrPath(void) { | |||||
| if (path_ptr != nullptr) { | if (path_ptr != nullptr) { | ||||
| path = path_ptr; | path = path_ptr; | ||||
| char real_path[PATH_MAX] = {0}; | char real_path[PATH_MAX] = {0}; | ||||
| #if defined(_WIN32) || defined(_WIN64) | |||||
| if (path.size() > PATH_MAX || _fullpath(real_path, path.c_str(), PATH_MAX) == nullptr) { | |||||
| MS_LOG(EXCEPTION) << "MS IR Path error, " << path_ptr; | |||||
| } | |||||
| #else | |||||
| if (path.size() > PATH_MAX || nullptr == realpath(path.c_str(), real_path)) { | if (path.size() > PATH_MAX || nullptr == realpath(path.c_str(), real_path)) { | ||||
| MS_LOG(EXCEPTION) << "MS IR path error, " << path_ptr; | MS_LOG(EXCEPTION) << "MS IR path error, " << path_ptr; | ||||
| } | } | ||||
| #endif | |||||
| path = real_path; | path = real_path; | ||||
| } | } | ||||
| return path; | return path; | ||||
| @@ -2247,8 +2253,14 @@ void DumpIRProto(const FuncGraphPtr& func_graph, const std::string& suffix) { | |||||
| return; | return; | ||||
| } | } | ||||
| char real_path[PATH_MAX] = {0}; | char real_path[PATH_MAX] = {0}; | ||||
| if (nullptr == realpath(file_path.c_str(), real_path)) { | |||||
| MS_LOG(DEBUG) << "Dir " << file_path << " does not exit."; | |||||
| char* real_path_ret = nullptr; | |||||
| #if defined(_WIN32) || defined(_WIN64) | |||||
| real_path_ret = _fullpath(real_path, file_path.c_str(), PATH_MAX); | |||||
| #else | |||||
| real_path_ret = realpath(file_path.c_str(), real_path); | |||||
| #endif | |||||
| if (nullptr == real_path_ret) { | |||||
| MS_LOG(DEBUG) << "dir " << file_path << " does not exit."; | |||||
| } else { | } else { | ||||
| std::string path_string = real_path; | std::string path_string = real_path; | ||||
| if (chmod(common::SafeCStr(path_string), S_IRUSR | S_IWUSR) == -1) { | if (chmod(common::SafeCStr(path_string), S_IRUSR | S_IWUSR) == -1) { | ||||
| @@ -53,10 +53,15 @@ std::string Location::ToString(SourceLineTip tip) { | |||||
| } | } | ||||
| char path[PATH_MAX + 1] = {0x00}; | char path[PATH_MAX + 1] = {0x00}; | ||||
| #if defined(_WIN32) || defined(_WIN64) | |||||
| if (file_name_.size() > PATH_MAX || _fullpath(path, file_name_.c_str(), PATH_MAX) == nullptr) { | |||||
| return debug_info_ss.str(); | |||||
| } | |||||
| #else | |||||
| if (file_name_.size() > PATH_MAX || realpath(file_name_.c_str(), path) == nullptr) { | if (file_name_.size() > PATH_MAX || realpath(file_name_.c_str(), path) == nullptr) { | ||||
| return debug_info_ss.str(); | return debug_info_ss.str(); | ||||
| } | } | ||||
| #endif | |||||
| auto src_path = std::string(path); | auto src_path = std::string(path); | ||||
| std::ifstream file(src_path); | std::ifstream file(src_path); | ||||
| if (!file.is_open()) { | if (!file.is_open()) { | ||||
| @@ -60,6 +60,7 @@ void CPUResourceManager::MemMalloc(const session::KernelGraph *graph) { | |||||
| void *CPUResourceManager::MemMalloc(size_t mem_size) { | void *CPUResourceManager::MemMalloc(size_t mem_size) { | ||||
| void *ptr = malloc(mem_size); | void *ptr = malloc(mem_size); | ||||
| if (ptr != nullptr) { | if (ptr != nullptr) { | ||||
| memset_s(ptr, mem_size, 0, mem_size); | |||||
| dynamic_mem_[ptr] = mem_size; | dynamic_mem_[ptr] = mem_size; | ||||
| return ptr; | return ptr; | ||||
| } else { | } else { | ||||
| @@ -0,0 +1,28 @@ | |||||
| /** | |||||
| * Copyright 2019 Huawei Technologies Co., Ltd | |||||
| * | |||||
| * Licensed under the Apache License, Version 2.0 (the "License"); | |||||
| * you may not use this file except in compliance with the License. | |||||
| * You may obtain a copy of the License at | |||||
| * | |||||
| * http://www.apache.org/licenses/LICENSE-2.0 | |||||
| * | |||||
| * Unless required by applicable law or agreed to in writing, software | |||||
| * distributed under the License is distributed on an "AS IS" BASIS, | |||||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
| * See the License for the specific language governing permissions and | |||||
| * limitations under the License. | |||||
| */ | |||||
| #include "device/gpu/distribution/collective_fake_init.h" | |||||
| #include "utils/log_adapter.h" | |||||
| namespace mindspore { | |||||
| namespace device { | |||||
| namespace gpu { | |||||
| void CollectiveFakeInitializer::InitCollective() { MS_LOG(EXCEPTION) << "build without enable gpu!"; } | |||||
| void CollectiveFakeInitializer::FinalizeCollective() { MS_LOG(EXCEPTION) << "build without enable gpu!"; } | |||||
| } // namespace gpu | |||||
| } // namespace device | |||||
| } // namespace mindspore | |||||
| @@ -0,0 +1,37 @@ | |||||
| /** | |||||
| * Copyright 2019 Huawei Technologies Co., Ltd | |||||
| * | |||||
| * Licensed under the Apache License, Version 2.0 (the "License"); | |||||
| * you may not use this file except in compliance with the License. | |||||
| * You may obtain a copy of the License at | |||||
| * | |||||
| * http://www.apache.org/licenses/LICENSE-2.0 | |||||
| * | |||||
| * Unless required by applicable law or agreed to in writing, software | |||||
| * distributed under the License is distributed on an "AS IS" BASIS, | |||||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
| * See the License for the specific language governing permissions and | |||||
| * limitations under the License. | |||||
| */ | |||||
| #ifndef MINDSPORE_CCSRC_DEVICE_GPU_DISTRIBUTION_COLLECTIVE_FAKE_INIT_H_ | |||||
| #define MINDSPORE_CCSRC_DEVICE_GPU_DISTRIBUTION_COLLECTIVE_FAKE_INIT_H_ | |||||
| namespace mindspore { | |||||
| namespace device { | |||||
| namespace gpu { | |||||
| class CollectiveFakeInitializer { | |||||
| public: | |||||
| CollectiveFakeInitializer() = default; | |||||
| ~CollectiveFakeInitializer() = default; | |||||
| CollectiveFakeInitializer(CollectiveFakeInitializer const &) = delete; | |||||
| CollectiveFakeInitializer &operator=(const CollectiveFakeInitializer &) = delete; | |||||
| static void InitCollective(); | |||||
| static void FinalizeCollective(); | |||||
| }; | |||||
| } // namespace gpu | |||||
| } // namespace device | |||||
| } // namespace mindspore | |||||
| #endif // MINDSPORE_CCSRC_DEVICE_GPU_DISTRIBUTION_COLLECTIVE_FAKE_INIT_H_ | |||||
| @@ -44,19 +44,29 @@ bool KernelRuntime::Run(session::KernelGraph *graph) { | |||||
| bool ret = false; | bool ret = false; | ||||
| auto context_ptr = MsContext::GetInstance(); | auto context_ptr = MsContext::GetInstance(); | ||||
| MS_EXCEPTION_IF_NULL(context_ptr); | MS_EXCEPTION_IF_NULL(context_ptr); | ||||
| #if defined(_WIN32) || defined(_WIN64) | |||||
| auto start_time = std::chrono::steady_clock::now(); | |||||
| #else | |||||
| struct timeval start_time, end_time; | struct timeval start_time, end_time; | ||||
| (void)gettimeofday(&start_time, nullptr); | (void)gettimeofday(&start_time, nullptr); | ||||
| #endif | |||||
| bool is_task_sink = context_ptr->enable_task_sink(); | bool is_task_sink = context_ptr->enable_task_sink(); | ||||
| if (is_task_sink) { | if (is_task_sink) { | ||||
| ret = RunTask(graph); | ret = RunTask(graph); | ||||
| } else { | } else { | ||||
| ret = LaunchKernel(graph); | ret = LaunchKernel(graph); | ||||
| } | } | ||||
| #if defined(_WIN32) || defined(_WIN64) | |||||
| auto end_time = std::chrono::steady_clock::now(); | |||||
| std::chrono::duration<double, std::ratio<1, 1000000>> cost = end_time - start_time; | |||||
| MS_LOG(INFO) << "Call MS Run Success in " << cost.count() << " us"; | |||||
| #else | |||||
| (void)gettimeofday(&end_time, nullptr); | (void)gettimeofday(&end_time, nullptr); | ||||
| const uint64_t kUSecondInSecond = 1000000; | const uint64_t kUSecondInSecond = 1000000; | ||||
| uint64_t cost = kUSecondInSecond * static_cast<uint64_t>(end_time.tv_sec - start_time.tv_sec); | uint64_t cost = kUSecondInSecond * static_cast<uint64_t>(end_time.tv_sec - start_time.tv_sec); | ||||
| cost += static_cast<uint64_t>(end_time.tv_usec - start_time.tv_usec); | cost += static_cast<uint64_t>(end_time.tv_usec - start_time.tv_usec); | ||||
| MS_LOG(INFO) << "Call MS Run Success in " << cost << " us"; | MS_LOG(INFO) << "Call MS Run Success in " << cost << " us"; | ||||
| #endif | |||||
| return ret; | return ret; | ||||
| } | } | ||||
| @@ -561,8 +571,12 @@ bool KernelRuntime::LaunchKernelMod(const session::KernelGraph &graph) { | |||||
| AddressPtrList kernel_workspaces; | AddressPtrList kernel_workspaces; | ||||
| AddressPtrList kernel_outputs; | AddressPtrList kernel_outputs; | ||||
| GenLaunchArgs(*kernel_mod, kernel, &kernel_inputs, &kernel_workspaces, &kernel_outputs); | GenLaunchArgs(*kernel_mod, kernel, &kernel_inputs, &kernel_workspaces, &kernel_outputs); | ||||
| #if defined(_WIN32) || defined(_WIN64) | |||||
| auto start_time = std::chrono::steady_clock::now(); | |||||
| #else | |||||
| struct timeval start_time, end_time; | struct timeval start_time, end_time; | ||||
| (void)gettimeofday(&start_time, nullptr); | (void)gettimeofday(&start_time, nullptr); | ||||
| #endif | |||||
| auto ret = | auto ret = | ||||
| kernel_mod->Launch(kernel_inputs, kernel_workspaces, kernel_outputs, reinterpret_cast<uintptr_t>(stream_)); | kernel_mod->Launch(kernel_inputs, kernel_workspaces, kernel_outputs, reinterpret_cast<uintptr_t>(stream_)); | ||||
| if (!ret) { | if (!ret) { | ||||
| @@ -572,11 +586,17 @@ bool KernelRuntime::LaunchKernelMod(const session::KernelGraph &graph) { | |||||
| if (AnfAlgo::GetKernelType(kernel) == TBE_KERNEL && !SyncStream()) { | if (AnfAlgo::GetKernelType(kernel) == TBE_KERNEL && !SyncStream()) { | ||||
| MS_LOG(EXCEPTION) << "SyncStream failed."; | MS_LOG(EXCEPTION) << "SyncStream failed."; | ||||
| } | } | ||||
| #if defined(_WIN32) || defined(_WIN64) | |||||
| auto end_time = std::chrono::steady_clock::now(); | |||||
| std::chrono::duration<double, std::ratio<1, 1000000>> cost = end_time - start_time; | |||||
| MS_LOG(DEBUG) << "d " << kernel->fullname_with_scope() << " in " << cost.count() << " us"; | |||||
| #else | |||||
| (void)gettimeofday(&end_time, nullptr); | (void)gettimeofday(&end_time, nullptr); | ||||
| const uint64_t kUSecondInSecond = 1000000; | const uint64_t kUSecondInSecond = 1000000; | ||||
| uint64_t cost = kUSecondInSecond * static_cast<uint64_t>(end_time.tv_sec - start_time.tv_sec); | uint64_t cost = kUSecondInSecond * static_cast<uint64_t>(end_time.tv_sec - start_time.tv_sec); | ||||
| cost += static_cast<uint64_t>(end_time.tv_usec - start_time.tv_usec); | cost += static_cast<uint64_t>(end_time.tv_usec - start_time.tv_usec); | ||||
| MS_LOG(DEBUG) << "d " << kernel->fullname_with_scope() << " in " << cost << " us"; | MS_LOG(DEBUG) << "d " << kernel->fullname_with_scope() << " in " << cost << " us"; | ||||
| #endif | |||||
| } | } | ||||
| } | } | ||||
| return true; | return true; | ||||
| @@ -117,7 +117,11 @@ bool IsAtomicNode(const CNodePtr &kernel_node) { | |||||
| bool KernelMeta::ReadIndex(const std::string &bin_dir) { | bool KernelMeta::ReadIndex(const std::string &bin_dir) { | ||||
| DIR *dir = opendir(bin_dir.c_str()); | DIR *dir = opendir(bin_dir.c_str()); | ||||
| if (dir == nullptr) { | if (dir == nullptr) { | ||||
| #if defined(_WIN32) || defined(_WIN64) | |||||
| auto ret = mkdir(bin_dir.c_str()); | |||||
| #else | |||||
| auto ret = mkdir(bin_dir.c_str(), S_IRWXG | S_IRWXU); | auto ret = mkdir(bin_dir.c_str(), S_IRWXG | S_IRWXU); | ||||
| #endif | |||||
| if (ret != 0) { | if (ret != 0) { | ||||
| MS_LOG(INFO) << "kernel dir not exist[" << bin_dir << "]."; | MS_LOG(INFO) << "kernel dir not exist[" << bin_dir << "]."; | ||||
| return false; | return false; | ||||
| @@ -500,10 +504,17 @@ void SaveJsonInfo(const std::string &json_name, const std::string &info) { | |||||
| } | } | ||||
| filewrite << info << std::endl; | filewrite << info << std::endl; | ||||
| filewrite.close(); | filewrite.close(); | ||||
| #if defined(_WIN32) || defined(_WIN64) | |||||
| if (nullptr == _fullpath(real_path, path.c_str(), PATH_MAX)) { | |||||
| MS_LOG(DEBUG) << "dir " << path << " does not exit."; | |||||
| return; | |||||
| } | |||||
| #else | |||||
| if (nullptr == realpath(path.c_str(), real_path)) { | if (nullptr == realpath(path.c_str(), real_path)) { | ||||
| MS_LOG(DEBUG) << "dir " << path << " does not exit."; | MS_LOG(DEBUG) << "dir " << path << " does not exit."; | ||||
| return; | return; | ||||
| } | } | ||||
| #endif | |||||
| MS_LOG(INFO) << "real path is :" << real_path; | MS_LOG(INFO) << "real path is :" << real_path; | ||||
| if (chmod(real_path, S_IRUSR) == -1) { | if (chmod(real_path, S_IRUSR) == -1) { | ||||
| MS_LOG(DEBUG) << "modify file:" << real_path << " to read only fail."; | MS_LOG(DEBUG) << "modify file:" << real_path << " to read only fail."; | ||||
| @@ -26,7 +26,11 @@ set_target_properties(_c_mindrecord PROPERTIES | |||||
| ) | ) | ||||
| # add link library | # add link library | ||||
| target_link_libraries(_c_mindrecord PRIVATE mindspore::sqlite ${PYTHON_LIB} ${SECUREC_LIBRARY} mindspore mindspore_gvar mindspore::protobuf) | |||||
| if (WIN32) | |||||
| target_link_libraries(_c_mindrecord PRIVATE mindspore::sqlite mindspore mindspore::protobuf) | |||||
| else() | |||||
| target_link_libraries(_c_mindrecord PRIVATE mindspore::sqlite ${PYTHON_LIB} ${SECUREC_LIBRARY} mindspore mindspore_gvar mindspore::protobuf) | |||||
| endif() | |||||
| if (USE_GLOG) | if (USE_GLOG) | ||||
| target_link_libraries(_c_mindrecord PRIVATE mindspore::glog) | target_link_libraries(_c_mindrecord PRIVATE mindspore::glog) | ||||
| @@ -65,6 +65,15 @@ std::pair<MSRStatus, std::string> GetFileName(const std::string &path) { | |||||
| return {FAILED, ""}; | return {FAILED, ""}; | ||||
| } | } | ||||
| char tmp[PATH_MAX] = {0}; | char tmp[PATH_MAX] = {0}; | ||||
| #if defined(_WIN32) || defined(_WIN64) | |||||
| if (_fullpath(tmp, dirname(&(buf[0])), PATH_MAX) == nullptr) { | |||||
| MS_LOG(ERROR) << "Invalid file path, path: " << buf; | |||||
| return {FAILED, ""}; | |||||
| } | |||||
| if (_fullpath(real_path, common::SafeCStr(path), PATH_MAX) == nullptr) { | |||||
| MS_LOG(DEBUG) << "Path: " << common::SafeCStr(path) << "check successfully"; | |||||
| } | |||||
| #else | |||||
| if (realpath(dirname(&(buf[0])), tmp) == nullptr) { | if (realpath(dirname(&(buf[0])), tmp) == nullptr) { | ||||
| MS_LOG(ERROR) << "Invalid file path, path: " << buf; | MS_LOG(ERROR) << "Invalid file path, path: " << buf; | ||||
| return {FAILED, ""}; | return {FAILED, ""}; | ||||
| @@ -72,6 +81,7 @@ std::pair<MSRStatus, std::string> GetFileName(const std::string &path) { | |||||
| if (realpath(common::SafeCStr(path), real_path) == nullptr) { | if (realpath(common::SafeCStr(path), real_path) == nullptr) { | ||||
| MS_LOG(DEBUG) << "Path: " << path << "check successfully"; | MS_LOG(DEBUG) << "Path: " << path << "check successfully"; | ||||
| } | } | ||||
| #endif | |||||
| std::string s = real_path; | std::string s = real_path; | ||||
| char sep = '/'; | char sep = '/'; | ||||
| size_t i = s.rfind(sep, s.length()); | size_t i = s.rfind(sep, s.length()); | ||||
| @@ -91,6 +101,15 @@ std::pair<MSRStatus, std::string> GetParentDir(const std::string &path) { | |||||
| return {FAILED, ""}; | return {FAILED, ""}; | ||||
| } | } | ||||
| char tmp[PATH_MAX] = {0}; | char tmp[PATH_MAX] = {0}; | ||||
| #if defined(_WIN32) || defined(_WIN64) | |||||
| if (_fullpath(tmp, dirname(&(buf[0])), PATH_MAX) == nullptr) { | |||||
| MS_LOG(ERROR) << "Invalid file path, path: " << buf; | |||||
| return {FAILED, ""}; | |||||
| } | |||||
| if (_fullpath(real_path, common::SafeCStr(path), PATH_MAX) == nullptr) { | |||||
| MS_LOG(DEBUG) << "Path: " << common::SafeCStr(path) << "check successfully"; | |||||
| } | |||||
| #else | |||||
| if (realpath(dirname(&(buf[0])), tmp) == nullptr) { | if (realpath(dirname(&(buf[0])), tmp) == nullptr) { | ||||
| MS_LOG(ERROR) << "Invalid file path, path: " << buf; | MS_LOG(ERROR) << "Invalid file path, path: " << buf; | ||||
| return {FAILED, ""}; | return {FAILED, ""}; | ||||
| @@ -98,6 +117,7 @@ std::pair<MSRStatus, std::string> GetParentDir(const std::string &path) { | |||||
| if (realpath(common::SafeCStr(path), real_path) == nullptr) { | if (realpath(common::SafeCStr(path), real_path) == nullptr) { | ||||
| MS_LOG(DEBUG) << "Path: " << path << "check successfully"; | MS_LOG(DEBUG) << "Path: " << path << "check successfully"; | ||||
| } | } | ||||
| #endif | |||||
| std::string s = real_path; | std::string s = real_path; | ||||
| if (s.rfind('/') + 1 <= s.size()) { | if (s.rfind('/') + 1 <= s.size()) { | ||||
| return {SUCCESS, s.substr(0, s.rfind('/') + 1)}; | return {SUCCESS, s.substr(0, s.rfind('/') + 1)}; | ||||
| @@ -144,6 +164,9 @@ bool IsLegalFile(const std::string &path) { | |||||
| } | } | ||||
| std::pair<MSRStatus, uint64_t> GetDiskSize(const std::string &str_dir, const DiskSizeType &disk_type) { | std::pair<MSRStatus, uint64_t> GetDiskSize(const std::string &str_dir, const DiskSizeType &disk_type) { | ||||
| #if defined(_WIN32) || defined(_WIN64) | |||||
| return {SUCCESS, 100}; | |||||
| #else | |||||
| uint64_t ll_count = 0; | uint64_t ll_count = 0; | ||||
| struct statfs disk_info; | struct statfs disk_info; | ||||
| if (statfs(common::SafeCStr(str_dir), &disk_info) == -1) { | if (statfs(common::SafeCStr(str_dir), &disk_info) == -1) { | ||||
| @@ -166,6 +189,7 @@ std::pair<MSRStatus, uint64_t> GetDiskSize(const std::string &str_dir, const Dis | |||||
| } | } | ||||
| return {SUCCESS, ll_count}; | return {SUCCESS, ll_count}; | ||||
| #endif | |||||
| } | } | ||||
| uint32_t GetMaxThreadNum() { | uint32_t GetMaxThreadNum() { | ||||
| @@ -21,8 +21,10 @@ | |||||
| #include <limits.h> | #include <limits.h> | ||||
| #include <stdlib.h> | #include <stdlib.h> | ||||
| #include <sys/stat.h> | #include <sys/stat.h> | ||||
| #if !defined(_WIN32) && !defined(_WIN64) | |||||
| #include <sys/statfs.h> | #include <sys/statfs.h> | ||||
| #include <sys/wait.h> | #include <sys/wait.h> | ||||
| #endif | |||||
| #include <unistd.h> | #include <unistd.h> | ||||
| #include <cassert> | #include <cassert> | ||||
| #include <cmath> | #include <cmath> | ||||
| @@ -19,7 +19,9 @@ | |||||
| #include <dirent.h> | #include <dirent.h> | ||||
| #include <signal.h> | #include <signal.h> | ||||
| #if !defined(_WIN32) && !defined(_WIN64) | |||||
| #include <sys/prctl.h> | #include <sys/prctl.h> | ||||
| #endif | |||||
| #include <sys/stat.h> | #include <sys/stat.h> | ||||
| #include <sys/types.h> | #include <sys/types.h> | ||||
| #include <unistd.h> | #include <unistd.h> | ||||
| @@ -987,8 +987,10 @@ TASK_RETURN_CONTENT ShardReader::ConsumerOneTask(int task_id, uint32_t consumer_ | |||||
| MSRStatus ShardReader::ConsumerByRow(int consumer_id) { | MSRStatus ShardReader::ConsumerByRow(int consumer_id) { | ||||
| // Set thread name | // Set thread name | ||||
| #if !defined(_WIN32) && !defined(_WIN64) | |||||
| auto thread_id = kThreadName + std::to_string(consumer_id); | auto thread_id = kThreadName + std::to_string(consumer_id); | ||||
| prctl(PR_SET_NAME, common::SafeCStr(thread_id), 0, 0, 0); | prctl(PR_SET_NAME, common::SafeCStr(thread_id), 0, 0, 0); | ||||
| #endif | |||||
| // Loop forever | // Loop forever | ||||
| for (;;) { | for (;;) { | ||||
| @@ -1040,8 +1042,10 @@ MSRStatus ShardReader::ReadBlob(const int &shard_id, const uint64_t &page_offset | |||||
| MSRStatus ShardReader::ConsumerByBlock(int consumer_id) { | MSRStatus ShardReader::ConsumerByBlock(int consumer_id) { | ||||
| // Set thread name | // Set thread name | ||||
| #if !defined(_WIN32) && !defined(_WIN64) | |||||
| auto thread_id = kThreadName + std::to_string(consumer_id); | auto thread_id = kThreadName + std::to_string(consumer_id); | ||||
| prctl(PR_SET_NAME, common::SafeCStr(thread_id), 0, 0, 0); | prctl(PR_SET_NAME, common::SafeCStr(thread_id), 0, 0, 0); | ||||
| #endif | |||||
| // Loop forever | // Loop forever | ||||
| for (;;) { | for (;;) { | ||||
| @@ -63,6 +63,15 @@ MSRStatus ShardWriter::Open(const std::vector<std::string> &paths, bool append) | |||||
| MS_LOG(ERROR) << "Securec func failed"; | MS_LOG(ERROR) << "Securec func failed"; | ||||
| return FAILED; | return FAILED; | ||||
| } | } | ||||
| #if defined(_WIN32) || defined(_WIN64) | |||||
| if (_fullpath(resolved_path, dirname(&(buf[0])), PATH_MAX) == nullptr) { | |||||
| MS_LOG(ERROR) << "Invalid file path"; | |||||
| return FAILED; | |||||
| } | |||||
| if (_fullpath(resolved_path, common::SafeCStr(path), PATH_MAX) == nullptr) { | |||||
| MS_LOG(DEBUG) << "Path " << resolved_path; | |||||
| } | |||||
| #else | |||||
| if (realpath(dirname(&(buf[0])), resolved_path) == nullptr) { | if (realpath(dirname(&(buf[0])), resolved_path) == nullptr) { | ||||
| MS_LOG(ERROR) << "Invalid file path"; | MS_LOG(ERROR) << "Invalid file path"; | ||||
| return FAILED; | return FAILED; | ||||
| @@ -70,6 +79,7 @@ MSRStatus ShardWriter::Open(const std::vector<std::string> &paths, bool append) | |||||
| if (realpath(common::SafeCStr(path), resolved_path) == nullptr) { | if (realpath(common::SafeCStr(path), resolved_path) == nullptr) { | ||||
| MS_LOG(DEBUG) << "Path " << resolved_path; | MS_LOG(DEBUG) << "Path " << resolved_path; | ||||
| } | } | ||||
| #endif | |||||
| file_paths_.emplace_back(string(resolved_path)); | file_paths_.emplace_back(string(resolved_path)); | ||||
| } | } | ||||
| @@ -38,10 +38,12 @@ bool StepAllreduceFusion(const FuncGraphPtr &root, const opt::OptimizerPtr &opti | |||||
| (root->has_flag(ALLREDUCE_FUSION_RUN_ONCE_ONLY))) { | (root->has_flag(ALLREDUCE_FUSION_RUN_ONCE_ONLY))) { | ||||
| return changes; | return changes; | ||||
| } | } | ||||
| #if defined(_WIN32) || defined(_WIN64) | |||||
| auto start_time = std::chrono::steady_clock::now(); | |||||
| #else | |||||
| struct timeval start_time, end_time; | struct timeval start_time, end_time; | ||||
| (void)gettimeofday(&start_time, nullptr); | (void)gettimeofday(&start_time, nullptr); | ||||
| #endif | |||||
| MS_LOG(INFO) << "Now entering allreduce fusion"; | MS_LOG(INFO) << "Now entering allreduce fusion"; | ||||
| DumpGraph(root, std::string(ALLREDUCE_FUSION_BEGIN)); | DumpGraph(root, std::string(ALLREDUCE_FUSION_BEGIN)); | ||||
| @@ -63,11 +65,16 @@ bool StepAllreduceFusion(const FuncGraphPtr &root, const opt::OptimizerPtr &opti | |||||
| // allreduce fusion only run once | // allreduce fusion only run once | ||||
| root->flags()[ALLREDUCE_FUSION_RUN_ONCE_ONLY] = true; | root->flags()[ALLREDUCE_FUSION_RUN_ONCE_ONLY] = true; | ||||
| res->results()[pipeline::kStepParallelGraph] = root; | res->results()[pipeline::kStepParallelGraph] = root; | ||||
| #if defined(_WIN32) || defined(_WIN64) | |||||
| auto end_time = std::chrono::steady_clock::now(); | |||||
| std::chrono::duration<double, std::ratio<1, 1000000>> cost = end_time - start_time; | |||||
| MS_LOG(INFO) << "Now leaving allreduce fusion, used time: " << cost.count() << " us"; | |||||
| #else | |||||
| (void)gettimeofday(&end_time, nullptr); | (void)gettimeofday(&end_time, nullptr); | ||||
| uint64_t time = 1000000 * static_cast<uint64_t>(end_time.tv_sec - start_time.tv_sec); | uint64_t time = 1000000 * static_cast<uint64_t>(end_time.tv_sec - start_time.tv_sec); | ||||
| time += static_cast<uint64_t>(end_time.tv_usec - start_time.tv_usec); | time += static_cast<uint64_t>(end_time.tv_usec - start_time.tv_usec); | ||||
| MS_LOG(INFO) << "Now leaving allreduce fusion, used time: " << time << " us"; | MS_LOG(INFO) << "Now leaving allreduce fusion, used time: " << time << " us"; | ||||
| #endif | |||||
| return changes; | return changes; | ||||
| } | } | ||||
| } // namespace parallel | } // namespace parallel | ||||
| @@ -229,7 +229,7 @@ size_t GetLengthOfDataType(const TypePtr &type) { | |||||
| case kNumberTypeInt: | case kNumberTypeInt: | ||||
| return sizeof(int); | return sizeof(int); | ||||
| case kNumberTypeUInt: | case kNumberTypeUInt: | ||||
| return sizeof(uint); | |||||
| return sizeof(unsigned int); | |||||
| case kNumberTypeFloat: | case kNumberTypeFloat: | ||||
| return sizeof(float); | return sizeof(float); | ||||
| default: | default: | ||||
| @@ -264,7 +264,7 @@ bool TaskEmitAction(const ResourcePtr& res) { | |||||
| auto bc_ptr = res->results()[kBackend].cast<compile::BackendPtr>(); | auto bc_ptr = res->results()[kBackend].cast<compile::BackendPtr>(); | ||||
| std::vector<PrimitivePtr> cut_list = compile::nonlinear_ops; | std::vector<PrimitivePtr> cut_list = compile::nonlinear_ops; | ||||
| if (bc_ptr->name() == kMsConvert) { | if (bc_ptr->name() == kMsConvert) { | ||||
| cut_list = compile::ms_nonlinear_ops; | |||||
| cut_list = compile::GetMsNonlinearOps(); | |||||
| } | } | ||||
| std::shared_ptr<CompileGraphs> compile = std::make_shared<CompileGraphs>(bc_ptr, cut_list); | std::shared_ptr<CompileGraphs> compile = std::make_shared<CompileGraphs>(bc_ptr, cut_list); | ||||
| res->results()[kOutput] = compile->CompileAndLink(func_graph); | res->results()[kOutput] = compile->CompileAndLink(func_graph); | ||||
| @@ -29,8 +29,11 @@ | |||||
| #include "parallel/context.h" | #include "parallel/context.h" | ||||
| #include "parallel/device_manager.h" | #include "parallel/device_manager.h" | ||||
| #include "parallel/costmodel_context.h" | #include "parallel/costmodel_context.h" | ||||
| #ifdef ENABLE_GPUQUE | |||||
| #include "device/gpu/distribution/collective_init.h" | #include "device/gpu/distribution/collective_init.h" | ||||
| #else | |||||
| #include "device/gpu/distribution/collective_fake_init.h" | |||||
| #endif | |||||
| namespace py = pybind11; | namespace py = pybind11; | ||||
| using FuncGraph = mindspore::FuncGraph; | using FuncGraph = mindspore::FuncGraph; | ||||
| @@ -297,9 +300,16 @@ PYBIND11_MODULE(_c_expression, m) { | |||||
| (void)py::class_<OpLib, std::shared_ptr<OpLib>>(m, "Oplib") | (void)py::class_<OpLib, std::shared_ptr<OpLib>>(m, "Oplib") | ||||
| .def(py::init()) | .def(py::init()) | ||||
| .def("reg_op", &OpLib::RegOp, "Register op info."); | .def("reg_op", &OpLib::RegOp, "Register op info."); | ||||
| #ifdef ENABLE_GPUQUE | |||||
| (void)m.def("init_gpu_collective", &mindspore::device::gpu::CollectiveInitializer::InitCollective, | (void)m.def("init_gpu_collective", &mindspore::device::gpu::CollectiveInitializer::InitCollective, | ||||
| "Init gpu collective communication mode."); | "Init gpu collective communication mode."); | ||||
| (void)m.def("finalize_gpu_collective", &mindspore::device::gpu::CollectiveInitializer::FinalizeCollective, | (void)m.def("finalize_gpu_collective", &mindspore::device::gpu::CollectiveInitializer::FinalizeCollective, | ||||
| "Finalize gpu collective communication mode."); | "Finalize gpu collective communication mode."); | ||||
| #else | |||||
| (void)m.def("init_gpu_collective", &mindspore::device::gpu::CollectiveFakeInitializer::InitCollective, | |||||
| "Init gpu collective communication mode."); | |||||
| (void)m.def("finalize_gpu_collective", &mindspore::device::gpu::CollectiveFakeInitializer::FinalizeCollective, | |||||
| "Finalize gpu collective communication mode."); | |||||
| #endif | |||||
| } | } | ||||
| @@ -52,17 +52,27 @@ bool PassManager::Run(const FuncGraphPtr &func_graph, const std::vector<PassPtr> | |||||
| size_t num = 0; | size_t num = 0; | ||||
| for (const auto &pass : passes) { | for (const auto &pass : passes) { | ||||
| if (pass != nullptr) { | if (pass != nullptr) { | ||||
| #if defined(_WIN32) || defined(_WIN64) | |||||
| auto start_time = std::chrono::steady_clock::now(); | |||||
| #else | |||||
| struct timeval start_time {}; | struct timeval start_time {}; | ||||
| struct timeval end_time {}; | struct timeval end_time {}; | ||||
| (void)gettimeofday(&start_time, nullptr); | (void)gettimeofday(&start_time, nullptr); | ||||
| #endif | |||||
| if (pass->Run(func_graph)) { | if (pass->Run(func_graph)) { | ||||
| changed = true; | changed = true; | ||||
| } | } | ||||
| #if defined(_WIN32) || defined(_WIN64) | |||||
| auto end_time = std::chrono::steady_clock::now(); | |||||
| std::chrono::duration<double, std::ratio<1, 1000000>> cost = end_time - start_time; | |||||
| MS_LOG(INFO) << "Run pass hwopt_" + name() + "_" << num << "_" + pass->name() + " in " << cost.count() << " us"; | |||||
| #else | |||||
| (void)gettimeofday(&end_time, nullptr); | (void)gettimeofday(&end_time, nullptr); | ||||
| const uint64_t kUSecondInSecond = 1000000; | const uint64_t kUSecondInSecond = 1000000; | ||||
| uint64_t cost = kUSecondInSecond * static_cast<uint64_t>(end_time.tv_sec - start_time.tv_sec); | uint64_t cost = kUSecondInSecond * static_cast<uint64_t>(end_time.tv_sec - start_time.tv_sec); | ||||
| cost += static_cast<uint64_t>(end_time.tv_usec - start_time.tv_usec); | cost += static_cast<uint64_t>(end_time.tv_usec - start_time.tv_usec); | ||||
| MS_LOG(INFO) << "Run pass hwopt_" + name() + "_" << num << "_" + pass->name() + " in " << cost << " us"; | MS_LOG(INFO) << "Run pass hwopt_" + name() + "_" << num << "_" + pass->name() + " in " << cost << " us"; | ||||
| #endif | |||||
| if (save_graphs) { | if (save_graphs) { | ||||
| auto dump_file_path = | auto dump_file_path = | ||||
| save_graphs_path + "/" + "hwopt_" + name() + "_" + std::to_string(num) + "_" + pass->name() + ".ir"; | save_graphs_path + "/" + "hwopt_" + name() + "_" + std::to_string(num) + "_" + pass->name() + ".ir"; | ||||
| @@ -26,12 +26,19 @@ namespace mindspore { | |||||
| #ifdef USE_GLOG | #ifdef USE_GLOG | ||||
| static std::string GetTime() { | static std::string GetTime() { | ||||
| #define BUFLEN 80 | #define BUFLEN 80 | ||||
| static char buf[BUFLEN]; | |||||
| #if defined(_WIN32) || defined(_WIN64) | |||||
| time_t time_seconds = time(0); | |||||
| struct tm now_time; | |||||
| localtime_s(&now_time, &time_seconds); | |||||
| sprintf_s(buf, BUFLEN, "%d-%d-%d %d:%d:%d", now_time.tm_year + 1900, now_time.tm_mon + 1, now_time.tm_mday, | |||||
| now_time.tm_hour, now_time.tm_min, now_time.tm_sec); | |||||
| #else | |||||
| struct timeval cur_time; | struct timeval cur_time; | ||||
| (void)gettimeofday(&cur_time, NULL); | (void)gettimeofday(&cur_time, NULL); | ||||
| struct tm now; | struct tm now; | ||||
| (void)localtime_r(&cur_time.tv_sec, &now); | (void)localtime_r(&cur_time.tv_sec, &now); | ||||
| static char buf[BUFLEN]; | |||||
| (void)strftime(buf, BUFLEN, "%Y-%m-%d-%H:%M:%S", &now); // format date and time | (void)strftime(buf, BUFLEN, "%Y-%m-%d-%H:%M:%S", &now); // format date and time | ||||
| // set micro-second | // set micro-second | ||||
| buf[27] = '\0'; | buf[27] = '\0'; | ||||
| @@ -44,6 +51,7 @@ static std::string GetTime() { | |||||
| buf[idx--] = '.'; | buf[idx--] = '.'; | ||||
| } | } | ||||
| } | } | ||||
| #endif | |||||
| return std::string(buf); | return std::string(buf); | ||||
| } | } | ||||
| @@ -41,8 +41,10 @@ using TypedPrimitiveAbstractClosurePtr = std::shared_ptr<abstract::TypedPrimitiv | |||||
| std::vector<PrimitivePtr> nonlinear_ops = {prim::kPrimReturn, prim::kPrimPartial, prim::kPrimSwitch, | std::vector<PrimitivePtr> nonlinear_ops = {prim::kPrimReturn, prim::kPrimPartial, prim::kPrimSwitch, | ||||
| prim::kPrimMakeTuple}; | prim::kPrimMakeTuple}; | ||||
| std::vector<PrimitivePtr> ms_nonlinear_ops = {prim::kPrimReturn, prim::kPrimPartial, prim::kPrimSwitch}; | |||||
| const std::vector<PrimitivePtr>& GetMsNonlinearOps() { | |||||
| static const std::vector<PrimitivePtr> ms_nonlinear_ops = {prim::kPrimReturn, prim::kPrimPartial, prim::kPrimSwitch}; | |||||
| return ms_nonlinear_ops; | |||||
| } | |||||
| CompileGraph::CompileGraph(const BackendPtr& backend, const std::vector<PrimitivePtr>& cut_list) | CompileGraph::CompileGraph(const BackendPtr& backend, const std::vector<PrimitivePtr>& cut_list) | ||||
| : backend_(backend), cut_list_(cut_list) { | : backend_(backend), cut_list_(cut_list) { | ||||
| @@ -42,7 +42,7 @@ extern const char kGeVm[]; | |||||
| // A sub namespace in ME to support compile related definition. | // A sub namespace in ME to support compile related definition. | ||||
| namespace compile { | namespace compile { | ||||
| extern std::vector<PrimitivePtr> nonlinear_ops; | extern std::vector<PrimitivePtr> nonlinear_ops; | ||||
| extern std::vector<PrimitivePtr> ms_nonlinear_ops; | |||||
| const std::vector<PrimitivePtr>& GetMsNonlinearOps(); | |||||
| using VmEvalFunc = std::function<BaseRef(const VectorRef&)>; | using VmEvalFunc = std::function<BaseRef(const VectorRef&)>; | ||||
| using VmEvalFuncPtr = std::shared_ptr<std::function<BaseRef(const VectorRef&)>>; | using VmEvalFuncPtr = std::shared_ptr<std::function<BaseRef(const VectorRef&)>>; | ||||
| @@ -19,11 +19,13 @@ import sys | |||||
| import os | import os | ||||
| import stat | import stat | ||||
| import time | import time | ||||
| import fcntl | |||||
| import logging | import logging | ||||
| from logging.handlers import RotatingFileHandler | from logging.handlers import RotatingFileHandler | ||||
| import traceback | import traceback | ||||
| import threading | import threading | ||||
| import platform | |||||
| if platform.system() != "Windows": | |||||
| import fcntl | |||||
| __all__ = ['get_level', 'get_log_config'] | __all__ = ['get_level', 'get_log_config'] | ||||
| @@ -90,7 +92,8 @@ class _MultiCompatibleRotatingFileHandler(RotatingFileHandler): | |||||
| # Attain an exclusive lock with bloking mode by `fcntl` module. | # Attain an exclusive lock with bloking mode by `fcntl` module. | ||||
| with open(self.baseFilename, 'a') as file_pointer: | with open(self.baseFilename, 'a') as file_pointer: | ||||
| fcntl.lockf(file_pointer.fileno(), fcntl.LOCK_EX) | |||||
| if platform.system() != "Windows": | |||||
| fcntl.lockf(file_pointer.fileno(), fcntl.LOCK_EX) | |||||
| if self.backupCount > 0: | if self.backupCount > 0: | ||||
| self.rolling_rename() | self.rolling_rename() | ||||
| @@ -1,10 +1,18 @@ | |||||
| SET(CMAKE_BUILD_TYPE "Debug") | SET(CMAKE_BUILD_TYPE "Debug") | ||||
| SET(CMAKE_C_FLAGS_DEBUG "$ENV{CFLAGS} -fPIC -O0 -Wall -Wno-deprecated-declarations -g2 -ggdb -fno-inline-functions -fno-omit-frame-pointer -D_LIBCPP_INLINE_VISIBILITY='' -D'_LIBCPP_EXTERN_TEMPLATE(...)='") | |||||
| if (WIN32) | |||||
| SET(CMAKE_C_FLAGS_DEBUG "$ENV{CFLAGS} -fPIC -O0 -Wall -Wno-deprecated-declarations -g2 -ggdb -fno-inline-functions -fno-omit-frame-pointer") | |||||
| else() | |||||
| SET(CMAKE_C_FLAGS_DEBUG "$ENV{CFLAGS} -fPIC -O0 -Wall -Wno-deprecated-declarations -g2 -ggdb -fno-inline-functions -fno-omit-frame-pointer -D_LIBCPP_INLINE_VISIBILITY='' -D'_LIBCPP_EXTERN_TEMPLATE(...)='") | |||||
| endif() | |||||
| SET(CMAKE_C_FLAGS_RELEASE "$ENV{CFLAGS} -fPIC -O3 -Wall -Wno-deprecated-declarations") | SET(CMAKE_C_FLAGS_RELEASE "$ENV{CFLAGS} -fPIC -O3 -Wall -Wno-deprecated-declarations") | ||||
| set(CMAKE_EXPORT_COMPILE_COMMANDS ON) | set(CMAKE_EXPORT_COMPILE_COMMANDS ON) | ||||
| #add flags | #add flags | ||||
| set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -I/usr/local/include -Werror") | |||||
| if (WIN32) | |||||
| set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -I/usr/local/include") | |||||
| else() | |||||
| set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -I/usr/local/include -Werror") | |||||
| endif() | |||||
| include_directories(./include) | include_directories(./include) | ||||