diff --git a/CMakeLists.txt b/CMakeLists.txt index d595437f3a..9018070688 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -108,6 +108,8 @@ if(MODE_ASCEND_ACL) include(cmake/package_tar.cmake) elseif(CMAKE_SYSTEM_NAME MATCHES "Windows") include(cmake/package_win.cmake) +elseif(CMAKE_SYSTEM_NAME MATCHES "Darwin") + include(cmake/package_mac.cmake) else() include(cmake/package.cmake) endif() diff --git a/cmake/external_libs/absl.cmake b/cmake/external_libs/absl.cmake index 60a174dfc1..bac9370d10 100644 --- a/cmake/external_libs/absl.cmake +++ b/cmake/external_libs/absl.cmake @@ -15,7 +15,11 @@ mindspore_add_pkg(absl LIBS absl_strings absl_throw_delegate absl_raw_logging_internal absl_int128 absl_bad_optional_access URL ${REQ_URL} MD5 ${MD5} - CMAKE_OPTION -DCMAKE_BUILD_TYPE:STRING=Release -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=TRUE) + CMAKE_OPTION + -DCMAKE_BUILD_TYPE:STRING=Release + -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=TRUE + -DCMAKE_CXX_STANDARD=11 + ) include_directories(${absl_INC}) diff --git a/cmake/external_libs/flatbuffers.cmake b/cmake/external_libs/flatbuffers.cmake index b194f20daf..15893c1b49 100644 --- a/cmake/external_libs/flatbuffers.cmake +++ b/cmake/external_libs/flatbuffers.cmake @@ -19,6 +19,9 @@ else() set(MD5 "a27992324c3cbf86dd888268a23d17bd") endif() +if(APPLE) + set(flatbuffers_CXXFLAGS "${flatbuffers_CXXFLAGS} -Wno-deprecated") +endif() if(APPLE) mindspore_add_pkg(flatbuffers VER 2.0.0 diff --git a/cmake/external_libs/grpc.cmake b/cmake/external_libs/grpc.cmake index 85663c6702..fd70480281 100644 --- a/cmake/external_libs/grpc.cmake +++ b/cmake/external_libs/grpc.cmake @@ -1,41 +1,44 @@ set(grpc_USE_STATIC_LIBS OFF) if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin") - set(grpc_CXXFLAGS "-fstack-protector-all -Wno-uninitialized -Wno-unused-parameter -fPIC -D_FORTIFY_SOURCE=2 -O2") + set(grpc_CXXFLAGS "-fstack-protector-all -Wno-uninitialized -Wno-unused-parameter -fPIC -D_FORTIFY_SOURCE=2 -O2 \ + -Dgrpc=mindspore_grpc -Dgrpc_impl=mindspore_grpc_impl -Dgrpc_core=mindspore_grpc_core") elseif(${CMAKE_SYSTEM_NAME} MATCHES "Windows") set(grpc_CXXFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -D_FORTIFY_SOURCE=2 -O2") else() set(grpc_CXXFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -D_FORTIFY_SOURCE=2 -O2 \ - -Dgrpc=mindspore_grpc -Dgrpc_impl=mindspore_grpc_impl -Dgrpc_core=mindspore_grpc_core") + -Dgrpc=mindspore_grpc -Dgrpc_impl=mindspore_grpc_impl -Dgrpc_core=mindspore_grpc_core") if(NOT ENABLE_GLIBCXX) set(grpc_CXXFLAGS "${grpc_CXXFLAGS} -D_GLIBCXX_USE_CXX11_ABI=0") endif() endif() -set(grpc_LDFLAGS "-Wl,-z,relro,-z,now,-z,noexecstack") +if(NOT ${CMAKE_SYSTEM_NAME} MATCHES "Darwin") + set(grpc_LDFLAGS "-Wl,-z,relro,-z,now,-z,noexecstack") +endif() if(EXISTS ${protobuf_ROOT}/lib64) - set(_FINDPACKAGE_PROTOBUF_CONFIG_DIR "${protobuf_ROOT}/lib64/cmake/protobuf") + set(_FINDPACKAGE_PROTOBUF_CONFIG_DIR "${protobuf_ROOT}/lib64/cmake/protobuf") else() - set(_FINDPACKAGE_PROTOBUF_CONFIG_DIR "${protobuf_ROOT}/lib/cmake/protobuf") + set(_FINDPACKAGE_PROTOBUF_CONFIG_DIR "${protobuf_ROOT}/lib/cmake/protobuf") endif() message("grpc using Protobuf_DIR : " ${_FINDPACKAGE_PROTOBUF_CONFIG_DIR}) if(EXISTS ${absl_ROOT}/lib64) - set(_FINDPACKAGE_ABSL_CONFIG_DIR "${absl_ROOT}/lib64/cmake/absl") + set(_FINDPACKAGE_ABSL_CONFIG_DIR "${absl_ROOT}/lib64/cmake/absl") else() - set(_FINDPACKAGE_ABSL_CONFIG_DIR "${absl_ROOT}/lib/cmake/absl") + set(_FINDPACKAGE_ABSL_CONFIG_DIR "${absl_ROOT}/lib/cmake/absl") endif() message("grpc using absl_DIR : " ${_FINDPACKAGE_ABSL_CONFIG_DIR}) if(EXISTS ${re2_ROOT}/lib64) - set(_FINDPACKAGE_RE2_CONFIG_DIR "${re2_ROOT}/lib64/cmake/re2") + set(_FINDPACKAGE_RE2_CONFIG_DIR "${re2_ROOT}/lib64/cmake/re2") else() - set(_FINDPACKAGE_RE2_CONFIG_DIR "${re2_ROOT}/lib/cmake/re2") + set(_FINDPACKAGE_RE2_CONFIG_DIR "${re2_ROOT}/lib/cmake/re2") endif() message("grpc using re2_DIR : " ${_FINDPACKAGE_RE2_CONFIG_DIR}) if(EXISTS ${openssl_ROOT}) - set(_CMAKE_ARGS_OPENSSL_ROOT_DIR "-DOPENSSL_ROOT_DIR:PATH=${openssl_ROOT}") + set(_CMAKE_ARGS_OPENSSL_ROOT_DIR "-DOPENSSL_ROOT_DIR:PATH=${openssl_ROOT}") endif() if(ENABLE_GITEE) @@ -78,7 +81,7 @@ add_library(mindspore::grpc++ ALIAS grpc::mindspore_grpc++) # link other grpc libs target_link_libraries(grpc::mindspore_grpc++ INTERFACE grpc::mindspore_grpc grpc::mindspore_gpr grpc::mindspore_upb - grpc::mindspore_address_sorting) + grpc::mindspore_address_sorting) # modify mindspore macro define add_compile_definitions(grpc=mindspore_grpc) diff --git a/cmake/external_libs/openssl.cmake b/cmake/external_libs/openssl.cmake index ff174ec9af..61c67be428 100644 --- a/cmake/external_libs/openssl.cmake +++ b/cmake/external_libs/openssl.cmake @@ -12,7 +12,7 @@ else() set(OPENSSL_PATCH_ROOT ${CMAKE_SOURCE_DIR}/third_party/patch/openssl) endif() -if(${CMAKE_SYSTEM_NAME} MATCHES "Linux") +if(${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR APPLE) mindspore_add_pkg(openssl VER 1.1.1k LIBS ssl crypto diff --git a/cmake/external_libs/sentencepiece.cmake b/cmake/external_libs/sentencepiece.cmake index 5427503ae7..f942d2c19a 100644 --- a/cmake/external_libs/sentencepiece.cmake +++ b/cmake/external_libs/sentencepiece.cmake @@ -27,7 +27,7 @@ else() LIBS sentencepiece sentencepiece_train URL ${REQ_URL} CMAKE_OPTION -DCMAKE_BUILD_TYPE=Release -DSPM_USE_BUILTIN_PROTOBUF=OFF -DSPM_ENABLE_SHARED=OFF - -DPROTOBUF_INC=${protobuf_INC} + -DPROTOBUF_INC=${protobuf_INC} -DCMAKE_CXX_STANDARD=11 MD5 ${MD5} PATCHES ${CMAKE_SOURCE_DIR}/third_party/patch/sentencepiece/sentencepiece.patch001_cpu ) diff --git a/cmake/options.cmake b/cmake/options.cmake index 66306dc983..d94fc1d64c 100644 --- a/cmake/options.cmake +++ b/cmake/options.cmake @@ -28,6 +28,11 @@ option(MODE_ASCEND_ACL "supports ascend acl mode only" OFF) option(ENABLE_SYM_FILE "enable sym file" OFF) option(BUILD_DEV_MODE "MindSpore build nightly dev mode" OFF) + +if(CMAKE_SYSTEM_NAME MATCHES "Darwin") + set(ENABLE_DEBUGGER OFF) +endif() + if(NOT ENABLE_D AND NOT ENABLE_TESTCASES AND NOT ENABLE_ACL AND NOT ENABLE_GE) set(ENABLE_GLIBCXX ON) endif() @@ -137,7 +142,7 @@ endif() if(ENABLE_DEBUGGER OR ENABLE_TESTCASES) set(MS_BUILD_GRPC ON) endif() -if(ENABLE_MINDDATA AND NOT CMAKE_SYSTEM_NAME MATCHES "Windows") +if(ENABLE_MINDDATA AND NOT CMAKE_SYSTEM_NAME MATCHES "Windows" AND NOT CMAKE_SYSTEM_NAME MATCHES "Darwin") set(MS_BUILD_GRPC ON) endif() @@ -149,7 +154,7 @@ if(ENABLE_ACL AND NOT ENABLE_D) set(MODE_ASCEND_ACL ON) endif() -if(ENABLE_CPU AND NOT WIN32) +if(ENABLE_CPU AND NOT WIN32 AND NOT CMAKE_SYSTEM_NAME MATCHES "Darwin") add_compile_definitions(ENABLE_ARMOUR) endif() diff --git a/cmake/package_mac.cmake b/cmake/package_mac.cmake new file mode 100644 index 0000000000..e273925c7f --- /dev/null +++ b/cmake/package_mac.cmake @@ -0,0 +1,247 @@ +# include dependency +include(CMakePackageConfigHelpers) +include(GNUInstallDirs) + +# set package information +set(CPACK_PACKAGE_NAME ${PROJECT_NAME}) +set(CPACK_GENERATOR "External") +set(CPACK_CMAKE_GENERATOR "Ninja") +set(CPACK_EXTERNAL_PACKAGE_SCRIPT ${CMAKE_SOURCE_DIR}/cmake/package_script.cmake) +set(CPACK_EXTERNAL_ENABLE_STAGING true) +set(CPACK_TEMPORARY_PACKAGE_FILE_NAME ${BUILD_PATH}/package/mindspore) +set(CPACK_TEMPORARY_INSTALL_DIRECTORY ${BUILD_PATH}/package/mindspore) +set(CPACK_PACK_ROOT_DIR ${BUILD_PATH}/package/) +set(CPACK_CMAKE_SOURCE_DIR ${CMAKE_SOURCE_DIR}) +set(CPACK_ENABLE_SYM_FILE ${ENABLE_SYM_FILE}) +set(CPACK_CMAKE_BUILD_TYPE ${CMAKE_BUILD_TYPE}) +set(CPACK_PYTHON_EXE ${Python3_EXECUTABLE}) +set(CPACK_PYTHON_VERSION ${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR}) + + +if(ENABLE_GPU) + set(CPACK_MS_BACKEND "ms") + set(CPACK_MS_TARGET "gpu or cpu") + set(CPACK_MS_PACKAGE_NAME "mindspore-gpu") +elseif(ENABLE_CPU) + set(CPACK_MS_BACKEND "ms") + set(CPACK_MS_TARGET "cpu") + set(CPACK_MS_PACKAGE_NAME "mindspore") +else() + set(CPACK_MS_BACKEND "debug") + set(CPACK_MS_TARGET "ascend or gpu or cpu") + set(CPACK_MS_PACKAGE_NAME "mindspore") +endif() +include(CPack) + +# set install path +set(INSTALL_LIB_DIR ${CMAKE_INSTALL_LIBDIR} CACHE PATH "Installation directory for libraries") +set(INSTALL_PY_DIR ".") +set(INSTALL_BASE_DIR ".") +set(INSTALL_BIN_DIR "bin") +set(INSTALL_CFG_DIR "config") +set(INSTALL_LIB_DIR "lib") + +message("offline debugger does not support macosx system temporarily") + +# set package files +install( + TARGETS _c_expression + DESTINATION ${INSTALL_BASE_DIR} + COMPONENT mindspore +) + +install( + TARGETS mindspore_shared_lib + DESTINATION ${INSTALL_LIB_DIR} + COMPONENT mindspore +) + +install( + TARGETS mindspore_gvar + DESTINATION ${INSTALL_LIB_DIR} + COMPONENT mindspore +) + +if(USE_GLOG) + file(GLOB_RECURSE GLOG_LIB_LIST ${glog_LIBPATH}/libmindspore_glog*) + install( + FILES ${GLOG_LIB_LIST} + DESTINATION ${INSTALL_LIB_DIR} + COMPONENT mindspore + ) +endif() + +if(MS_BUILD_GRPC) + file(GLOB_RECURSE GRPC_LIB_LIST + ${grpc_LIBPATH}/libmindspore* + ) + install( + FILES ${GRPC_LIB_LIST} + DESTINATION ${INSTALL_LIB_DIR} + COMPONENT mindspore + ) +endif() + +if(ENABLE_MINDDATA) + install( + TARGETS _c_dataengine _c_mindrecord + DESTINATION ${INSTALL_BASE_DIR} + COMPONENT mindspore + ) + + file(GLOB_RECURSE OPENCV_LIB_LIST + ${opencv_LIBPATH}/libopencv_core* + ${opencv_LIBPATH}/libopencv_imgcodecs* + ${opencv_LIBPATH}/libopencv_imgproc* + ) + install( + FILES ${OPENCV_LIB_LIST} + DESTINATION ${INSTALL_LIB_DIR} + COMPONENT mindspore + ) + file(GLOB_RECURSE JPEG_LIB_LIST + ${jpeg_turbo_LIBPATH}/*.dylib + ) + install( + FILES ${JPEG_LIB_LIST} + DESTINATION ${INSTALL_LIB_DIR} + COMPONENT mindspore + ) + file(GLOB_RECURSE TINYXML2_LIB_LIST ${tinyxml2_LIBPATH}/libtinyxml2*) + install( + FILES ${TINYXML2_LIB_LIST} + DESTINATION ${INSTALL_LIB_DIR} + COMPONENT mindspore + ) + file(GLOB_RECURSE ICU4C_LIB_LIST + ${icu4c_LIBPATH}/libicuuc* + ${icu4c_LIBPATH}/libicudata* + ${icu4c_LIBPATH}/libicui18n* + ) + install( + FILES ${ICU4C_LIB_LIST} + DESTINATION ${INSTALL_LIB_DIR} + COMPONENT mindspore + ) +endif() + +if(ENABLE_CPU) + file(GLOB_RECURSE DNNL_LIB_LIST ${onednn_LIBPATH}/libdnnl*) + install( + FILES ${DNNL_LIB_LIST} + DESTINATION ${INSTALL_LIB_DIR} + COMPONENT mindspore + ) + install( + TARGETS nnacl + DESTINATION ${INSTALL_LIB_DIR} + COMPONENT mindspore + ) +endif() + +if(ENABLE_MPI) + if(ENABLE_GPU) + install( + TARGETS _ms_mpi + DESTINATION ${INSTALL_BASE_DIR} + COMPONENT mindspore + ) + endif() + if(ENABLE_CPU) + install( + TARGETS mpi_adapter + DESTINATION ${INSTALL_LIB_DIR} + COMPONENT mindspore + ) + endif() +endif() + +if(ENABLE_GPU) + if(ENABLE_MPI) + install( + TARGETS gpu_collective + DESTINATION ${INSTALL_LIB_DIR} + COMPONENT mindspore + ) + endif() + install( + TARGETS gpu_queue + DESTINATION ${INSTALL_LIB_DIR} + COMPONENT mindspore + ) +endif() + +get_filename_component(CXX_DIR ${CMAKE_CXX_COMPILER} PATH) +file(GLOB CXX_LIB_LIST ${CXX_DIR}/*.dylib) + +file(GLOB JPEG_LIB_LIST ${jpeg_turbo_LIBPATH}/*.dylib) +file(GLOB SQLITE_LIB_LIST ${sqlite_LIBPATH}/*.dylib) +install( + FILES ${CXX_LIB_LIST} ${SQLITE_LIB_LIST} + DESTINATION ${INSTALL_LIB_DIR} + COMPONENT mindspore +) + +# set python files +file(GLOB MS_PY_LIST ${CMAKE_SOURCE_DIR}/mindspore/*.py) +install( + FILES ${MS_PY_LIST} + DESTINATION ${INSTALL_PY_DIR} + COMPONENT mindspore +) + +install( + DIRECTORY + ${CMAKE_SOURCE_DIR}/mindspore/nn + ${CMAKE_SOURCE_DIR}/mindspore/_extends + ${CMAKE_SOURCE_DIR}/mindspore/parallel + ${CMAKE_SOURCE_DIR}/mindspore/mindrecord + ${CMAKE_SOURCE_DIR}/mindspore/numpy + ${CMAKE_SOURCE_DIR}/mindspore/scipy + ${CMAKE_SOURCE_DIR}/mindspore/train + ${CMAKE_SOURCE_DIR}/mindspore/boost + ${CMAKE_SOURCE_DIR}/mindspore/common + ${CMAKE_SOURCE_DIR}/mindspore/ops + ${CMAKE_SOURCE_DIR}/mindspore/communication + ${CMAKE_SOURCE_DIR}/mindspore/profiler + ${CMAKE_SOURCE_DIR}/mindspore/compression + ${CMAKE_SOURCE_DIR}/mindspore/run_check + DESTINATION ${INSTALL_PY_DIR} + COMPONENT mindspore +) + +if(EXISTS ${CMAKE_SOURCE_DIR}/mindspore/dataset) + install( + DIRECTORY ${CMAKE_SOURCE_DIR}/mindspore/dataset + DESTINATION ${INSTALL_PY_DIR} + COMPONENT mindspore + ) +endif() + +## Public header files +install( + DIRECTORY ${CMAKE_SOURCE_DIR}/include + DESTINATION ${INSTALL_BASE_DIR} + COMPONENT mindspore +) + +## Public header files for minddata +install( + FILES ${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/include/dataset/config.h + ${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/include/dataset/constants.h + ${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/include/dataset/execute.h + ${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/include/dataset/text.h + ${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/include/dataset/transforms.h + ${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/include/dataset/vision.h + ${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/include/dataset/vision_lite.h + ${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/include/dataset/vision_ascend.h + DESTINATION ${INSTALL_BASE_DIR}/include/dataset + COMPONENT mindspore +) + +## config files +install( + FILES ${CMAKE_SOURCE_DIR}/config/op_info.config + DESTINATION ${INSTALL_CFG_DIR} + COMPONENT mindspore +) diff --git a/mindspore/ccsrc/CMakeLists.txt b/mindspore/ccsrc/CMakeLists.txt index cca4a5bdac..1363726398 100644 --- a/mindspore/ccsrc/CMakeLists.txt +++ b/mindspore/ccsrc/CMakeLists.txt @@ -17,6 +17,8 @@ if(ENABLE_CPU) if(${CMAKE_HOST_SYSTEM_PROCESSOR} MATCHES "aarch64") set(PLATFORM_ARM64 "on") set(X86_64_SIMD "off") + elseif(CMAKE_SYSTEM_NAME MATCHES "Darwin") + set(X86_64_SIMD "off") elseif("${X86_64_SIMD}" STREQUAL "off" AND NOT ${ENABLE_ASAN}) set(X86_64_SIMD "avx") endif() @@ -24,10 +26,13 @@ if(ENABLE_CPU) add_subdirectory(backend/kernel_compiler/cpu/nnacl) endif() -if(NOT(CMAKE_SYSTEM_NAME MATCHES "Darwin")) +if(NOT CMAKE_SYSTEM_NAME MATCHES "Darwin") link_directories(${CMAKE_SOURCE_DIR}/build/mindspore/graphengine) else() - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-delete-non-abstract-non-virtual-dtor") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 \ + -Wno-delete-non-abstract-non-virtual-dtor -Wno-unused-private-field -Wno-overloaded-virtual \ + -Wno-unused-const-variable -Wno-pessimizing-move -Wno-range-loop-analysis -Wno-mismatched-tags \ + -Wno-c++11-narrowing") endif() if(CMAKE_SYSTEM_NAME MATCHES "Windows") @@ -240,7 +245,11 @@ target_link_libraries(mindspore mindspore_core) if(ENABLE_DEBUGGER) # debugger: link grpc - target_link_libraries(mindspore -Wl,--no-as-needed mindspore::grpc++) + if(CMAKE_SYSTEM_NAME MATCHES "Darwin") + target_link_libraries(mindspore mindspore::grpc++) + else() + target_link_libraries(mindspore -Wl,--no-as-needed mindspore::grpc++) + endif() endif() target_link_libraries(mindspore securec mindspore::flatbuffers) @@ -351,7 +360,7 @@ if(CMAKE_SYSTEM_NAME MATCHES "Windows") elseif(CMAKE_SYSTEM_NAME MATCHES "Darwin") target_link_libraries(mindspore mindspore::pybind11_module) target_link_libraries(mindspore mindspore_gvar) - target_link_libraries(_c_expression PRIVATE -Wl,-force_load mindspore mindspore_core -Wl,-noall_load) + target_link_libraries(_c_expression PRIVATE -Wl,-all_load mindspore mindspore_core -Wl,-noall_load) else() if(ENABLE_CPU AND NOT WIN32) target_link_libraries(mindspore proto_input mindspore::protobuf diff --git a/mindspore/ccsrc/backend/kernel_compiler/CMakeLists.txt b/mindspore/ccsrc/backend/kernel_compiler/CMakeLists.txt index 1438c94328..21f2ccfbc3 100644 --- a/mindspore/ccsrc/backend/kernel_compiler/CMakeLists.txt +++ b/mindspore/ccsrc/backend/kernel_compiler/CMakeLists.txt @@ -6,7 +6,7 @@ file(GLOB_RECURSE KERNEL_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} ) if(CMAKE_SYSTEM_NAME MATCHES "Darwin") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-delete-non-abstract-non-virtual-dtor") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-delete-non-abstract-non-virtual-dtor -Wno-overloaded-virtual") endif() if(ENABLE_D) @@ -44,7 +44,7 @@ if(ENABLE_CPU) endif() endif() -if(NOT ENABLE_CPU OR WIN32) +if(NOT ENABLE_CPU OR WIN32 OR APPLE) list(REMOVE_ITEM CPU_SRC_LIST "cpu/ps/apply_momentum_ps_kernel.cc") list(REMOVE_ITEM CPU_SRC_LIST "cpu/ps/embedding_look_up_proxy_kernel.cc") list(REMOVE_ITEM CPU_SRC_LIST "cpu/ps/embedding_look_up_ps_kernel.cc") diff --git a/mindspore/ccsrc/backend/kernel_compiler/cpu/cpu_kernel_factory.cc b/mindspore/ccsrc/backend/kernel_compiler/cpu/cpu_kernel_factory.cc index 7e01e30bdf..4952a5ab91 100644 --- a/mindspore/ccsrc/backend/kernel_compiler/cpu/cpu_kernel_factory.cc +++ b/mindspore/ccsrc/backend/kernel_compiler/cpu/cpu_kernel_factory.cc @@ -37,7 +37,7 @@ CPUKernelFactory &CPUKernelFactory::GetInstance() { void CPUKernelFactory::Register(const std::string &kernel_name, const KernelAttr &kernel_attr, CPUKernelCreator &&kernel_creator) { (void)name_to_attr_creator_[kernel_name].emplace_back(kernel_attr, kernel_creator); -#if !defined(_WIN32) && !defined(_WIN64) +#if !defined(_WIN32) && !defined(_WIN64) && !defined(__APPLE__) MS_LOG(DEBUG) << "CPUKernelFactory register operator: " << kernel_name; #endif } diff --git a/mindspore/ccsrc/backend/kernel_compiler/cpu/eigen/qr_cpu_kernel.cc b/mindspore/ccsrc/backend/kernel_compiler/cpu/eigen/qr_cpu_kernel.cc index 472d26d4a9..1c8d9a124c 100644 --- a/mindspore/ccsrc/backend/kernel_compiler/cpu/eigen/qr_cpu_kernel.cc +++ b/mindspore/ccsrc/backend/kernel_compiler/cpu/eigen/qr_cpu_kernel.cc @@ -77,10 +77,10 @@ bool QRCPUKernel::Launch(const std::vector &inputs, const auto householder_qr = input_a.householderQr(); if (economic_) { // r_row_ = std::min(a_row_, a_col_) - output_r = Eigen::MatrixXd::Identity(r_row_, a_row_).cast() * + output_r = Eigen::MatrixXd::Identity(r_row_, a_row_).template cast() * householder_qr.matrixQR().template triangularView(); // q_col_ = std::min(a_row_, a_col_) - output_q = householder_qr.householderQ() * Eigen::MatrixXd::Identity(q_row_, q_col_).cast(); + output_q = householder_qr.householderQ() * Eigen::MatrixXd::Identity(q_row_, q_col_).template cast(); } else { output_r = householder_qr.matrixQR().template triangularView(); output_q = householder_qr.householderQ(); diff --git a/mindspore/ccsrc/backend/kernel_compiler/cpu/nnacl/CMakeLists.txt b/mindspore/ccsrc/backend/kernel_compiler/cpu/nnacl/CMakeLists.txt index 5d2ce73305..53be8014c9 100644 --- a/mindspore/ccsrc/backend/kernel_compiler/cpu/nnacl/CMakeLists.txt +++ b/mindspore/ccsrc/backend/kernel_compiler/cpu/nnacl/CMakeLists.txt @@ -136,7 +136,9 @@ if(ENABLE_CPU) target_compile_options(nnacl_mid PRIVATE -fPIC) add_library(nnacl SHARED $) if(NOT CMAKE_SYSTEM_NAME MATCHES "Windows") - target_link_options(nnacl PRIVATE -Wl,-z,relro,-z,now,-z,noexecstack) + if(NOT CMAKE_SYSTEM_NAME MATCHES "Darwin") + target_link_options(nnacl PRIVATE -Wl,-z,relro,-z,now,-z,noexecstack) + endif() if("${CMAKE_BUILD_TYPE}" STREQUAL "Release") target_link_options(nnacl PRIVATE -s) endif() diff --git a/mindspore/ccsrc/backend/kernel_compiler/cpu/rl/buffer_sample_cpu_kernel.h b/mindspore/ccsrc/backend/kernel_compiler/cpu/rl/buffer_sample_cpu_kernel.h index bca2b885d9..0be355bc59 100644 --- a/mindspore/ccsrc/backend/kernel_compiler/cpu/rl/buffer_sample_cpu_kernel.h +++ b/mindspore/ccsrc/backend/kernel_compiler/cpu/rl/buffer_sample_cpu_kernel.h @@ -81,7 +81,11 @@ class BufferCPUSampleKernel : public CPUKernel { for (size_t i = 0; i < IntToSize(count_addr[0]); ++i) { (void)indexes.emplace_back(i); } +#if !defined(__APPLE__) random_shuffle(indexes.begin(), indexes.end(), [&](int i) { return std::rand() % i; }); +#else + std::shuffle(indexes.begin(), indexes.end(), generator_); +#endif } else { std::uniform_int_distribution<> distrib(0, count_addr[0]); for (size_t i = 0; i < batch_size_; ++i) { diff --git a/mindspore/ccsrc/backend/optimizer/CMakeLists.txt b/mindspore/ccsrc/backend/optimizer/CMakeLists.txt index 1f43c2099f..0529aa6125 100644 --- a/mindspore/ccsrc/backend/optimizer/CMakeLists.txt +++ b/mindspore/ccsrc/backend/optimizer/CMakeLists.txt @@ -30,7 +30,7 @@ if(ENABLE_GPU_INFER) endif() if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin") - set(CMAKE_CXX_FLAGS_RELEASE "$ENV{CXXFLAGS} -Wno-user-defined-warnings -Wno-inconsistent-missing-override + set(CMAKE_CXX_FLAGS_RELEASE "$ENV{CXXFLAGS} -Wno-user-defined-warnings -Wno-inconsistent-missing-override \ -Wno-overloaded-virtual -Wno-unused-const-variable -Wno-pessimizing-move") endif() diff --git a/mindspore/ccsrc/backend/session/CMakeLists.txt b/mindspore/ccsrc/backend/session/CMakeLists.txt index aec86d2bca..136880690d 100644 --- a/mindspore/ccsrc/backend/session/CMakeLists.txt +++ b/mindspore/ccsrc/backend/session/CMakeLists.txt @@ -15,7 +15,7 @@ if("${ENABLE_HIDDEN}" STREQUAL "OFF") endif() if(CMAKE_SYSTEM_NAME MATCHES "Darwin") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-overloaded-virtual") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-overloaded-virtual -Wno-delete-abstract-non-virtual-dtor") endif() if(ENABLE_GPU) diff --git a/mindspore/ccsrc/backend/session/cpu_session.cc b/mindspore/ccsrc/backend/session/cpu_session.cc index be776b7146..064a971e77 100644 --- a/mindspore/ccsrc/backend/session/cpu_session.cc +++ b/mindspore/ccsrc/backend/session/cpu_session.cc @@ -85,7 +85,7 @@ void CPUSession::Reorder(std::vector *node_list) { AnfAlgo::ReorderPos void CPUSession::Optimize(const std::shared_ptr &kernel_graph) { auto optimizer = std::make_shared(); auto pm = std::make_shared(); -#if ((defined ENABLE_CPU) && (!defined _WIN32)) +#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__)) auto ms_context = MsContext::GetInstance(); MS_EXCEPTION_IF_NULL(ms_context); if (ms_context->get_param(MS_CTX_EXECUTION_MODE) != kPynativeMode && ps::PSContext::instance()->is_ps_mode()) { @@ -205,7 +205,7 @@ void CPUSession::PreExecuteGraph(const std::shared_ptr &kernel_grap MS_LOG(INFO) << "Bind input output address"; runtime_.BindInputOutput(kernel_graph.get(), inputs, outputs); -#if ((defined ENABLE_CPU) && (!defined _WIN32)) +#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__)) InitPSParamAndOptim(kernel_graph, inputs); #endif } diff --git a/mindspore/ccsrc/backend/session/executor.cc b/mindspore/ccsrc/backend/session/executor.cc index 801114a1d8..c85ba3be61 100644 --- a/mindspore/ccsrc/backend/session/executor.cc +++ b/mindspore/ccsrc/backend/session/executor.cc @@ -23,7 +23,7 @@ #include "utils/comm_manager.h" #include "utils/scoped_long_running.h" #include "pybind_api/ir/tensor_py.h" -#if ((defined ENABLE_CPU) && (!defined _WIN32)) +#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__)) #include "ps/ps_cache/ps_cache_manager.h" #endif diff --git a/mindspore/ccsrc/backend/session/session_basic.cc b/mindspore/ccsrc/backend/session/session_basic.cc index 4aefd6bae9..9f7bdb3c8a 100644 --- a/mindspore/ccsrc/backend/session/session_basic.cc +++ b/mindspore/ccsrc/backend/session/session_basic.cc @@ -44,7 +44,7 @@ #include "utils/file_utils.h" #include "utils/trace_base.h" #include "frontend/parallel/context.h" -#if ((defined ENABLE_CPU) && (!defined _WIN32)) +#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__)) #include "ps/ps_cache/ps_cache_manager.h" #include "ps/constants.h" #include "ps/util.h" @@ -2569,7 +2569,7 @@ void SessionBasic::DumpGraph(const std::shared_ptr &kernel_graph) { void SessionBasic::UnifyMindIR(const KernelGraphPtr &graph) { opt::CommonUnifyMindIR(graph); } -#if ((defined ENABLE_CPU) && (!defined _WIN32)) +#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__)) void SessionBasic::InitPsWorker(const KernelGraphPtr &kernel_graph) { if (!ps::PSContext::instance()->is_worker()) { return; diff --git a/mindspore/ccsrc/backend/session/session_basic.h b/mindspore/ccsrc/backend/session/session_basic.h index e8e0e19906..40b50f41f5 100644 --- a/mindspore/ccsrc/backend/session/session_basic.h +++ b/mindspore/ccsrc/backend/session/session_basic.h @@ -309,7 +309,7 @@ class SessionBasic : public std::enable_shared_from_this { std::vector GetAllReduceSplitIndex(); virtual std::string GetCommWorldGroup() { return std::string(); } void DumpGraph(const std::shared_ptr &kernel_graph); -#if ((defined ENABLE_CPU) && (!defined _WIN32)) +#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__)) void CheckPSModeConsistence(const KernelGraphPtr &kernel_graph) const; void GetBatchElements(const AnfNodePtr &kernel_node) const; void InitPsWorker(const KernelGraphPtr &kernel_graph); diff --git a/mindspore/ccsrc/cxx_api/CMakeLists.txt b/mindspore/ccsrc/cxx_api/CMakeLists.txt index 5de840e99c..180e27508e 100644 --- a/mindspore/ccsrc/cxx_api/CMakeLists.txt +++ b/mindspore/ccsrc/cxx_api/CMakeLists.txt @@ -88,21 +88,16 @@ if(NOT BUILD_LITE) set_target_properties(mindspore_shared_lib PROPERTIES OUTPUT_NAME mindspore) endif() -if(CMAKE_SYSTEM_NAME MATCHES "Darwin") - target_link_libraries(mindspore_shared_lib PRIVATE ${PYTHON_LIBRARIES} ${SECUREC_LIBRARY} - -Wl,-force_load mindspore -Wl,-noall_load mindspore_core proto_input mindspore_gvar mindspore::protobuf) +if(ENABLE_D OR ENABLE_GPU) + target_link_libraries(mindspore_shared_lib PRIVATE -Wl,--as-needed ${PYTHON_LIBRARIES} ${SECUREC_LIBRARY} + -Wl,--whole-archive mindspore -Wl,--no-whole-archive mindspore_core proto_input mindspore_gvar + mindspore::protobuf) else() - if(ENABLE_D OR ENABLE_GPU) - target_link_libraries(mindspore_shared_lib PRIVATE -Wl,--as-needed ${PYTHON_LIBRARIES} ${SECUREC_LIBRARY} - -Wl,--whole-archive mindspore -Wl,--no-whole-archive mindspore_core proto_input mindspore_gvar - mindspore::protobuf) + if(BUILD_LITE) + target_link_libraries(mindspore_shared_lib PRIVATE ${SECUREC_LIBRARY}) else() - if(BUILD_LITE) - target_link_libraries(mindspore_shared_lib PRIVATE ${SECUREC_LIBRARY}) - else() - target_link_libraries(mindspore_shared_lib PRIVATE ${PY3_LIBG} ${SECUREC_LIBRARY} - mindspore mindspore_core proto_input mindspore_gvar mindspore::protobuf) - endif() + target_link_libraries(mindspore_shared_lib PRIVATE ${PY3_LIBG} ${SECUREC_LIBRARY} + mindspore mindspore_core proto_input mindspore_gvar mindspore::protobuf) endif() endif() diff --git a/mindspore/ccsrc/debug/CMakeLists.txt b/mindspore/ccsrc/debug/CMakeLists.txt index fa2e8e74f8..1b057c8c6f 100644 --- a/mindspore/ccsrc/debug/CMakeLists.txt +++ b/mindspore/ccsrc/debug/CMakeLists.txt @@ -33,6 +33,10 @@ if("${ENABLE_HIDDEN}" STREQUAL "OFF") string(REPLACE " -fvisibility=hidden" " -fvisibility=default" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") endif() +if(CMAKE_SYSTEM_NAME MATCHES "Darwin") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-private-field") +endif() + if(ENABLE_DEBUGGER) list(APPEND _DEBUG_SRC_LIST "${CMAKE_CURRENT_SOURCE_DIR}/debugger/debugger.cc" @@ -52,7 +56,7 @@ if(NOT ENABLE_SECURITY) "${CMAKE_CURRENT_SOURCE_DIR}/data_dump/dump_utils.cc" "${CMAKE_CURRENT_SOURCE_DIR}/data_dump/npy_header.cc" ) - if(NOT CMAKE_SYSTEM_NAME MATCHES "Windows") + if(NOT CMAKE_SYSTEM_NAME MATCHES "Windows" AND NOT CMAKE_SYSTEM_NAME MATCHES "Darwin") list(APPEND _DEBUG_SRC_LIST "${CMAKE_CURRENT_SOURCE_DIR}/common.cc" "${CMAKE_CURRENT_SOURCE_DIR}/data_dump/e2e_dump.cc" @@ -69,7 +73,11 @@ if(ENABLE_DEBUGGER) set_property(SOURCE ${_OFFLINE_SRC_LIST} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_OFFLINE_DEBUG) add_library(_mindspore_offline_debug SHARED ${_OFFLINE_SRC_LIST}) - target_link_libraries(_mindspore_offline_debug PRIVATE mindspore::glog mindspore_gvar) + target_link_libraries(_mindspore_offline_debug PRIVATE + mindspore::pybind11_module + mindspore::glog + mindspore_gvar + ) set_target_properties(_mindspore_offline_debug PROPERTIES PREFIX "${PYTHON_MODULE_PREFIX}" SUFFIX "${PYTHON_MODULE_EXTENSION}" diff --git a/mindspore/ccsrc/debug/common.h b/mindspore/ccsrc/debug/common.h index 1d63e5afa9..2786ec34d4 100644 --- a/mindspore/ccsrc/debug/common.h +++ b/mindspore/ccsrc/debug/common.h @@ -69,7 +69,11 @@ inline std::string GetSaveGraphsPathName(const std::string &file_name, const std inline std::string ErrnoToString(const int error_number) { std::ostringstream ret_info; ret_info << " Errno: " << error_number; -#if defined(SYSTEM_ENV_POSIX) +#if defined(__APPLE__) + char err_info[MAX_FILENAME_LENGTH]; + (void)strerror_r(error_number, err_info, sizeof(err_info)); + ret_info << ", ErrInfo: " << err_info; +#elif defined(SYSTEM_ENV_POSIX) char err_info[MAX_FILENAME_LENGTH]; char *ret = strerror_r(error_number, err_info, sizeof(err_info)); if (ret != nullptr) { diff --git a/mindspore/ccsrc/debug/debug_services.cc b/mindspore/ccsrc/debug/debug_services.cc index 98e81213a4..f2a7b7589e 100644 --- a/mindspore/ccsrc/debug/debug_services.cc +++ b/mindspore/ccsrc/debug/debug_services.cc @@ -1715,7 +1715,11 @@ std::string DebugServices::RealPath(const std::string &input_path) { } uint64_t DebugServices::BytestoUInt64(const std::vector &buffer) { +#if defined(__APPLE__) + return *reinterpret_cast(buffer.data()); +#else return le64toh(*reinterpret_cast(buffer.data())); +#endif } bool DebugServices::TensorExistsInCurrent(const std::string &tensor_name) { diff --git a/mindspore/ccsrc/debug/debugger/offline_debug/dbg_services.cc b/mindspore/ccsrc/debug/debugger/offline_debug/dbg_services.cc index 62904d515c..b8eb393986 100644 --- a/mindspore/ccsrc/debug/debugger/offline_debug/dbg_services.cc +++ b/mindspore/ccsrc/debug/debugger/offline_debug/dbg_services.cc @@ -33,10 +33,17 @@ DbgServices &DbgServices::operator=(const DbgServices &other) { return *this; } +#if !defined(__APPLE__) DbgServices::~DbgServices() noexcept { MS_LOG(INFO) << "cpp DbgServices object is deleted"; debug_services_ = nullptr; } +#else +DbgServices::~DbgServices() { + MS_LOG(INFO) << "cpp DbgServices object is deleted"; + debug_services_ = nullptr; +} +#endif std::string DbgServices::GetVersion() const { MS_LOG(INFO) << "get version is called"; diff --git a/mindspore/ccsrc/fl/CMakeLists.txt b/mindspore/ccsrc/fl/CMakeLists.txt index 11cbf4247e..67c438f851 100644 --- a/mindspore/ccsrc/fl/CMakeLists.txt +++ b/mindspore/ccsrc/fl/CMakeLists.txt @@ -1,6 +1,6 @@ file(GLOB_RECURSE _FL_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc") -if(NOT ENABLE_CPU OR WIN32) +if(NOT ENABLE_CPU OR WIN32 OR APPLE) list(REMOVE_ITEM _FL_SRC_FILES "server/kernel/apply_momentum_kernel.cc") list(REMOVE_ITEM _FL_SRC_FILES "server/kernel/aggregation_kernel_factory.cc") list(REMOVE_ITEM _FL_SRC_FILES "server/kernel/dense_grad_accum_kernel.cc") diff --git a/mindspore/ccsrc/frontend/optimizer/irpass/symbol_resolver.cc b/mindspore/ccsrc/frontend/optimizer/irpass/symbol_resolver.cc index 4e2b211225..7bec784f77 100644 --- a/mindspore/ccsrc/frontend/optimizer/irpass/symbol_resolver.cc +++ b/mindspore/ccsrc/frontend/optimizer/irpass/symbol_resolver.cc @@ -28,12 +28,12 @@ namespace irpass { // {prim::kPrimGetAttr, bool, attr} // {prim::kPrimResolve, namespace, symbol} AnfNodePtr ResolverGetAttrResolve::operator()(const OptimizerPtr &optimizer, const AnfNodePtr &node) { - constexpr char PARSE_SUPER_NAME[] = "namespace"; + constexpr std::string_view PARSE_SUPER_NAME = "namespace"; constexpr size_t namespace_index = 1; constexpr size_t symbol_index = 2; PatternNode resolve_node, ns_node, sym_node, attr_node, bool_node; - auto GetAttrResolveLambda = [&node, &resolve_node, &attr_node, &optimizer]() -> AnfNodePtr { + auto GetAttrResolveLambda = [&node, &resolve_node, &attr_node, &optimizer, &PARSE_SUPER_NAME]() -> AnfNodePtr { auto inner = resolve_node.GetNode(node); auto attr = attr_node.GetNode(node); if (IsPrimitiveCNode(inner, prim::kPrimResolve)) { diff --git a/mindspore/ccsrc/frontend/parallel/cache_embedding/cache_embedding.cc b/mindspore/ccsrc/frontend/parallel/cache_embedding/cache_embedding.cc index dd7bdf5ae4..88b0dbf76c 100644 --- a/mindspore/ccsrc/frontend/parallel/cache_embedding/cache_embedding.cc +++ b/mindspore/ccsrc/frontend/parallel/cache_embedding/cache_embedding.cc @@ -216,7 +216,13 @@ void InitHashMapData(void *data, const int64_t host_size, const int64_t cache_si for (int64_t i = 0; i < host_size; ++i) { host_range.emplace_back(static_cast(i)); } +#if defined(__APPLE__) + std::random_device rd; + std::mt19937 rng(rd()); + std::shuffle(host_range.begin(), host_range.end(), rng); +#else std::random_shuffle(host_range.begin(), host_range.end()); +#endif size_t size = static_cast(cache_size); size_t hashmap_count = 0; for (size_t i = 0; i < size; ++i) { diff --git a/mindspore/ccsrc/frontend/parallel/ops_info/gather_info.cc b/mindspore/ccsrc/frontend/parallel/ops_info/gather_info.cc index 04ec7a88eb..a111b9abe8 100644 --- a/mindspore/ccsrc/frontend/parallel/ops_info/gather_info.cc +++ b/mindspore/ccsrc/frontend/parallel/ops_info/gather_info.cc @@ -25,7 +25,7 @@ #include "frontend/parallel/device_matrix.h" #include "frontend/parallel/graph_util/generate_graph.h" #include "frontend/parallel/context.h" -#if ((defined ENABLE_CPU) && (!defined _WIN32)) +#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__)) #include "ps/ps_cache/ps_cache_manager.h" #include "utils/ms_context.h" #endif @@ -160,7 +160,7 @@ Status GatherInfo::GetAttrs() { if (std::find(inputs_shape_[1].begin(), inputs_shape_[1].end(), -1) != inputs_shape_[1].end()) { dynamic_shape_indices_ = true; } -#if ((defined ENABLE_CPU) && (!defined _WIN32)) +#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__)) MS_EXCEPTION_IF_NULL(MsContext::GetInstance()); bool enable_sparse = MsContext::GetInstance()->get_param(MS_CTX_ENABLE_SPARSE); if (ps::PsDataPrefetch::GetInstance().cache_enable() && enable_sparse) { @@ -645,7 +645,7 @@ Status GatherInfo::InferBias() { rank = rank % (params_strategy[0] * params_strategy[1]); } } -#if ((defined ENABLE_CPU) && (!defined _WIN32)) +#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__)) if (ps::PsDataPrefetch::GetInstance().cache_enable()) { bias_ = static_cast(ps::PsCacheManager::GetInstance().cache_indices_lower_bound()); return SUCCESS; diff --git a/mindspore/ccsrc/frontend/parallel/ops_info/unique_info.cc b/mindspore/ccsrc/frontend/parallel/ops_info/unique_info.cc index bc250a3c53..2d6219a89a 100644 --- a/mindspore/ccsrc/frontend/parallel/ops_info/unique_info.cc +++ b/mindspore/ccsrc/frontend/parallel/ops_info/unique_info.cc @@ -28,7 +28,7 @@ #include "frontend/parallel/strategy.h" #include "frontend/parallel/context.h" #include "frontend/parallel/tensor_layout/tensor_redistribution.h" -#if ((defined ENABLE_CPU) && (!defined _WIN32)) +#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__))) #include "ps/ps_cache/ps_cache_manager.h" #endif @@ -101,7 +101,7 @@ std::vector UniqueInfo::GenerateOpStrategies(int64_t stage_id) { return sp_vector; } -#if ((defined ENABLE_CPU) && (!defined _WIN32)) +#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__))) Status UniqueInfo::ComputeReplaceGraph(const CNodePtr &cnode) { GenerateGraph gen_g = GenerateGraph(attrs_); if (gen_g.Init(cnode) != SUCCESS) { @@ -138,7 +138,7 @@ Status UniqueInfo::ComputeReplaceGraph(const CNodePtr &cnode) { #endif ReplaceGraphPtr UniqueInfo::replace_graph(const CNodePtr &cnode) { -#if ((defined ENABLE_CPU) && (!defined _WIN32)) +#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__)) if (ps::PsDataPrefetch::GetInstance().cache_enable()) { auto inputs = cnode->inputs(); if (inputs.empty()) { diff --git a/mindspore/ccsrc/frontend/parallel/ops_info/unique_info.h b/mindspore/ccsrc/frontend/parallel/ops_info/unique_info.h index 9f5cd4c895..5056c73ab1 100644 --- a/mindspore/ccsrc/frontend/parallel/ops_info/unique_info.h +++ b/mindspore/ccsrc/frontend/parallel/ops_info/unique_info.h @@ -47,7 +47,7 @@ class UniqueInfo : public OperatorInfo { Status InferDevMatrixShape() override; Status InferForwardCommunication() override { return SUCCESS; } Status InferAsLossDivisor() override { return SUCCESS; } -#if ((defined ENABLE_CPU) && (!defined _WIN32)) +#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__))) Status ComputeReplaceGraph(const CNodePtr &cnode); #endif diff --git a/mindspore/ccsrc/frontend/parallel/step_auto_parallel.cc b/mindspore/ccsrc/frontend/parallel/step_auto_parallel.cc index 35affcbf70..d6a3394a60 100644 --- a/mindspore/ccsrc/frontend/parallel/step_auto_parallel.cc +++ b/mindspore/ccsrc/frontend/parallel/step_auto_parallel.cc @@ -55,7 +55,7 @@ namespace mindspore { namespace parallel { bool StepAutoParallel(const FuncGraphPtr &root, const opt::OptimizerPtr &) { -#if ((defined ENABLE_CPU) && (!defined _WIN32)) +#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__)) if (ps::Util::IsRoleOfPServer() || ps::Util::IsRoleOfScheduler()) { return false; } diff --git a/mindspore/ccsrc/frontend/parallel/step_parallel.cc b/mindspore/ccsrc/frontend/parallel/step_parallel.cc index 18049c6edd..7ce0ddddac 100644 --- a/mindspore/ccsrc/frontend/parallel/step_parallel.cc +++ b/mindspore/ccsrc/frontend/parallel/step_parallel.cc @@ -3182,7 +3182,7 @@ static void PipelinePostProcess(const FuncGraphPtr &root, const std::vectoris_server() || ps::PSContext::instance()->is_scheduler()) { return false; } diff --git a/mindspore/ccsrc/minddata/dataset/CMakeLists.txt b/mindspore/ccsrc/minddata/dataset/CMakeLists.txt index 600428aa5c..1238bd27d6 100644 --- a/mindspore/ccsrc/minddata/dataset/CMakeLists.txt +++ b/mindspore/ccsrc/minddata/dataset/CMakeLists.txt @@ -3,9 +3,10 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-reorder") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-switch") if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-uninitialized -Wno-deprecated-declarations - -Wno-delete-non-abstract-non-virtual-dtor - -Wno-constant-conversion -Wno-unknown-warning-option") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-uninitialized -Wno-deprecated-declarations \ + -Wno-delete-non-abstract-non-virtual-dtor \ + -Wno-unused-comparison -Wno-constant-conversion \ + -Wno-unknown-warning-option -Wno-macro-redefined") else() set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-maybe-uninitialized") endif() @@ -73,7 +74,9 @@ add_subdirectory(core) add_subdirectory(kernels) add_subdirectory(engine) add_subdirectory(api) -add_subdirectory(audio) +if(NOT APPLE) + add_subdirectory(audio) +endif() add_subdirectory(text) add_subdirectory(callback) add_subdirectory(plugin) @@ -88,12 +91,16 @@ add_dependencies(engine-datasetops-source-sampler core) add_dependencies(engine-datasetops core) add_dependencies(engine-datasetops-mapop core) add_dependencies(engine-opt core) -add_dependencies(engine-gnn core) +if(NOT APPLE) + add_dependencies(engine-gnn core) +endif() add_dependencies(engine core) add_dependencies(callback core) -add_dependencies(audio-kernels core) -add_dependencies(audio-ir core) -add_dependencies(audio-ir-kernels core) +if(NOT APPLE) + add_dependencies(audio-kernels core) + add_dependencies(audio-ir core) + add_dependencies(audio-ir-kernels core) +endif() add_dependencies(text core) add_dependencies(text-kernels core) add_dependencies(text-ir core) @@ -154,14 +161,10 @@ set(submodules $ $ $ - $ $ $ $ $ - $ - $ - $ $ $ $ @@ -172,6 +175,14 @@ set(submodules $ ) +if(NOT APPLE) + set(submodules + ${submodules} + $ + $ + $ + $) +endif() if(ENABLE_ACL) set(submodules @@ -212,12 +223,6 @@ endif() ################# Link with external libraries ######################## target_link_libraries(_c_dataengine PRIVATE mindspore mindspore_gvar) -if(CMAKE_SYSTEM_NAME MATCHES "Darwin") - set(CMAKE_MACOSX_RPATH 1) - set(CMAKE_INSTALL_RPATH "@loader_path/lib;@loader_path") - set_target_properties(_c_dataengine PROPERTIES INSTALL_RPATH "${CMAKE_INSTALL_RPATH}") -endif() - set(targetso $) set(target_tiny "libtinyxml") set(target_icuuc "libicuuc") @@ -315,7 +320,9 @@ else() if(${ENABLE_IBVERBS} STREQUAL "ON") target_link_libraries(_c_dataengine PRIVATE ibverbs rdmacm) endif() - target_link_libraries(_c_dataengine PRIVATE ps_cache) + if(NOT CMAKE_SYSTEM_NAME MATCHES "Darwin") + target_link_libraries(_c_dataengine PRIVATE ps_cache) + endif() endif() endif() @@ -332,7 +339,13 @@ else() endif() if(MS_BUILD_GRPC) - target_link_libraries(_c_dataengine PRIVATE -Wl,--no-as-needed mindspore::grpc++) + if(NOT CMAKE_SYSTEM_NAME MATCHES "Darwin") + target_link_libraries(_c_dataengine PRIVATE -Wl,--no-as-needed mindspore::grpc++) + else() + target_link_libraries(_c_dataengine PRIVATE mindspore::grpc++) + endif() endif() -set_target_properties(_c_dataengine PROPERTIES INSTALL_RPATH ${MINDSPORE_RPATH}) +if(NOT CMAKE_SYSTEM_NAME MATCHES "Darwin") + set_target_properties(_c_dataengine PROPERTIES INSTALL_RPATH ${MINDSPORE_RPATH}) +endif() diff --git a/mindspore/ccsrc/minddata/dataset/api/CMakeLists.txt b/mindspore/ccsrc/minddata/dataset/api/CMakeLists.txt index 3e0cb3d36d..54ec3ce350 100644 --- a/mindspore/ccsrc/minddata/dataset/api/CMakeLists.txt +++ b/mindspore/ccsrc/minddata/dataset/api/CMakeLists.txt @@ -1,42 +1,55 @@ file(GLOB_RECURSE _CURRENT_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc") set_property(SOURCE ${_CURRENT_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_MD) if(ENABLE_PYTHON) - add_library(APItoPython OBJECT + set(API_PYTHON_SRC_FILES + python/bindings/dataset/audio/bindings.cc + python/bindings/dataset/audio/kernels/ir/bindings.cc + python/bindings/dataset/callback/bindings.cc + python/bindings/dataset/core/bindings.cc + python/bindings/dataset/engine/cache/bindings.cc + python/bindings/dataset/engine/datasetops/bindings.cc + python/bindings/dataset/engine/gnn/bindings.cc + python/bindings/dataset/engine/ir/consumer/bindings.cc + python/bindings/dataset/engine/ir/datasetops/bindings.cc + python/bindings/dataset/engine/ir/datasetops/source/bindings.cc + python/bindings/dataset/engine/ir/datasetops/source/samplers/bindings.cc + python/bindings/dataset/engine/ir/execute/bindings.cc + python/bindings/dataset/engine/ir/schema/bindings.cc + python/bindings/dataset/engine/perf/bindings.cc + python/bindings/dataset/kernels/bindings.cc + python/bindings/dataset/kernels/ir/bindings.cc + python/bindings/dataset/kernels/ir/image/bindings.cc + python/bindings/dataset/text/bindings.cc + python/bindings/dataset/text/kernels/ir/bindings.cc + python/bindings/mindrecord/include/bindings.cc + python/pybind_conversion.cc + python/pybind_register.cc + ) + if(APPLE) + list(REMOVE_ITEM API_PYTHON_SRC_FILES python/bindings/dataset/audio/bindings.cc python/bindings/dataset/audio/kernels/ir/bindings.cc - python/bindings/dataset/callback/bindings.cc - python/bindings/dataset/core/bindings.cc - python/bindings/dataset/engine/cache/bindings.cc - python/bindings/dataset/engine/datasetops/bindings.cc python/bindings/dataset/engine/gnn/bindings.cc - python/bindings/dataset/engine/ir/consumer/bindings.cc - python/bindings/dataset/engine/ir/datasetops/bindings.cc - python/bindings/dataset/engine/ir/datasetops/source/bindings.cc - python/bindings/dataset/engine/ir/datasetops/source/samplers/bindings.cc - python/bindings/dataset/engine/ir/execute/bindings.cc - python/bindings/dataset/engine/ir/schema/bindings.cc - python/bindings/dataset/engine/perf/bindings.cc - python/bindings/dataset/kernels/bindings.cc - python/bindings/dataset/kernels/ir/bindings.cc - python/bindings/dataset/kernels/ir/image/bindings.cc - python/bindings/dataset/text/bindings.cc - python/bindings/dataset/text/kernels/ir/bindings.cc - python/bindings/mindrecord/include/bindings.cc - python/pybind_conversion.cc - python/pybind_register.cc ) + endif() + add_library(APItoPython OBJECT ${API_PYTHON_SRC_FILES}) target_include_directories(APItoPython PRIVATE ${pybind11_INCLUDE_DIRS}) endif() -add_library(cpp-API OBJECT - audio.cc - config.cc - data_helper.cc - datasets.cc - execute.cc - iterator.cc - transforms.cc - samplers.cc - text.cc - vision.cc - ) +set(API_CPP_SRC_FILES + audio.cc + config.cc + data_helper.cc + datasets.cc + execute.cc + iterator.cc + transforms.cc + samplers.cc + text.cc + vision.cc + ) + +if(APPLE) + list(REMOVE_ITEM API_CPP_SRC_FILES audio.cc) +endif() +add_library(cpp-API OBJECT ${API_CPP_SRC_FILES}) \ No newline at end of file diff --git a/mindspore/ccsrc/minddata/dataset/engine/CMakeLists.txt b/mindspore/ccsrc/minddata/dataset/engine/CMakeLists.txt index 884f294439..5bc5ce87af 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/CMakeLists.txt +++ b/mindspore/ccsrc/minddata/dataset/engine/CMakeLists.txt @@ -1,6 +1,8 @@ add_subdirectory(datasetops) add_subdirectory(opt) -add_subdirectory(gnn) +if(NOT APPLE) + add_subdirectory(gnn) +endif() add_subdirectory(ir) message("ENABLE_SECURITY = ${ENABLE_SECURITY}") @@ -43,13 +45,16 @@ if(ENABLE_PYTHON) target_include_directories(engine PRIVATE ${pybind11_INCLUDE_DIRS}) endif() -add_dependencies(engine engine-datasetops - engine-datasetops-source - engine-opt - engine-gnn - engine-cache-client - engine-datasetops-mapop - ) +add_dependencies(engine + engine-datasetops + engine-datasetops-source + engine-opt + engine-cache-client + engine-datasetops-mapop + ) +if(NOT APPLE) + add_dependencies(engine engine-gnn) +endif() if(NOT ENABLE_SECURITY) add_dependencies(engine engine-perf) diff --git a/mindspore/ccsrc/minddata/dataset/engine/cache/CMakeLists.txt b/mindspore/ccsrc/minddata/dataset/engine/cache/CMakeLists.txt index 6f733ce09d..feee86342d 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/cache/CMakeLists.txt +++ b/mindspore/ccsrc/minddata/dataset/engine/cache/CMakeLists.txt @@ -96,10 +96,15 @@ if(ENABLE_CACHE) ${PYTHON_LIBRARIES} ${SECUREC_LIBRARY} pthread - -ldl - -Wl,--no-as-needed - mindspore::grpc++ - -Wl,--as-needed) + -ldl) + if(CMAKE_SYSTEM_NAME MATCHES "Darwin") + target_link_libraries(cache_server mindspore::grpc++) + else() + target_link_libraries(cache_server + -Wl,--no-as-needed + mindspore::grpc++ + -Wl,--as-needed) + endif() endif() if(USE_GLOG) @@ -116,8 +121,12 @@ if(ENABLE_CACHE) target_link_libraries(cache_admin -Wl,--start-group mindspore::protobuf mindspore_core mindspore_shared_lib ${ACL_TDT_CHANNEL} -Wl,--end-group) else() - target_link_libraries(cache_admin - -Wl,--start-group mindspore::protobuf mindspore_core mindspore_shared_lib -Wl,--end-group) + if(CMAKE_SYSTEM_NAME MATCHES "Darwin") + target_link_libraries(cache_admin mindspore::protobuf mindspore_core mindspore_shared_lib) + else() + target_link_libraries(cache_admin + -Wl,--start-group mindspore::protobuf mindspore_core mindspore_shared_lib -Wl,--end-group) + endif() endif() if(USE_GLOG) diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/batch_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/batch_op.cc index 0352f58839..bc4285a8ff 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/batch_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/batch_op.cc @@ -130,7 +130,7 @@ Status BatchOp::operator()() { RETURN_IF_NOT_OK(GetBatchSize(&cur_batch_size, CBatchInfo(epoch_num, batch_num, cnt - epoch_num))); RETURN_IF_NOT_OK(child_iterator_->FetchNextTensorRow(&new_row)); -#if !defined(_WIN32) && !defined(_WIN64) && ENABLE_PYTHON +#if !defined(_WIN32) && !defined(_WIN64) && !defined(__APPLE__) && ENABLE_PYTHON if ((num_workers_ > 1 || batch_map_func_) && GetMemoryUsage() > MAX_MEMORY_USAGE_THRESHOLD) { MS_LOG(WARNING) << "Memory consumption is more than " << MAX_MEMORY_USAGE_THRESHOLD * 100 << "%, " << "which may cause oom error. Please reduce num_parallel_workers size / " diff --git a/mindspore/ccsrc/minddata/dataset/util/status.cc b/mindspore/ccsrc/minddata/dataset/util/status.cc index a58e93799a..a4d6c7ed5d 100644 --- a/mindspore/ccsrc/minddata/dataset/util/status.cc +++ b/mindspore/ccsrc/minddata/dataset/util/status.cc @@ -28,7 +28,7 @@ namespace mindspore { namespace dataset { -#if !defined(_WIN32) && !defined(_WIN64) +#if !defined(_WIN32) && !defined(_WIN64) && !defined(__APPLE__) float GetMemoryUsage() { char buf[128] = {0}; diff --git a/mindspore/ccsrc/minddata/dataset/util/status.h b/mindspore/ccsrc/minddata/dataset/util/status.h index f7f19b6511..b55783ae9f 100644 --- a/mindspore/ccsrc/minddata/dataset/util/status.h +++ b/mindspore/ccsrc/minddata/dataset/util/status.h @@ -89,7 +89,7 @@ namespace dataset { } \ } while (false) -#if !defined(_WIN32) && !defined(_WIN64) +#if !defined(_WIN32) && !defined(_WIN64) && !defined(__APPLE__) const float MAX_MEMORY_USAGE_THRESHOLD = 0.95; float GetMemoryUsage(); #endif diff --git a/mindspore/ccsrc/minddata/dataset/util/task.cc b/mindspore/ccsrc/minddata/dataset/util/task.cc index bb2f55331d..ecc009eca1 100644 --- a/mindspore/ccsrc/minddata/dataset/util/task.cc +++ b/mindspore/ccsrc/minddata/dataset/util/task.cc @@ -39,7 +39,7 @@ void Task::operator()() { id_ = this_thread::get_id(); std::stringstream ss; ss << id_; -#if defined(__ANDROID__) || defined(ANDROID) +#if defined(__ANDROID__) || defined(ANDROID) || defined(__APPLE__) // The thread id in Linux may be duplicate ss << Services::GetUniqueID(); #endif diff --git a/mindspore/ccsrc/minddata/mindrecord/io/shard_index_generator.cc b/mindspore/ccsrc/minddata/mindrecord/io/shard_index_generator.cc index 0433dd0ec6..c76097e672 100644 --- a/mindspore/ccsrc/minddata/mindrecord/io/shard_index_generator.cc +++ b/mindspore/ccsrc/minddata/mindrecord/io/shard_index_generator.cc @@ -225,7 +225,7 @@ Status ShardIndexGenerator::CreateShardNameTable(sqlite3 *db, const std::string Status ShardIndexGenerator::CreateDatabase(int shard_no, sqlite3 **db) { std::string shard_address = shard_header_.GetShardAddressByID(shard_no); - CHECK_FAIL_RETURN_UNEXPECTED(!shard_address.empty(), "Shard address is empty, shard No: " + shard_no); + CHECK_FAIL_RETURN_UNEXPECTED(!shard_address.empty(), "Shard address is empty, shard No: " + std::to_string(shard_no)); std::shared_ptr fn_ptr; RETURN_IF_NOT_OK(GetFileName(shard_address, &fn_ptr)); shard_address += ".db"; diff --git a/mindspore/ccsrc/pipeline/jit/action.cc b/mindspore/ccsrc/pipeline/jit/action.cc index 55d38453ed..81fbdc4301 100644 --- a/mindspore/ccsrc/pipeline/jit/action.cc +++ b/mindspore/ccsrc/pipeline/jit/action.cc @@ -46,7 +46,7 @@ #include "frontend/optimizer/py_pass_manager.h" #include "utils/ms_context.h" #include "vm/transform.h" -#if ((defined ENABLE_CPU) && (!defined _WIN32)) +#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__))) #include "ps/parameter_server.h" #include "ps/scheduler.h" #include "ps/worker.h" @@ -623,7 +623,7 @@ bool OptInlineAction(const ResourcePtr &res) { bool GeOptimizeAction(const ResourcePtr &res) { return OptimizeAction(res, kGePasses); } bool VmOptimizeAction(const ResourcePtr &res) { -#if ((defined ENABLE_CPU) && (!defined _WIN32)) +#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__))) if (ps::PSContext::instance()->is_ps_mode()) { kVmPasses.push_back({"server_communication_op_fusion", ps::Util::FuseServerCommOps}); } @@ -823,7 +823,7 @@ bool ExecuteAction(const ResourcePtr &res) { return true; } -#if ((defined ENABLE_CPU) && (!defined _WIN32)) +#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__))) bool StartPSWorkerAction(const ResourcePtr &) { ps::Worker::GetInstance().Run(); return true; @@ -1156,7 +1156,7 @@ std::vector VmPipeline() { (void)actions.emplace_back(std::make_pair("eliminate_forward_cnode", EliminateForwardCNode)); (void)actions.emplace_back(std::make_pair("validate", ValidateAction)); -#if ((defined ENABLE_CPU) && (!defined _WIN32)) +#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__)) if (ps::PSContext::instance()->is_worker()) { std::string server_mode = ps::PSContext::instance()->server_mode(); if (server_mode == ps::kServerModeFL || server_mode == ps::kServerModeHybrid) { @@ -1194,7 +1194,7 @@ std::vector MindIRPipeline() { (void)actions.emplace_back(std::make_pair("execute", ExecuteAction)); return actions; } -#if ((defined ENABLE_CPU) && (!defined _WIN32)) +#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__)) std::vector ServerPipeline() { auto actions = CommonPipeline(); (void)actions.emplace_back(std::make_pair("optimize", VmOptimizeAction)); diff --git a/mindspore/ccsrc/pipeline/jit/init.cc b/mindspore/ccsrc/pipeline/jit/init.cc index 05374bc581..d56888ddba 100644 --- a/mindspore/ccsrc/pipeline/jit/init.cc +++ b/mindspore/ccsrc/pipeline/jit/init.cc @@ -36,7 +36,7 @@ #else #include "runtime/device/gpu/distribution/collective_fake_init.h" #endif -#if ((defined ENABLE_CPU) && (!defined _WIN32)) +#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__)) #include "ps/util.h" #endif #include "ps/ps_context.h" diff --git a/mindspore/ccsrc/pipeline/jit/pass.cc b/mindspore/ccsrc/pipeline/jit/pass.cc index 1d9e4317ed..ff04ccd9b9 100644 --- a/mindspore/ccsrc/pipeline/jit/pass.cc +++ b/mindspore/ccsrc/pipeline/jit/pass.cc @@ -48,7 +48,7 @@ #include "frontend/optimizer/irpass/gradient_eliminate.h" #include "frontend/optimizer/irpass/parameter_eliminate.h" #include "frontend/optimizer/irpass/updatestate_eliminate.h" -#if ((defined ENABLE_CPU) && (!defined _WIN32)) +#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__)) #include "ps/util.h" #include "ps/ps_context.h" #endif @@ -217,7 +217,7 @@ namespace { bool ReAutoMonadWrapper(const FuncGraphPtr &root, const opt::OptimizerPtr &) { return ReAutoMonad(root); } bool parallel_mode() { -#if ((defined ENABLE_CPU) && (!defined _WIN32)) +#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__)) if (ps::PSContext::instance()->is_server() || ps::PSContext::instance()->is_scheduler()) { return false; } @@ -596,7 +596,7 @@ bool AddRecomputationPass(const ResourcePtr &res) { bool AddCacheEmbeddingPass(const ResourcePtr &res) { MS_EXCEPTION_IF_NULL(res); -#if ((defined ENABLE_CPU) && (!defined _WIN32)) +#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__)) if (ps::PSContext::instance()->is_ps_mode()) { return true; } diff --git a/mindspore/ccsrc/pipeline/jit/pipeline.cc b/mindspore/ccsrc/pipeline/jit/pipeline.cc index ee23b24e2d..c13d3820db 100644 --- a/mindspore/ccsrc/pipeline/jit/pipeline.cc +++ b/mindspore/ccsrc/pipeline/jit/pipeline.cc @@ -64,7 +64,7 @@ #endif #endif -#if ((defined ENABLE_CPU) && (!defined _WIN32)) +#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__)) #include "ps/constants.h" #include "ps/util.h" #include "ps/worker.h" @@ -707,7 +707,7 @@ std::vector GetPipeline(const ResourcePtr &resource, const std::stri std::string backend = MsContext::GetInstance()->backend_policy(); -#if ((defined ENABLE_CPU) && (!defined _WIN32)) +#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__)) const std::string &server_mode = ps::PSContext::instance()->server_mode(); if ((server_mode == ps::kServerModeFL || server_mode == ps::kServerModeHybrid) && ps::PSContext::instance()->is_server()) { @@ -1228,7 +1228,7 @@ bool InitExecDataset(const std::string &queue_name, int64_t iter_num, int64_t ba bool InitExecDatasetVm(const std::string &queue_name, int64_t size, int64_t batch_size, const std::vector &types, const std::vector> &shapes, const std::vector &input_indexes, bool need_run) { -#if ((defined ENABLE_CPU) && (!defined _WIN32)) +#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__))) if ((ps::PSContext::instance()->is_ps_mode()) && (!ps::PSContext::instance()->is_worker())) { return true; } @@ -1297,7 +1297,7 @@ bool InitExecDatasetVm(const std::string &queue_name, int64_t size, int64_t batc auto runner = convert_fn(segment, ""); ConfigManager::GetInstance().set_iter_num(size); // PS cache does not support loop sink. -#if ((defined ENABLE_CPU) && (!defined _WIN32)) +#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__)) if (ps::PSContext::instance()->is_worker() && ps::PsDataPrefetch::GetInstance().cache_enable()) { ps::PsDataPrefetch::GetInstance().CreateDataChannel(queue_name, LongToSize(size)); ConfigManager::GetInstance().set_iter_num(1); @@ -1482,7 +1482,7 @@ void FinalizeBackend() { void ClearResAtexit() { MS_LOG(DEBUG) << "Pipeline clear all resource"; RecordExitStatus(); -#if ((defined ENABLE_CPU) && (!defined _WIN32)) +#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__)) if (ps::PSContext::instance()->is_ps_mode() && ps::PSContext::instance()->is_worker()) { if (ps::PsDataPrefetch::GetInstance().cache_enable()) { ps::ps_cache_instance.Finalize(); diff --git a/mindspore/ccsrc/pipeline/jit/static_analysis/static_analysis.h b/mindspore/ccsrc/pipeline/jit/static_analysis/static_analysis.h index 73a3bad5af..10177265f9 100644 --- a/mindspore/ccsrc/pipeline/jit/static_analysis/static_analysis.h +++ b/mindspore/ccsrc/pipeline/jit/static_analysis/static_analysis.h @@ -136,11 +136,14 @@ class AnfNodeConfig : public Config { bool operator==(const AnfNodeConfig &other) const { // compare node with pointer, context with pointer except DummyContext as it's created by make_shared; // context should not be nullptr; + if (node_ != other.node_) { + return false; + } if (context_->IsDummyContext() && other.context_->IsDummyContext()) { return true; } // Don't check `func_graph_` equality. - return (node_ == other.node_) && (context_ == other.context_); + return context_ == other.context_; } std::string ToString() const override { diff --git a/mindspore/ccsrc/ps/CMakeLists.txt b/mindspore/ccsrc/ps/CMakeLists.txt index f8868d8f5d..796723f13f 100644 --- a/mindspore/ccsrc/ps/CMakeLists.txt +++ b/mindspore/ccsrc/ps/CMakeLists.txt @@ -1,6 +1,6 @@ file(GLOB_RECURSE _PS_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc") -if(NOT ENABLE_CPU OR WIN32) +if(NOT ENABLE_CPU OR WIN32 OR APPLE) list(REMOVE_ITEM _PS_SRC_FILES "optimizer_info_builder.cc") list(REMOVE_ITEM _PS_SRC_FILES "optimizer_info.cc") list(REMOVE_ITEM _PS_SRC_FILES "scheduler.cc") diff --git a/mindspore/ccsrc/ps/ps_cache/CMakeLists.txt b/mindspore/ccsrc/ps/ps_cache/CMakeLists.txt index 524f448c16..ebc8d36a51 100644 --- a/mindspore/ccsrc/ps/ps_cache/CMakeLists.txt +++ b/mindspore/ccsrc/ps/ps_cache/CMakeLists.txt @@ -1,4 +1,4 @@ -if(ENABLE_CPU AND NOT WIN32) +if(ENABLE_CPU AND NOT WIN32 AND NOT APPLE) file(GLOB_RECURSE _PS_CACHE_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "ps_data/*.cc") set_property(SOURCE ${_PS_CACHE_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_PS) add_library(ps_cache SHARED ${_PS_CACHE_SRC_FILES}) diff --git a/mindspore/ccsrc/ps/ps_context.cc b/mindspore/ccsrc/ps/ps_context.cc index 033af166db..90574143dc 100644 --- a/mindspore/ccsrc/ps/ps_context.cc +++ b/mindspore/ccsrc/ps/ps_context.cc @@ -18,7 +18,7 @@ #include "utils/log_adapter.h" #include "utils/ms_utils.h" #include "backend/kernel_compiler/kernel.h" -#if ((defined ENABLE_CPU) && (!defined _WIN32)) +#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__))) #include "ps/ps_cache/ps_cache_manager.h" #include "ps/ps_cache/ps_data/ps_data_prefetch.h" #endif @@ -88,7 +88,7 @@ void PSContext::Reset() { is_worker_ = false; is_pserver_ = false; is_sched_ = false; -#if ((defined ENABLE_CPU) && (!defined _WIN32)) +#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__))) if (ps::PsDataPrefetch::GetInstance().cache_enable()) { ps_cache_instance.Finalize(); set_cache_enable(false); @@ -144,44 +144,44 @@ uint32_t PSContext::ps_rank_id() const { return rank_id_; } void PSContext::InsertHashTableSize(const std::string ¶m_name, size_t cache_vocab_size, size_t embedding_size, size_t vocab_size) const { -#if ((defined ENABLE_CPU) && (!defined _WIN32)) +#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__))) ps_cache_instance.InsertHashTableSize(param_name, cache_vocab_size, embedding_size, vocab_size); #endif } void PSContext::ReInsertHashTableSize(const std::string &new_param_name, const std::string &cur_param_name, size_t cache_vocab_size, size_t embedding_size) const { -#if ((defined ENABLE_CPU) && (!defined _WIN32)) +#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__))) ps_cache_instance.ReInsertHashTableSize(new_param_name, cur_param_name, cache_vocab_size, embedding_size); #endif } void PSContext::InsertWeightInitInfo(const std::string ¶m_name, size_t global_seed, size_t op_seed) const { -#if ((defined ENABLE_CPU) && (!defined _WIN32)) +#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__))) ps_cache_instance.InsertWeightInitInfo(param_name, global_seed, op_seed); #endif } void PSContext::InsertAccumuInitInfo(const std::string ¶m_name, float init_val) const { -#if ((defined ENABLE_CPU) && (!defined _WIN32)) +#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__))) ps_cache_instance.InsertAccumuInitInfo(param_name, init_val); #endif } void PSContext::CloneHashTable(const std::string &dest_param_name, const std::string &src_param_name) const { -#if ((defined ENABLE_CPU) && (!defined _WIN32)) +#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__))) ps_cache_instance.CloneHashTable(dest_param_name, src_param_name); #endif } void PSContext::set_cache_enable(bool cache_enable) const { -#if ((defined ENABLE_CPU) && (!defined _WIN32)) +#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__))) PsDataPrefetch::GetInstance().set_cache_enable(cache_enable); #endif } void PSContext::set_rank_id(uint32_t rank_id) const { -#if ((defined ENABLE_CPU) && (!defined _WIN32)) +#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__))) ps_cache_instance.set_rank_id(rank_id); #endif } diff --git a/mindspore/ccsrc/runtime/device/kernel_runtime.cc b/mindspore/ccsrc/runtime/device/kernel_runtime.cc index 9fe8cc8897..b04d279472 100644 --- a/mindspore/ccsrc/runtime/device/kernel_runtime.cc +++ b/mindspore/ccsrc/runtime/device/kernel_runtime.cc @@ -33,7 +33,7 @@ #include "frontend/parallel/context.h" #include "debug/env_config_parser.h" #include "pipeline/pynative/pynative_profiling.h" -#if ((defined ENABLE_CPU) && (!defined _WIN32)) +#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__)) #include "ps/ps_cache/ps_cache_manager.h" #endif @@ -579,7 +579,7 @@ void KernelRuntime::AssignStaticMemoryInput(const session::KernelGraph &graph) { } add_need_alloc_nodes(input_node); } -#if ((defined ENABLE_CPU) && (!defined _WIN32)) +#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__))) bool ps_cache_check = false; #endif for (auto &item : need_alloc_nodes) { @@ -593,7 +593,7 @@ void KernelRuntime::AssignStaticMemoryInput(const session::KernelGraph &graph) { continue; } DeviceAddressPtr device_address = nullptr; -#if ((defined ENABLE_CPU) && (!defined _WIN32)) +#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__)) const std::string ¶m_name = item->fullname_with_scope(); if (ps::ps_cache_instance.IsHashTable(param_name)) { MS_LOG(INFO) << "Parameter(" << param_name << ")" @@ -1606,7 +1606,7 @@ void KernelRuntime::ClearGraphRuntimeResource(uint32_t graph_id) { MS_LOG(INFO) << "Clear graph:" << graph_id << " runtime resource"; } -#if ((defined ENABLE_CPU) && (!defined _WIN32)) +#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__)) void KernelRuntime::GetFirstPSEmbeddingCache(const session::KernelGraph &graph, AnfNodePtr *const first_cache_input_index, size_t *const first_cache_size) { diff --git a/mindspore/ccsrc/runtime/device/kernel_runtime.h b/mindspore/ccsrc/runtime/device/kernel_runtime.h index e2714c984b..957ef05cd5 100644 --- a/mindspore/ccsrc/runtime/device/kernel_runtime.h +++ b/mindspore/ccsrc/runtime/device/kernel_runtime.h @@ -173,7 +173,7 @@ class KernelRuntime { void RunOpAssignOutputNodeMemory(const ValuePtr &pre_output_value, const session::KernelGraph &graph); void AssignValueNodeTensor(const ValueNodePtr &value_node, const ValuePtr &node_value, size_t output_idx); DeviceAddressPtr PreAssignCNodeMemory(const AnfNodePtr &anf_node, size_t index) const; -#if ((defined ENABLE_CPU) && (!defined _WIN32)) +#if ((defined ENABLE_CPU) && (!defined _WIN32) && (!defined(__APPLE__))) void GetFirstPSEmbeddingCache(const session::KernelGraph &graph, AnfNodePtr *const first_cache_input_index, size_t *const first_cache_size); void CheckIfSupportPSEmbeddingCache(const session::KernelGraph &graph); diff --git a/mindspore/ccsrc/runtime/device/kernel_runtime_manager.cc b/mindspore/ccsrc/runtime/device/kernel_runtime_manager.cc index 5dfe3741c0..5b93251992 100644 --- a/mindspore/ccsrc/runtime/device/kernel_runtime_manager.cc +++ b/mindspore/ccsrc/runtime/device/kernel_runtime_manager.cc @@ -16,7 +16,7 @@ #include "runtime/device/kernel_runtime_manager.h" #include "utils/log_adapter.h" -#if ((defined ENABLE_CPU) && (!defined _WIN32)) +#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__)) #include "ps/ps_cache/ps_cache_manager.h" #endif #include "backend/session/pynative_task_manager.h" @@ -26,7 +26,7 @@ namespace device { void KernelRuntimeManager::ClearRuntimeResource() { // Just remove PyNative tasks before runtime resource release. session::PynativeTaskManager::GetInstance().Reset(); -#if ((defined ENABLE_CPU) && (!defined _WIN32)) +#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__)) if (ps::PSContext::instance()->is_worker() && ps::PsDataPrefetch::GetInstance().cache_enable()) { ps::ps_cache_instance.SyncEmbeddingTable(); } @@ -132,7 +132,7 @@ void KernelRuntimeManager::ReleaseKernelRuntime(const std::string &device_name, if (runtime == nullptr) { return; } -#if ((defined ENABLE_CPU) && (!defined _WIN32)) +#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__)) if (ps::PSContext::instance()->is_worker() && ps::PsDataPrefetch::GetInstance().cache_enable()) { ps::ps_cache_instance.SyncEmbeddingTable(); } diff --git a/mindspore/ccsrc/runtime/framework/actor/super_kernel_actor.cc b/mindspore/ccsrc/runtime/framework/actor/super_kernel_actor.cc index 541b840cac..a754644eb1 100644 --- a/mindspore/ccsrc/runtime/framework/actor/super_kernel_actor.cc +++ b/mindspore/ccsrc/runtime/framework/actor/super_kernel_actor.cc @@ -52,21 +52,22 @@ void SuperKernelActor::Run(OpContext *const context) { MS_EXCEPTION_IF_NULL(context); MS_EXCEPTION_IF_NULL(graph_); MS_EXCEPTION_IF_NULL(device_contexts_[0]); - MS_LOG(INFO) << "Super kernel actor(" << GetAID().Name() << ") launches graph: " << graph_->graph_id(); + MS_LOG(INFO) << "Super kernel actor(" << GetAID().Name() + << ") launches graph: " << std::to_string(graph_->graph_id()); if (!CheckInputData(context)) { - std::string error_info = "Check the input data invalid, graph id: " + graph_->graph_id(); + std::string error_info = "Check the input data invalid, graph id: " + std::to_string(graph_->graph_id()); SET_OPCONTEXT_FAIL_RET_WITH_ERROR((*context), error_info); } try { auto ret = device_contexts_[0]->LaunchGraph(graph_); if (!ret) { - std::string error_info = "Launch graph failed, graph id: " + graph_->graph_id(); + std::string error_info = "Launch graph failed, graph id: " + std::to_string(graph_->graph_id()); SET_OPCONTEXT_FAIL_RET_WITH_ERROR((*context), error_info); } } catch (const std::exception &e) { MsException::Instance().SetException(); - std::string error_info = "Launch graph exception, graph id: " + graph_->graph_id(); + std::string error_info = "Launch graph exception, graph id: " + std::to_string(graph_->graph_id()); SET_OPCONTEXT_FAIL_RET_WITH_ERROR((*context), error_info); } diff --git a/mindspore/ccsrc/runtime/hardware/CMakeLists.txt b/mindspore/ccsrc/runtime/hardware/CMakeLists.txt index 9592f7cef1..c58bb11ed6 100644 --- a/mindspore/ccsrc/runtime/hardware/CMakeLists.txt +++ b/mindspore/ccsrc/runtime/hardware/CMakeLists.txt @@ -35,6 +35,10 @@ if(ENABLE_CPU) endif() endif() +if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-delete-abstract-non-virtual-dtor") +endif() + set_property(SOURCE ${HARDWARE_SRC_LIST} ${HARDWARE_D_SRC_LIST} ${HARDWARE_GPU_SRC_LIST} ${HARDWARE_CPU_SRC_LIST} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_DEVICE) diff --git a/mindspore/ccsrc/runtime/hardware/cpu/cpu_memory_pool.cc b/mindspore/ccsrc/runtime/hardware/cpu/cpu_memory_pool.cc index caa39dcad5..82af0c4156 100644 --- a/mindspore/ccsrc/runtime/hardware/cpu/cpu_memory_pool.cc +++ b/mindspore/ccsrc/runtime/hardware/cpu/cpu_memory_pool.cc @@ -27,7 +27,7 @@ const size_t kKBToByte = 1024; const size_t kLineMaxSize = 1024; size_t GetSystemMemorySize(const std::string &key) { -#if defined(_WIN32) || defined(_WIN64) +#if defined(_WIN32) || defined(_WIN64) || defined(__APPLE__) return SIZE_MAX; #else FILE *file = fopen("/proc/meminfo", "r"); diff --git a/mindspore/ccsrc/vm/transform.cc b/mindspore/ccsrc/vm/transform.cc index 54eb336a0d..a5b7511acf 100644 --- a/mindspore/ccsrc/vm/transform.cc +++ b/mindspore/ccsrc/vm/transform.cc @@ -632,7 +632,7 @@ void SetMindRTEnable() { return; #endif -#if ((defined ENABLE_CPU) && (!defined _WIN32)) +#if ((defined ENABLE_CPU) && (!defined _WIN32) && !defined(__APPLE__)) if (ps::PSContext::instance()->is_ps_mode()) { return; } diff --git a/mindspore/core/CMakeLists.txt b/mindspore/core/CMakeLists.txt index 1ad522bda9..3cc9b6ffa7 100644 --- a/mindspore/core/CMakeLists.txt +++ b/mindspore/core/CMakeLists.txt @@ -42,7 +42,8 @@ if(CMAKE_SYSTEM_NAME MATCHES "Windows" AND NOT MSVC) add_compile_definitions(BUILDING_DLL) elseif(CMAKE_SYSTEM_NAME MATCHES "Darwin") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} \ - -Wuser-defined-warnings -Winconsistent-missing-override -Wno-delete-non-abstract-non-virtual-dtor") + -Wno-overloaded-virtual -Wno-user-defined-warnings -Winconsistent-missing-override -Wno-delete-non-virtual-dtor \ + -Wno-unused-private-field") endif() set_property(SOURCE ${CORE_SRC_LIST} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_CORE) diff --git a/mindspore/core/gvar/log_adapter_common.cc b/mindspore/core/gvar/log_adapter_common.cc index 5e8efa9843..1c4312afdd 100644 --- a/mindspore/core/gvar/log_adapter_common.cc +++ b/mindspore/core/gvar/log_adapter_common.cc @@ -76,7 +76,12 @@ std::string GetTimeString() { constexpr int64_t time_convert_unit = 1000; (void)localtime_r(&cur_time.tv_sec, &now); (void)strftime(buf, BUFLEN, "%Y-%m-%d-%H:%M:%S", &now); // format date and time - (void)snprintf(buf + time_str_len, BUFLEN - time_str_len, ".%03ld.%03ld", cur_time.tv_usec / time_convert_unit, +#ifdef __APPLE__ + const std::string fmt_str = ".%03lld.%03lld"; +#else + const std::string fmt_str = ".%03ld.%03ld"; +#endif + (void)snprintf(buf + time_str_len, BUFLEN - time_str_len, fmt_str.c_str(), cur_time.tv_usec / time_convert_unit, cur_time.tv_usec % time_convert_unit); #endif return std::string(buf); diff --git a/mindspore/core/ir/anf.h b/mindspore/core/ir/anf.h index 331e85ef08..c3ca1dbf12 100644 --- a/mindspore/core/ir/anf.h +++ b/mindspore/core/ir/anf.h @@ -74,8 +74,8 @@ using FuncGraphVector = std::vector; class Primitive; using PrimitivePtr = std::shared_ptr; -class PrimitiveHasher; -class PrimitiveEqual; +struct PrimitiveHasher; +struct PrimitiveEqual; using PrimitiveSet = std::unordered_set; class BaseRef; diff --git a/mindspore/core/ir/scalar.h b/mindspore/core/ir/scalar.h index 15e4e5caa0..06d5b073de 100644 --- a/mindspore/core/ir/scalar.h +++ b/mindspore/core/ir/scalar.h @@ -435,6 +435,12 @@ class MS_CORE_API UInt64Imm final : public IntergerImm { }; using UInt64ImmPtr = std::shared_ptr; IMM_TRAITS(UInt64ImmPtr, uint64_t); + +#if defined(__APPLE__) +using SizetImmPtr = std::shared_ptr; +IMM_TRAITS(SizetImmPtr, size_t); +#endif + /// \beief FloatImm defines interface for float data. class MS_CORE_API FloatImm : public Scalar { public: diff --git a/mindspore/core/utils/log_adapter.cc b/mindspore/core/utils/log_adapter.cc index aa4ee8be8c..68f2cd6d52 100644 --- a/mindspore/core/utils/log_adapter.cc +++ b/mindspore/core/utils/log_adapter.cc @@ -422,7 +422,7 @@ void InitSubModulesLogLevel() { } // namespace mindspore extern "C" { -#if defined(_WIN32) || defined(_WIN64) +#if defined(_WIN32) || defined(_WIN64) || defined(__APPLE__) #ifdef _MSC_VER void common_log_init(void) { #else @@ -483,7 +483,7 @@ MS_CORE_API void common_log_init(void) { } // shared lib init hook -#if defined(_WIN32) || defined(_WIN64) +#if defined(_WIN32) || defined(_WIN64) || defined(__APPLE__) #ifdef _MSC_VER void mindspore_log_init(void) { #else @@ -496,7 +496,7 @@ void mindspore_log_init(void) { #define google mindspore_private static bool is_glog_initialzed = false; if (!is_glog_initialzed) { -#if !defined(_WIN32) && !defined(_WIN64) +#if !defined(_WIN32) && !defined(_WIN64) && !defined(__APPLE__) google::InitGoogleLogging("mindspore"); #endif is_glog_initialzed = true; diff --git a/mindspore/core/utils/log_adapter.h b/mindspore/core/utils/log_adapter.h index fc312674d3..e54dc50360 100644 --- a/mindspore/core/utils/log_adapter.h +++ b/mindspore/core/utils/log_adapter.h @@ -80,6 +80,11 @@ struct LocationInfo { const char *func_; }; +template ::value, int>::type = 0> +constexpr std::ostream &operator<<(std::ostream &stream, const T &value) { + return stream << static_cast::type>(value); +} + class LogStream { public: LogStream() { sstream_ = std::make_shared(); } @@ -102,11 +107,6 @@ class LogStream { std::shared_ptr sstream_; }; -template ::value, int>::type = 0> -constexpr std::ostream &operator<<(std::ostream &stream, const T &value) { - return stream << static_cast::type>(value); -} - enum MsLogLevel : int { DEBUG = 0, INFO, WARNING, ERROR, EXCEPTION }; enum SubModuleId : int { diff --git a/mindspore/dataset/__init__.py b/mindspore/dataset/__init__.py index 5cd7640e21..9bb4d6a71a 100644 --- a/mindspore/dataset/__init__.py +++ b/mindspore/dataset/__init__.py @@ -31,13 +31,15 @@ Common imported modules in corresponding API examples are as follows: from mindspore.dataset.transforms import c_transforms """ +import platform from .core import config from .engine import * from .engine.cache_client import DatasetCache from .engine.datasets import * -from .engine.graphdata import GraphData from .engine.samplers import * from .engine.serializer_deserializer import compare, deserialize, serialize, show +if platform.system().lower() != "darwin": + from .engine.graphdata import GraphData __all__ = [] __all__.extend(engine.__all__) diff --git a/mindspore/dataset/core/config.py b/mindspore/dataset/core/config.py index b280fcd215..5396383936 100644 --- a/mindspore/dataset/core/config.py +++ b/mindspore/dataset/core/config.py @@ -422,9 +422,9 @@ def get_enable_shared_mem(): >>> # Get the flag of shared memory feature. >>> shared_mem_flag = ds.config.get_enable_shared_mem() """ - # For windows we forbid shared mem function temporarily - if platform.system().lower() == 'windows': - logger.warning("For windows we forbid shared mem function temporarily.") + # For windows and macos we forbid shared mem function temporarily + if platform.system().lower() in {"windows", "darwin"}: + logger.warning("For windows and macos we forbid shared mem function temporarily.") return False return _config.get_enable_shared_mem() diff --git a/mindspore/dataset/engine/__init__.py b/mindspore/dataset/engine/__init__.py index ac104f232d..67fff2bc0c 100644 --- a/mindspore/dataset/engine/__init__.py +++ b/mindspore/dataset/engine/__init__.py @@ -22,19 +22,23 @@ high performance and parse data precisely. It also provides the following operations for users to preprocess data: shuffle, batch, repeat, map, and zip. """ +import platform from ..callback import DSCallback, WaitedDSCallback from ..core import config from .cache_client import DatasetCache from .datasets import * -from .graphdata import GraphData, SamplingStrategy, OutputFormat from .iterators import * from .samplers import * from .serializer_deserializer import compare, deserialize, serialize, show +if platform.system().lower() != "darwin": + from .graphdata import GraphData, SamplingStrategy, OutputFormat __all__ = ["CelebADataset", "Cifar100Dataset", "Cifar10Dataset", "CLUEDataset", "CocoDataset", "CSVDataset", - "GeneratorDataset", "GraphData", "ImageFolderDataset", "ManifestDataset", "MindDataset", "MnistDataset", + "GeneratorDataset", "ImageFolderDataset", "ManifestDataset", "MindDataset", "MnistDataset", "NumpySlicesDataset", "PaddedDataset", "TextFileDataset", "TFRecordDataset", "VOCDataset", "DistributedSampler", "PKSampler", "RandomSampler", "SequentialSampler", "SubsetRandomSampler", "WeightedRandomSampler", "SubsetSampler", "DatasetCache", "DSCallback", "Schema", "WaitedDSCallback", "compare", "deserialize", "serialize", "show", "zip"] +if platform.system().lower() != "darwin": + __all__.append("GraphData") diff --git a/mindspore/dataset/engine/datasets.py b/mindspore/dataset/engine/datasets.py index d4e7fdf3e4..078f7dbe2b 100644 --- a/mindspore/dataset/engine/datasets.py +++ b/mindspore/dataset/engine/datasets.py @@ -81,6 +81,8 @@ try: except ModuleNotFoundError: context = None +if platform.system().lower() == "darwin": + multiprocessing.set_start_method("fork") class Shuffle(str, Enum): GLOBAL: str = "global" @@ -4384,7 +4386,7 @@ def _check_shm_usage(num_worker, queue_size, max_rowsize, num_queues=1): when training in parallel mode. """ threshold_ratio = 0.8 - if platform.system() != "Windows": + if platform.system().lower() not in {"windows", "darwin"}: shm_estimate_usage = _get_device_num() * num_worker * num_queues * \ (queue_size + 2) * max_rowsize * 1024 * 1024 try: diff --git a/third_party/patch/sentencepiece/sentencepiece.patch001_cpu b/third_party/patch/sentencepiece/sentencepiece.patch001_cpu index cea482b7e3..1fb9c2f75f 100644 --- a/third_party/patch/sentencepiece/sentencepiece.patch001_cpu +++ b/third_party/patch/sentencepiece/sentencepiece.patch001_cpu @@ -88,4 +88,20 @@ diff -Npur sentencepiece-0.1.92/src/CMakeLists.txt sentencepiece-0.1.92_bak/src/ +install(FILES sentencepiece_trainer.h sentencepiece_processor.h "${CMAKE_BINARY_DIR}/sentencepiece_model.pb.h" DESTINATION ${CMAKE_INSTALL_INCDIR}) - file(TO_NATIVE_PATH "${PROJECT_SOURCE_DIR}/data" data_dir) \ No newline at end of file + file(TO_NATIVE_PATH "${PROJECT_SOURCE_DIR}/data" data_dir) +diff -Npur sentencepiece-0.1.92/src/VERSION sentencepiece-0.1.92_bak/src/VERSION +--- a/VERSION 2021-10-29 14:48:06.343250100 +0800 ++++ b/VERSION 1970-01-01 08:00:00.000000000 +0800 +@@ -1 +0,0 @@ +-0.1.92 +diff -Npur sentencepiece-0.1.92/src/VERSION.txt sentencepiece-0.1.92_bak/src/VERSION.txt +--- a/VERSION.txt 1970-01-01 08:00:00.000000000 +0800 ++++ b/VERSION.txt 2021-10-29 14:48:09.481862200 +0800 +@@ -0,0 +1 @@ ++0.1.92 +diff -Npur sentencepiece-0.1.92/CMakeLists.txt sentencepiece-0.1.92_bak/CMakeLists.txt +--- sentencepiece-0.1.92/CMakeLists.txt 2020-06-08 16:25:01.000000000 +0800 ++++ sentencepiece-0.1.92_bak/CMakeLists.txt 2020-07-02 17:42:33.306933546 +0800 +@@ -16,1 +16,1 @@ +-file(STRINGS "VERSION" SPM_VERSION) ++file(STRINGS "VERSION.txt" SPM_VERSION)