| @@ -13,7 +13,8 @@ if (NOT CMAKE_SYSTEM_NAME MATCHES "Windows") | |||
| endif () | |||
| if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") | |||
| set(CMAKE_CXX_FLAGS_RELEASE "$ENV{CXXFLAGS} -O2 -Werror -Wno-return-std-move -Wno-unused-private-field -Wno-unused-lambda-capture -Wno-sign-compare -Wno-overloaded-virtual -Wno-unneeded-internal-declaration -Wno-unused-variable -Wno-pessimizing-move -Wno-inconsistent-missing-override -DHALF_ENABLE_CPP11_USER_LITERALS=0 -D_FORTIFY_SOURCE=2") | |||
| set(CMAKE_OSX_SYSROOT "") | |||
| set(CMAKE_CXX_FLAGS_RELEASE "$ENV{CXXFLAGS} -O2 -Winconsistent-missing-override -Wuser-defined-warnings -Wno-return-std-move -Wno-unused-private-field -Wno-unused-lambda-capture -Wno-sign-compare -Wno-overloaded-virtual -Wno-unneeded-internal-declaration -Wno-unused-variable -Wno-pessimizing-move -Wno-inconsistent-missing-override -DHALF_ENABLE_CPP11_USER_LITERALS=0 -D_FORTIFY_SOURCE=2") | |||
| else() | |||
| set(CMAKE_CXX_FLAGS_RELEASE "$ENV{CXXFLAGS} -O2 -Wl,--allow-shlib-undefined -DHALF_ENABLE_CPP11_USER_LITERALS=0 -D_FORTIFY_SOURCE=2") | |||
| endif() | |||
| @@ -5,14 +5,14 @@ set(LIB_ICU_I18N icui18n) | |||
| if (ENABLE_GITEE) | |||
| set(REQ_URL "https://gitee.com/mirrors/icu/repository/archive/release-67-1.tar.gz") | |||
| set(MD5 "72415ffd1af3acf19f9aa3fa82c7b5bc") | |||
| else() | |||
| else () | |||
| set(REQ_URL "https://github.com/unicode-org/icu/archive/release-67-1.tar.gz") | |||
| set(MD5 "fd525fb47d8827b0b7da78b51dd2d93f") | |||
| endif () | |||
| if (CMAKE_SYSTEM_NAME MATCHES "Windows") | |||
| message("icu4c thirdparty do not support windows currently.") | |||
| else() | |||
| else () | |||
| set(JSON_FILE "{ \n\ | |||
| \"strategy\": \"additive\",\n\ | |||
| \"featureFilters\": {\n\ | |||
| @@ -21,17 +21,28 @@ else() | |||
| }\ | |||
| ") | |||
| file(WRITE ${CMAKE_BINARY_DIR}/icu4c_filter.json ${JSON_FILE}) | |||
| mindspore_add_pkg(icu4c | |||
| VER 67.1 | |||
| LIBS ${LIB_ICU_COMMON} ${LIB_ICU_DATA} ${LIB_ICU_I18N} | |||
| URL ${REQ_URL} | |||
| MD5 ${MD5} | |||
| PATCHES ${CMAKE_SOURCE_DIR}/third_party/patch/icu4c/icu4c.patch01 | |||
| CONFIGURE_COMMAND ./icu4c/source/runConfigureICU Linux --enable-rpath --disable-tests --disable-samples --disable-icuio --disable-extras ICU_DATA_FILTER_FILE=${CMAKE_BINARY_DIR}/icu4c_filter.json | |||
| ) | |||
| if (CMAKE_SYSTEM_NAME MATCHES "Darwin") | |||
| mindspore_add_pkg(icu4c | |||
| VER 67.1 | |||
| LIBS ${LIB_ICU_COMMON} ${LIB_ICU_DATA} ${LIB_ICU_I18N} | |||
| URL ${REQ_URL} | |||
| MD5 ${MD5} | |||
| PATCHES ${CMAKE_SOURCE_DIR}/third_party/patch/icu4c/icu4c.patch01 | |||
| CONFIGURE_COMMAND ./icu4c/source/runConfigureICU MacOSX --enable-rpath --disable-tests --disable-samples --disable-icuio --disable-extras ICU_DATA_FILTER_FILE=${CMAKE_BINARY_DIR}/icu4c_filter.json | |||
| ) | |||
| else () | |||
| mindspore_add_pkg(icu4c | |||
| VER 67.1 | |||
| LIBS ${LIB_ICU_COMMON} ${LIB_ICU_DATA} ${LIB_ICU_I18N} | |||
| URL ${REQ_URL} | |||
| MD5 ${MD5} | |||
| PATCHES ${CMAKE_SOURCE_DIR}/third_party/patch/icu4c/icu4c.patch01 | |||
| CONFIGURE_COMMAND ./icu4c/source/runConfigureICU Linux --enable-rpath --disable-tests --disable-samples --disable-icuio --disable-extras ICU_DATA_FILTER_FILE=${CMAKE_BINARY_DIR}/icu4c_filter.json | |||
| ) | |||
| endif () | |||
| include_directories(${icu4c_INC}) | |||
| add_library(mindspore::icuuc ALIAS icu4c::${LIB_ICU_COMMON}) | |||
| add_library(mindspore::icudata ALIAS icu4c::${LIB_ICU_DATA}) | |||
| add_library(mindspore::icui18n ALIAS icu4c::${LIB_ICU_I18N}) | |||
| add_definitions(-D ENABLE_ICU4C) | |||
| endif() | |||
| endif () | |||
| @@ -1,5 +1,7 @@ | |||
| set(libevent_CFLAGS "-fstack-protector-all -D_FORTIFY_SOURCE=2 -O2") | |||
| set(libevent_LDFLAGS "-Wl,-z,now") | |||
| if (NOT CMAKE_SYSTEM_NAME MATCHES "Darwin") | |||
| set(libevent_LDFLAGS "-Wl,-z,now") | |||
| endif () | |||
| if (ENABLE_GITEE) | |||
| set(REQ_URL "https://gitee.com/mirrors/libevent/repository/archive/release-2.1.12-stable.tar.gz") | |||
| @@ -24,8 +24,8 @@ else () | |||
| set(sqlite_CFLAGS "-fstack-protector-all -Wno-uninitialized -Wno-unused-parameter -fPIC -D_FORTIFY_SOURCE=2 -O2") | |||
| else() | |||
| set(sqlite_CFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -fPIC -D_FORTIFY_SOURCE=2 -O2") | |||
| set(sqlite_LDFLAGS "-Wl,-z,relro,-z,now,-z,noexecstack") | |||
| endif() | |||
| set(sqlite_LDFLAGS "-Wl,-z,relro,-z,now,-z,noexecstack") | |||
| mindspore_add_pkg(sqlite | |||
| VER 3.32.2 | |||
| LIBS sqlite3 | |||
| @@ -4,15 +4,15 @@ set(tinyxml2_CFLAGS "-fstack-protector -D_FORTIFY_SOURCE=2 -O2") | |||
| if (ENABLE_GITEE) | |||
| set(REQ_URL "https://gitee.com/mirrors/tinyxml2/repository/archive/8.0.0.tar.gz") | |||
| set(MD5 "6a70cea637d0b17179e8bfd77860f811") | |||
| else() | |||
| else () | |||
| set(REQ_URL "https://github.com/leethomason/tinyxml2/archive/8.0.0.tar.gz") | |||
| set(MD5 "5dc535c8b34ee621fe2128f072d275b5") | |||
| endif () | |||
| if (NOT WIN32) | |||
| set(tinyxml2_LDFLAGS "-Wl,-z,relro,-z,now,-z,noexecstack") | |||
| endif() | |||
| if (NOT WIN32 AND NOT APPLE) | |||
| set(tinyxml2_LDFLAGS "-Wl,-z,relro,-z,now,-z,noexecstack") | |||
| endif () | |||
| mindspore_add_pkg(tinyxml2 | |||
| VER 8.0.0 | |||
| @@ -2,7 +2,10 @@ | |||
| include_directories(${CMAKE_SOURCE_DIR}/mindspore/core) | |||
| include_directories(${CMAKE_CURRENT_SOURCE_DIR}) | |||
| include_directories(${CMAKE_BINARY_DIR}) | |||
| link_directories(${CMAKE_SOURCE_DIR}/build/mindspore/graphengine) | |||
| if (NOT(CMAKE_SYSTEM_NAME MATCHES "Darwin")) | |||
| link_directories(${CMAKE_SOURCE_DIR}/build/mindspore/graphengine) | |||
| endif () | |||
| if (CMAKE_SYSTEM_NAME MATCHES "Windows") | |||
| set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-attributes -DHAVE_SNPRINTF") | |||
| @@ -252,9 +255,11 @@ if (ENABLE_D) | |||
| target_link_libraries(mindspore -Wl,--start-group proto_input mindspore::protobuf -Wl,--end-group) | |||
| elseif (CMAKE_SYSTEM_NAME MATCHES "Windows") | |||
| target_link_libraries(mindspore -Wl,--start-group proto_input mindspore::protobuf mindspore::sentencepiece -Wl,--end-group) | |||
| elseif (CMAKE_SYSTEM_NAME MATCHES "Darwin") | |||
| target_link_libraries(mindspore -Wl proto_input mindspore::protobuf mindspore::sentencepiece -Wl) | |||
| else () | |||
| target_link_libraries(mindspore -Wl,--start-group proto_input mindspore::protobuf -Wl,--end-group) | |||
| endif() | |||
| endif () | |||
| # set c_expression building | |||
| set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE) | |||
| @@ -296,6 +301,10 @@ if (CMAKE_SYSTEM_NAME MATCHES "Windows") | |||
| target_link_libraries(mindspore mindspore::pybind11_module) | |||
| target_link_libraries(mindspore mindspore_gvar) | |||
| target_link_libraries(_c_expression PRIVATE -Wl,--whole-archive mindspore -Wl,--no-whole-archive) | |||
| elseif (CMAKE_SYSTEM_NAME MATCHES "Darwin") | |||
| target_link_libraries(mindspore mindspore::pybind11_module) | |||
| target_link_libraries(mindspore mindspore_gvar) | |||
| target_link_libraries(_c_expression PRIVATE -Wl,-force_load mindspore -Wl,-noall_load) | |||
| else () | |||
| if (ENABLE_CPU AND (ENABLE_D OR ENABLE_GPU)) | |||
| target_link_libraries(mindspore mindspore::pslite proto_input mindspore::protobuf mindspore::event mindspore::event_pthreads ${zeromq_DIRPATH}/zmq_install/lib/libzmq.a) | |||
| @@ -326,7 +335,7 @@ if (ENABLE_GPU) | |||
| ${CUDA_PATH}/lib64/libcusolver.so) | |||
| if (ENABLE_MPI) | |||
| set_target_properties(_ms_mpi PROPERTIES INSTALL_RPATH ${MINDSPORE_RPATH}) | |||
| endif() | |||
| endif () | |||
| endif () | |||
| if (ENABLE_CPU) | |||
| @@ -349,8 +358,14 @@ add_library(inference SHARED | |||
| ) | |||
| set_target_properties(inference PROPERTIES INSTALL_RPATH ${MINDSPORE_RPATH}) | |||
| target_link_libraries(inference PRIVATE ${PYTHON_LIBRARIES} ${SECUREC_LIBRARY} | |||
| -Wl,--whole-archive mindspore proto_input -Wl,--no-whole-archive mindspore_gvar) | |||
| if (CMAKE_SYSTEM_NAME MATCHES "Darwin") | |||
| target_link_libraries(inference PRIVATE ${PYTHON_LIBRARIES} ${SECUREC_LIBRARY} | |||
| -Wl,-force_load mindspore proto_input -Wl,-noall_load mindspore_gvar) | |||
| else() | |||
| target_link_libraries(inference PRIVATE ${PYTHON_LIBRARIES} ${SECUREC_LIBRARY} | |||
| -Wl,--whole-archive mindspore proto_input -Wl,--no-whole-archive mindspore_gvar) | |||
| endif() | |||
| if (ENABLE_D) | |||
| find_library(adump_server libadump_server.a ${ASCEND_RUNTIME_PATH} ${ASCEND_TOOLKIT_RUNTIME_PATH}) | |||
| @@ -5,6 +5,10 @@ file(GLOB_RECURSE KERNEL_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} | |||
| "oplib/*.cc" | |||
| ) | |||
| if (CMAKE_SYSTEM_NAME MATCHES "Darwin") | |||
| set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-delete-non-abstract-non-virtual-dtor") | |||
| endif () | |||
| if (ENABLE_D) | |||
| file(GLOB_RECURSE D_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} | |||
| "kernel_query.cc" | |||
| @@ -22,7 +22,7 @@ namespace kernel { | |||
| void SigmoidCrossEntropyWithLogitsCPUKernel::InitKernel(const CNodePtr &kernel_node) { | |||
| CheckParam(kernel_node); | |||
| dtype_ = AnfAlgo::GetPrevNodeOutputInferDataType(kernel_node, 0); | |||
| std::vector<uint64_t> x_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 0); | |||
| std::vector<size_t> x_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 0); | |||
| for (const uint64_t &d : x_shape) { | |||
| tensor_size_ *= d; | |||
| } | |||
| @@ -22,7 +22,7 @@ namespace kernel { | |||
| void SigmoidCrossEntropyWithLogitsGradCPUKernel::InitKernel(const CNodePtr &kernel_node) { | |||
| CheckParam(kernel_node); | |||
| dtype_ = AnfAlgo::GetPrevNodeOutputInferDataType(kernel_node, 0); | |||
| std::vector<uint64_t> x_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 0); | |||
| std::vector<size_t> x_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 0); | |||
| for (const uint64_t &d : x_shape) { | |||
| tensor_size_ *= d; | |||
| } | |||
| @@ -23,7 +23,7 @@ void SmoothL1LossCPUKernel::InitKernel(const CNodePtr &kernel_node) { | |||
| beta_ = AnfAlgo::GetNodeAttr<float>(kernel_node, "beta"); | |||
| CheckParam(kernel_node); | |||
| dtype_ = AnfAlgo::GetPrevNodeOutputInferDataType(kernel_node, 0); | |||
| std::vector<uint64_t> x_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 0); | |||
| std::vector<size_t> x_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 0); | |||
| for (const uint64_t &d : x_shape) { | |||
| tensor_size_ *= d; | |||
| } | |||
| @@ -23,7 +23,7 @@ void SmoothL1LossGradCPUKernel::InitKernel(const CNodePtr &kernel_node) { | |||
| beta_ = AnfAlgo::GetNodeAttr<float>(kernel_node, "beta"); | |||
| CheckParam(kernel_node); | |||
| dtype_ = AnfAlgo::GetPrevNodeOutputInferDataType(kernel_node, 0); | |||
| std::vector<uint64_t> x_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 0); | |||
| std::vector<size_t> x_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 0); | |||
| for (const uint64_t &d : x_shape) { | |||
| tensor_size_ *= d; | |||
| } | |||
| @@ -21,5 +21,9 @@ if (ENABLE_GPU) | |||
| list(APPEND _PREACTIVATE_SRC_LIST ${_GPU_SRC_LIST}) | |||
| endif () | |||
| if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") | |||
| set(CMAKE_CXX_FLAGS_RELEASE "$ENV{CXXFLAGS} -Wno-user-defined-warnings -Wno-inconsistent-missing-override -Wno-overloaded-virtual -Wno-unused-const-variable -Wno-pessimizing-move") | |||
| endif () | |||
| set_property(SOURCE ${_PREACTIVATE_SRC_LIST} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_PRE_ACT) | |||
| add_library(_mindspore_backend_optimizer_obj OBJECT ${_PREACTIVATE_SRC_LIST}) | |||
| @@ -8,17 +8,17 @@ file(GLOB_RECURSE _SESSION_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} | |||
| "anf_runtime_algorithm.cc" | |||
| ) | |||
| if (CMAKE_SYSTEM_NAME MATCHES "Darwin") | |||
| set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-overloaded-virtual") | |||
| endif () | |||
| if (ENABLE_GPU) | |||
| file(GLOB_RECURSE _GPU_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} | |||
| "gpu_session.cc" | |||
| ) | |||
| file(GLOB_RECURSE _GPU_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "gpu_session.cc") | |||
| list(APPEND _SESSION_SRC_LIST ${_GPU_SRC_LIST}) | |||
| endif () | |||
| if (ENABLE_CPU) | |||
| file(GLOB_RECURSE _CPU_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} | |||
| "cpu_session.cc" | |||
| ) | |||
| file(GLOB_RECURSE _CPU_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "cpu_session.cc") | |||
| list(APPEND _SESSION_SRC_LIST ${_CPU_SRC_LIST}) | |||
| endif () | |||
| @@ -257,7 +257,7 @@ GraphId Executor::CompileGraph(const SessionPtr &session, const GraphSegmentPtr | |||
| GraphId Executor::CompileGraph(const SessionPtr &session, NotNull<FuncGraphPtr> func_graph) { | |||
| auto task = std::make_shared<CompileGraphTask>(); | |||
| task->session_ = session; | |||
| task->func_graph_ = func_graph; | |||
| task->func_graph_ = func_graph.get(); | |||
| SyncRunTask(task); | |||
| return task->graph_id_; | |||
| } | |||
| @@ -472,7 +472,11 @@ void KernelGraph::SetKernelInfoForNode(const AnfNodePtr &node) const { | |||
| if (kOpAssignKernelNameList.find(AnfAlgo::GetCNodeName(node)) != kOpAssignKernelNameList.end()) { | |||
| ResetAssignInputFeaatureMapFlag(node->cast<CNodePtr>()); | |||
| } | |||
| #ifdef __APPLE__ | |||
| std::vector<int> feature_map_input_indexs; | |||
| #else | |||
| std::vector<size_t> feature_map_input_indexs; | |||
| #endif | |||
| kernel_info->set_feature_map_flag(false); | |||
| for (size_t index = 0; index < AnfAlgo::GetInputTensorNum(node); ++index) { | |||
| if (AnfAlgo::IsFeatureMapInput(node, index)) { | |||
| @@ -53,7 +53,11 @@ struct OpRunInfo { | |||
| bool is_dynamic_shape = false; | |||
| bool is_auto_mixed_precision = false; | |||
| std::string next_op_name = ""; | |||
| #ifdef __APPLE__ | |||
| int next_input_index = 0; | |||
| #else | |||
| size_t next_input_index = 0; | |||
| #endif | |||
| }; | |||
| using OpRunInfoPtr = std::shared_ptr<OpRunInfo>; | |||
| class Executor; | |||
| @@ -36,8 +36,13 @@ set(MSLIB_SRC ${CMAKE_CURRENT_SOURCE_DIR}/types.cc | |||
| add_library(mindspore_shared_lib SHARED ${MSLIB_SRC}) | |||
| set_target_properties(mindspore_shared_lib PROPERTIES OUTPUT_NAME mindspore) | |||
| target_link_libraries(mindspore_shared_lib PRIVATE ${PYTHON_LIBRARIES} ${SECUREC_LIBRARY} | |||
| -Wl,--whole-archive mindspore -Wl,--no-whole-archive proto_input mindspore_gvar mindspore::protobuf) | |||
| if (CMAKE_SYSTEM_NAME MATCHES "Darwin") | |||
| target_link_libraries(mindspore_shared_lib PRIVATE ${PYTHON_LIBRARIES} ${SECUREC_LIBRARY} | |||
| -Wl,-force_load mindspore -Wl,-noall_load proto_input mindspore_gvar mindspore::protobuf) | |||
| else () | |||
| target_link_libraries(mindspore_shared_lib PRIVATE ${PYTHON_LIBRARIES} ${SECUREC_LIBRARY} | |||
| -Wl,--whole-archive mindspore -Wl,--no-whole-archive proto_input mindspore_gvar mindspore::protobuf) | |||
| endif () | |||
| if (ENABLE_CPU) | |||
| target_link_libraries(mindspore_shared_lib PRIVATE mindspore::dnnl mindspore::mkldnn) | |||
| @@ -1,3 +1,8 @@ | |||
| file(GLOB_RECURSE _OPTIMIZER_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc") | |||
| if (CMAKE_SYSTEM_NAME MATCHES "Darwin") | |||
| set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-delete-non-abstract-non-virtual-dtor") | |||
| endif () | |||
| set_property(SOURCE ${_OPTIMIZER_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_OPTIMIZER) | |||
| add_library(_mindspore_frontend_optimizer_obj OBJECT ${_OPTIMIZER_SRC_FILES}) | |||
| @@ -4,5 +4,9 @@ if (ENABLE_DUMP_PROTO) | |||
| list(REMOVE_ITEM _PARALLEL_SRC_FILES "parallel/strategy_checkpoint/parallel_strategy_checkpoint.cc") | |||
| endif () | |||
| if (CMAKE_SYSTEM_NAME MATCHES "Darwin") | |||
| set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-delete-non-abstract-non-virtual-dtor") | |||
| endif () | |||
| set_property(SOURCE ${_PARALLEL_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_PARALLEL) | |||
| add_library(_mindspore_frontend_parallel_obj OBJECT ${_PARALLEL_SRC_FILES}) | |||
| @@ -111,7 +111,7 @@ size_t GetLengthOfDataType(const TypePtr &type) { | |||
| case kNumberTypeInt: | |||
| return sizeof(int64_t); | |||
| case kNumberTypeUInt: | |||
| return sizeof(unsigned int64_t); | |||
| return sizeof(unsigned); | |||
| case kNumberTypeFloat: | |||
| return sizeof(float); | |||
| default: | |||
| @@ -3,7 +3,7 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-reorder") | |||
| set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-switch") | |||
| if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") | |||
| set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-uninitialized") | |||
| set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-uninitialized -Wno-deprecated-declarations -Wno-delete-non-abstract-non-virtual-dtor -Wno-constant-conversion") | |||
| else() | |||
| set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-maybe-uninitialized") | |||
| endif() | |||
| @@ -49,16 +49,20 @@ include_directories(${CMAKE_SOURCE_DIR}/mindspore/ccsrc/minddata/dataset/kernels | |||
| ####################### Flags ######################################## | |||
| # compile flags | |||
| set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wl,-rpath,$ORIGIN:$ORIGIN/lib") | |||
| set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility=default") | |||
| if (CMAKE_SYSTEM_NAME MATCHES "Darwin") | |||
| set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-delete-abstract-non-virtual-dtor") | |||
| else () | |||
| set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wl,-rpath,$ORIGIN:$ORIGIN/lib") | |||
| endif() | |||
| ################## Include sub-modules ############################### | |||
| add_subdirectory(util) | |||
| add_subdirectory(core) | |||
| add_subdirectory(kernels) | |||
| add_subdirectory(engine) | |||
| add_subdirectory(api) | |||
| add_subdirectory(text) | |||
| add_subdirectory(callback) | |||
| ###################################################################### | |||
| add_dependencies(utils core) | |||
| @@ -75,16 +79,20 @@ add_dependencies(engine-perf core) | |||
| add_dependencies(engine-gnn core) | |||
| add_dependencies(engine core) | |||
| add_dependencies(callback core) | |||
| add_dependencies(text core) | |||
| add_dependencies(text-kernels core) | |||
| add_dependencies(cpp-API core) | |||
| add_dependencies(engine-ir-datasetops core) | |||
| add_dependencies(engine-ir-datasetops-source core) | |||
| add_dependencies(engine-ir-cache core) | |||
| if (NOT(${CMAKE_SYSTEM_NAME} MATCHES "Darwin")) | |||
| add_subdirectory(text) | |||
| add_dependencies(text core) | |||
| add_dependencies(text-kernels core) | |||
| endif () | |||
| if (ENABLE_PYTHON) | |||
| add_dependencies(APItoPython core) | |||
| endif() | |||
| endif () | |||
| if (ENABLE_TDTQUE) | |||
| add_dependencies(engine-tdt core) | |||
| endif () | |||
| @@ -100,37 +108,42 @@ if (ENABLE_CACHE) | |||
| endif () | |||
| ################### Create _c_dataengine Library ###################### | |||
| set(submodules | |||
| $<TARGET_OBJECTS:core> | |||
| $<TARGET_OBJECTS:callback> | |||
| $<TARGET_OBJECTS:utils> | |||
| $<TARGET_OBJECTS:kernels> | |||
| $<TARGET_OBJECTS:kernels-image> | |||
| $<TARGET_OBJECTS:lite-cv> | |||
| $<TARGET_OBJECTS:kernels-data> | |||
| $<TARGET_OBJECTS:cpp-API> | |||
| $<TARGET_OBJECTS:engine-ir-datasetops> | |||
| $<TARGET_OBJECTS:core> | |||
| $<TARGET_OBJECTS:callback> | |||
| $<TARGET_OBJECTS:utils> | |||
| $<TARGET_OBJECTS:kernels> | |||
| $<TARGET_OBJECTS:kernels-image> | |||
| $<TARGET_OBJECTS:lite-cv> | |||
| $<TARGET_OBJECTS:kernels-data> | |||
| $<TARGET_OBJECTS:cpp-API> | |||
| $<TARGET_OBJECTS:engine-ir-datasetops> | |||
| $<TARGET_OBJECTS:engine-ir-datasetops-source> | |||
| $<TARGET_OBJECTS:engine-ir-cache> | |||
| $<TARGET_OBJECTS:kernels-soft-dvpp-image> | |||
| $<TARGET_OBJECTS:soft-dvpp-utils> | |||
| $<TARGET_OBJECTS:engine-datasetops-source> | |||
| $<TARGET_OBJECTS:engine-datasetops-source-sampler> | |||
| $<TARGET_OBJECTS:engine-datasetops-mapop> | |||
| $<TARGET_OBJECTS:engine-gnn> | |||
| $<TARGET_OBJECTS:engine-perf> | |||
| $<TARGET_OBJECTS:engine-datasetops> | |||
| $<TARGET_OBJECTS:engine-opt> | |||
| $<TARGET_OBJECTS:engine-cache-client> | |||
| $<TARGET_OBJECTS:engine> | |||
| $<TARGET_OBJECTS:text> | |||
| $<TARGET_OBJECTS:text-kernels> | |||
| ) | |||
| $<TARGET_OBJECTS:soft-dvpp-utils> | |||
| $<TARGET_OBJECTS:engine-datasetops-source> | |||
| $<TARGET_OBJECTS:engine-datasetops-source-sampler> | |||
| $<TARGET_OBJECTS:engine-datasetops-mapop> | |||
| $<TARGET_OBJECTS:engine-gnn> | |||
| $<TARGET_OBJECTS:engine-perf> | |||
| $<TARGET_OBJECTS:engine-datasetops> | |||
| $<TARGET_OBJECTS:engine-opt> | |||
| $<TARGET_OBJECTS:engine-cache-client> | |||
| $<TARGET_OBJECTS:engine> | |||
| ) | |||
| if (NOT(${CMAKE_SYSTEM_NAME} MATCHES "Darwin")) | |||
| set(submodules | |||
| ${submodules} | |||
| $<TARGET_OBJECTS:text> | |||
| $<TARGET_OBJECTS:text-kernels> | |||
| ) | |||
| endif () | |||
| if (ENABLE_PYTHON) | |||
| set(submodules | |||
| ${submodules} | |||
| $<TARGET_OBJECTS:APItoPython>) | |||
| endif() | |||
| endif () | |||
| if (ENABLE_TDTQUE) | |||
| add_library(_c_dataengine SHARED ${submodules} $<TARGET_OBJECTS:engine-tdt>) | |||
| @@ -143,7 +156,7 @@ set_target_properties(_c_dataengine PROPERTIES | |||
| PREFIX "${PYTHON_MODULE_PREFIX}" | |||
| SUFFIX "${PYTHON_MODULE_EXTENSION}" | |||
| ) | |||
| endif() | |||
| endif () | |||
| ###################################################################### | |||
| @@ -152,18 +165,18 @@ target_link_libraries(_c_dataengine PRIVATE mindspore mindspore_gvar) | |||
| if (${CMAKE_SYSTEM_NAME} MATCHES "Windows") | |||
| if (ENABLE_PYTHON) | |||
| target_link_libraries(_c_dataengine PRIVATE mindspore::pybind11_module ${PYTHON_LIBRARIES} ${SECUREC_LIBRARY}) | |||
| else() | |||
| else () | |||
| target_link_libraries(_c_dataengine PRIVATE ${SECUREC_LIBRARY}) | |||
| endif() | |||
| else() | |||
| endif () | |||
| else () | |||
| set(ICU_LIB mindspore::icuuc mindspore::icudata mindspore::icui18n) | |||
| if (ENABLE_PYTHON) | |||
| target_link_libraries(_c_dataengine PRIVATE mindspore::pybind11_module -ldl ${SECUREC_LIBRARY}) | |||
| else() | |||
| else () | |||
| target_link_libraries(_c_dataengine PRIVATE -ldl ${SECUREC_LIBRARY}) | |||
| endif() | |||
| endif () | |||
| target_link_libraries(_c_dataengine PUBLIC mindspore::sentencepiece) | |||
| endif() | |||
| endif () | |||
| target_link_libraries(_c_dataengine PUBLIC mindspore::jpeg_turbo mindspore::turbojpeg mindspore::opencv_core mindspore::opencv_imgcodecs | |||
| mindspore::opencv_imgproc mindspore::tinyxml2 mindspore::sentencepiece_train ${ICU_LIB}) | |||
| if (ENABLE_GPUQUE) | |||
| @@ -181,26 +194,26 @@ add_dependencies(_c_dataengine _c_mindrecord) | |||
| if (${CMAKE_SYSTEM_NAME} MATCHES "Windows") | |||
| set(MINDRECORD_LINK_OBJECT ${CMAKE_BINARY_DIR}/mindspore/ccsrc/minddata/mindrecord/CMakeFiles/_c_mindrecord.dir/objects.a) | |||
| target_link_libraries(_c_dataengine PRIVATE _c_mindrecord ${MINDRECORD_LINK_OBJECT} mindspore::sqlite) | |||
| else() | |||
| else () | |||
| target_link_libraries(_c_dataengine PRIVATE _c_mindrecord) | |||
| if (ENABLE_CPU AND (ENABLE_D OR ENABLE_GPU)) | |||
| target_link_libraries(_c_dataengine PRIVATE mindspore::pslite ${zeromq_DIRPATH}/zmq_install/lib/libzmq.a) | |||
| if (${ENABLE_IBVERBS} STREQUAL "ON") | |||
| target_link_libraries(_c_dataengine PRIVATE ibverbs rdmacm) | |||
| endif() | |||
| endif() | |||
| endif() | |||
| endif () | |||
| endif () | |||
| endif () | |||
| if (USE_GLOG) | |||
| target_link_libraries(_c_dataengine PRIVATE mindspore::glog) | |||
| else() | |||
| else () | |||
| if (CMAKE_SYSTEM_NAME MATCHES "Linux") | |||
| target_link_options(_c_dataengine PRIVATE -Wl,-init,mindspore_log_init) | |||
| elseif (CMAKE_SYSTEM_NAME MATCHES "Darwin") | |||
| set_target_properties(_c_dataengine PROPERTIES MACOSX_RPATH ON) | |||
| endif () | |||
| endif() | |||
| endif () | |||
| if (MS_BUILD_GRPC) | |||
| target_link_libraries(_c_dataengine PRIVATE mindspore::grpc++) | |||
| endif() | |||
| endif () | |||
| @@ -1,37 +1,71 @@ | |||
| file(GLOB_RECURSE _CURRENT_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc") | |||
| set_property(SOURCE ${_CURRENT_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_MD) | |||
| if (ENABLE_PYTHON) | |||
| add_library(APItoPython OBJECT | |||
| python/pybind_register.cc | |||
| python/pybind_conversion.cc | |||
| python/bindings/dataset/include/datasets_bindings.cc | |||
| python/bindings/dataset/include/iterator_bindings.cc | |||
| python/bindings/dataset/include/execute_binding.cc | |||
| python/bindings/dataset/include/schema_bindings.cc | |||
| python/bindings/dataset/engine/cache/bindings.cc | |||
| python/bindings/dataset/core/bindings.cc | |||
| python/bindings/dataset/callback/bindings.cc | |||
| python/bindings/dataset/kernels/data/bindings.cc | |||
| python/bindings/dataset/kernels/bindings.cc | |||
| python/bindings/dataset/engine/datasetops/bindings.cc | |||
| python/bindings/dataset/engine/datasetops/source/bindings.cc | |||
| python/bindings/dataset/engine/gnn/bindings.cc | |||
| python/bindings/dataset/kernels/image/bindings.cc | |||
| python/bindings/dataset/engine/datasetops/source/sampler/bindings.cc | |||
| python/bindings/dataset/text/bindings.cc | |||
| python/bindings/dataset/text/kernels/bindings.cc | |||
| python/bindings/mindrecord/include/bindings.cc | |||
| ) | |||
| if (APPLE) | |||
| add_library(APItoPython OBJECT | |||
| python/pybind_register.cc | |||
| python/pybind_conversion.cc | |||
| python/bindings/dataset/include/datasets_bindings.cc | |||
| python/bindings/dataset/include/iterator_bindings.cc | |||
| python/bindings/dataset/include/execute_binding.cc | |||
| python/bindings/dataset/include/schema_bindings.cc | |||
| python/bindings/dataset/engine/cache/bindings.cc | |||
| python/bindings/dataset/core/bindings.cc | |||
| python/bindings/dataset/callback/bindings.cc | |||
| python/bindings/dataset/kernels/data/bindings.cc | |||
| python/bindings/dataset/kernels/bindings.cc | |||
| python/bindings/dataset/engine/datasetops/bindings.cc | |||
| python/bindings/dataset/engine/datasetops/source/bindings.cc | |||
| python/bindings/dataset/engine/gnn/bindings.cc | |||
| python/bindings/dataset/kernels/image/bindings.cc | |||
| python/bindings/dataset/engine/datasetops/source/sampler/bindings.cc | |||
| python/bindings/mindrecord/include/bindings.cc | |||
| ) | |||
| else() | |||
| add_library(APItoPython OBJECT | |||
| python/pybind_register.cc | |||
| python/pybind_conversion.cc | |||
| python/bindings/dataset/include/datasets_bindings.cc | |||
| python/bindings/dataset/include/iterator_bindings.cc | |||
| python/bindings/dataset/include/execute_binding.cc | |||
| python/bindings/dataset/include/schema_bindings.cc | |||
| python/bindings/dataset/engine/cache/bindings.cc | |||
| python/bindings/dataset/core/bindings.cc | |||
| python/bindings/dataset/callback/bindings.cc | |||
| python/bindings/dataset/kernels/data/bindings.cc | |||
| python/bindings/dataset/kernels/bindings.cc | |||
| python/bindings/dataset/engine/datasetops/bindings.cc | |||
| python/bindings/dataset/engine/datasetops/source/bindings.cc | |||
| python/bindings/dataset/engine/gnn/bindings.cc | |||
| python/bindings/dataset/kernels/image/bindings.cc | |||
| python/bindings/dataset/engine/datasetops/source/sampler/bindings.cc | |||
| python/bindings/dataset/text/bindings.cc | |||
| python/bindings/dataset/text/kernels/bindings.cc | |||
| python/bindings/mindrecord/include/bindings.cc | |||
| ) | |||
| endif() | |||
| target_include_directories(APItoPython PRIVATE ${pybind11_INCLUDE_DIRS}) | |||
| endif () | |||
| add_library(cpp-API OBJECT | |||
| config.cc | |||
| datasets.cc | |||
| execute.cc | |||
| iterator.cc | |||
| transforms.cc | |||
| samplers.cc | |||
| text.cc | |||
| vision.cc | |||
| ) | |||
| if (APPLE) | |||
| add_library(cpp-API OBJECT | |||
| config.cc | |||
| datasets.cc | |||
| execute.cc | |||
| iterator.cc | |||
| transforms.cc | |||
| samplers.cc | |||
| vision.cc | |||
| ) | |||
| else() | |||
| add_library(cpp-API OBJECT | |||
| config.cc | |||
| datasets.cc | |||
| execute.cc | |||
| iterator.cc | |||
| transforms.cc | |||
| samplers.cc | |||
| text.cc | |||
| vision.cc | |||
| ) | |||
| endif() | |||
| @@ -695,13 +695,17 @@ Status Tensor::GetDataAsNumpy(py::array *data) { | |||
| } | |||
| Status Tensor::GetDataAsNumpyStrings(py::array *data) { | |||
| auto itr = begin<std::string_view>(); | |||
| uint64_t max = 0; | |||
| uint64_t max_value = 0; | |||
| for (; itr != end<std::string_view>(); itr++) { | |||
| max = std::max((*itr).length(), max); | |||
| #ifdef __APPLE__ | |||
| max_value = fmax((*itr).length(), max_value); | |||
| #else | |||
| max_value = std::max((*itr).length(), max_value); | |||
| #endif | |||
| } | |||
| // if all strings are empty, numpy stores a byte for each string |S1 | |||
| max = (max == 0 ? 1 : max); | |||
| uint64_t total_size = shape_.NumOfElements() * max; | |||
| max_value = (max_value == 0 ? 1 : max_value); | |||
| uint64_t total_size = shape_.NumOfElements() * max_value; | |||
| char *tmp_data = reinterpret_cast<char *>(data_allocator_->allocate(total_size)); | |||
| if (tmp_data == nullptr) RETURN_STATUS_UNEXPECTED("Cannot create temp array."); | |||
| int ret_code = memset_s(tmp_data, total_size, 0, total_size); | |||
| @@ -711,13 +715,14 @@ Status Tensor::GetDataAsNumpyStrings(py::array *data) { | |||
| uint64_t i = 0; | |||
| for (; itr != end<std::string_view>(); itr++, i++) { | |||
| if (!(*itr).empty()) { | |||
| ret_code = memcpy_s(tmp_data + i * max, total_size, (*itr).data(), (*itr).length()); | |||
| ret_code = memcpy_s(tmp_data + i * max_value, total_size, (*itr).data(), (*itr).length()); | |||
| CHECK_FAIL_RETURN_UNEXPECTED(ret_code == 0, "Failed to copy string data."); | |||
| } | |||
| } | |||
| auto strides = shape_.Strides(); | |||
| std::transform(strides.begin(), strides.end(), strides.begin(), [&max](const auto &s) { return s * max; }); | |||
| *data = py::array(py::dtype("S" + std::to_string(max)), shape_.AsVector(), strides, tmp_data); | |||
| std::transform(strides.begin(), strides.end(), strides.begin(), | |||
| [&max_value](const auto &s) { return s * max_value; }); | |||
| *data = py::array(py::dtype("S" + std::to_string(max_value)), shape_.AsVector(), strides, tmp_data); | |||
| data_allocator_->deallocate(reinterpret_cast<uchar *>(tmp_data)); | |||
| return Status::OK(); | |||
| } | |||
| @@ -154,6 +154,9 @@ Status CacheServerHW::GetNumaNodeInfo() { | |||
| } | |||
| Status CacheServerHW::SetAffinity(const Task &tk, numa_id_t numa_node) { | |||
| #ifdef __APPLE__ | |||
| return Status::OK(); | |||
| #else | |||
| auto r = numa_cpuset_.find(numa_node); | |||
| if (r != numa_cpuset_.end()) { | |||
| auto err = pthread_setaffinity_np(tk.GetNativeHandle(), sizeof(r->second), &r->second); | |||
| @@ -165,6 +168,7 @@ Status CacheServerHW::SetAffinity(const Task &tk, numa_id_t numa_node) { | |||
| RETURN_STATUS_UNEXPECTED("Numa node " + std::to_string(numa_node) + " not found"); | |||
| } | |||
| return Status::OK(); | |||
| #endif | |||
| } | |||
| std::vector<cpu_id_t> CacheServerHW::GetCpuList(numa_id_t numa_id) { | |||
| @@ -182,6 +186,9 @@ std::vector<cpu_id_t> CacheServerHW::GetCpuList(numa_id_t numa_id) { | |||
| } | |||
| numa_id_t CacheServerHW::GetMyNode() const { | |||
| #ifdef __APPLE__ | |||
| numa_id_t node_id = -1; | |||
| #else | |||
| numa_id_t node_id = 0; | |||
| auto cpu = sched_getcpu(); | |||
| #ifdef NUMA_ENABLED | |||
| @@ -197,7 +204,8 @@ numa_id_t CacheServerHW::GetMyNode() const { | |||
| } | |||
| } | |||
| MS_LOG(DEBUG) << "cpu id " << cpu << " found : " << std::boolalpha << found; | |||
| #endif | |||
| #endif // end NUMA_ENABLED | |||
| #endif // end __APPLE__ | |||
| return node_id; | |||
| } | |||
| @@ -18,7 +18,7 @@ | |||
| #ifdef NUMA_ENABLED | |||
| #include <numa.h> | |||
| #endif | |||
| #endif // NUMA_ENABLED | |||
| #include <sched.h> | |||
| #include <stdlib.h> | |||
| #include <map> | |||
| @@ -32,6 +32,23 @@ | |||
| #include "minddata/dataset/util/status.h" | |||
| #include "minddata/dataset/util/task.h" | |||
| #ifdef __APPLE__ | |||
| #define SYSCTL_CORE_COUNT "machdep.cpu.core_count" | |||
| #include <sys/sysctl.h> | |||
| #include <mach/thread_policy.h> | |||
| typedef struct cpu_set { | |||
| uint32_t count; | |||
| } cpu_set_t; | |||
| static inline void CPU_ZERO(cpu_set_t *cs) { cs->count = 0; } | |||
| static inline void CPU_SET(int num, cpu_set_t *cs) { cs->count |= (1 << num); } | |||
| static inline int CPU_ISSET(int num, cpu_set_t *cs) { return (cs->count & (1 << num)); } | |||
| #endif // __APPLE__ | |||
| namespace mindspore { | |||
| namespace dataset { | |||
| class CacheServerHW { | |||
| @@ -235,7 +235,7 @@ Status CreateCacheRequest::PostReply() { | |||
| // Next is a set of cpu id that we should re-adjust ourselves for better affinity. | |||
| auto sz = p->cpu_id()->size(); | |||
| cc_->cpu_list_.reserve(sz); | |||
| #if !defined(_WIN32) && !defined(_WIN64) && !defined(__ANDROID__) && !defined(ANDROID) | |||
| #if !defined(_WIN32) && !defined(_WIN64) && !defined(__ANDROID__) && !defined(ANDROID) && !defined(__APPLE__) | |||
| std::string c_list; | |||
| cpu_set_t cpu_set; | |||
| CPU_ZERO(&cpu_set); | |||
| @@ -243,13 +243,13 @@ Status CreateCacheRequest::PostReply() { | |||
| for (auto i = 0; i < sz; ++i) { | |||
| auto cpu_id = p->cpu_id()->Get(i); | |||
| cc_->cpu_list_.push_back(cpu_id); | |||
| #if !defined(_WIN32) && !defined(_WIN64) && !defined(__ANDROID__) && !defined(ANDROID) | |||
| #if !defined(_WIN32) && !defined(_WIN64) && !defined(__ANDROID__) && !defined(ANDROID) && !defined(__APPLE__) | |||
| c_list += std::to_string(cpu_id) + " "; | |||
| CPU_SET(cpu_id, &cpu_set); | |||
| #endif | |||
| } | |||
| #if !defined(_WIN32) && !defined(_WIN64) && !defined(__ANDROID__) && !defined(ANDROID) | |||
| #if !defined(_WIN32) && !defined(_WIN64) && !defined(__ANDROID__) && !defined(ANDROID) && !defined(__APPLE__) | |||
| if (sz > 0) { | |||
| auto err = sched_setaffinity(getpid(), sizeof(cpu_set), &cpu_set); | |||
| if (err == -1) { | |||
| @@ -71,6 +71,8 @@ Status StorageContainer::Read(WritableSlice *dest, off64_t offset) const noexcep | |||
| RETURN_STATUS_UNEXPECTED(strerror(errno)); | |||
| } | |||
| auto r_sz = read(fd_, dest->GetMutablePointer(), sz); | |||
| #elif defined(__APPLE__) | |||
| auto r_sz = pread(fd_, dest->GetMutablePointer(), sz, offset); | |||
| #else | |||
| auto r_sz = pread64(fd_, dest->GetMutablePointer(), sz, offset); | |||
| #endif | |||
| @@ -94,6 +96,8 @@ Status StorageContainer::Write(const ReadableSlice &dest, off64_t offset) const | |||
| RETURN_STATUS_UNEXPECTED(strerror(errno)); | |||
| } | |||
| auto r_sz = write(fd_, dest.GetPointer(), sz); | |||
| #elif defined(__APPLE__) | |||
| auto r_sz = pwrite(fd_, dest.GetPointer(), sz, offset); | |||
| #else | |||
| auto r_sz = pwrite64(fd_, dest.GetPointer(), sz, offset); | |||
| #endif | |||
| @@ -745,7 +745,13 @@ Status TFReaderOp::LoadBytesList(const ColDescriptor ¤t_col, const dataeng | |||
| } | |||
| uint64_t max_size = 0; | |||
| for (uint32_t i = 0; i < bytes_list.value_size(); ++i) max_size = std::max(max_size, bytes_list.value(i).size()); | |||
| for (uint32_t i = 0; i < bytes_list.value_size(); ++i) { | |||
| #ifdef __APPLE__ | |||
| max_size = fmax(max_size, bytes_list.value(i).size()); | |||
| #else | |||
| max_size = std::max(max_size, bytes_list.value(i).size()); | |||
| #endif | |||
| } | |||
| int64_t pad_size = max_size; | |||
| @@ -136,7 +136,7 @@ void ExecutionTree::PrintNode(std::ostream &out, const std::shared_ptr<DatasetOp | |||
| Status ExecutionTree::Launch() { | |||
| // opencv limit too many threads | |||
| #ifndef ENABLE_ANDROID | |||
| #if !defined(_WIN32) && !defined(_WIN64) | |||
| #if !defined(_WIN32) && !defined(_WIN64) && !defined(__APPLE__) | |||
| int32_t thread_num = get_nprocs(); | |||
| if (thread_num == 0) { | |||
| std::string err_msg = "Invalid thread number."; | |||
| @@ -22,7 +22,7 @@ | |||
| #include <string> | |||
| #include <vector> | |||
| #ifndef ENABLE_ANDROID | |||
| #if !defined(_WIN32) && !defined(_WIN64) | |||
| #if !defined(_WIN32) && !defined(_WIN64) && !defined(__APPLE__) | |||
| #include <sys/sysinfo.h> | |||
| #include <opencv2/imgproc/imgproc.hpp> | |||
| #endif | |||
| @@ -193,7 +193,7 @@ void JpegSetSource(j_decompress_ptr cinfo, const void *data, int64_t datasize) { | |||
| (*cinfo->mem->alloc_small)(reinterpret_cast<j_common_ptr>(cinfo), JPOOL_PERMANENT, sizeof(struct jpeg_source_mgr))); | |||
| cinfo->src->init_source = JpegInitSource; | |||
| cinfo->src->fill_input_buffer = JpegFillInputBuffer; | |||
| #if defined(_WIN32) || defined(_WIN64) || defined(ENABLE_ARM32) | |||
| #if defined(_WIN32) || defined(_WIN64) || defined(ENABLE_ARM32) || defined(__APPLE__) | |||
| cinfo->src->skip_input_data = reinterpret_cast<void (*)(j_decompress_ptr, long)>(JpegSkipInputData); | |||
| #else | |||
| cinfo->src->skip_input_data = JpegSkipInputData; | |||
| @@ -25,6 +25,9 @@ | |||
| #if defined(_WIN32) || defined(_WIN64) | |||
| #undef HAVE_STDDEF_H | |||
| #undef HAVE_STDLIB_H | |||
| #elif __APPLE__ | |||
| #include <sys/param.h> | |||
| #include <sys/mount.h> | |||
| #endif | |||
| #include "./jpeglib.h" | |||
| #include "./jerror.h" | |||
| @@ -18,7 +18,7 @@ | |||
| #include <string> | |||
| #include <memory> | |||
| #if !defined(__APPLE__) | |||
| #include "cppjieba/Jieba.hpp" | |||
| #include "minddata/dataset/core/constants.h" | |||
| #include "minddata/dataset/kernels/tensor_op.h" | |||
| @@ -67,3 +67,4 @@ class JiebaTokenizerOp : public TensorOp { | |||
| } // namespace dataset | |||
| } // namespace mindspore | |||
| #endif // MINDSPORE_CCSRC_MINDDATA_DATASET_ENGINE_TEXT_JIEBA_OP_H_ | |||
| #endif | |||
| @@ -27,6 +27,10 @@ | |||
| #include "minddata/dataset/util/random.h" | |||
| #include "minddata/dataset/util/task_manager.h" | |||
| #if defined(__APPLE__) | |||
| #define LOGIN_NAME_MAX 256 | |||
| #endif | |||
| namespace mindspore { | |||
| namespace dataset { | |||
| std::unique_ptr<Services> Services::instance_ = nullptr; | |||
| @@ -16,7 +16,7 @@ | |||
| #include "minddata/dataset/util/sig_handler.h" | |||
| #include <signal.h> | |||
| #include <sys/types.h> | |||
| #if !defined(_WIN32) && !defined(_WIN64) && !defined(__ANDROID__) && !defined(ANDROID) | |||
| #if !defined(_WIN32) && !defined(_WIN64) && !defined(__ANDROID__) && !defined(ANDROID) && !defined(__APPLE__) | |||
| #include <ucontext.h> | |||
| #endif | |||
| #include <unistd.h> | |||
| @@ -22,6 +22,11 @@ | |||
| #include "./securec.h" | |||
| #include "minddata/dataset/util/allocator.h" | |||
| #include "minddata/dataset/util/status.h" | |||
| #ifdef __APPLE__ | |||
| #define off64_t off_t | |||
| #endif | |||
| namespace mindspore { | |||
| namespace dataset { | |||
| /// \brief A ReadableSlice wraps a const pointer in memory and its size. | |||
| @@ -10,7 +10,11 @@ aux_source_directory(common DIR_LIB_SRCS) | |||
| # set(CMAKE_CXX_COMPILER "g++") | |||
| # set(CMAKE_CXX_FLAGS "-Wall -fvisibility=hidden") | |||
| set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wl,-rpath,$ORIGIN:$ORIGIN/lib") | |||
| if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") | |||
| set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-command-line-argument") | |||
| else() | |||
| set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wl,-rpath,$ORIGIN:$ORIGIN/lib") | |||
| endif() | |||
| set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-format") | |||
| set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-return-type") | |||
| set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-variable") | |||
| @@ -164,7 +164,7 @@ bool IsLegalFile(const std::string &path) { | |||
| } | |||
| std::pair<MSRStatus, uint64_t> GetDiskSize(const std::string &str_dir, const DiskSizeType &disk_type) { | |||
| #if defined(_WIN32) || defined(_WIN64) | |||
| #if defined(_WIN32) || defined(_WIN64) || defined(__APPLE__) | |||
| return {SUCCESS, 100}; | |||
| #else | |||
| uint64_t ll_count = 0; | |||
| @@ -21,7 +21,7 @@ | |||
| #include <limits.h> | |||
| #include <stdlib.h> | |||
| #include <sys/stat.h> | |||
| #if !defined(_WIN32) && !defined(_WIN64) | |||
| #if !defined(_WIN32) && !defined(_WIN64) && !defined(__APPLE__) | |||
| #include <sys/statfs.h> | |||
| #include <sys/wait.h> | |||
| #endif | |||
| @@ -19,7 +19,7 @@ | |||
| #include <dirent.h> | |||
| #include <signal.h> | |||
| #if !defined(_WIN32) && !defined(_WIN64) | |||
| #if !defined(_WIN32) && !defined(_WIN64) && !defined(__APPLE__) | |||
| #include <sys/prctl.h> | |||
| #endif | |||
| #include <sys/stat.h> | |||
| @@ -1173,7 +1173,7 @@ TASK_RETURN_CONTENT ShardReader::ConsumerOneTask(int task_id, uint32_t consumer_ | |||
| MSRStatus ShardReader::ConsumerByRow(int consumer_id) { | |||
| // Set thread name | |||
| #if !defined(_WIN32) && !defined(_WIN64) | |||
| #if !defined(_WIN32) && !defined(_WIN64) && !defined(__APPLE__) | |||
| auto thread_id = kThreadName + std::to_string(consumer_id); | |||
| prctl(PR_SET_NAME, common::SafeCStr(thread_id), 0, 0, 0); | |||
| #endif | |||
| @@ -1337,6 +1337,15 @@ py::object PynativeExecutor::RunOpInMs(const OpExecInfoPtr &op_exec_info, Pynati | |||
| ConstructInputTensor(op_exec_info, &tensors_mask, &input_tensors); | |||
| // get graph info for checking it whether existing in the cache | |||
| std::string graph_info = GetSingleOpGraphInfo(op_exec_info, input_tensors); | |||
| #ifdef __APPLE__ | |||
| session::OpRunInfo op_run_info = {op_exec_info->op_name, | |||
| op_exec_info->py_primitive, | |||
| op_exec_info->abstract, | |||
| op_exec_info->is_dynamic_shape, | |||
| op_exec_info->is_mixed_precision_cast, | |||
| op_exec_info->next_op_name, | |||
| static_cast<int>(op_exec_info->next_input_index)}; | |||
| #else | |||
| session::OpRunInfo op_run_info = {op_exec_info->op_name, | |||
| op_exec_info->py_primitive, | |||
| op_exec_info->abstract, | |||
| @@ -1344,6 +1353,7 @@ py::object PynativeExecutor::RunOpInMs(const OpExecInfoPtr &op_exec_info, Pynati | |||
| op_exec_info->is_mixed_precision_cast, | |||
| op_exec_info->next_op_name, | |||
| op_exec_info->next_input_index}; | |||
| #endif | |||
| VectorRef outputs; | |||
| session->RunOp(&op_run_info, graph_info, &input_tensors, &outputs, tensors_mask); | |||
| if (op_exec_info->is_dynamic_shape) { | |||
| @@ -847,7 +847,11 @@ void KernelRuntime::GenAddrCleanLaunchArgs(const CNodePtr &cnode, AddressPtrList | |||
| auto pre_node = (cnode->inputs()[1])->cast<CNodePtr>(); | |||
| // set clean output address | |||
| if (AnfAlgo::HasNodeAttr(kAttrAtomicOutputIndexs, pre_node)) { | |||
| #if defined(__APPLE__) | |||
| auto clean_output_indexes = AnfAlgo::GetNodeAttr<std::vector<int>>(pre_node, kAttrAtomicOutputIndexs); | |||
| #else | |||
| auto clean_output_indexes = AnfAlgo::GetNodeAttr<std::vector<size_t>>(pre_node, kAttrAtomicOutputIndexs); | |||
| #endif | |||
| for (auto index : clean_output_indexes) { | |||
| auto device_address = AnfAlgo::GetOutputAddr(pre_node, index); | |||
| kernel::AddressPtr input = std::make_shared<kernel::Address>(); | |||
| @@ -861,7 +865,11 @@ void KernelRuntime::GenAddrCleanLaunchArgs(const CNodePtr &cnode, AddressPtrList | |||
| } | |||
| // set clean workspace address | |||
| if (AnfAlgo::HasNodeAttr(kAttrAtomicWorkspaceIndexs, pre_node)) { | |||
| #if defined(__APPLE__) | |||
| auto clean_workspaces_indexes = AnfAlgo::GetNodeAttr<std::vector<int>>(pre_node, kAttrAtomicWorkspaceIndexs); | |||
| #else | |||
| auto clean_workspaces_indexes = AnfAlgo::GetNodeAttr<std::vector<size_t>>(pre_node, kAttrAtomicWorkspaceIndexs); | |||
| #endif | |||
| for (const auto &index : clean_workspaces_indexes) { | |||
| auto device_address = AnfAlgo::GetWorkspaceAddr(pre_node, index); | |||
| kernel::AddressPtr workspace = std::make_shared<kernel::Address>(); | |||
| @@ -21,6 +21,7 @@ | |||
| #include <iomanip> | |||
| #include <fstream> | |||
| #include <memory> | |||
| #include <vector> | |||
| namespace mindspore { | |||
| namespace system { | |||
| @@ -100,9 +101,9 @@ bool ProcessInner(const std::string &message, const int &bias, uint32_t *digest, | |||
| w[i] = sigma3(w[i - 2]) + w[i - 7] + sigma2(w[i - 15]) + w[i - 16]; | |||
| } | |||
| std::shared_ptr<uint32_t[]> hash(new uint32_t[digest_size]()); | |||
| std::vector<uint32_t> hash(digest_size); | |||
| size_t mem_size = digest_size * sizeof(uint32_t); | |||
| auto ret = memcpy_s(hash.get(), mem_size, digest, mem_size); | |||
| auto ret = memcpy_s(hash.data(), mem_size, digest, mem_size); | |||
| if (ret != EOK) { | |||
| return false; | |||
| } | |||
| @@ -131,7 +131,7 @@ std::tuple<FuncGraphPtr, AnfNodePtrList, AnfNodePtrList> TransformSegmentToAnfGr | |||
| if (!IsValueNode<Primitive>(inps[0]) && | |||
| !(IsValueNode<FuncGraph>(inps[0]) && | |||
| inps[0]->cast<ValueNodePtr>()->value()->cast<FuncGraphPtr>()->has_attr(FUNC_GRAPH_ATTR_GRAPH_KERNEL))) { | |||
| MS_LOG(EXCEPTION) << "Input[0] Must be a Primitive valuenode"; | |||
| MS_LOG(EXCEPTION) << "Input[0] Must be a Primitive ValueNode"; | |||
| } | |||
| auto fn = inps[0]; | |||
| std::vector<AnfNodePtr> args{fn}; | |||
| @@ -141,7 +141,7 @@ std::tuple<FuncGraphPtr, AnfNodePtrList, AnfNodePtrList> TransformSegmentToAnfGr | |||
| } else if (IsPrimitive(fn, prim::kPrimControlDepend) && inps.size() == 3) { | |||
| for (size_t i = 1; i < inps.size(); ++i) { | |||
| if (inps[i]->isa<CNode>() && std::find(lst.begin(), lst.end(), inps[i]) == lst.end()) { | |||
| args.emplace_back(NewValueNode(MakeValue(i))); | |||
| args.emplace_back(NewValueNode(MakeValue(static_cast<int>(i)))); | |||
| } else { | |||
| args.emplace_back(RefSubGraphNode(fg, inps[i], &inputs, &eqv)); | |||
| } | |||
| @@ -2,12 +2,9 @@ include_directories(${CMAKE_CURRENT_SOURCE_DIR}) | |||
| include_directories(${CMAKE_BINARY_DIR}) | |||
| include_directories(${CMAKE_SOURCE_DIR}/mindspore/core) | |||
| add_subdirectory(gvar) | |||
| message("************build core***************") | |||
| message("************ build core ***************") | |||
| if (CMAKE_SYSTEM_NAME MATCHES "Windows") | |||
| set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-attributes -DHAVE_SNPRINTF") | |||
| add_compile_definitions(BUILDING_DLL) | |||
| endif() | |||
| file(GLOB_RECURSE CORE_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} | |||
| "abstract/*.cc" | |||
| @@ -17,9 +14,17 @@ file(GLOB_RECURSE CORE_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} | |||
| "utils/*.cc" | |||
| "load_mindir/*.cc" | |||
| ) | |||
| if (CMAKE_SYSTEM_NAME MATCHES "Windows") | |||
| set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-attributes -DHAVE_SNPRINTF") | |||
| add_compile_definitions(BUILDING_DLL) | |||
| elseif (CMAKE_SYSTEM_NAME MATCHES "Darwin") | |||
| set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wuser-defined-warnings -Winconsistent-missing-override") | |||
| endif () | |||
| set_property(SOURCE ${CORE_SRC_LIST} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_CORE) | |||
| add_library(mindspore_core STATIC ${CORE_SRC_LIST}) | |||
| target_link_libraries(mindspore_core PRIVATE mindspore_gvar) | |||
| if (USE_GLOG) | |||
| target_link_libraries(mindspore_core PRIVATE mindspore::glog) | |||
| endif () | |||
| @@ -1,12 +1,16 @@ | |||
| file(GLOB_RECURSE MS_GVAR_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} *.cc) | |||
| set_property(SOURCE ${MS_GVAR_SRC_LIST} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_COMMON) | |||
| set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D_FORTIFY_SOURCE=2 -O2") | |||
| if (CMAKE_SYSTEM_NAME MATCHES "Windows") | |||
| set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-attributes -DHAVE_SNPRINTF") | |||
| add_compile_definitions(BUILDING_DLL) | |||
| endif() | |||
| elseif (CMAKE_SYSTEM_NAME MATCHES "Darwin") | |||
| set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wuser-defined-warnings -Winconsistent-missing-override") | |||
| endif () | |||
| add_library(mindspore_gvar SHARED ${MS_GVAR_SRC_LIST}) | |||
| if (APPLE) | |||
| set_target_properties(mindspore_gvar PROPERTIES MACOSX_RPATH ON) | |||
| endif () | |||
| @@ -5,8 +5,8 @@ | |||
| THE_OS="Linux" | |||
| THE_COMP="the clang or else GNU C++" | |||
| - RELEASE_CFLAGS='-O3' | |||
| + RELEASE_CFLAGS='-fstack-protector -Wl,-z,now -D_FORTIFY_SOURCE=2 -O3' | |||
| + RELEASE_CFLAGS='-fstack-protector -D_FORTIFY_SOURCE=2 -O3' | |||
| - RELEASE_CXXFLAGS='-O3' | |||
| + RELEASE_CXXFLAGS='-fstack-protector -Wl,-z,now -D_FORTIFY_SOURCE=2 -O3' | |||
| + RELEASE_CXXFLAGS='-fstack-protector -D_FORTIFY_SOURCE=2 -O3' | |||
| DEBUG_CFLAGS='-g' | |||
| DEBUG_CXXFLAGS='-g' | |||
| @@ -1,9 +1,9 @@ | |||
| SET(CMAKE_BUILD_TYPE "Debug") | |||
| if (CMAKE_SYSTEM_NAME MATCHES "Windows") | |||
| SET(CMAKE_C_FLAGS_DEBUG "$ENV{CFLAGS} -fPIC -O0 -Wall -Wno-deprecated-declarations -g2 -ggdb -fno-inline-functions -fno-omit-frame-pointer -fstack-protector-all") | |||
| else() | |||
| SET(CMAKE_C_FLAGS_DEBUG "$ENV{CFLAGS} -fPIC -O0 -Wall -Wno-deprecated-declarations -g2 -ggdb -fno-inline-functions -fno-omit-frame-pointer -fstack-protector-all -D_LIBCPP_INLINE_VISIBILITY='' -D'_LIBCPP_EXTERN_TEMPLATE(...)='") | |||
| endif() | |||
| else () | |||
| SET(CMAKE_C_FLAGS_DEBUG "$ENV{CFLAGS} -Wno-nullability-completeness -fPIC -O0 -Wall -Wno-deprecated-declarations -g2 -ggdb -fno-inline-functions -fno-omit-frame-pointer -fstack-protector-all -D_LIBCPP_INLINE_VISIBILITY='' -D'_LIBCPP_EXTERN_TEMPLATE(...)='") | |||
| endif () | |||
| SET(CMAKE_C_FLAGS_RELEASE "$ENV{CFLAGS} -fPIC -O3 -Wall -Wno-deprecated-declarations -fstack-protector-all") | |||
| set(CMAKE_EXPORT_COMPILE_COMMANDS ON) | |||