diff --git a/.gitignore b/.gitignore index 77ff222a1a..057169ec42 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,7 @@ cmake-build-debug *_pb2.py *.pb.h *.pb.cc +*.pb # Object files *.o diff --git a/CMakeLists.txt b/CMakeLists.txt index 37c3288f12..987e4ae709 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -12,7 +12,7 @@ if (NOT CMAKE_SYSTEM_NAME MATCHES "Windows") endif () if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") - set(CMAKE_CXX_FLAGS_RELEASE "$ENV{CXXFLAGS} -O2 -Werror -Wno-return-std-move -Wno-unused-private-field -Wno-unused-lambda-capture -Wno-sign-compare -Wno-overloaded-virtual -Wno-unneeded-internal-declaration -Wno-unused-variable -Wno-pessimizing-move -Wno-inconsistent-missing-override -DHALF_ENABLE_CPP11_USER_LITERALS=0 -D_FORTIFY_SOURCE=2") + set(CMAKE_CXX_FLAGS_RELEASE "$ENV{CXXFLAGS} -O2 -Werror -Wno-return-std-move -Wno-unused-private-field -Wno-unused-lambda-capture -Wno-sign-compare -Wno-overloaded-virtual -Wno-unneeded-internal-declaration -Wno-unused-variable -Wno-pessimizing-move -Wno-inconsistent-missing-override -DHALF_ENABLE_CPP11_USER_LITERALS=0 -D_FORTIFY_SOURCE=2") else() set(CMAKE_CXX_FLAGS_RELEASE "$ENV{CXXFLAGS} -O2 -Wl,--allow-shlib-undefined -DHALF_ENABLE_CPP11_USER_LITERALS=0 -D_FORTIFY_SOURCE=2") endif() @@ -38,6 +38,10 @@ if (NOT Patch_FOUND) endif () message(PATCH_EXECUTABLE = ${Patch_EXECUTABLE}) +if (ENABLE_AKG AND ENABLE_D) + add_subdirectory("${CMAKE_SOURCE_DIR}/akg") +endif() + include(${CMAKE_SOURCE_DIR}/cmake/mind_expression.cmake) include_directories(${CMAKE_CURRENT_SOURCE_DIR}) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/third_party/flatbuffers/include) @@ -86,10 +90,6 @@ if (ENABLE_GE OR ENABLE_D OR ENABLE_TESTCASES) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/graphengine/third_party/fwkacllib/inc/toolchain) endif() -if (ENABLE_AKG AND ENABLE_D) - add_subdirectory("${CMAKE_SOURCE_DIR}/akg") -endif() - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility=hidden") add_subdirectory(mindspore/ccsrc) if (ENABLE_TESTCASES) diff --git a/README.md b/README.md index a6bfd1ebbb..25abdd6fcb 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ enrichment of the AI software/hardware application ecosystem. MindSpore Architecture -For more details please check out our [Architecture Guide](https://www.mindspore.cn/docs/en/0.3.0-alpha/architecture.html). +For more details please check out our [Architecture Guide](https://www.mindspore.cn/docs/en/master/architecture.html). ### Automatic Differentiation @@ -66,7 +66,6 @@ MindSpore offers build options across multiple backends: | Ascend910 | Ubuntu-x86 | ✔️ | | | EulerOS-x86 | ✔️ | | | EulerOS-aarch64 | ✔️ | -| GPU CUDA 9.2 | Ubuntu-x86 | ✔️ | | GPU CUDA 10.1 | Ubuntu-x86 | ✔️ | | CPU | Ubuntu-x86 | ✔️ | | | Windows-x86 | ✔️ | @@ -76,7 +75,7 @@ For installation using `pip`, take `CPU` and `Ubuntu-x86` build version as an ex 1. Download whl from [MindSpore download page](https://www.mindspore.cn/versions/en), and install the package. ``` - pip install https://ms-release.obs.cn-north-4.myhuaweicloud.com/0.3.0-alpha/MindSpore/cpu/ubuntu_x86/mindspore-0.3.0-cp37-cp37m-linux_x86_64.whl + pip install https://ms-release.obs.cn-north-4.myhuaweicloud.com/0.5.0-beta/MindSpore/cpu/ubuntu_x86/mindspore-0.5.0-cp37-cp37m-linux_x86_64.whl ``` 2. Run the following command to verify the install. @@ -133,8 +132,8 @@ currently the containerized build options are supported as follows: For `CPU` backend, you can directly pull and run the latest stable image using the below command: ``` - docker pull mindspore/mindspore-cpu:0.3.0-alpha - docker run -it mindspore/mindspore-cpu:0.3.0-alpha /bin/bash + docker pull mindspore/mindspore-cpu:0.5.0-beta + docker run -it mindspore/mindspore-cpu:0.5.0-beta /bin/bash ``` * GPU @@ -151,8 +150,8 @@ currently the containerized build options are supported as follows: Then you can pull and run the latest stable image using the below command: ``` - docker pull mindspore/mindspore-gpu:0.3.0-alpha - docker run -it --runtime=nvidia --privileged=true mindspore/mindspore-gpu:0.3.0-alpha /bin/bash + docker pull mindspore/mindspore-gpu:0.5.0-beta + docker run -it --runtime=nvidia --privileged=true mindspore/mindspore-gpu:0.5.0-beta /bin/bash ``` To test if the docker image works, please execute the python code below and check the output: @@ -187,7 +186,7 @@ please check out [docker](docker/README.md) repo for the details. ## Quickstart -See the [Quick Start](https://www.mindspore.cn/tutorial/en/0.3.0-alpha/quick_start/quick_start.html) +See the [Quick Start](https://www.mindspore.cn/tutorial/en/master/quick_start/quick_start.html) to implement the image classification. ## Docs diff --git a/RELEASE.md b/RELEASE.md index 9824f803f0..4b829152a2 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -1,3 +1,75 @@ +# Release 0.5.0-beta + +## Major Features and Improvements + +### Ascend 910 Training and Inference Framework +* New models + * ResNext50: a simple, highly modularized network architecture using aggregated resdiual transformations for image classification on ImageNet 2012 dataset. + * MASS: a pre-training method for sequence to sequence based language generation tasks on Text Summarization and Conversational Response Generation using News Crawls 2007-2017 dataset, Gigaword corpus and Cornell movie dialog corpus. + * Transformer: a neural network architecture for language understanding on WMT 2014 English-German dataset. + * GCN：Graph Convolutional Networks for the task of classification of nodes in a graph on Cora and Citeseer datasets. + * GAT：an attention-based graph neural network for node classification on Cora and CiteSeer dataset. +* Frontend and user interface + * Support tensor value and assignment of mixed tensor index in graph mode. + * Support tensor comparison, len operator, constexpr syntax, value and assignment of tensor index in pynative mode. + * Support converting MindSpore IR to pb format for infer model. + * Support print operator to write data directly on the hard disk. + * Add the double recursive programming solution for very high speed parallel strategy search in automatic parallel. + * User interfaces change log + * Allow the learning rate of AdamWeightDecayDynamicLR and Lamb to be 0([!1826](https://gitee.com/mindspore/mindspore/pulls/1826)) + * Restricting the entire network input parameter is Tensor([!1967](https://gitee.com/mindspore/mindspore/pulls/1967)) + * Turn shape and dtype into attributes instead of interfaces([!1919](https://gitee.com/mindspore/mindspore/pulls/1919)) + * Delete multitypefungraph([!2116](https://gitee.com/mindspore/mindspore/pulls/2116)) + * Refactor the callback module in an encapsulated way, use _CallbackManager instead of _build_callbacks([!2236](https://gitee.com/mindspore/mindspore/pulls/2236)) + * Delete EmbeddingLookup([!2163](https://gitee.com/mindspore/mindspore/pulls/2163)) + * Checkpoint add model_type([!2517](https://gitee.com/mindspore/mindspore/pulls/2517)) +* Executor and performance optimization + * Heterogeneous execution on CPU and Ascend devices supported, and is verified in Wide&Deep model. + * Quantitative training of MobileNetV2, Lenet and Resnet50 on Ascend-910 are supported. + * Support new fusion architecture, which can do fusion optimization across graphs and kernels to improve execution speed. +* Data processing, augmentation, and save format + * Support data processing pipeline performance profiling. + * Support public dataset loading, such as CLUE and Coco. + * Support more text processing, such as more tokenizers and vocab data. + * Support MindRecord padded data. +### Other Hardware Support +* GPU platform + * New model supported: Bert / Wide&Deep. + * Support setting max device memory. +* CPU platform + * New model supported: LSTM. + +## Bugfixes +* Models + * Bert, Move Bert from `example` to `model_zoo`, optimize network for better performance. ([!1902](https://gitee.com/mindspore/mindspore/pulls/1902)) + * VGG16, Move VGG16 from `example` to `model_zoo`, optimize network for better accuracy. ([!2645](https://gitee.com/mindspore/mindspore/pulls/2645)) + * Alexnet, modify parameter setting to improve accuracy ([!1364](https://gitee.com/mindspore/mindspore/pulls/2370)) + * Wide&Deep, Move Wide&Deep from `example` to `model_zoo`, optimize network for better performance. ([!2221](https://gitee.com/mindspore/mindspore/pulls/2221)) +* Python API + * Fix bug in auto cast([!1766](https://gitee.com/mindspore/mindspore/pulls/1766)) + * Fix bug of register_backward_hook([!2148](https://gitee.com/mindspore/mindspore/pulls/2148)) + * Fix bug of tuple args in pynative mode([!1878](https://gitee.com/mindspore/mindspore/pulls/1878)) + * Fix bug of checking numbers of arguments and graph parameters([!1701](https://gitee.com/mindspore/mindspore/pulls/1701)) +* Executor + * Fix bug of loading input data repeatedly in pynative mode([!1966](https://gitee.com/mindspore/mindspore/pulls/1966)) + * Fix bug of list cannot be used as input in pynative mode([!1765](https://gitee.com/mindspore/mindspore/pulls/1765)) + * Fix bug of kernel select ([!2103](https://gitee.com/mindspore/mindspore/pulls/2103)) + * Fix bug of pattern matching for batchnorm fusion in the case of auto mix precision.([!1851](https://gitee.com/mindspore/mindspore/pulls/1851)) + * Fix bug of generate hccl's kernel info.([!2393](https://gitee.com/mindspore/mindspore/mindspore/pulls/2393)) +* GPU platform + * Fix bug of summary feature invalid([!2173](https://gitee.com/mindspore/mindspore/pulls/2173)) +* Data processing + * Fix bug of Cifar dataset reading([!2096](https://gitee.com/mindspore/mindspore/pulls/2096)) + * Fix bug of C++ behavior in RandomCropAndResize([!2026](https://gitee.com/mindspore/mindspore/pulls/2026)) + * Fix the bug of mindrecord shuffle([!2420](https://gitee.com/mindspore/mindspore/pulls/2420)) + +## Contributors +Thanks goes to these wonderful people: + +Alexey Shevlyakov, avakh, baihuawei, BowenK, buxue, caifubi, caojian05, Cathy Wong, changzherui, chenfei, chengxianbin, chenhaozhe, chenjianping, chentingting, chenzomi, chujinjin, Danish Farid, dayschan, dengwentao, dinghao, etone-chan, fangzehua, fary86, geekun, Giancarlo Colmenares, gong chen, gukecai, guohongzilong, hangangqiang, heleiwang, hesham, He Wei, hexia, hongxing, huangdongrun, huanghui, islam_amin, Jamie Nisbet, Jesse Lee, jiangjinsheng, jiangzhiwen, jinyaohui, jjfeing, jojobugfree, Jonathan Yan, jonyguo, Junhan Hu, Kang, kingfo, kouzhenzhong, kpy, kswang, laiyongqiang, leopz, liangzelang, lichenever, lihongkang, Li Hongzhang, lilei, limingqi107, lirongzhen1, liubuyu, liuchongming74, liuwenhao4, liuxiao, Lixia Chen, liyanliu, liyong, lizhenyu, lvliang, Mahdi, Margaret_wangrui, meixiaowei, ms_yan, nhussain, ougongchang, panfengfeng, panyifeng, peilinwang, Peilin Wang, pkuliuliu, qianlong, rick_sanchez, shibeiji, Shida He, shijianning, simson, sunsuodong, suteng, Tinazhang, Tron Zhang, unknown, VectorSL, wandongdong, wangcong, wangdongxu, wangdongxu6, wanghua, wangnan39, Wei Luning, wenchunjiang, wenkai, wilfChen, WilliamLian, wukesong, Xian Weizhao, Xiaoda Zhang, xiefangqi, xulei2020, xunxue, xutianchun, Yang, yanghaitao, yanghaitao1, yanghaoran, yangjie, yangjie159, YangLuo, Yanjun Peng, yankai, yanzhenxiang2020, yao_yf, Yi Huaijie, yoonlee666, yuchaojie, yujianfeng, zhangzhongpeng, zhangdengcheng, Zhang Qinghua, zhangyinxia, zhangz0911gm, zhaojichen, zhaoting, zhaozhenlong, zhoufeng, zhouneng, zhousiyi, Zirui Wu, Ziyan, zjun, ZPaC, lihongzhang, wangdongxu + +Contributions of any kind are welcome! + # Release 0.3.0-alpha ## Major Features and Improvements diff --git a/akg b/akg index c460176523..df57a6cf94 160000 --- a/akg +++ b/akg @@ -1 +1 @@ -Subproject commit c460176523d039c8995f1d71089753725ebc0792 +Subproject commit df57a6cf9450e347d1854687d1fe66a420ee3b35 diff --git a/build.sh b/build.sh index 70718bf89b..059478b9af 100755 --- a/build.sh +++ b/build.sh @@ -25,7 +25,7 @@ usage() echo "Usage:" echo "bash build.sh [-d] [-r] [-v] [-c on|off] [-t on|off] [-g on|off] [-h] [-b ge] [-m infer|train] \\" echo " [-a on|off] [-Q on|off] [-p on|off] [-i] [-L] [-R] [-D on|off] [-j[n]] [-e gpu|d|cpu] \\" - echo " [-P on|off] [-z [on|off]] [-M on|off] [-V 9.2|10.1] [-I] [-K]" + echo " [-P on|off] [-z [on|off]] [-M on|off] [-V 9.2|10.1] [-I] [-K] [-B on|off] [-E]" echo "" echo "Options:" echo " -d Debug mode" @@ -50,10 +50,12 @@ usage() echo " -D Enable dumping of function graph ir, default on" echo " -z Compile dataset & mindrecord, default on" echo " -M Enable MPI and NCCL for GPU training, gpu default on" - echo " -V Specify the minimum required cuda version, default CUDA 9.2" + echo " -V Specify the minimum required cuda version, default CUDA 10.1" echo " -I Compile predict, default off" - echo " -K Compile with AKG, default off" + echo " -K Compile with AKG, default on" echo " -s Enable serving module, default off" + echo " -B Enable debugger, default off" + echo " -E Enable IBVERBS for parameter server, default off" } # check value of input is 'on' or 'off' @@ -88,14 +90,17 @@ checkopts() ENABLE_DUMP_IR="on" COMPILE_MINDDATA="on" ENABLE_MPI="off" - CUDA_VERSION="9.2" + CUDA_VERSION="10.1" COMPILE_PREDICT="off" USE_GLOG="on" PREDICT_PLATFORM="" ENABLE_AKG="on" ENABLE_SERVING="off" + ENABLE_DEBUGGER="off" + ENABLE_IBVERBS="off" + # Process the options - while getopts 'drvj:c:t:hsb:a:g:p:ie:m:I:LRP:Q:D:zM:V:K:s' opt + while getopts 'drvj:c:t:hsb:a:g:p:ie:m:I:LRP:Q:D:zM:V:K:sB:E' opt do OPTARG=$(echo ${OPTARG} | tr '[A-Z]' '[a-z]') case "${opt}" in @@ -191,6 +196,10 @@ checkopts() usage exit 1 fi + if [[ "X$OPTARG" == "X9.2" ]]; then + echo "Unsupported CUDA version 9.2" + exit 1 + fi CUDA_VERSION="$OPTARG" ;; P) @@ -240,6 +249,15 @@ checkopts() ENABLE_SERVING="on" echo "enable serving" ;; + B) + check_on_off $OPTARG B + ENABLE_DEBUGGER="on" + echo "enable debugger" + ;; + E) + ENABLE_IBVERBS="on" + echo "enable IBVERBS for parameter server" + ;; *) echo "Unknown option ${opt}!" usage @@ -322,7 +340,13 @@ build_mindspore() if [[ "X$ENABLE_SERVING" = "Xon" ]]; then CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_SERVING=ON" fi + if [[ "X$ENABLE_DEBUGGER" = "Xon" ]]; then + CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_DEBUGGER=ON" + fi + if [[ "X$ENABLE_IBVERBS" = "Xon" ]]; then + CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_IBVERBS=ON" + fi echo "${CMAKE_ARGS}" if [[ "X$INC_BUILD" = "Xoff" ]]; then cmake ${CMAKE_ARGS} ../.. @@ -446,9 +470,9 @@ build_predict() cd "${BASEPATH}/predict/output/" if [[ "$PREDICT_PLATFORM" == "x86_64" ]]; then - tar -cf MSPredict-0.3.0-linux_x86_64.tar.gz include/ lib/ --warning=no-file-changed + tar -cf MSPredict-0.5.0-linux_x86_64.tar.gz include/ lib/ --warning=no-file-changed elif [[ "$PREDICT_PLATFORM" == "arm64" ]]; then - tar -cf MSPredict-0.3.0-linux_aarch64.tar.gz include/ lib/ --warning=no-file-changed + tar -cf MSPredict-0.5.0-linux_aarch64.tar.gz include/ lib/ --warning=no-file-changed fi echo "success to build predict project!" } diff --git a/cmake/external_libs/absl.cmake b/cmake/external_libs/absl.cmake new file mode 100644 index 0000000000..6087b65128 --- /dev/null +++ b/cmake/external_libs/absl.cmake @@ -0,0 +1,14 @@ +mindspore_add_pkg(absl + VER 20200225.2 + LIBS absl_strings absl_throw_delegate absl_raw_logging_internal absl_int128 absl_bad_optional_access + URL https://github.com/abseil/abseil-cpp/archive/20200225.2.tar.gz + MD5 73f2b6e72f1599a9139170c29482ddc4 + CMAKE_OPTION -DCMAKE_BUILD_TYPE:STRING=Release -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=TRUE) + +include_directories(${absl_INC}) + +add_library(mindspore::absl_strings ALIAS absl::absl_strings) +add_library(mindspore::absl_throw_delegate ALIAS absl::absl_throw_delegate) +add_library(mindspore::absl_raw_logging_internal ALIAS absl::absl_raw_logging_internal) +add_library(mindspore::absl_int128 ALIAS absl::absl_int128) +add_library(mindspore::absl_bad_optional_access ALIAS absl::absl_bad_optional_access) diff --git a/cmake/external_libs/c-ares.cmake b/cmake/external_libs/c-ares.cmake new file mode 100644 index 0000000000..9bb547f2db --- /dev/null +++ b/cmake/external_libs/c-ares.cmake @@ -0,0 +1,12 @@ +mindspore_add_pkg(c-ares + VER 1.15.0 + LIBS cares + URL https://github.com/c-ares/c-ares/releases/download/cares-1_15_0/c-ares-1.15.0.tar.gz + MD5 d2391da274653f7643270623e822dff7 + CMAKE_OPTION -DCMAKE_BUILD_TYPE:STRING=Release + -DCARES_SHARED:BOOL=OFF + -DCARES_STATIC:BOOL=ON + -DCARES_STATIC_PIC:BOOL=ON) + +include_directories(${c-ares_INC}) +add_library(mindspore::cares ALIAS c-ares::cares) diff --git a/cmake/external_libs/grpc.cmake b/cmake/external_libs/grpc.cmake new file mode 100644 index 0000000000..7496cfd88e --- /dev/null +++ b/cmake/external_libs/grpc.cmake @@ -0,0 +1,110 @@ +set(grpc_USE_STATIC_LIBS ON) +if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") + set(grpc_CXXFLAGS "-fstack-protector-all -Wno-uninitialized -Wno-unused-parameter -fPIC -fvisibility=hidden -D_FORTIFY_SOURCE=2 -O2") +elseif (${CMAKE_SYSTEM_NAME} MATCHES "Windows") + set(grpc_CXXFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -fPIC -fvisibility=hidden -D_FORTIFY_SOURCE=2 -O2") +else() + set(grpc_CXXFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -fPIC -fvisibility=hidden -D_FORTIFY_SOURCE=2 -D_GLIBCXX_USE_CXX11_ABI=0 -O2") +endif() + +set(grpc_LDFLAGS "-Wl,-z,relro,-z,now,-z,noexecstack") + + +if (EXISTS ${protobuf_ROOT}/lib64) + set(_FINDPACKAGE_PROTOBUF_CONFIG_DIR "${protobuf_ROOT}/lib64/cmake/protobuf") +else() + set(_FINDPACKAGE_PROTOBUF_CONFIG_DIR "${protobuf_ROOT}/lib/cmake/protobuf") +endif() +message("grpc using Protobuf_DIR : " ${_FINDPACKAGE_PROTOBUF_CONFIG_DIR}) + +if (EXISTS ${absl_ROOT}/lib64) + set(_FINDPACKAGE_ABSL_CONFIG_DIR "${absl_ROOT}/lib64/cmake/absl") +else() + set(_FINDPACKAGE_ABSL_CONFIG_DIR "${absl_ROOT}/lib/cmake/absl") +endif() +message("grpc using absl_DIR : " ${_FINDPACKAGE_ABSL_CONFIG_DIR}) + +set(_CMAKE_ARGS_OPENSSL_ROOT_DIR "") +if (OPENSSL_ROOT_DIR) + set(_CMAKE_ARGS_OPENSSL_ROOT_DIR "-DOPENSSL_ROOT_DIR:PATH=${OPENSSL_ROOT_DIR}") +endif() + +mindspore_add_pkg(grpc + VER 1.27.3 + LIBS grpc++ grpc gpr upb address_sorting + EXE grpc_cpp_plugin + URL https://github.com/grpc/grpc/archive/v1.27.3.tar.gz + MD5 0c6c3fc8682d4262dd0e5e6fabe1a7e2 + CMAKE_OPTION -DCMAKE_BUILD_TYPE:STRING=Release + -DgRPC_INSTALL:BOOL=ON + -DgRPC_BUILD_TESTS:BOOL=OFF + -DgRPC_PROTOBUF_PROVIDER:STRING=package + -DgRPC_PROTOBUF_PACKAGE_TYPE:STRING=CONFIG + -DProtobuf_DIR:PATH=${_FINDPACKAGE_PROTOBUF_CONFIG_DIR} + -DgRPC_ZLIB_PROVIDER:STRING=package + -DZLIB_ROOT:PATH=${zlib_ROOT} + -DgRPC_ABSL_PROVIDER:STRING=package + -Dabsl_DIR:PATH=${_FINDPACKAGE_ABSL_CONFIG_DIR} + -DgRPC_CARES_PROVIDER:STRING=package + -Dc-ares_DIR:PATH=${c-ares_ROOT}/lib/cmake/c-ares + -DgRPC_SSL_PROVIDER:STRING=package + ${_CMAKE_ARGS_OPENSSL_ROOT_DIR} + ) + +include_directories(${grpc_INC}) + +add_library(mindspore::grpc++ ALIAS grpc::grpc++) + +# link other grpc libs +target_link_libraries(grpc::grpc++ INTERFACE grpc::grpc grpc::gpr grpc::upb grpc::address_sorting) + +# link built dependencies +target_link_libraries(grpc::grpc++ INTERFACE mindspore::z) +target_link_libraries(grpc::grpc++ INTERFACE mindspore::cares) +target_link_libraries(grpc::grpc++ INTERFACE mindspore::absl_strings mindspore::absl_throw_delegate + mindspore::absl_raw_logging_internal mindspore::absl_int128 mindspore::absl_bad_optional_access) + +# link system openssl +find_package(OpenSSL REQUIRED) +target_link_libraries(grpc::grpc++ INTERFACE OpenSSL::SSL OpenSSL::Crypto) + + +function(ms_grpc_generate c_var h_var) + if(NOT ARGN) + message(SEND_ERROR "Error: ms_grpc_generate() called without any proto files") + return() + endif() + + set(${c_var}) + set(${h_var}) + + foreach(file ${ARGN}) + get_filename_component(abs_file ${file} ABSOLUTE) + get_filename_component(file_name ${file} NAME_WE) + get_filename_component(file_dir ${abs_file} PATH) + file(RELATIVE_PATH rel_path ${CMAKE_CURRENT_SOURCE_DIR} ${file_dir}) + + list(APPEND ${c_var} "${CMAKE_BINARY_DIR}/proto/${file_name}.pb.cc") + list(APPEND ${h_var} "${CMAKE_BINARY_DIR}/proto/${file_name}.pb.h") + list(APPEND ${c_var} "${CMAKE_BINARY_DIR}/proto/${file_name}.grpc.pb.cc") + list(APPEND ${h_var} "${CMAKE_BINARY_DIR}/proto/${file_name}.grpc.pb.h") + + add_custom_command( + OUTPUT "${CMAKE_BINARY_DIR}/proto/${file_name}.pb.cc" + "${CMAKE_BINARY_DIR}/proto/${file_name}.pb.h" + "${CMAKE_BINARY_DIR}/proto/${file_name}.grpc.pb.cc" + "${CMAKE_BINARY_DIR}/proto/${file_name}.grpc.pb.h" + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} + COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_BINARY_DIR}/proto" + COMMAND protobuf::protoc --version + COMMAND protobuf::protoc -I${file_dir} --cpp_out=${CMAKE_BINARY_DIR}/proto + --grpc_out=${CMAKE_BINARY_DIR}/proto --plugin=protoc-gen-grpc=$ ${abs_file} + DEPENDS protobuf::protoc grpc::grpc_cpp_plugin ${abs_file} + COMMENT "Running C++ gRPC compiler on ${file}" VERBATIM) + endforeach() + + set_source_files_properties(${${c_var}} ${${h_var}} PROPERTIES GENERATED TRUE) + set(${c_var} ${${c_var}} PARENT_SCOPE) + set(${h_var} ${${h_var}} PARENT_SCOPE) + +endfunction() diff --git a/cmake/external_libs/pslite.cmake b/cmake/external_libs/pslite.cmake new file mode 100644 index 0000000000..28c851b094 --- /dev/null +++ b/cmake/external_libs/pslite.cmake @@ -0,0 +1,14 @@ +set(pslite_USE_STATIC_LIBS ON) +if (${ENABLE_IBVERBS} STREQUAL "ON") + set(pslite_CXXFLAGS "USE_IBVERBS=1") +endif() +mindspore_add_pkg(pslite + LIBS ps + URL https://github.com/dmlc/ps-lite/archive/34fd45cae457d59850fdcb2066467778d0673f21.zip + MD5 393c0e27b68bfaf96718caa3aa96f5a3 + PATCHES ${CMAKE_SOURCE_DIR}/third_party/patch/pslite/ps_lite.patch001 + ONLY_MAKE True + ONLY_MAKE_INCS include/* + ONLY_MAKE_LIBS build/*) +include_directories(${pslite_INC}) +add_library(mindspore::pslite ALIAS pslite::ps) diff --git a/cmake/external_libs/zeromq.cmake b/cmake/external_libs/zeromq.cmake new file mode 100644 index 0000000000..122f1ee90c --- /dev/null +++ b/cmake/external_libs/zeromq.cmake @@ -0,0 +1,5 @@ +mindspore_add_pkg(zeromq + VER 4.1.4 + HEAD_ONLY ./ + URL https://raw.githubusercontent.com/mli/deps/master/build/zeromq-4.1.4.tar.gz + MD5 a611ecc93fffeb6d058c0e6edf4ad4fb) diff --git a/cmake/external_libs/zlib.cmake b/cmake/external_libs/zlib.cmake new file mode 100644 index 0000000000..06532ed8d7 --- /dev/null +++ b/cmake/external_libs/zlib.cmake @@ -0,0 +1,9 @@ +mindspore_add_pkg(zlib + VER 1.2.11 + LIBS z + URL https://github.com/madler/zlib/archive/v1.2.11.tar.gz + MD5 0095d2d2d1f3442ce1318336637b695f + CMAKE_OPTION -DCMAKE_BUILD_TYPE:STRING=Release) + +include_directories(${zlib_INC}) +add_library(mindspore::z ALIAS zlib::z) diff --git a/cmake/mind_expression.cmake b/cmake/mind_expression.cmake index 86337c1dd2..63a65cd533 100644 --- a/cmake/mind_expression.cmake +++ b/cmake/mind_expression.cmake @@ -14,12 +14,26 @@ include(${CMAKE_SOURCE_DIR}/cmake/external_libs/eigen.cmake) include(${CMAKE_SOURCE_DIR}/cmake/external_libs/json.cmake) include(${CMAKE_SOURCE_DIR}/cmake/dependency_securec.cmake) include(${CMAKE_SOURCE_DIR}/cmake/external_libs/protobuf.cmake) + +if (ENABLE_DEBUGGER) + # build dependencies of gRPC + include(${CMAKE_SOURCE_DIR}/cmake/external_libs/absl.cmake) + include(${CMAKE_SOURCE_DIR}/cmake/external_libs/c-ares.cmake) + include(${CMAKE_SOURCE_DIR}/cmake/external_libs/zlib.cmake) + # build gRPC + include(${CMAKE_SOURCE_DIR}/cmake/external_libs/grpc.cmake) +endif() + include(${CMAKE_SOURCE_DIR}/cmake/external_libs/pybind11.cmake) MESSAGE("go to link flatbuffers") include(${CMAKE_SOURCE_DIR}/cmake/external_libs/flatbuffers.cmake) if(USE_GLOG) include(${CMAKE_SOURCE_DIR}/cmake/external_libs/glog.cmake) endif() +if (NOT ${CMAKE_SYSTEM_NAME} MATCHES "Windows") + include(${CMAKE_SOURCE_DIR}/cmake/external_libs/zeromq.cmake) + include(${CMAKE_SOURCE_DIR}/cmake/external_libs/pslite.cmake) +endif() find_package(Python3) include_directories(${Python3_INCLUDE_DIRS}) diff --git a/cmake/options.cmake b/cmake/options.cmake index 3e03ed3339..18db942d68 100644 --- a/cmake/options.cmake +++ b/cmake/options.cmake @@ -17,6 +17,8 @@ option(ENABLE_DUMP_E2E "Enable dump e2e file, default on" OFF) option(ENABLE_DUMP_IR "Enable dump funciton graph ir, default on" ON) option(ENABLE_MPI "enable mpi" OFF) option(ENABLE_AKG "enable akg" OFF) +option(ENABLE_DEBUGGER "enable debugger" OFF) +option(ENABLE_IBVERBS "enable IBVERBS for parameter server" OFF) if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") if (WIN32) @@ -112,3 +114,7 @@ endif() if(ENABLE_DUMP_E2E) add_compile_definitions(ENABLE_DUMP_E2E) endif() + +if(ENABLE_DEBUGGER) + add_compile_definitions(ENABLE_DEBUGGER) +endif() diff --git a/cmake/package.cmake b/cmake/package.cmake index 1cff396ef1..42821cf41d 100644 --- a/cmake/package.cmake +++ b/cmake/package.cmake @@ -128,6 +128,11 @@ if (ENABLE_MPI) DESTINATION ${INSTALL_BASE_DIR} COMPONENT mindspore ) + install( + TARGETS mpi_adapter + DESTINATION ${INSTALL_LIB_DIR} + COMPONENT mindspore + ) endif () if (ENABLE_GPU) diff --git a/cmake/utils.cmake b/cmake/utils.cmake index f0a5dc594c..cf8f6ebb46 100644 --- a/cmake/utils.cmake +++ b/cmake/utils.cmake @@ -206,7 +206,7 @@ function(mindspore_add_pkg pkg_name ) set(options ) set(oneValueArgs URL MD5 GIT_REPOSITORY GIT_TAG VER EXE DIR HEAD_ONLY CMAKE_PATH RELEASE LIB_PATH CUSTOM_CMAKE) - set(multiValueArgs CMAKE_OPTION LIBS PRE_CONFIGURE_COMMAND CONFIGURE_COMMAND BUILD_OPTION INSTALL_INCS INSTALL_LIBS PATCHES SUBMODULES SOURCEMODULES) + set(multiValueArgs CMAKE_OPTION LIBS PRE_CONFIGURE_COMMAND CONFIGURE_COMMAND BUILD_OPTION INSTALL_INCS INSTALL_LIBS PATCHES SUBMODULES SOURCEMODULES ONLY_MAKE ONLY_MAKE_INCS ONLY_MAKE_LIBS) cmake_parse_arguments(PKG "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN} ) if (NOT PKG_LIB_PATH) @@ -290,7 +290,7 @@ function(mindspore_add_pkg pkg_name ) foreach(_PATCH_FILE ${PKG_PATCHES}) get_filename_component(_PATCH_FILE_NAME ${_PATCH_FILE} NAME) set(_LF_PATCH_FILE ${CMAKE_BINARY_DIR}/_ms_patch/${_PATCH_FILE_NAME}) - configure_file(${_PATCH_FILE} ${_LF_PATCH_FILE} NEWLINE_STYLE LF) + configure_file(${_PATCH_FILE} ${_LF_PATCH_FILE} NEWLINE_STYLE LF @ONLY) message("patching ${${pkg_name}_SOURCE_DIR} -p1 < ${_LF_PATCH_FILE}") execute_process(COMMAND ${Patch_EXECUTABLE} -p1 INPUT_FILE ${_LF_PATCH_FILE} @@ -324,6 +324,16 @@ function(mindspore_add_pkg pkg_name ) target_include_directories(${pkg_name} INTERFACE ${${pkg_name}_INC}) endif () + elseif (PKG_ONLY_MAKE) + __exec_cmd(COMMAND ${CMAKE_MAKE_PROGRAM} ${${pkg_name}_CXXFLAGS} -j${THNUM} + WORKING_DIRECTORY ${${pkg_name}_SOURCE_DIR}) + set(PKG_INSTALL_INCS ${PKG_ONLY_MAKE_INCS}) + set(PKG_INSTALL_LIBS ${PKG_ONLY_MAKE_LIBS}) + file(GLOB ${pkg_name}_INSTALL_INCS ${${pkg_name}_SOURCE_DIR}/${PKG_INSTALL_INCS}) + file(GLOB ${pkg_name}_INSTALL_LIBS ${${pkg_name}_SOURCE_DIR}/${PKG_INSTALL_LIBS}) + file(COPY ${${pkg_name}_INSTALL_INCS} DESTINATION ${${pkg_name}_BASE_DIR}/include) + file(COPY ${${pkg_name}_INSTALL_LIBS} DESTINATION ${${pkg_name}_BASE_DIR}/lib) + elseif (PKG_CMAKE_OPTION) # in cmake file(MAKE_DIRECTORY ${${pkg_name}_SOURCE_DIR}/_build) diff --git a/docker/mindspore-cpu/0.5.0-beta/Dockerfile b/docker/mindspore-cpu/0.5.0-beta/Dockerfile new file mode 100644 index 0000000000..4da6294296 --- /dev/null +++ b/docker/mindspore-cpu/0.5.0-beta/Dockerfile @@ -0,0 +1,67 @@ +FROM ubuntu:18.04 + +MAINTAINER leonwanghui + +# Set env +ENV PYTHON_ROOT_PATH /usr/local/python-3.7.5 +ENV PATH /usr/local/bin:$PATH + +# Install base tools +RUN apt update \ + && DEBIAN_FRONTEND=noninteractive apt install -y \ + vim \ + wget \ + curl \ + xz-utils \ + net-tools \ + openssh-client \ + git \ + ntpdate \ + tzdata \ + tcl \ + sudo \ + bash-completion + +# Install compile tools +RUN DEBIAN_FRONTEND=noninteractive apt install -y \ + gcc \ + g++ \ + zlibc \ + make \ + libgmp-dev \ + patch \ + autoconf \ + libtool \ + automake \ + flex + +# Set bash +RUN echo "dash dash/sh boolean false" | debconf-set-selections +RUN DEBIAN_FRONTEND=noninteractive dpkg-reconfigure dash + +# Install python (v3.7.5) +RUN apt install -y libffi-dev libssl-dev zlib1g-dev libbz2-dev libncurses5-dev \ + libgdbm-dev libgdbm-compat-dev liblzma-dev libreadline-dev libsqlite3-dev \ + && cd /tmp \ + && wget https://github.com/python/cpython/archive/v3.7.5.tar.gz \ + && tar -xvf v3.7.5.tar.gz \ + && cd /tmp/cpython-3.7.5 \ + && mkdir -p ${PYTHON_ROOT_PATH} \ + && ./configure --prefix=${PYTHON_ROOT_PATH} \ + && make -j4 \ + && make install -j4 \ + && rm -f /usr/local/bin/python \ + && rm -f /usr/local/bin/pip \ + && ln -s ${PYTHON_ROOT_PATH}/bin/python3.7 /usr/local/bin/python \ + && ln -s ${PYTHON_ROOT_PATH}/bin/pip3.7 /usr/local/bin/pip \ + && rm -rf /tmp/cpython-3.7.5 \ + && rm -f /tmp/v3.7.5.tar.gz + +# Set pip source +RUN mkdir -pv /root/.pip \ + && echo "[global]" > /root/.pip/pip.conf \ + && echo "trusted-host=mirrors.aliyun.com" >> /root/.pip/pip.conf \ + && echo "index-url=http://mirrors.aliyun.com/pypi/simple/" >> /root/.pip/pip.conf + +# Install MindSpore cpu whl package +RUN pip install --no-cache-dir https://ms-release.obs.cn-north-4.myhuaweicloud.com/0.5.0-beta/MindSpore/cpu/ubuntu_x86/mindspore-0.5.0-cp37-cp37m-linux_x86_64.whl diff --git a/docker/mindspore-gpu/0.5.0-beta/Dockerfile b/docker/mindspore-gpu/0.5.0-beta/Dockerfile new file mode 100644 index 0000000000..dae6d16370 --- /dev/null +++ b/docker/mindspore-gpu/0.5.0-beta/Dockerfile @@ -0,0 +1,83 @@ +FROM nvidia/cuda:10.1-cudnn7-runtime-ubuntu18.04 + +MAINTAINER leonwanghui + +# Set env +ENV PYTHON_ROOT_PATH /usr/local/python-3.7.5 +ENV OMPI_ROOT_PATH /usr/local/openmpi-3.1.5 +ENV PATH ${OMPI_ROOT_PATH}/bin:/usr/local/bin:$PATH +ENV LD_LIBRARY_PATH ${OMPI_ROOT_PATH}/lib:$LD_LIBRARY_PATH + +# Install base tools +RUN apt update \ + && DEBIAN_FRONTEND=noninteractive apt install -y \ + vim \ + wget \ + curl \ + xz-utils \ + net-tools \ + openssh-client \ + git \ + ntpdate \ + tzdata \ + tcl \ + sudo \ + bash-completion + +# Install compile tools +RUN DEBIAN_FRONTEND=noninteractive apt install -y \ + gcc \ + g++ \ + zlibc \ + make \ + libgmp-dev \ + patch \ + autoconf \ + libtool \ + automake \ + flex \ + libnccl2=2.4.8-1+cuda10.1 \ + libnccl-dev=2.4.8-1+cuda10.1 + +# Set bash +RUN echo "dash dash/sh boolean false" | debconf-set-selections +RUN DEBIAN_FRONTEND=noninteractive dpkg-reconfigure dash + +# Install python (v3.7.5) +RUN apt install -y libffi-dev libssl-dev zlib1g-dev libbz2-dev libncurses5-dev \ + libgdbm-dev libgdbm-compat-dev liblzma-dev libreadline-dev libsqlite3-dev \ + && cd /tmp \ + && wget https://github.com/python/cpython/archive/v3.7.5.tar.gz \ + && tar -xvf v3.7.5.tar.gz \ + && cd /tmp/cpython-3.7.5 \ + && mkdir -p ${PYTHON_ROOT_PATH} \ + && ./configure --prefix=${PYTHON_ROOT_PATH} \ + && make -j4 \ + && make install -j4 \ + && rm -f /usr/local/bin/python \ + && rm -f /usr/local/bin/pip \ + && ln -s ${PYTHON_ROOT_PATH}/bin/python3.7 /usr/local/bin/python \ + && ln -s ${PYTHON_ROOT_PATH}/bin/pip3.7 /usr/local/bin/pip \ + && rm -rf /tmp/cpython-3.7.5 \ + && rm -f /tmp/v3.7.5.tar.gz + +# Set pip source +RUN mkdir -pv /root/.pip \ + && echo "[global]" > /root/.pip/pip.conf \ + && echo "trusted-host=mirrors.aliyun.com" >> /root/.pip/pip.conf \ + && echo "index-url=http://mirrors.aliyun.com/pypi/simple/" >> /root/.pip/pip.conf + +# Install openmpi (v3.1.5) +RUN cd /tmp \ + && wget https://download.open-mpi.org/release/open-mpi/v3.1/openmpi-3.1.5.tar.gz \ + && tar -xvf openmpi-3.1.5.tar.gz \ + && cd /tmp/openmpi-3.1.5 \ + && mkdir -p ${OMPI_ROOT_PATH} \ + && ./configure --prefix=${OMPI_ROOT_PATH} \ + && make -j4 \ + && make install -j4 \ + && rm -rf /tmp/openmpi-3.1.5 \ + && rm -f /tmp/openmpi-3.1.5.tar.gz + +# Install MindSpore cuda-10.1 whl package +RUN pip install --no-cache-dir https://ms-release.obs.cn-north-4.myhuaweicloud.com/0.5.0-beta/MindSpore/gpu/ubuntu_x86/cuda-10.1/mindspore_gpu-0.5.0-cp37-cp37m-linux_x86_64.whl diff --git a/example/nlp_to_mindrecord/CLUERNER2020/README.md b/example/nlp_to_mindrecord/CLUERNER2020/README.md deleted file mode 100644 index c862156a47..0000000000 --- a/example/nlp_to_mindrecord/CLUERNER2020/README.md +++ /dev/null @@ -1,82 +0,0 @@ -# Guideline to Convert Training Data CLUERNER2020 to MindRecord For Bert Fine Tuning - - - -- [What does the example do](#what-does-the-example-do) -- [How to use the example to process CLUERNER2020](#how-to-use-the-example-to-process-cluerner2020) - - [Download CLUERNER2020 and unzip](#download-cluerner2020-and-unzip) - - [Generate MindRecord](#generate-mindrecord) - - [Create MindDataset By MindRecord](#create-minddataset-by-mindrecord) - - - - -## What does the example do - -This example is based on [CLUERNER2020](https://www.cluebenchmarks.com/introduce.html) training data, generating MindRecord file, and finally used for Bert Fine Tuning progress. - -1. run.sh: generate MindRecord entry script -2. run_read.py: create MindDataset by MindRecord entry script. - - create_dataset.py: use MindDataset to read MindRecord to generate dataset. - -## How to use the example to process CLUERNER2020 - -Download CLUERNER2020, convert it to MindRecord, use MindDataset to read MindRecord. - -### Download CLUERNER2020 and unzip - -1. Download the training data zip. - > [CLUERNER2020 dataset download address](https://www.cluebenchmarks.com/introduce.html) **-> 任务介绍 -> CLUENER 细粒度命名实体识别 -> cluener下载链接** - -2. Unzip the training data to dir example/nlp_to_mindrecord/CLUERNER2020/cluener_public. - ``` - unzip -d {your-mindspore}/example/nlp_to_mindrecord/CLUERNER2020/data/cluener_public cluener_public.zip - ``` - -### Generate MindRecord - -1. Run the run.sh script. - ```bash - bash run.sh - ``` - -2. Output like this: - ``` - ... - [INFO] ME(17603:139620983514944,MainProcess):2020-04-28-16:56:12.498.235 [mindspore/mindrecord/filewriter.py:313] The list of mindrecord files created are: ['data/train.mindrecord'], and the list of index files are: ['data/train.mindrecord.db'] - ... - [INFO] ME(17603,python):2020-04-28-16:56:13.400.175 [mindspore/ccsrc/mindrecord/io/shard_writer.cc:667] WriteRawData] Write 1 records successfully. - [INFO] ME(17603,python):2020-04-28-16:56:13.400.863 [mindspore/ccsrc/mindrecord/io/shard_writer.cc:667] WriteRawData] Write 1 records successfully. - [INFO] ME(17603,python):2020-04-28-16:56:13.401.534 [mindspore/ccsrc/mindrecord/io/shard_writer.cc:667] WriteRawData] Write 1 records successfully. - [INFO] ME(17603,python):2020-04-28-16:56:13.402.179 [mindspore/ccsrc/mindrecord/io/shard_writer.cc:667] WriteRawData] Write 1 records successfully. - [INFO] ME(17603,python):2020-04-28-16:56:13.402.702 [mindspore/ccsrc/mindrecord/io/shard_writer.cc:667] WriteRawData] Write 1 records successfully. - ... - [INFO] ME(17603:139620983514944,MainProcess):2020-04-28-16:56:13.431.208 [mindspore/mindrecord/filewriter.py:313] The list of mindrecord files created are: ['data/dev.mindrecord'], and the list of index files are: ['data/dev.mindrecord.db'] - ``` - -3. Generate files like this: - ```bash - $ ls output/ - dev.mindrecord dev.mindrecord.db README.md train.mindrecord train.mindrecord.db - ``` - -### Create MindDataset By MindRecord - -1. Run the run_read.sh script. - ```bash - bash run_read.sh - ``` - -2. Output like this: - ``` - ... - example 1340: input_ids: [ 101 3173 1290 4852 7676 3949 122 3299 123 126 3189 4510 8020 6381 5442 7357 2590 3636 8021 7676 3949 4294 1166 6121 3124 1277 6121 3124 7270 2135 3295 5789 3326 123 126 3189 1355 6134 1093 1325 3173 2399 6590 6791 8024 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0] - example 1340: input_mask: [1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0] - example 1340: segment_ids: [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0] - example 1340: label_ids: [ 0 18 19 20 2 4 0 0 0 0 0 0 0 34 36 26 27 28 0 34 35 35 35 35 35 35 35 35 35 36 26 27 28 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0] - example 1341: input_ids: [ 101 1728 711 4293 3868 1168 2190 2150 3791 934 3633 3428 4638 6237 7025 8024 3297 1400 5310 3362 6206 5023 5401 1744 3297 7770 3791 7368 976 1139 1104 2137 511 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0] - example 1341: input_mask: [1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0] - example 1341: segment_ids: [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0] - example 1341: label_ids: [ 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 18 19 19 19 19 20 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0] - ... - ``` diff --git a/example/nlp_to_mindrecord/CLUERNER2020/create_dataset.py b/example/nlp_to_mindrecord/CLUERNER2020/create_dataset.py deleted file mode 100644 index 616bc71028..0000000000 --- a/example/nlp_to_mindrecord/CLUERNER2020/create_dataset.py +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -"""create MindDataset by MindRecord""" -import mindspore.dataset as ds - -def create_dataset(data_file): - """create MindDataset""" - num_readers = 4 - data_set = ds.MindDataset(dataset_file=data_file, num_parallel_workers=num_readers, shuffle=True) - index = 0 - for item in data_set.create_dict_iterator(): - # print("example {}: {}".format(index, item)) - print("example {}: input_ids: {}".format(index, item['input_ids'])) - print("example {}: input_mask: {}".format(index, item['input_mask'])) - print("example {}: segment_ids: {}".format(index, item['segment_ids'])) - print("example {}: label_ids: {}".format(index, item['label_ids'])) - index += 1 - if index % 1000 == 0: - print("read rows: {}".format(index)) - print("total rows: {}".format(index)) - -if __name__ == '__main__': - create_dataset('output/train.mindrecord') - create_dataset('output/dev.mindrecord') diff --git a/example/nlp_to_mindrecord/CLUERNER2020/data/.gitignore b/example/nlp_to_mindrecord/CLUERNER2020/data/.gitignore deleted file mode 100644 index cbbd6256c0..0000000000 --- a/example/nlp_to_mindrecord/CLUERNER2020/data/.gitignore +++ /dev/null @@ -1 +0,0 @@ -cluener_public diff --git a/example/nlp_to_mindrecord/CLUERNER2020/output/README.md b/example/nlp_to_mindrecord/CLUERNER2020/output/README.md deleted file mode 100644 index 7904933f43..0000000000 --- a/example/nlp_to_mindrecord/CLUERNER2020/output/README.md +++ /dev/null @@ -1 +0,0 @@ -## output dir diff --git a/example/nlp_to_mindrecord/CLUERNER2020/run.sh b/example/nlp_to_mindrecord/CLUERNER2020/run.sh deleted file mode 100644 index 15c6aa4362..0000000000 --- a/example/nlp_to_mindrecord/CLUERNER2020/run.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ - -rm -f output/train.mindrecord* -rm -f output/dev.mindrecord* - -if [ ! -d "../../../third_party/to_mindrecord/CLUERNER2020" ]; then - echo "The patch base dir ../../../third_party/to_mindrecord/CLUERNER2020 is not exist." - exit 1 -fi - -if [ ! -f "../../../third_party/patch/to_mindrecord/CLUERNER2020/data_processor_seq.patch" ]; then - echo "The patch file ../../../third_party/patch/to_mindrecord/CLUERNER2020/data_processor_seq.patch is not exist." - exit 1 -fi - -# patch for data_processor_seq.py -patch -p0 -d ../../../third_party/to_mindrecord/CLUERNER2020/ -o data_processor_seq_patched.py < ../../../third_party/patch/to_mindrecord/CLUERNER2020/data_processor_seq.patch -if [ $? -ne 0 ]; then - echo "Patch ../../../third_party/to_mindrecord/CLUERNER2020/data_processor_seq.py failed" - exit 1 -fi - -# use patched script -python ../../../third_party/to_mindrecord/CLUERNER2020/data_processor_seq_patched.py \ ---vocab_file=../../../third_party/to_mindrecord/CLUERNER2020/vocab.txt \ ---label2id_file=../../../third_party/to_mindrecord/CLUERNER2020/label2id.json diff --git a/example/nlp_to_mindrecord/aclImdb_preprocess/data/README.md b/example/nlp_to_mindrecord/aclImdb_preprocess/data/README.md deleted file mode 100644 index b54948808e..0000000000 --- a/example/nlp_to_mindrecord/aclImdb_preprocess/data/README.md +++ /dev/null @@ -1 +0,0 @@ -## The input dataset diff --git a/example/nlp_to_mindrecord/enwiki/README.md b/example/nlp_to_mindrecord/enwiki/README.md deleted file mode 100644 index e92e8dbcc6..0000000000 --- a/example/nlp_to_mindrecord/enwiki/README.md +++ /dev/null @@ -1,173 +0,0 @@ -# Guideline to Convert Training Data enwiki to MindRecord For Bert Pre Training - - - -- [What does the example do](#what-does-the-example-do) -- [How to use the example to process enwiki](#how-to-use-the-example-to-process-enwiki) - - [Download enwiki training data](#download-enwiki-training-data) - - [Process the enwiki](#process-the-enwiki) - - [Generate MindRecord](#generate-mindrecord) - - [Create MindDataset By MindRecord](#create-minddataset-by-mindrecord) - - - - -## What does the example do - -This example is based on [enwiki](https://dumps.wikimedia.org/enwiki) training data, generating MindRecord file, and finally used for Bert network training. - -1. run.sh: generate MindRecord entry script. -2. run_read.py: create MindDataset by MindRecord entry script. - - create_dataset.py: use MindDataset to read MindRecord to generate dataset. - -## How to use the example to process enwiki - -Download enwiki data, process it, convert it to MindRecord, use MindDataset to read MindRecord. - -### Download enwiki training data - -> [enwiki dataset download address](https://dumps.wikimedia.org/enwiki) **-> 20200501 -> enwiki-20200501-pages-articles-multistream.xml.bz2** - -### Process the enwiki - -1. Please follow the steps in [process enwiki](https://github.com/mlperf/training/tree/master/language_model/tensorflow/bert) -- All permissions of this step belong to the link address website. - -### Generate MindRecord - -1. Run the run.sh script. - ``` - bash run.sh input_dir output_dir vocab_file - ``` - - input_dir: the directory which contains files like 'part-00251-of-00500'. - - output_dir: which will store the output mindrecord files. - - vocab_file: the vocab file which you can download from other opensource project. - -2. The output like this: - ``` - ... - Begin preprocess Wed Jun 10 09:21:23 CST 2020 - Begin preprocess input file: /mnt/data/results/part-00000-of-00500 - Begin output file: part-00000-of-00500.mindrecord - Total task: 510, processing: 1 - Begin preprocess input file: /mnt/data/results/part-00001-of-00500 - Begin output file: part-00001-of-00500.mindrecord - Total task: 510, processing: 2 - Begin preprocess input file: /mnt/data/results/part-00002-of-00500 - Begin output file: part-00002-of-00500.mindrecord - Total task: 510, processing: 3 - Begin preprocess input file: /mnt/data/results/part-00003-of-00500 - Begin output file: part-00003-of-00500.mindrecord - Total task: 510, processing: 4 - Begin preprocess input file: /mnt/data/results/part-00004-of-00500 - Begin output file: part-00004-of-00500.mindrecord - Total task: 510, processing: 4 - ... - ``` - -3. Generate files like this: - ```bash - $ ls {your_output_dir}/ - part-00000-of-00500.mindrecord part-00000-of-00500.mindrecord.db part-00001-of-00500.mindrecord part-00001-of-00500.mindrecord.db part-00002-of-00500.mindrecord part-00002-of-00500.mindrecord.db ... - ``` - -### Create MindDataset By MindRecord - -1. Run the run_read.sh script. - ```bash - bash run_read.sh input_dir - ``` - - input_dir: the directory which contains mindrecord files. - -2. The output like this: - ``` - ... - example 633: input_ids: [ 101 2043 19781 4305 2140 4520 2041 1010 103 2034 2455 2002 - 7879 2003 1996 2455 1997 103 26378 4160 1012 102 7291 2001 - 1996 103 1011 2343 1997 6327 1010 3423 1998 103 4262 2005 - 1996 2118 1997 2329 3996 103 102 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0] - example 633: input_mask: [1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 - 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0] - example 633: segment_ids: [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 - 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0] - example 633: masked_lm_positions: [ 8 17 20 25 33 41 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0] - example 633: masked_lm_ids: [ 1996 16137 1012 3580 2451 1012 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0] - example 633: masked_lm_weights: [1. 1. 1. 1. 1. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. - 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. - 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. - 0. 0. 0. 0.] - example 633: next_sentence_labels: [1] - ... - ``` diff --git a/example/nlp_to_mindrecord/enwiki/create_dataset.py b/example/nlp_to_mindrecord/enwiki/create_dataset.py deleted file mode 100644 index d90d12b7f2..0000000000 --- a/example/nlp_to_mindrecord/enwiki/create_dataset.py +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -"""create MindDataset by MindRecord""" -import argparse -import mindspore.dataset as ds - -def create_dataset(data_file): - """create MindDataset""" - num_readers = 4 - data_set = ds.MindDataset(dataset_file=data_file, num_parallel_workers=num_readers, shuffle=True) - index = 0 - for item in data_set.create_dict_iterator(): - # print("example {}: {}".format(index, item)) - print("example {}: input_ids: {}".format(index, item['input_ids'])) - print("example {}: input_mask: {}".format(index, item['input_mask'])) - print("example {}: segment_ids: {}".format(index, item['segment_ids'])) - print("example {}: masked_lm_positions: {}".format(index, item['masked_lm_positions'])) - print("example {}: masked_lm_ids: {}".format(index, item['masked_lm_ids'])) - print("example {}: masked_lm_weights: {}".format(index, item['masked_lm_weights'])) - print("example {}: next_sentence_labels: {}".format(index, item['next_sentence_labels'])) - index += 1 - if index % 1000 == 0: - print("read rows: {}".format(index)) - print("total rows: {}".format(index)) - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument("-i", "--input_file", nargs='+', type=str, help='Input mindreord file') - args = parser.parse_args() - - create_dataset(args.input_file) diff --git a/example/nlp_to_mindrecord/enwiki/run.sh b/example/nlp_to_mindrecord/enwiki/run.sh deleted file mode 100644 index cf66bed0fd..0000000000 --- a/example/nlp_to_mindrecord/enwiki/run.sh +++ /dev/null @@ -1,133 +0,0 @@ -#!/bin/bash -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ - -if [ $# -ne 3 ]; then - echo "Usage: $0 input_dir output_dir vocab_file" - exit 1 -fi - -if [ ! -d $1 ]; then - echo "The input dir: $1 is not exist." - exit 1 -fi - -if [ ! -d $2 ]; then - echo "The output dir: $2 is not exist." - exit 1 -fi -rm -fr $2/*.mindrecord* - -if [ ! -f $3 ]; then - echo "The vocab file: $3 is not exist." - exit 1 -fi - -data_dir=$1 -output_dir=$2 -vocab_file=$3 -file_list=() -output_filename=() -file_index=0 - -function getdir() { - elements=`ls $1` - for element in ${elements[*]}; - do - dir_or_file=$1"/"$element - if [ -d $dir_or_file ]; - then - getdir $dir_or_file - else - file_list[$file_index]=$dir_or_file - echo "${dir_or_file}" | tr '/' '\n' > dir_file_list.txt # dir dir file to mapfile - mapfile parent_dir < dir_file_list.txt - rm dir_file_list.txt >/dev/null 2>&1 - tmp_output_filename=${parent_dir[${#parent_dir[@]}-1]}".mindrecord" - output_filename[$file_index]=`echo ${tmp_output_filename} | sed 's/ //g'` - file_index=`expr $file_index + 1` - fi - done -} - -getdir "${data_dir}" -# echo "The input files: "${file_list[@]} -# echo "The output files: "${output_filename[@]} - -if [ ! -d "../../../third_party/to_mindrecord/zhwiki" ]; then - echo "The patch base dir ../../../third_party/to_mindrecord/zhwiki is not exist." - exit 1 -fi - -if [ ! -f "../../../third_party/patch/to_mindrecord/zhwiki/create_pretraining_data.patch" ]; then - echo "The patch file ../../../third_party/patch/to_mindrecord/zhwiki/create_pretraining_data.patch is not exist." - exit 1 -fi - -# patch for create_pretraining_data.py -patch -p0 -d ../../../third_party/to_mindrecord/zhwiki/ -o create_pretraining_data_patched.py < ../../../third_party/patch/to_mindrecord/zhwiki/create_pretraining_data.patch -if [ $? -ne 0 ]; then - echo "Patch ../../../third_party/to_mindrecord/zhwiki/create_pretraining_data.py failed" - exit 1 -fi - -# get the cpu core count -num_cpu_core=`cat /proc/cpuinfo | grep "processor" | wc -l` -avaiable_core_size=`expr $num_cpu_core / 3 \* 2` - -echo "Begin preprocess `date`" - -# using patched script to generate mindrecord -file_list_len=`expr ${#file_list[*]} - 1` -for index in $(seq 0 $file_list_len); do - echo "Begin preprocess input file: ${file_list[$index]}" - echo "Begin output file: ${output_filename[$index]}" - python ../../../third_party/to_mindrecord/zhwiki/create_pretraining_data_patched.py \ - --input_file=${file_list[$index]} \ - --output_file=${output_dir}/${output_filename[$index]} \ - --partition_number=1 \ - --vocab_file=${vocab_file} \ - --do_lower_case=True \ - --max_seq_length=512 \ - --max_predictions_per_seq=76 \ - --masked_lm_prob=0.15 \ - --random_seed=12345 \ - --dupe_factor=10 >/tmp/${output_filename[$index]}.log 2>&1 & - process_count=`ps -ef | grep create_pretraining_data_patched | grep -v grep | wc -l` - echo "Total task: ${#file_list[*]}, processing: ${process_count}" - if [ $process_count -ge $avaiable_core_size ]; then - while [ 1 ]; do - process_num=`ps -ef | grep create_pretraining_data_patched | grep -v grep | wc -l` - if [ $process_count -gt $process_num ]; then - process_count=$process_num - break; - fi - sleep 2 - done - fi -done - -process_num=`ps -ef | grep create_pretraining_data_patched | grep -v grep | wc -l` -while [ 1 ]; do - if [ $process_num -eq 0 ]; then - break; - fi - echo "There are still ${process_num} preprocess running ..." - sleep 2 - process_num=`ps -ef | grep create_pretraining_data_patched | grep -v grep | wc -l` -done - -echo "Preprocess all the data success." -echo "End preprocess `date`" diff --git a/example/nlp_to_mindrecord/enwiki/run_read.sh b/example/nlp_to_mindrecord/enwiki/run_read.sh deleted file mode 100644 index 737e9375c4..0000000000 --- a/example/nlp_to_mindrecord/enwiki/run_read.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ - -if [ $# -ne 1 ]; then - echo "Usage: $0 input_dir" - exit 1 -fi - -if [ ! -d $1 ]; then - echo "The input dir: $1 is not exist." - exit 1 -fi - -file_list=() -file_index=0 - -# get all the mindrecord file from output dir -function getdir() { - elements=`ls $1/part-*.mindrecord` - for element in ${elements[*]}; - do - file_list[$file_index]=$element - file_index=`expr $file_index + 1` - done -} - -getdir $1 -echo "Get all the mindrecord files: "${file_list[*]} - -# create dataset for train -python create_dataset.py --input_file ${file_list[*]} diff --git a/example/nlp_to_mindrecord/zhwiki/README.md b/example/nlp_to_mindrecord/zhwiki/README.md deleted file mode 100644 index 1a9de05114..0000000000 --- a/example/nlp_to_mindrecord/zhwiki/README.md +++ /dev/null @@ -1,113 +0,0 @@ -# Guideline to Convert Training Data zhwiki to MindRecord For Bert Pre Training - - - -- [What does the example do](#what-does-the-example-do) -- [Run simple test](#run-simple-test) -- [How to use the example to process zhwiki](#how-to-use-the-example-to-process-zhwiki) - - [Download zhwiki training data](#download-zhwiki-training-data) - - [Extract the zhwiki](#extract-the-zhwiki) - - [Generate MindRecord](#generate-mindrecord) - - [Create MindDataset By MindRecord](#create-minddataset-by-mindrecord) - - - - -## What does the example do - -This example is based on [zhwiki](https://dumps.wikimedia.org/zhwiki) training data, generating MindRecord file, and finally used for Bert network training. - -1. run.sh: generate MindRecord entry script. -2. run_read.py: create MindDataset by MindRecord entry script. - - create_dataset.py: use MindDataset to read MindRecord to generate dataset. - -## Run simple test - -Follow the step: - -```bash -bash run_simple.sh # generate output/simple.mindrecord* by ../../../third_party/to_mindrecord/zhwiki/sample_text.txt -bash run_read_simple.sh # use MindDataset to read output/simple.mindrecord* -``` - -## How to use the example to process zhwiki - -Download zhwiki data, extract it, convert it to MindRecord, use MindDataset to read MindRecord. - -### Download zhwiki training data - -> [zhwiki dataset download address](https://dumps.wikimedia.org/zhwiki) **-> 20200401 -> zhwiki-20200401-pages-articles-multistream.xml.bz2** - -- put the zhwiki-20200401-pages-articles-multistream.xml.bz2 in {your-mindspore}/example/nlp_to_mindrecord/zhwiki/data directory. - -### Extract the zhwiki - -1. Download [wikiextractor](https://github.com/attardi/wikiextractor) script to {your-mindspore}/example/nlp_to_mindrecord/zhwiki/data directory. - - ``` - $ ls data/ - README.md wikiextractor zhwiki-20200401-pages-articles-multistream.xml.bz2 - ``` - -2. Extract the zhwiki. - ```python - python data/wikiextractor/WikiExtractor.py data/zhwiki-20200401-pages-articles-multistream.xml.bz2 --processes 4 --templates data/template --bytes 8M --min_text_length 0 --filter_disambig_pages --output data/extract - ``` - -3. Generate like this: - ``` - $ ls data/extract - AA AB - ``` - -### Generate MindRecord - -1. Run the run.sh script. - ``` - bash run.sh - ``` - > Caution: This process maybe slow, please wait patiently. If you do not have a machine with enough memory and cpu, it is recommended that you modify the script to generate mindrecord in step by step. - -2. The output like this: - ``` - patching file create_pretraining_data_patched.py (read from create_pretraining_data.py) - Begin preprocess input file: ./data/extract/AA/wiki_00 - Begin output file: AAwiki_00.mindrecord - Total task: 5, processing: 1 - Begin preprocess input file: ./data/extract/AA/wiki_01 - Begin output file: AAwiki_01.mindrecord - Total task: 5, processing: 2 - Begin preprocess input file: ./data/extract/AA/wiki_02 - Begin output file: AAwiki_02.mindrecord - Total task: 5, processing: 3 - Begin preprocess input file: ./data/extract/AB/wiki_02 - Begin output file: ABwiki_02.mindrecord - Total task: 5, processing: 4 - ... - ``` - -3. Generate files like this: - ```bash - $ ls output/ - AAwiki_00.mindrecord AAwiki_00.mindrecord.db AAwiki_01.mindrecord AAwiki_01.mindrecord.db AAwiki_02.mindrecord AAwiki_02.mindrecord.db ... ABwiki_00.mindrecord ABwiki_00.mindrecord.db ... - ``` - -### Create MindDataset By MindRecord - -1. Run the run_read.sh script. - ```bash - bash run_read.sh - ``` - -2. The output like this: - ``` - ... - example 74: input_ids: [ 101 8168 118 12847 8783 9977 15908 117 8256 9245 11643 8168 8847 8588 11575 8154 8228 143 8384 8376 9197 10241 103 10564 11421 8199 12268 112 161 8228 11541 9586 8436 8174 8363 9864 9702 103 103 119 103 9947 10564 103 8436 8806 11479 103 8912 119 103 103 103 12209 8303 103 8757 8824 117 8256 103 8619 8168 11541 102 11684 8196 103 8228 8847 11523 117 9059 9064 12410 8358 8181 10764 117 11167 11706 9920 148 8332 11390 8936 8205 10951 11997 103 8154 117 103 8670 10467 112 161 10951 13139 12413 117 10288 143 10425 8205 152 10795 8472 8196 103 161 12126 9172 13129 12106 8217 8174 12244 8205 143 103 8461 8277 10628 160 8221 119 102] - example 74: input_mask: [1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1] - example 74: segment_ids: [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1] - example 74: masked_lm_positions: [ 6 22 37 38 40 43 47 50 51 52 55 60 67 76 89 92 98 109 120 0] - example 74: masked_lm_ids: [ 8118 8165 8329 8890 8554 8458 119 8850 8565 10392 8174 11467 10291 8181 8549 12718 13139 112 158 0] - example 74: masked_lm_weights: [1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 0.] - example 74: next_sentence_labels: [0] - ... - ``` diff --git a/example/nlp_to_mindrecord/zhwiki/create_dataset.py b/example/nlp_to_mindrecord/zhwiki/create_dataset.py deleted file mode 100644 index d90d12b7f2..0000000000 --- a/example/nlp_to_mindrecord/zhwiki/create_dataset.py +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -"""create MindDataset by MindRecord""" -import argparse -import mindspore.dataset as ds - -def create_dataset(data_file): - """create MindDataset""" - num_readers = 4 - data_set = ds.MindDataset(dataset_file=data_file, num_parallel_workers=num_readers, shuffle=True) - index = 0 - for item in data_set.create_dict_iterator(): - # print("example {}: {}".format(index, item)) - print("example {}: input_ids: {}".format(index, item['input_ids'])) - print("example {}: input_mask: {}".format(index, item['input_mask'])) - print("example {}: segment_ids: {}".format(index, item['segment_ids'])) - print("example {}: masked_lm_positions: {}".format(index, item['masked_lm_positions'])) - print("example {}: masked_lm_ids: {}".format(index, item['masked_lm_ids'])) - print("example {}: masked_lm_weights: {}".format(index, item['masked_lm_weights'])) - print("example {}: next_sentence_labels: {}".format(index, item['next_sentence_labels'])) - index += 1 - if index % 1000 == 0: - print("read rows: {}".format(index)) - print("total rows: {}".format(index)) - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument("-i", "--input_file", nargs='+', type=str, help='Input mindreord file') - args = parser.parse_args() - - create_dataset(args.input_file) diff --git a/example/nlp_to_mindrecord/zhwiki/data/.gitignore b/example/nlp_to_mindrecord/zhwiki/data/.gitignore deleted file mode 100644 index f15cab0c89..0000000000 --- a/example/nlp_to_mindrecord/zhwiki/data/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -wikiextractor/ -zhwiki-20200401-pages-articles-multistream.xml.bz2 -extract/ diff --git a/example/nlp_to_mindrecord/zhwiki/data/README.md b/example/nlp_to_mindrecord/zhwiki/data/README.md deleted file mode 100644 index b54948808e..0000000000 --- a/example/nlp_to_mindrecord/zhwiki/data/README.md +++ /dev/null @@ -1 +0,0 @@ -## The input dataset diff --git a/example/nlp_to_mindrecord/zhwiki/output/README.md b/example/nlp_to_mindrecord/zhwiki/output/README.md deleted file mode 100644 index b7cfba1b47..0000000000 --- a/example/nlp_to_mindrecord/zhwiki/output/README.md +++ /dev/null @@ -1 +0,0 @@ -## Output the mindrecord diff --git a/example/nlp_to_mindrecord/zhwiki/run.sh b/example/nlp_to_mindrecord/zhwiki/run.sh deleted file mode 100644 index a057031e6b..0000000000 --- a/example/nlp_to_mindrecord/zhwiki/run.sh +++ /dev/null @@ -1,112 +0,0 @@ -#!/bin/bash -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ - -rm -f output/*.mindrecord* - -data_dir="./data/extract" -file_list=() -output_filename=() -file_index=0 - -function getdir() { - elements=`ls $1` - for element in ${elements[*]}; - do - dir_or_file=$1"/"$element - if [ -d $dir_or_file ]; - then - getdir $dir_or_file - else - file_list[$file_index]=$dir_or_file - echo "${dir_or_file}" | tr '/' '\n' > dir_file_list.txt # dir dir file to mapfile - mapfile parent_dir < dir_file_list.txt - rm dir_file_list.txt >/dev/null 2>&1 - tmp_output_filename=${parent_dir[${#parent_dir[@]}-2]}${parent_dir[${#parent_dir[@]}-1]}".mindrecord" - output_filename[$file_index]=`echo ${tmp_output_filename} | sed 's/ //g'` - file_index=`expr $file_index + 1` - fi - done -} - -getdir "${data_dir}" -# echo "The input files: "${file_list[@]} -# echo "The output files: "${output_filename[@]} - -if [ ! -d "../../../third_party/to_mindrecord/zhwiki" ]; then - echo "The patch base dir ../../../third_party/to_mindrecord/zhwiki is not exist." - exit 1 -fi - -if [ ! -f "../../../third_party/patch/to_mindrecord/zhwiki/create_pretraining_data.patch" ]; then - echo "The patch file ../../../third_party/patch/to_mindrecord/zhwiki/create_pretraining_data.patch is not exist." - exit 1 -fi - -# patch for create_pretraining_data.py -patch -p0 -d ../../../third_party/to_mindrecord/zhwiki/ -o create_pretraining_data_patched.py < ../../../third_party/patch/to_mindrecord/zhwiki/create_pretraining_data.patch -if [ $? -ne 0 ]; then - echo "Patch ../../../third_party/to_mindrecord/zhwiki/create_pretraining_data.py failed" - exit 1 -fi - -# get the cpu core count -num_cpu_core=`cat /proc/cpuinfo | grep "processor" | wc -l` -avaiable_core_size=`expr $num_cpu_core / 3 \* 2` - -echo "Begin preprocess `date`" - -# using patched script to generate mindrecord -file_list_len=`expr ${#file_list[*]} - 1` -for index in $(seq 0 $file_list_len); do - echo "Begin preprocess input file: ${file_list[$index]}" - echo "Begin output file: ${output_filename[$index]}" - python ../../../third_party/to_mindrecord/zhwiki/create_pretraining_data_patched.py \ - --input_file=${file_list[$index]} \ - --output_file=output/${output_filename[$index]} \ - --partition_number=1 \ - --vocab_file=../../../third_party/to_mindrecord/zhwiki/vocab.txt \ - --do_lower_case=True \ - --max_seq_length=128 \ - --max_predictions_per_seq=20 \ - --masked_lm_prob=0.15 \ - --random_seed=12345 \ - --dupe_factor=10 >/tmp/${output_filename[$index]}.log 2>&1 & # user defined - process_count=`ps -ef | grep create_pretraining_data_patched | grep -v grep | wc -l` - echo "Total task: ${#file_list[*]}, processing: ${process_count}" - if [ $process_count -ge $avaiable_core_size ]; then - while [ 1 ]; do - process_num=`ps -ef | grep create_pretraining_data_patched | grep -v grep | wc -l` - if [ $process_count -gt $process_num ]; then - process_count=$process_num - break; - fi - sleep 2 - done - fi -done - -process_num=`ps -ef | grep create_pretraining_data_patched | grep -v grep | wc -l` -while [ 1 ]; do - if [ $process_num -eq 0 ]; then - break; - fi - echo "There are still ${process_num} preprocess running ..." - sleep 2 - process_num=`ps -ef | grep create_pretraining_data_patched | grep -v grep | wc -l` -done - -echo "Preprocess all the data success." -echo "End preprocess `date`" diff --git a/example/nlp_to_mindrecord/zhwiki/run_read_simple.sh b/example/nlp_to_mindrecord/zhwiki/run_read_simple.sh deleted file mode 100644 index 1c26dec449..0000000000 --- a/example/nlp_to_mindrecord/zhwiki/run_read_simple.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ - -# create dataset for train -python create_dataset.py --input_file=output/simple.mindrecord0 diff --git a/example/nlp_to_mindrecord/zhwiki/run_simple.sh b/example/nlp_to_mindrecord/zhwiki/run_simple.sh deleted file mode 100644 index 20c1d98d66..0000000000 --- a/example/nlp_to_mindrecord/zhwiki/run_simple.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ - -rm -f output/simple.mindrecord* - -if [ ! -d "../../../third_party/to_mindrecord/zhwiki" ]; then - echo "The patch base dir ../../../third_party/to_mindrecord/zhwiki is not exist." - exit 1 -fi - -if [ ! -f "../../../third_party/patch/to_mindrecord/zhwiki/create_pretraining_data.patch" ]; then - echo "The patch file ../../../third_party/patch/to_mindrecord/zhwiki/create_pretraining_data.patch is not exist." - exit 1 -fi - -# patch for create_pretraining_data.py -patch -p0 -d ../../../third_party/to_mindrecord/zhwiki/ -o create_pretraining_data_patched.py < ../../../third_party/patch/to_mindrecord/zhwiki/create_pretraining_data.patch -if [ $? -ne 0 ]; then - echo "Patch ../../../third_party/to_mindrecord/zhwiki/create_pretraining_data.py failed" - exit 1 -fi - -# using patched script to generate mindrecord -python ../../../third_party/to_mindrecord/zhwiki/create_pretraining_data_patched.py \ ---input_file=../../../third_party/to_mindrecord/zhwiki/sample_text.txt \ ---output_file=output/simple.mindrecord \ ---partition_number=4 \ ---vocab_file=../../../third_party/to_mindrecord/zhwiki/vocab.txt \ ---do_lower_case=True \ ---max_seq_length=128 \ ---max_predictions_per_seq=20 \ ---masked_lm_prob=0.15 \ ---random_seed=12345 \ ---dupe_factor=10 # user defined diff --git a/example/resnet50_cifar10/README.md b/example/resnet50_cifar10/README.md deleted file mode 100644 index abb0ba4090..0000000000 --- a/example/resnet50_cifar10/README.md +++ /dev/null @@ -1,137 +0,0 @@ -# ResNet-50 Example - -## Description - -This is an example of training ResNet-50 with CIFAR-10 dataset in MindSpore. - -## Requirements - -- Install [MindSpore](https://www.mindspore.cn/install/en). - -- Download the dataset CIFAR-10 - -> Unzip the CIFAR-10 dataset to any path you want and the folder structure should include train and eval dataset as follows: -> ``` -> . -> ├── cifar-10-batches-bin # train dataset -> └── cifar-10-verify-bin # infer dataset -> ``` - - -## Example structure - -```shell -. -├── config.py # parameter configuration -├── dataset.py # data preprocessing -├── eval.py # infer script -├── lr_generator.py # generate learning rate for each step -├── run_distribute_train.sh # launch distributed training(8 pcs) -├── run_infer.sh # launch infering -├── run_standalone_train.sh # launch standalone training(1 pcs) -└── train.py # train script -``` - - -## Parameter configuration - -Parameters for both training and inference can be set in config.py. - -``` -"class_num": 10, # dataset class num -"batch_size": 32, # batch size of input tensor -"loss_scale": 1024, # loss scale -"momentum": 0.9, # momentum -"weight_decay": 1e-4, # weight decay -"epoch_size": 90, # only valid for taining, which is always 1 for inference -"buffer_size": 100, # number of queue size in data preprocessing -"image_height": 224, # image height -"image_width": 224, # image width -"save_checkpoint": True, # whether save checkpoint or not -"save_checkpoint_steps": 195, # the step interval between two checkpoints. By default, the last checkpoint will be saved after the last step -"keep_checkpoint_max": 10, # only keep the last keep_checkpoint_max checkpoint -"save_checkpoint_path": "./", # path to save checkpoint -"warmup_epochs": 5, # number of warmup epoch -"lr_decay_mode": "poly" # decay mode can be selected in steps, ploy and default -"lr_init": 0.01, # initial learning rate -"lr_end": 0.00001, # final learning rate -"lr_max": 0.1, # maximum learning rate -``` - -## Running the example - -### Train - -#### Usage - -``` -# distributed training -Usage: sh run_distribute_train.sh [MINDSPORE_HCCL_CONFIG_PATH] [DATASET_PATH] - -# standalone training -Usage: sh run_standalone_train.sh [DATASET_PATH] -``` - - -#### Launch - -``` -# distribute training example -sh run_distribute_train.sh rank_table.json ~/cifar-10-batches-bin - -# standalone training example -sh run_standalone_train.sh ~/cifar-10-batches-bin -``` - -> About rank_table.json, you can refer to the [distributed training tutorial](https://www.mindspore.cn/tutorial/en/master/advanced_use/distributed_training.html). - -#### Result - -Training result will be stored in the example path, whose folder name begins with "train" or "train_parallel". Under this, you can find checkpoint file together with result like the followings in log. - -``` -# distribute training result(8 pcs) -epoch: 1 step: 195, loss is 1.9601055 -epoch: 2 step: 195, loss is 1.8555021 -epoch: 3 step: 195, loss is 1.6707983 -epoch: 4 step: 195, loss is 1.8162166 -epoch: 5 step: 195, loss is 1.393667 -``` - -### Infer - -#### Usage - -``` -# infer -Usage: sh run_infer.sh [DATASET_PATH] [CHECKPOINT_PATH] -``` - -#### Launch - -``` -# infer example -sh run_infer.sh ~/cifar10-10-verify-bin ~/resnet50_cifar10/train_parallel0/resnet-90_195.ckpt -``` - -> checkpoint can be produced in training process. - -#### Result - -Inference result will be stored in the example path, whose folder name is "infer". Under this, you can find result like the followings in log. - -``` -result: {'acc': 0.91446314102564111} ckpt=~/resnet50_cifar10/train_parallel0/resnet-90_195.ckpt -``` - -### Running on GPU -``` -# distributed training example -mpirun -n 8 python train.py --dataset_path=~/cifar-10-batches-bin --device_target="GPU" --run_distribute=True - -# standalone training example -python train.py --dataset_path=~/cifar-10-batches-bin --device_target="GPU" - -# infer example -python eval.py --dataset_path=~/cifar10-10-verify-bin --device_target="GPU" --checkpoint_path=resnet-90_195.ckpt -``` \ No newline at end of file diff --git a/example/resnet50_cifar10/dataset.py b/example/resnet50_cifar10/dataset.py deleted file mode 100755 index 8a66ec573a..0000000000 --- a/example/resnet50_cifar10/dataset.py +++ /dev/null @@ -1,81 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -""" -create train or eval dataset. -""" -import os -import mindspore.common.dtype as mstype -import mindspore.dataset.engine as de -import mindspore.dataset.transforms.vision.c_transforms as C -import mindspore.dataset.transforms.c_transforms as C2 -from mindspore.communication.management import init, get_rank, get_group_size -from config import config - - -def create_dataset(dataset_path, do_train, repeat_num=1, batch_size=32, target="Ascend"): - """ - create a train or eval dataset - - Args: - dataset_path(string): the path of dataset. - do_train(bool): whether dataset is used for train or eval. - repeat_num(int): the repeat times of dataset. Default: 1 - batch_size(int): the batch size of dataset. Default: 32 - target(str): the device target. Default: Ascend - - Returns: - dataset - """ - if target == "Ascend": - device_num = int(os.getenv("DEVICE_NUM")) - rank_id = int(os.getenv("RANK_ID")) - else: - init("nccl") - rank_id = get_rank() - device_num = get_group_size() - - if device_num == 1: - ds = de.Cifar10Dataset(dataset_path, num_parallel_workers=8, shuffle=True) - else: - ds = de.Cifar10Dataset(dataset_path, num_parallel_workers=8, shuffle=True, - num_shards=device_num, shard_id=rank_id) - - # define map operations - trans = [] - if do_train: - trans += [ - C.RandomCrop((32, 32), (4, 4, 4, 4)), - C.RandomHorizontalFlip(prob=0.5) - ] - - trans += [ - C.Resize((config.image_height, config.image_width)), - C.Rescale(1.0 / 255.0, 0.0), - C.Normalize([0.4914, 0.4822, 0.4465], [0.2023, 0.1994, 0.2010]), - C.HWC2CHW() - ] - - type_cast_op = C2.TypeCast(mstype.int32) - - ds = ds.map(input_columns="label", num_parallel_workers=8, operations=type_cast_op) - ds = ds.map(input_columns="image", num_parallel_workers=8, operations=trans) - - # apply batch operations - ds = ds.batch(batch_size, drop_remainder=True) - - # apply dataset repeat operation - ds = ds.repeat(repeat_num) - - return ds diff --git a/example/resnet50_cifar10/eval.py b/example/resnet50_cifar10/eval.py deleted file mode 100755 index f7d71c8d29..0000000000 --- a/example/resnet50_cifar10/eval.py +++ /dev/null @@ -1,72 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -""" -eval. -""" -import os -import argparse -from dataset import create_dataset -from config import config -from mindspore import context -from mindspore.model_zoo.resnet import resnet50 -from mindspore.parallel._auto_parallel_context import auto_parallel_context -from mindspore.nn.loss import SoftmaxCrossEntropyWithLogits -from mindspore.train.model import Model, ParallelMode -from mindspore.train.serialization import load_checkpoint, load_param_into_net -from mindspore.communication.management import init, get_group_size - -parser = argparse.ArgumentParser(description='Image classification') -parser.add_argument('--run_distribute', type=bool, default=False, help='Run distribute') -parser.add_argument('--device_num', type=int, default=1, help='Device num.') -parser.add_argument('--do_train', type=bool, default=False, help='Do train or not.') -parser.add_argument('--do_eval', type=bool, default=True, help='Do eval or not.') -parser.add_argument('--checkpoint_path', type=str, default=None, help='Checkpoint file path') -parser.add_argument('--dataset_path', type=str, default=None, help='Dataset path') -parser.add_argument('--device_target', type=str, default='Ascend', help='Device target') -args_opt = parser.parse_args() - -if __name__ == '__main__': - target = args_opt.device_target - context.set_context(mode=context.GRAPH_MODE, device_target=target, save_graphs=False) - if not args_opt.do_eval and args_opt.run_distribute: - if target == "Ascend": - device_id = int(os.getenv('DEVICE_ID')) - context.set_context(device_id=device_id) - context.set_auto_parallel_context(device_num=args_opt.device_num, parallel_mode=ParallelMode.DATA_PARALLEL, - mirror_mean=True) - auto_parallel_context().set_all_reduce_fusion_split_indices([140]) - init() - elif target == "GPU": - init("nccl") - context.set_auto_parallel_context(device_num=get_group_size(), parallel_mode=ParallelMode.DATA_PARALLEL, - mirror_mean=True) - - epoch_size = config.epoch_size - net = resnet50(class_num=config.class_num) - loss = SoftmaxCrossEntropyWithLogits(sparse=True) - - if args_opt.do_eval: - dataset = create_dataset(dataset_path=args_opt.dataset_path, do_train=False, batch_size=config.batch_size, - target=target) - step_size = dataset.get_dataset_size() - - if args_opt.checkpoint_path: - param_dict = load_checkpoint(args_opt.checkpoint_path) - load_param_into_net(net, param_dict) - net.set_train(False) - - model = Model(net, loss_fn=loss, metrics={'acc'}) - res = model.eval(dataset) - print("result:", res, "ckpt=", args_opt.checkpoint_path) diff --git a/example/resnet50_cifar10/run_infer.sh b/example/resnet50_cifar10/run_infer.sh deleted file mode 100755 index 14d7faf981..0000000000 --- a/example/resnet50_cifar10/run_infer.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/bash -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ - -if [ $# != 2 ] -then - echo "Usage: sh run_infer.sh [DATASET_PATH] [CHECKPOINT_PATH]" -exit 1 -fi - -get_real_path(){ - if [ "${1:0:1}" == "/" ]; then - echo "$1" - else - echo "$(realpath -m $PWD/$1)" - fi -} - -PATH1=$(get_real_path $1) -PATH2=$(get_real_path $2) - - -if [ ! -d $PATH1 ] -then - echo "error: DATASET_PATH=$1 is not a directory" -exit 1 -fi - -if [ ! -f $PATH2 ] -then - echo "error: CHECKPOINT_PATH=$2 is not a file" -exit 1 -fi - -ulimit -u unlimited -export DEVICE_NUM=1 -export DEVICE_ID=0 -export RANK_SIZE=$DEVICE_NUM -export RANK_ID=0 - -if [ -d "infer" ]; -then - rm -rf ./infer -fi -mkdir ./infer -cp *.py ./infer -cp *.sh ./infer -cd ./infer || exit -env > env.log -echo "start infering for device $DEVICE_ID" -python eval.py --do_eval=True --dataset_path=$PATH1 --checkpoint_path=$PATH2 &> log & -cd .. diff --git a/example/resnet50_cifar10/train.py b/example/resnet50_cifar10/train.py deleted file mode 100755 index 323695ae29..0000000000 --- a/example/resnet50_cifar10/train.py +++ /dev/null @@ -1,97 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -"""train_imagenet.""" -import os -import argparse -import numpy as np -from dataset import create_dataset -from lr_generator import get_lr -from config import config -from mindspore import context -from mindspore import Tensor -from mindspore.model_zoo.resnet import resnet50 -from mindspore.parallel._auto_parallel_context import auto_parallel_context -from mindspore.nn.optim.momentum import Momentum -from mindspore.nn.loss import SoftmaxCrossEntropyWithLogits - -from mindspore.train.model import Model, ParallelMode - -from mindspore.train.callback import ModelCheckpoint, CheckpointConfig, LossMonitor, TimeMonitor -from mindspore.train.loss_scale_manager import FixedLossScaleManager -from mindspore.communication.management import init, get_rank, get_group_size - -parser = argparse.ArgumentParser(description='Image classification') -parser.add_argument('--run_distribute', type=bool, default=False, help='Run distribute') -parser.add_argument('--device_num', type=int, default=1, help='Device num.') -parser.add_argument('--do_train', type=bool, default=True, help='Do train or not.') -parser.add_argument('--do_eval', type=bool, default=False, help='Do eval or not.') -parser.add_argument('--dataset_path', type=str, default=None, help='Dataset path') -parser.add_argument('--device_target', type=str, default='Ascend', help='Device target') -args_opt = parser.parse_args() - - -if __name__ == '__main__': - target = args_opt.device_target - ckpt_save_dir = config.save_checkpoint_path - context.set_context(mode=context.GRAPH_MODE, device_target=target, save_graphs=False) - np.random.seed(1) - if not args_opt.do_eval and args_opt.run_distribute: - if target == "Ascend": - device_id = int(os.getenv('DEVICE_ID')) - context.set_context(mode=context.GRAPH_MODE, device_target="Ascend", save_graphs=False, device_id=device_id, - enable_auto_mixed_precision=True) - init() - context.set_auto_parallel_context(device_num=args_opt.device_num, parallel_mode=ParallelMode.DATA_PARALLEL, - mirror_mean=True) - auto_parallel_context().set_all_reduce_fusion_split_indices([107, 160]) - ckpt_save_dir = config.save_checkpoint_path - elif target == "GPU": - context.set_context(mode=context.GRAPH_MODE, device_target="GPU", save_graphs=False) - init("nccl") - context.set_auto_parallel_context(device_num=get_group_size(), parallel_mode=ParallelMode.DATA_PARALLEL, - mirror_mean=True) - ckpt_save_dir = config.save_checkpoint_path + "ckpt_" + str(get_rank()) + "/" - epoch_size = config.epoch_size - net = resnet50(class_num=config.class_num) - - if args_opt.do_train: - dataset = create_dataset(dataset_path=args_opt.dataset_path, do_train=True, - repeat_num=epoch_size, batch_size=config.batch_size, target=target) - step_size = dataset.get_dataset_size() - - loss_scale = FixedLossScaleManager(config.loss_scale, drop_overflow_update=False) - lr = Tensor(get_lr(global_step=0, lr_init=config.lr_init, lr_end=config.lr_end, lr_max=config.lr_max, - warmup_epochs=config.warmup_epochs, total_epochs=epoch_size, steps_per_epoch=step_size, - lr_decay_mode='poly')) - opt = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), lr, config.momentum, - config.weight_decay, config.loss_scale) - if target == 'GPU': - loss = SoftmaxCrossEntropyWithLogits(sparse=True, is_grad=False, reduction='mean') - opt = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), lr, config.momentum) - model = Model(net, loss_fn=loss, optimizer=opt, metrics={'acc'}) - else: - loss = SoftmaxCrossEntropyWithLogits(sparse=True, reduction='mean') - model = Model(net, loss_fn=loss, optimizer=opt, loss_scale_manager=loss_scale, metrics={'acc'}, - amp_level="O2", keep_batchnorm_fp32=False) - - time_cb = TimeMonitor(data_size=step_size) - loss_cb = LossMonitor() - cb = [time_cb, loss_cb] - if config.save_checkpoint: - config_ck = CheckpointConfig(save_checkpoint_steps=config.save_checkpoint_epochs*step_size, - keep_checkpoint_max=config.keep_checkpoint_max) - ckpt_cb = ModelCheckpoint(prefix="resnet", directory=ckpt_save_dir, config=config_ck) - cb += [ckpt_cb] - model.train(epoch_size, dataset, callbacks=cb) diff --git a/example/resnet50_imagenet2012/README.md b/example/resnet50_imagenet2012/README.md deleted file mode 100644 index 6baf863544..0000000000 --- a/example/resnet50_imagenet2012/README.md +++ /dev/null @@ -1,150 +0,0 @@ -# ResNet-50 Example - -## Description - -This is an example of training ResNet-50 with ImageNet2012 dataset in MindSpore. - -## Requirements - -- Install [MindSpore](https://www.mindspore.cn/install/en). - -- Download the dataset ImageNet2012 - -> Unzip the ImageNet2012 dataset to any path you want and the folder structure should include train and eval dataset as follows: -> ``` -> . -> ├── ilsvrc # train dataset -> └── ilsvrc_eval # infer dataset -> ``` - - -## Example structure - -```shell -. -├── crossentropy.py # CrossEntropy loss function -├── config.py # parameter configuration -├── dataset.py # data preprocessing -├── eval.py # infer script -├── lr_generator.py # generate learning rate for each step -├── run_distribute_train.sh # launch distributed training(8 pcs) -├── run_infer.sh # launch infering -├── run_standalone_train.sh # launch standalone training(1 pcs) -└── train.py # train script -``` - - -## Parameter configuration - -Parameters for both training and inference can be set in config.py. - -``` -"class_num": 1001, # dataset class number -"batch_size": 32, # batch size of input tensor -"loss_scale": 1024, # loss scale -"momentum": 0.9, # momentum optimizer -"weight_decay": 1e-4, # weight decay -"epoch_size": 90, # only valid for taining, which is always 1 for inference -"pretrained_epoch_size": 1, # epoch size that model has been trained before load pretrained checkpoint -"buffer_size": 1000, # number of queue size in data preprocessing -"image_height": 224, # image height -"image_width": 224, # image width -"save_checkpoint": True, # whether save checkpoint or not -"save_checkpoint_epochs": 1, # the epoch interval between two checkpoints. By default, the last checkpoint will be saved after the last epoch -"keep_checkpoint_max": 10, # only keep the last keep_checkpoint_max checkpoint -"save_checkpoint_path": "./", # path to save checkpoint relative to the executed path -"warmup_epochs": 0, # number of warmup epoch -"lr_decay_mode": "cosine", # decay mode for generating learning rate -"label_smooth": True, # label smooth -"label_smooth_factor": 0.1, # label smooth factor -"lr_init": 0, # initial learning rate -"lr_max": 0.1, # maximum learning rate -``` - -## Running the example - -### Train - -#### Usage - -``` -# distributed training -Usage: sh run_distribute_train.sh [MINDSPORE_HCCL_CONFIG_PATH] [DATASET_PATH] [PRETRAINED_CKPT_PATH](optional) - -# standalone training -Usage: sh run_standalone_train.sh [DATASET_PATH] [PRETRAINED_CKPT_PATH](optional) - -``` - - -#### Launch - -```bash -# distributed training example(8 pcs) -sh run_distribute_train.sh rank_table_8p.json dataset/ilsvrc - -# If you want to load pretrained ckpt file -sh run_distribute_train.sh rank_table_8p.json dataset/ilsvrc ./pretrained.ckpt - -# standalone training example(1 pcs) -sh run_standalone_train.sh dataset/ilsvrc - -# If you want to load pretrained ckpt file -sh run_standalone_train.sh dataset/ilsvrc ./pretrained.ckpt -``` - -> About rank_table.json, you can refer to the [distributed training tutorial](https://www.mindspore.cn/tutorial/en/master/advanced_use/distributed_training.html). - -#### Result - -Training result will be stored in the example path, whose folder name begins with "train" or "train_parallel". Under this, you can find checkpoint file together with result like the followings in log. - -``` -# distribute training result(8 pcs) -epoch: 1 step: 5004, loss is 4.8995576 -epoch: 2 step: 5004, loss is 3.9235563 -epoch: 3 step: 5004, loss is 3.833077 -epoch: 4 step: 5004, loss is 3.2795618 -epoch: 5 step: 5004, loss is 3.1978393 -``` - -### Infer - -#### Usage - -``` -# infer -Usage: sh run_infer.sh [DATASET_PATH] [CHECKPOINT_PATH] -``` - -#### Launch - -```bash -# infer with checkpoint -sh run_infer.sh dataset/ilsvrc_eval train_parallel0/resnet-90_5004.ckpt -``` - -> checkpoint can be produced in training process. - -#### Result - -Inference result will be stored in the example path, whose folder name is "infer". Under this, you can find result like the followings in log. - -``` -result: {'acc': 0.7671054737516005} ckpt=train_parallel0/resnet-90_5004.ckpt -``` - -### Running on GPU -``` -# distributed training example -mpirun -n 8 python train.py --dataset_path=dataset/ilsvrc/train --device_target="GPU" --run_distribute=True - -# standalone training example -python train.py --dataset_path=dataset/ilsvrc/train --device_target="GPU" - -# standalone training example with pretrained checkpoint -python train.py --dataset_path=dataset/ilsvrc/train --device_target="GPU" --pre_trained=pretrained.ckpt - -# infer example -python eval.py --dataset_path=dataset/ilsvrc/val --device_target="GPU" --checkpoint_path=resnet-90_5004ss.ckpt -``` \ No newline at end of file diff --git a/example/resnet50_imagenet2012/dataset.py b/example/resnet50_imagenet2012/dataset.py deleted file mode 100755 index 0691985e0b..0000000000 --- a/example/resnet50_imagenet2012/dataset.py +++ /dev/null @@ -1,85 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -""" -create train or eval dataset. -""" -import os -import mindspore.common.dtype as mstype -import mindspore.dataset.engine as de -import mindspore.dataset.transforms.vision.c_transforms as C -import mindspore.dataset.transforms.c_transforms as C2 -from mindspore.communication.management import init, get_rank, get_group_size - -def create_dataset(dataset_path, do_train, repeat_num=1, batch_size=32, target="Ascend"): - """ - create a train or eval dataset - - Args: - dataset_path(string): the path of dataset. - do_train(bool): whether dataset is used for train or eval. - repeat_num(int): the repeat times of dataset. Default: 1 - batch_size(int): the batch size of dataset. Default: 32 - target(str): the device target. Default: Ascend - - Returns: - dataset - """ - if target == "Ascend": - device_num = int(os.getenv("DEVICE_NUM")) - rank_id = int(os.getenv("RANK_ID")) - else: - init("nccl") - rank_id = get_rank() - device_num = get_group_size() - - if device_num == 1: - ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True) - else: - ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True, - num_shards=device_num, shard_id=rank_id) - - image_size = 224 - mean = [0.485 * 255, 0.456 * 255, 0.406 * 255] - std = [0.229 * 255, 0.224 * 255, 0.225 * 255] - - # define map operations - if do_train: - trans = [ - C.RandomCropDecodeResize(image_size, scale=(0.08, 1.0), ratio=(0.75, 1.333)), - C.RandomHorizontalFlip(prob=0.5), - C.Normalize(mean=mean, std=std), - C.HWC2CHW() - ] - else: - trans = [ - C.Decode(), - C.Resize((256, 256)), - C.CenterCrop(image_size), - C.Normalize(mean=mean, std=std), - C.HWC2CHW() - ] - - type_cast_op = C2.TypeCast(mstype.int32) - - ds = ds.map(input_columns="image", num_parallel_workers=8, operations=trans) - ds = ds.map(input_columns="label", num_parallel_workers=8, operations=type_cast_op) - - # apply batch operations - ds = ds.batch(batch_size, drop_remainder=True) - - # apply dataset repeat operation - ds = ds.repeat(repeat_num) - - return ds diff --git a/example/resnet50_imagenet2012/eval.py b/example/resnet50_imagenet2012/eval.py deleted file mode 100755 index 3f7961e786..0000000000 --- a/example/resnet50_imagenet2012/eval.py +++ /dev/null @@ -1,62 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -""" -eval. -""" -import os -import argparse -from dataset import create_dataset -from config import config -from mindspore import context -from mindspore.model_zoo.resnet import resnet50 -from mindspore.train.model import Model -from mindspore.train.serialization import load_checkpoint, load_param_into_net -from crossentropy import CrossEntropy - -parser = argparse.ArgumentParser(description='Image classification') -parser.add_argument('--run_distribute', type=bool, default=False, help='Run distribute') -parser.add_argument('--device_num', type=int, default=1, help='Device num.') -parser.add_argument('--do_train', type=bool, default=False, help='Do train or not.') -parser.add_argument('--do_eval', type=bool, default=True, help='Do eval or not.') -parser.add_argument('--checkpoint_path', type=str, default=None, help='Checkpoint file path') -parser.add_argument('--dataset_path', type=str, default=None, help='Dataset path') -parser.add_argument('--device_target', type=str, default='Ascend', help='Device target') -args_opt = parser.parse_args() -target = args_opt.device_target -context.set_context(mode=context.GRAPH_MODE, device_target=target, save_graphs=False) -if target == "Ascend": - device_id = int(os.getenv('DEVICE_ID')) - context.set_context(device_id=device_id) - -if __name__ == '__main__': - - net = resnet50(class_num=config.class_num) - if not config.use_label_smooth: - config.label_smooth_factor = 0.0 - loss = CrossEntropy(smooth_factor=config.label_smooth_factor, num_classes=config.class_num) - - if args_opt.do_eval: - dataset = create_dataset(dataset_path=args_opt.dataset_path, do_train=False, batch_size=config.batch_size, - target=target) - step_size = dataset.get_dataset_size() - - if args_opt.checkpoint_path: - param_dict = load_checkpoint(args_opt.checkpoint_path) - load_param_into_net(net, param_dict) - net.set_train(False) - - model = Model(net, loss_fn=loss, metrics={'acc'}) - res = model.eval(dataset) - print("result:", res, "ckpt=", args_opt.checkpoint_path) diff --git a/example/resnet50_imagenet2012/train.py b/example/resnet50_imagenet2012/train.py deleted file mode 100755 index 6896320ece..0000000000 --- a/example/resnet50_imagenet2012/train.py +++ /dev/null @@ -1,122 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -"""train_imagenet.""" -import os -import argparse -import numpy as np -from dataset import create_dataset -from lr_generator import get_lr -from config import config -from mindspore import context -from mindspore import Tensor -from mindspore.model_zoo.resnet import resnet50 -from mindspore.parallel._auto_parallel_context import auto_parallel_context -from mindspore.nn.optim.momentum import Momentum - -from mindspore.train.model import Model, ParallelMode - -from mindspore.train.callback import ModelCheckpoint, CheckpointConfig, LossMonitor, TimeMonitor -from mindspore.train.loss_scale_manager import FixedLossScaleManager -from mindspore.train.serialization import load_checkpoint, load_param_into_net -from mindspore.communication.management import init, get_rank, get_group_size -import mindspore.nn as nn -import mindspore.common.initializer as weight_init -from crossentropy import CrossEntropy - -parser = argparse.ArgumentParser(description='Image classification') -parser.add_argument('--run_distribute', type=bool, default=False, help='Run distribute') -parser.add_argument('--device_num', type=int, default=1, help='Device num.') -parser.add_argument('--do_train', type=bool, default=True, help='Do train or not.') -parser.add_argument('--do_eval', type=bool, default=False, help='Do eval or not.') -parser.add_argument('--dataset_path', type=str, default=None, help='Dataset path') -parser.add_argument('--device_target', type=str, default='Ascend', help='Device target') -parser.add_argument('--pre_trained', type=str, default=None, help='Pretrained checkpoint path') -args_opt = parser.parse_args() - -if __name__ == '__main__': - target = args_opt.device_target - ckpt_save_dir = config.save_checkpoint_path - context.set_context(mode=context.GRAPH_MODE, device_target=target, save_graphs=False) - np.random.seed(1) - if not args_opt.do_eval and args_opt.run_distribute: - if target == "Ascend": - device_id = int(os.getenv('DEVICE_ID')) - context.set_context(mode=context.GRAPH_MODE, device_target="Ascend", save_graphs=False, device_id=device_id, - enable_auto_mixed_precision=True) - init() - context.set_auto_parallel_context(device_num=args_opt.device_num, parallel_mode=ParallelMode.DATA_PARALLEL, - mirror_mean=True) - auto_parallel_context().set_all_reduce_fusion_split_indices([107, 160]) - ckpt_save_dir = config.save_checkpoint_path - elif target == "GPU": - context.set_context(mode=context.GRAPH_MODE, device_target="GPU", save_graphs=False) - init("nccl") - context.set_auto_parallel_context(device_num=get_group_size(), parallel_mode=ParallelMode.DATA_PARALLEL, - mirror_mean=True) - ckpt_save_dir = config.save_checkpoint_path + "ckpt_" + str(get_rank()) + "/" - - epoch_size = config.epoch_size - net = resnet50(class_num=config.class_num) - - # weight init - if args_opt.pre_trained: - param_dict = load_checkpoint(args_opt.pre_trained) - load_param_into_net(net, param_dict) - epoch_size = config.epoch_size - config.pretrained_epoch_size - else: - for _, cell in net.cells_and_names(): - if isinstance(cell, nn.Conv2d): - cell.weight.default_input = weight_init.initializer(weight_init.XavierUniform(), - cell.weight.default_input.shape, - cell.weight.default_input.dtype).to_tensor() - if isinstance(cell, nn.Dense): - cell.weight.default_input = weight_init.initializer(weight_init.TruncatedNormal(), - cell.weight.default_input.shape, - cell.weight.default_input.dtype).to_tensor() - if not config.use_label_smooth: - config.label_smooth_factor = 0.0 - - loss = CrossEntropy(smooth_factor=config.label_smooth_factor, num_classes=config.class_num) - - if args_opt.do_train: - dataset = create_dataset(dataset_path=args_opt.dataset_path, do_train=True, - repeat_num=epoch_size, batch_size=config.batch_size, target=target) - step_size = dataset.get_dataset_size() - - loss_scale = FixedLossScaleManager(config.loss_scale, drop_overflow_update=False) - lr = get_lr(lr_init=config.lr_init, lr_end=0.0, lr_max=config.lr_max, warmup_epochs=config.warmup_epochs, - total_epochs=config.epoch_size, steps_per_epoch=step_size, lr_decay_mode='cosine') - if args_opt.pre_trained: - lr = lr[config.pretrained_epoch_size * step_size:] - lr = Tensor(lr) - - opt = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), lr, config.momentum, - config.weight_decay, config.loss_scale) - if target == "Ascend": - model = Model(net, loss_fn=loss, optimizer=opt, loss_scale_manager=loss_scale, metrics={'acc'}, - amp_level="O2", keep_batchnorm_fp32=False) - elif target == "GPU": - model = Model(net, loss_fn=loss, optimizer=opt, loss_scale_manager=loss_scale, metrics={'acc'}) - - - time_cb = TimeMonitor(data_size=step_size) - loss_cb = LossMonitor() - cb = [time_cb, loss_cb] - if config.save_checkpoint: - config_ck = CheckpointConfig(save_checkpoint_steps=config.save_checkpoint_epochs*step_size, - keep_checkpoint_max=config.keep_checkpoint_max) - ckpt_cb = ModelCheckpoint(prefix="resnet", directory=ckpt_save_dir, config=config_ck) - cb += [ckpt_cb] - model.train(epoch_size, dataset, callbacks=cb) diff --git a/mindspore/_checkparam.py b/mindspore/_checkparam.py index 880d26bfad..d5ac7c3e33 100644 --- a/mindspore/_checkparam.py +++ b/mindspore/_checkparam.py @@ -593,6 +593,17 @@ def check_bool(input_param): raise TypeError("Input type must be bool!") +def check_string(input_param, valid_values): + """String type judgment.""" + if isinstance(input_param, str) and input_param in valid_values: + return input_param + if len(valid_values) == 1: + raise ValueError(f'Input should be str and must be {valid_values[0]},' + f' but got {input_param}.') + raise ValueError(f'Input should be str and must be one of {valid_values},' + f' but got {input_param}.') + + def check_input_format(input_param): """Judge input format.""" if input_param == "NCHW": diff --git a/mindspore/_extends/parse/parser.py b/mindspore/_extends/parse/parser.py index 2a1c9e0943..a6043eb787 100644 --- a/mindspore/_extends/parse/parser.py +++ b/mindspore/_extends/parse/parser.py @@ -19,6 +19,7 @@ import ast import types import inspect +import hashlib from textwrap import dedent from dataclasses import is_dataclass import asttokens @@ -319,7 +320,6 @@ def get_dataclass_methods(cls): if isinstance(getattr(cls, name), (types.FunctionType,))} return methods - class Parser: """ Parser python code to ast tree. @@ -327,7 +327,10 @@ class Parser: Args: fn(FunctionType/MethodType): Need parse object instance. parse_method(ExtendInfoOfParseObj): Extend information for parse the function. + ast_cache: Dictionary for caching ast tree. """ + ast_cache = {} + def __init__(self, fn: (types.FunctionType, types.MethodType), parse_method=None) -> None: self.fn = fn self.parse_method = parse_method @@ -348,11 +351,15 @@ class Parser: tree = None if isinstance(self.fn, (types.FunctionType, types.MethodType)): original_src = inspect.getsource(self.fn) - src = dedent(original_src) - self.col_offset = \ - len(original_src.split('\n')[0]) - len(src.split('\n')[0]) - logger.debug("get source = %s", src) - tree = asttokens.ASTTokens(src, parse=True).tree + hexstr = hashlib.sha256(original_src.encode()).hexdigest() + tree = Parser.ast_cache.get(hexstr) + if not tree: + src = dedent(original_src) + self.col_offset = \ + len(original_src.split('\n')[0]) - len(src.split('\n')[0]) + logger.debug("get source = %s", src) + tree = asttokens.ASTTokens(src, parse=True).tree + Parser.ast_cache[hexstr] = tree else: logger.error("Fn type is invalid") return tree diff --git a/mindspore/_extends/parse/resources.py b/mindspore/_extends/parse/resources.py index 2ae8b7172f..e60b70efac 100644 --- a/mindspore/_extends/parse/resources.py +++ b/mindspore/_extends/parse/resources.py @@ -17,6 +17,7 @@ """Resources for ast tree parse.""" import ast import math +from mindspore import IndexedSlices from mindspore.ops.composite import multitype_ops from mindspore.ops import functional as F, composite as C from . import standard_method as M @@ -111,10 +112,11 @@ convert_object_map = { # system function T.len: M.ms_len, T.bool: M.bool_, - T.map: C.HyperMap(), + T.map: C.Map(), T.partial: F.partial, T.zip: C.zip_operation, T.print: F.print_, + T.enumerate: M.enumerate_, # custom define operation T.iter: M.ms_iter, @@ -135,4 +137,7 @@ convert_object_map = { math.sin: NO_IMPLEMENT, math.cos: NO_IMPLEMENT, math.tan: NO_IMPLEMENT, + + # user defined + IndexedSlices: F.make_indexed_slices, } diff --git a/mindspore/_extends/parse/standard_method.py b/mindspore/_extends/parse/standard_method.py index 0f3f843b63..936099a4fb 100644 --- a/mindspore/_extends/parse/standard_method.py +++ b/mindspore/_extends/parse/standard_method.py @@ -104,6 +104,15 @@ def bool_(x): return x.__bool__() +def enumerate_(x, start=0): + """Enumerate list or tuple.""" + x_type = F.typeof(x) + ret = () + if check_is_tuple_or_list(x_type, "enumerate"): + ret = zip(range(start, start + len(x)), x) + return ret + + def while_cond(x): """For while condtion, if the condition is a tensor, the loop will not be unrolled""" if F.issubclass_(F.typeof(x), F.typeof(mstype.tensor)): @@ -113,6 +122,13 @@ def while_cond(x): return x +@constexpr +def check_is_tuple_or_list(x, op_name): + """check whether x is list or tuple.""" + if isinstance(x, (mstype.list_type, mstype.tuple_type)): + return True + raise TypeError(f"For '{op_name}', the input parameter should be tuple or list, but got {x}.") + @constexpr def check_is_tensor_bool_cond(shp): """check if tensor is a bool condition""" diff --git a/mindspore/_extends/parse/trope.py b/mindspore/_extends/parse/trope.py index f169c58fb9..28f3196975 100644 --- a/mindspore/_extends/parse/trope.py +++ b/mindspore/_extends/parse/trope.py @@ -27,7 +27,7 @@ from operator import ( # noqa # support system function call from builtins import ( # noqa - bool, getattr, setattr, len, iter, next, pow, range, map, zip, print + bool, getattr, setattr, len, iter, next, pow, range, map, zip, print, enumerate ) # support functools @@ -44,7 +44,7 @@ __all__ = ['add', 'sub', 'mul', 'truediv', 'floordiv', 'mod', 'eq', 'ne', 'lt', 'not_', 'and_', 'or_', 'xor', 'lshift', 'rshift', 'invert', 'is_', 'is_not', 'contains', 'matmul', 'getitem', 'setitem', 'bool', 'getattr', 'setattr', 'len', 'iter', 'next', 'pow', 'range', 'map', 'zip', - 'partial', 'print', + 'partial', 'print', 'enumerate', 'exp', 'log', 'sin', 'cos', 'tan'] diff --git a/mindspore/ccsrc/CMakeLists.txt b/mindspore/ccsrc/CMakeLists.txt index c435672bde..3f9965c042 100644 --- a/mindspore/ccsrc/CMakeLists.txt +++ b/mindspore/ccsrc/CMakeLists.txt @@ -71,6 +71,17 @@ message("onnx proto path is :" ${ONNX_PROTO}) ms_protobuf_generate(ONNX_PROTO_SRCS ONNX_PROTO_HDRS ${ONNX_PROTO}) list(APPEND MINDSPORE_PROTO_LIST ${ONNX_PROTO_SRCS}) +if (ENABLE_DEBUGGER) + # debugger: compile proto files + include_directories("${CMAKE_BINARY_DIR}/debug/debugger") + file(GLOB_RECURSE DEBUGGER_PROTO_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "debug/debugger/debug_graph.proto") + ms_protobuf_generate(DEBUGGER_PROTO_SRCS DEBUGGER_PROTO_HDRS ${DEBUGGER_PROTO_LIST}) + file(GLOB_RECURSE DEBUGGER_GRPC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "debug/debugger/debug_grpc.proto") + ms_grpc_generate(DEBUGGER_GRPC_SRCS DEBUGGER_GRPC_HDRS ${DEBUGGER_GRPC_LIST}) + list(APPEND MINDSPORE_PROTO_LIST ${DEBUGGER_PROTO_SRCS}) + list(APPEND MINDSPORE_PROTO_LIST ${DEBUGGER_GRPC_SRCS}) +endif () + if (ENABLE_DUMP_PROTO) include_directories(${CMAKE_BINARY_DIR}) file(GLOB_RECURSE PROTO_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "utils/node_strategy.proto") @@ -125,12 +136,21 @@ endforeach () set_property(SOURCE ${SUB_OBJECTS_SRC} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_ME) add_library(mindspore STATIC ${SUB_OBJECTS_SRC}) + +target_link_libraries(proto_input mindspore::protobuf) + +if (ENABLE_DEBUGGER) + # debugger: link grpc + target_link_libraries(proto_input mindspore::grpc++) +endif() + target_link_libraries(mindspore proto_input) -if (ENABLE_CPU AND ENABLE_MPI) - target_link_libraries(mindspore securec mindspore::flatbuffers mindspore::ompi) +if (ENABLE_MPI) + target_link_libraries(mindspore securec mindspore::flatbuffers mpi_adapter) else () target_link_libraries(mindspore securec mindspore::flatbuffers) endif () + if (NOT WIN32) target_link_libraries(mindspore dl) endif() @@ -210,6 +230,10 @@ else () target_link_libraries(_c_expression PRIVATE -Wl,--whole-archive mindspore -Wl,--no-whole-archive) target_link_libraries(_c_expression PRIVATE mindspore::pybind11_module) target_link_libraries(_c_expression PRIVATE mindspore_gvar) + target_link_libraries(_c_expression PRIVATE mindspore::pslite mindspore::protobuf ${zeromq_DIRPATH}/zmq_install/lib/libzmq.a) + if (${ENABLE_IBVERBS} STREQUAL "ON") + target_link_libraries(_c_expression PRIVATE ibverbs rdmacm) + endif() endif () if (USE_GLOG) @@ -217,6 +241,7 @@ if (USE_GLOG) endif () if (ENABLE_DUMP_PROTO) + message("add protobuf lib to c_expression") target_link_libraries(_c_expression PRIVATE mindspore::protobuf) endif () @@ -256,10 +281,11 @@ endif () if (USE_GLOG) target_link_libraries(inference PRIVATE mindspore::glog) -else() - if (CMAKE_SYSTEM_NAME MATCHES "Linux") - target_link_options(inference PRIVATE -Wl,-init,mindspore_log_init) - elseif (CMAKE_SYSTEM_NAME MATCHES "Darwin") - set_target_properties(inference PROPERTIES MACOSX_RPATH ON) - endif () endif() + +if (CMAKE_SYSTEM_NAME MATCHES "Linux") + target_link_options(inference PRIVATE -Wl,-init,common_log_init) +elseif (CMAKE_SYSTEM_NAME MATCHES "Darwin") + set_target_properties(inference PROPERTIES MACOSX_RPATH ON) +endif () + diff --git a/mindspore/ccsrc/dataset/api/de_pipeline.cc b/mindspore/ccsrc/dataset/api/de_pipeline.cc index ce70476423..78fcdb7dd4 100644 --- a/mindspore/ccsrc/dataset/api/de_pipeline.cc +++ b/mindspore/ccsrc/dataset/api/de_pipeline.cc @@ -15,6 +15,7 @@ */ #include "dataset/api/de_pipeline.h" +#include #include #include @@ -45,7 +46,7 @@ namespace mindspore { namespace dataset { -using pFunction = Status (DEPipeline::*)(const py::dict &, std::shared_ptr *); +using pFunction = Status (DEPipeline::*)(const py::dict &, std::shared_ptr *, std::shared_ptr *); static std::unordered_map g_parse_op_func_ = { {kShuffle, &DEPipeline::ParseShuffleOp}, @@ -107,18 +108,44 @@ DEPipeline::~DEPipeline() { } // Function to add a Node to the Execution Tree. -Status DEPipeline::AddNodeToTree(const OpName &op_name, const py::dict &args, DsOpPtr *out) { - // For each operator, Parse through the list of arguments, - // then call the respective builder/constructor. +Status DEPipeline::AddNodeToTree(const OpName &op_name, const py::dict &args, py::dict *output) { + // For each operator, Parse through the list of arguments, then call the respective builder/constructor. + // Note that each call to the parse function may result in building more than one dataset operator. + // For example, one call to ParseNNNOp may result in multiple internal C nodes: + // nodeA + // | + // nodeB + // | + // nodeC + // However, the python side dataset is more abstract, and it does not know about the potential subtree that + // is being built here. Since the python api is hooking tree nodes together (parent/child hookups), the + // python side needs to know about nodeA and NodeC to be able to appropriately hook up parents and child + // to this subtee. + // Thus, it is required that both the top-most parent and bottom-most child are returned from the parse + // function. + DsOpPtr top = nullptr; + DsOpPtr bottom = nullptr; auto iter = g_parse_op_func_.find(op_name); if (iter != g_parse_op_func_.end()) { pFunction func = iter->second; - RETURN_IF_NOT_OK((this->*func)(args, out)); + RETURN_IF_NOT_OK((this->*func)(args, &top, &bottom)); + + if (top == nullptr) { + RETURN_STATUS_UNEXPECTED("An operator was parsed but it did not produce a C node."); + } + + // It is not required that the parse function always produces the bottom pointer. If it's still null, + // then set top and bottom to be the same operator + if (bottom == nullptr) bottom = top; + + // Pack these pointers into a py dict so that we can return both back to python. + (*output)["top"] = top; + (*output)["bottom"] = bottom; } else { RETURN_STATUS_UNEXPECTED("No such Op"); } // Associate current dataset op node with the tree. - RETURN_IF_NOT_OK(tree_->AssociateNode(*out)); + RETURN_IF_NOT_OK(tree_->AssociateNode(top)); return Status::OK(); } // Function to add a child and parent relationship. @@ -300,7 +327,8 @@ Status DEPipeline::SetBatchParameters(const py::dict &args) { return Status::OK(); } -Status DEPipeline::ParseShuffleOp(const py::dict &args, std::shared_ptr *ptr) { +Status DEPipeline::ParseShuffleOp(const py::dict &args, std::shared_ptr *top, + std::shared_ptr *bottom) { std::shared_ptr builder = std::make_shared(); if (!args["buffer_size"].is_none()) { (void)builder->SetShuffleSize(ToInt(args["buffer_size"])); @@ -322,7 +350,7 @@ Status DEPipeline::ParseShuffleOp(const py::dict &args, std::shared_ptr op; RETURN_IF_NOT_OK(builder->Build(&op)); - *ptr = op; + *top = op; return Status::OK(); } @@ -350,7 +378,8 @@ Status DEPipeline::BuildMindrecordSamplerChain(const py::handle &handle, return Status::OK(); } -Status DEPipeline::ParseMindRecordOp(const py::dict &args, std::shared_ptr *ptr) { +Status DEPipeline::ParseMindRecordOp(const py::dict &args, std::shared_ptr *top, + std::shared_ptr *bottom) { if (args["dataset_file"].is_none()) { std::string err_msg = "Error: at least one of dataset_files is missing"; RETURN_STATUS_UNEXPECTED(err_msg); @@ -403,13 +432,15 @@ Status DEPipeline::ParseMindRecordOp(const py::dict &args, std::shared_ptr op; RETURN_IF_NOT_OK(builder->Build(&op)); num_rows_ = op->num_rows(); - *ptr = op; + *top = op; return Status::OK(); } -Status DEPipeline::ParseMapOp(const py::dict &args, std::shared_ptr *ptr) { - std::shared_ptr builder = std::make_shared(); +Status DEPipeline::ParseMapOp(const py::dict &args, std::shared_ptr *top, + std::shared_ptr *bottom) { + MapOp::Builder map_builder; std::vector> tensor_op_list; + std::vector project_columns; if (args["operations"].is_none()) RETURN_STATUS_UNEXPECTED("Error: 'operations' is not set. \n"); @@ -419,15 +450,15 @@ Status DEPipeline::ParseMapOp(const py::dict &args, std::shared_ptr * if (!value.is_none()) { if (key == "input_columns") { std::vector in_col_names = ToStringVector(args["input_columns"]); - (void)builder->SetInColNames(in_col_names); + (void)map_builder.SetInColNames(in_col_names); } else if (key == "output_columns") { - (void)builder->SetOutColNames(ToStringVector(value)); + (void)map_builder.SetOutColNames(ToStringVector(value)); } else if (key == "columns_order") { - (void)builder->SetColOrder(ToStringVector(value)); + project_columns = ToStringVector(value); } else if (key == "num_parallel_workers") { - (void)builder->SetNumWorkers(ToInt(value)); + (void)map_builder.SetNumWorkers(ToInt(value)); } else if (key == "prefetch_size") { - (void)builder->SetOpConnectorSize(ToInt(value)); + (void)map_builder.SetOpConnectorSize(ToInt(value)); } else if (key == "operations") { py::handle tensor_ops = args["operations"]; // operation can be a list of TensorOps or a single TensorOp. @@ -445,20 +476,34 @@ Status DEPipeline::ParseMapOp(const py::dict &args, std::shared_ptr * } } if (tensor_op_list.empty()) RETURN_STATUS_UNEXPECTED("Error: tensor_op is invalid or not set."); - (void)builder->SetTensorFuncs(std::move(tensor_op_list)); + (void)map_builder.SetTensorFuncs(std::move(tensor_op_list)); } else { RETURN_STATUS_UNEXPECTED("Error: Unhandled key: " + key); } } } - std::shared_ptr op; - RETURN_IF_NOT_OK(builder->Build(&op)); - *ptr = op; + std::shared_ptr map_op; + RETURN_IF_NOT_OK(map_builder.Build(&map_op)); + RETURN_IF_NOT_OK(tree_->AssociateNode(map_op)); + *top = map_op; + + // Add a project op over top of the map if the user wanted to reposition the columns + if (!project_columns.empty()) { + ProjectOp::Builder proj_builder(project_columns); + std::shared_ptr proj_op; + RETURN_IF_NOT_OK(proj_builder.Build(&proj_op)); + RETURN_IF_NOT_OK(tree_->AssociateNode(proj_op)); + RETURN_IF_NOT_OK(proj_op->AddChild(map_op)); + *top = proj_op; + *bottom = map_op; + } + return Status::OK(); } -Status DEPipeline::ParseFilterOp(const py::dict &args, std::shared_ptr *ptr) { +Status DEPipeline::ParseFilterOp(const py::dict &args, std::shared_ptr *top, + std::shared_ptr *bottom) { std::shared_ptr builder = std::make_shared(); if (args["predicate"].is_none()) { @@ -489,11 +534,12 @@ Status DEPipeline::ParseFilterOp(const py::dict &args, std::shared_ptr op; RETURN_IF_NOT_OK(builder->Build(&op)); - *ptr = op; + *top = op; return Status::OK(); } -Status DEPipeline::ParseRepeatOp(const py::dict &args, std::shared_ptr *ptr) { +Status DEPipeline::ParseRepeatOp(const py::dict &args, std::shared_ptr *top, + std::shared_ptr *bottom) { if (args["count"].is_none()) { std::string err_msg = "Error: count is invalid or not set."; RETURN_STATUS_UNEXPECTED(err_msg); @@ -501,22 +547,24 @@ Status DEPipeline::ParseRepeatOp(const py::dict &args, std::shared_ptr op; RETURN_IF_NOT_OK(RepeatOp::Builder(ToInt(args["count"])).Build(&op)); - *ptr = op; + *top = op; return Status::OK(); } -Status DEPipeline::ParseSkipOp(const py::dict &args, std::shared_ptr *ptr) { +Status DEPipeline::ParseSkipOp(const py::dict &args, std::shared_ptr *top, + std::shared_ptr *bottom) { if (args["count"].is_none()) { std::string err_msg = "Error: count is invalid or not set."; RETURN_STATUS_UNEXPECTED(err_msg); } std::shared_ptr op; RETURN_IF_NOT_OK(SkipOp::Builder(ToInt(args["count"])).Build(&op)); - *ptr = op; + *top = op; return Status::OK(); } -Status DEPipeline::ParseGeneratorOp(const py::dict &args, std::shared_ptr *ptr) { +Status DEPipeline::ParseGeneratorOp(const py::dict &args, std::shared_ptr *top, + std::shared_ptr *bottom) { std::shared_ptr builder = std::make_shared(); for (auto arg : args) { std::string key = py::str(arg.first); @@ -538,11 +586,12 @@ Status DEPipeline::ParseGeneratorOp(const py::dict &args, std::shared_ptr op; RETURN_IF_NOT_OK(builder->Build(&op)); - *ptr = op; + *top = op; return Status::OK(); } -Status DEPipeline::ParseBatchOp(const py::dict &args, std::shared_ptr *ptr) { +Status DEPipeline::ParseBatchOp(const py::dict &args, std::shared_ptr *top, + std::shared_ptr *bottom) { std::shared_ptr builder; if (py::isinstance(args["batch_size"])) { batch_size_ = ToInt(args["batch_size"]); @@ -582,11 +631,12 @@ Status DEPipeline::ParseBatchOp(const py::dict &args, std::shared_ptr std::shared_ptr op; RETURN_IF_NOT_OK(builder->Build(&op)); - *ptr = op; + *top = op; return Status::OK(); } -Status DEPipeline::ParseBucketBatchByLengthOp(const py::dict &args, std::shared_ptr *ptr) { +Status DEPipeline::ParseBucketBatchByLengthOp(const py::dict &args, std::shared_ptr *top, + std::shared_ptr *bottom) { std::vector mandatory_arguments = {"length_dependent_columns", "bucket_boundaries", "bucket_batch_sizes"}; for (auto name : mandatory_arguments) { @@ -632,11 +682,12 @@ Status DEPipeline::ParseBucketBatchByLengthOp(const py::dict &args, std::shared_ std::shared_ptr op; RETURN_IF_NOT_OK(builder->Build(&op)); - *ptr = op; + *top = op; return Status::OK(); } -Status DEPipeline::ParseBarrierOp(const py::dict &args, std::shared_ptr *ptr) { +Status DEPipeline::ParseBarrierOp(const py::dict &args, std::shared_ptr *top, + std::shared_ptr *bottom) { std::shared_ptr builder = std::make_shared(); // Right now barrier should only take num_rows_per_buffer = 1 // The reason for this is because having it otherwise can lead to blocking issues @@ -656,11 +707,12 @@ Status DEPipeline::ParseBarrierOp(const py::dict &args, std::shared_ptr op; RETURN_IF_NOT_OK(builder->Build(&op)); - *ptr = op; + *top = op; return Status::OK(); } -Status DEPipeline::ParseDeviceQueueOp(const py::dict &args, std::shared_ptr *ptr) { +Status DEPipeline::ParseDeviceQueueOp(const py::dict &args, std::shared_ptr *top, + std::shared_ptr *bottom) { int32_t prefetch_size = 0; if (args.contains("prefetch_size")) { if (args["prefetch_size"].is_none()) { @@ -687,11 +739,12 @@ Status DEPipeline::ParseDeviceQueueOp(const py::dict &args, std::shared_ptr op; RETURN_IF_NOT_OK(builder->Build(&op)); - *ptr = op; + *top = op; return Status::OK(); } -Status DEPipeline::ParseRenameOp(const py::dict &args, std::shared_ptr *ptr) { +Status DEPipeline::ParseRenameOp(const py::dict &args, std::shared_ptr *top, + std::shared_ptr *bottom) { std::vector in_col_names; std::vector out_col_names; std::shared_ptr builder = std::make_shared(); @@ -718,48 +771,57 @@ Status DEPipeline::ParseRenameOp(const py::dict &args, std::shared_ptrSetOutColNames(out_col_names); std::shared_ptr op; RETURN_IF_NOT_OK(builder->Build(&op)); - *ptr = op; + *top = op; return Status::OK(); } -Status DEPipeline::ParseTakeOp(const py::dict &args, std::shared_ptr *ptr) { +Status DEPipeline::ParseTakeOp(const py::dict &args, std::shared_ptr *top, + std::shared_ptr *bottom) { if (args["count"].is_none()) { std::string err_msg = "Error: count is invalid or not set."; RETURN_STATUS_UNEXPECTED(err_msg); } std::shared_ptr op; RETURN_IF_NOT_OK(TakeOp::Builder(ToInt(args["count"])).Build(&op)); - *ptr = op; + *top = op; return Status::OK(); } -Status DEPipeline::ParseZipOp(const py::dict &args, std::shared_ptr *ptr) { +Status DEPipeline::ParseZipOp(const py::dict &args, std::shared_ptr *top, + std::shared_ptr *bottom) { std::shared_ptr builder = std::make_shared(); std::shared_ptr op; RETURN_IF_NOT_OK(builder->Build(&op)); - *ptr = op; + *top = op; return Status::OK(); } -Status DEPipeline::ParseConcatOp(const py::dict &args, std::shared_ptr *ptr) { +Status DEPipeline::ParseConcatOp(const py::dict &args, std::shared_ptr *top, + std::shared_ptr *bottom) { std::shared_ptr builder = std::make_shared(); std::shared_ptr op; RETURN_IF_NOT_OK(builder->Build(&op)); - *ptr = op; + *top = op; return Status::OK(); } -Status DEPipeline::ParseTFReaderOp(const py::dict &args, std::shared_ptr *ptr) { +Status DEPipeline::ParseTFReaderOp(const py::dict &args, std::shared_ptr *top, + std::shared_ptr *bottom) { // Required arguments + std::vector files_list; std::shared_ptr builder = std::make_shared(); if (!args["dataset_files"].is_none()) { - (void)builder->SetDatasetFilesList(ToStringVector(args["dataset_files"])); + files_list = ToStringVector(args["dataset_files"]); + (void)builder->SetDatasetFilesList(files_list); } else { std::string err_msg = "Error: at least one of dataset_files or schema_file is missing"; RETURN_STATUS_UNEXPECTED(err_msg); } std::vector columns_to_load; bool schema_exists = false; + bool shuffle_required = false; + int64_t num_devices = 0; + int64_t total_rows = 0; // Optional arguments for (auto arg : args) { std::string key = py::str(arg.first); @@ -773,13 +835,15 @@ Status DEPipeline::ParseTFReaderOp(const py::dict &args, std::shared_ptrSetShuffleFiles(ToBool(value)); } else if (key == "shuffle_global") { - (void)builder->SetShuffleGlobal(ToBool(value)); + shuffle_required = ToBool(value); } else if (key == "schema_file_path" || key == "schema_json_string") { schema_exists = true; } else if (key == "num_samples") { - (void)builder->setTotalRows(ToInt(value)); + total_rows = ToInt(value); + (void)builder->setTotalRows(total_rows); } else if (key == "num_shards") { - (void)builder->SetNumDevices(ToInt(value)); + num_devices = ToInt(value); + (void)builder->SetNumDevices(num_devices); } else if (key == "shard_id") { (void)builder->SetDeviceId(ToInt(value)); } else if (key == "shard_equal_rows") { @@ -796,13 +860,33 @@ Status DEPipeline::ParseTFReaderOp(const py::dict &args, std::shared_ptrSetDataSchema(std::move(schema)); } - std::shared_ptr op; - RETURN_IF_NOT_OK(builder->Build(&op)); - *ptr = op; + std::shared_ptr tf_op; + RETURN_IF_NOT_OK(builder->Build(&tf_op)); + RETURN_IF_NOT_OK(tree_->AssociateNode(tf_op)); + *top = tf_op; + + if (shuffle_required) { + const boolean estimate = true; + const int64_t workers = 8; + std::shared_ptr shuffle_op = nullptr; + int64_t shuffle_size = 0; + int64_t num_rows = 0; + + // First, get the number of rows in the dataset via estimate and then compute the shuffle size + RETURN_IF_NOT_OK(TFReaderOp::CountTotalRows(&num_rows, files_list, workers, estimate)); + RETURN_IF_NOT_OK(ComputeShuffleSize(files_list.size(), num_devices, num_rows, total_rows, &shuffle_size)); + + // Add the shuffle op over top of this op and return the subtree (top/bottom) to caller + RETURN_IF_NOT_OK(AddShuffleOp(shuffle_size, tf_op, &shuffle_op)); + *top = shuffle_op; + *bottom = tf_op; + } + return Status::OK(); } -Status DEPipeline::ParseProjectOp(const py::dict &args, std::shared_ptr *ptr) { +Status DEPipeline::ParseProjectOp(const py::dict &args, std::shared_ptr *top, + std::shared_ptr *bottom) { if (args["columns"].is_none()) { std::string err_msg = "Error: columns is missing"; RETURN_STATUS_UNEXPECTED(err_msg); @@ -811,11 +895,12 @@ Status DEPipeline::ParseProjectOp(const py::dict &args, std::shared_ptr builder = std::make_shared(columns_to_project); std::shared_ptr op; RETURN_IF_NOT_OK(builder->Build(&op)); - *ptr = op; + *top = op; return Status::OK(); } -Status DEPipeline::ParseImageFolderOp(const py::dict &args, std::shared_ptr *ptr) { +Status DEPipeline::ParseImageFolderOp(const py::dict &args, std::shared_ptr *top, + std::shared_ptr *bottom) { // Required arguments if (args["dataset_dir"].is_none()) { std::string err_msg = "Error: No dataset path specified"; @@ -846,11 +931,12 @@ Status DEPipeline::ParseImageFolderOp(const py::dict &args, std::shared_ptr op; RETURN_IF_NOT_OK(builder->Build(&op)); - *ptr = op; + *top = op; return Status::OK(); } -Status DEPipeline::ParseManifestOp(const py::dict &args, std::shared_ptr *ptr) { +Status DEPipeline::ParseManifestOp(const py::dict &args, std::shared_ptr *top, + std::shared_ptr *bottom) { // Required arguments if (args["dataset_file"].is_none()) { std::string err_msg = "Error: No dataset files specified for manifest"; @@ -881,11 +967,12 @@ Status DEPipeline::ParseManifestOp(const py::dict &args, std::shared_ptr op; RETURN_IF_NOT_OK(builder->Build(&op)); - *ptr = op; + *top = op; return Status::OK(); } -Status DEPipeline::ParseVOCOp(const py::dict &args, std::shared_ptr *ptr) { +Status DEPipeline::ParseVOCOp(const py::dict &args, std::shared_ptr *top, + std::shared_ptr *bottom) { if (args["dataset_dir"].is_none()) { std::string err_msg = "Error: No dataset path specified"; RETURN_STATUS_UNEXPECTED(err_msg); @@ -924,11 +1011,13 @@ Status DEPipeline::ParseVOCOp(const py::dict &args, std::shared_ptr * } std::shared_ptr op; RETURN_IF_NOT_OK(builder->Build(&op)); - *ptr = op; + *top = op; + return Status::OK(); } -Status DEPipeline::ParseCocoOp(const py::dict &args, std::shared_ptr *ptr) { +Status DEPipeline::ParseCocoOp(const py::dict &args, std::shared_ptr *top, + std::shared_ptr *bottom) { if (args["dataset_dir"].is_none()) { std::string err_msg = "Error: No dataset path specified"; RETURN_STATUS_UNEXPECTED(err_msg); @@ -965,11 +1054,12 @@ Status DEPipeline::ParseCocoOp(const py::dict &args, std::shared_ptr } std::shared_ptr op; RETURN_IF_NOT_OK(builder->Build(&op)); - *ptr = op; + *top = op; return Status::OK(); } -Status DEPipeline::ParseCifar10Op(const py::dict &args, std::shared_ptr *ptr) { +Status DEPipeline::ParseCifar10Op(const py::dict &args, std::shared_ptr *top, + std::shared_ptr *bottom) { // Required arguments if (args["dataset_dir"].is_none()) { std::string err_msg = "Error: No dataset path specified"; @@ -998,11 +1088,12 @@ Status DEPipeline::ParseCifar10Op(const py::dict &args, std::shared_ptr op; RETURN_IF_NOT_OK(builder->Build(&op)); - *ptr = op; + *top = op; return Status::OK(); } -Status DEPipeline::ParseCifar100Op(const py::dict &args, std::shared_ptr *ptr) { +Status DEPipeline::ParseCifar100Op(const py::dict &args, std::shared_ptr *top, + std::shared_ptr *bottom) { // Required arguments if (args["dataset_dir"].is_none()) { std::string err_msg = "Error: No dataset path specified"; @@ -1031,11 +1122,12 @@ Status DEPipeline::ParseCifar100Op(const py::dict &args, std::shared_ptr op; RETURN_IF_NOT_OK(builder->Build(&op)); - *ptr = op; + *top = op; return Status::OK(); } -Status DEPipeline::ParseRandomDataOp(const py::dict &args, std::shared_ptr *ptr) { +Status DEPipeline::ParseRandomDataOp(const py::dict &args, std::shared_ptr *top, + std::shared_ptr *bottom) { // Required arguments RandomDataOp::Builder builder; @@ -1072,13 +1164,14 @@ Status DEPipeline::ParseRandomDataOp(const py::dict &args, std::shared_ptr op; RETURN_IF_NOT_OK(builder.Build(&op)); - *ptr = op; + *top = op; return Status::OK(); } int32_t DEPipeline::GetNumClasses() const { return num_classes_; } -Status DEPipeline::ParseMnistOp(const py::dict &args, std::shared_ptr *ptr) { +Status DEPipeline::ParseMnistOp(const py::dict &args, std::shared_ptr *top, + std::shared_ptr *bottom) { // Required arguments if (args["dataset_dir"].is_none()) { std::string err_msg = "Error: No dataset path specified"; @@ -1104,11 +1197,12 @@ Status DEPipeline::ParseMnistOp(const py::dict &args, std::shared_ptr } std::shared_ptr op; RETURN_IF_NOT_OK(builder->Build(&op)); - *ptr = op; + *top = op; return Status::OK(); } -Status DEPipeline::ParseCelebAOp(const py::dict &args, std::shared_ptr *ptr) { +Status DEPipeline::ParseCelebAOp(const py::dict &args, std::shared_ptr *top, + std::shared_ptr *bottom) { // Required arguments if (args["dataset_dir"].is_none()) { std::string err_msg = "Error: No dataset path specified"; @@ -1143,19 +1237,24 @@ Status DEPipeline::ParseCelebAOp(const py::dict &args, std::shared_ptr op; RETURN_IF_NOT_OK(builder->Build(&op)); - *ptr = op; + *top = op; return Status::OK(); } -Status DEPipeline::ParseTextFileOp(const py::dict &args, std::shared_ptr *ptr) { +Status DEPipeline::ParseTextFileOp(const py::dict &args, std::shared_ptr *top, + std::shared_ptr *bottom) { // Required arguments + std::vector files_list; std::shared_ptr builder = std::make_shared(); if (!args["dataset_files"].is_none()) { - (void)builder->SetTextFilesList(ToStringVector(args["dataset_files"])); + files_list = ToStringVector(args["dataset_files"]); + (void)builder->SetTextFilesList(files_list); } else { RETURN_STATUS_UNEXPECTED("Error: dataset_files is missing"); } // Optional arguments + bool shuffle_required = false; + int64_t num_devices = 0; for (auto arg : args) { std::string key = py::str(arg.first); py::handle value = arg.second; @@ -1165,19 +1264,38 @@ Status DEPipeline::ParseTextFileOp(const py::dict &args, std::shared_ptrSetShuffleFiles(ToBool(value)); } else if (key == "shuffle_global") { - (void)builder->SetShuffleGlobal(ToBool(value)); + shuffle_required = ToBool(value); } else if (key == "num_samples") { (void)builder->SetTotalRows(ToInt(value)); } else if (key == "num_shards") { - (void)builder->SetNumDevices(ToInt(value)); + num_devices = ToInt(value); + (void)builder->SetNumDevices(num_devices); } else if (key == "shard_id") { (void)builder->SetDeviceId(ToInt(value)); } } } - std::shared_ptr op; - RETURN_IF_NOT_OK(builder->Build(&op)); - *ptr = op; + + std::shared_ptr txt_op; + RETURN_IF_NOT_OK(builder->Build(&txt_op)); + RETURN_IF_NOT_OK(tree_->AssociateNode(txt_op)); + *top = txt_op; + + if (shuffle_required) { + std::shared_ptr shuffle_op = nullptr; + int64_t shuffle_size = 0; + int64_t num_rows = 0; + + // First, get the number of rows in the dataset and then compute the shuffle size + RETURN_IF_NOT_OK(TextFileOp::CountAllFileRows(files_list, &num_rows)); + RETURN_IF_NOT_OK(ComputeShuffleSize(files_list.size(), num_devices, num_rows, 0, &shuffle_size)); + + // Add the shuffle op over top of this op and return the subtree (top/bottom) to caller + RETURN_IF_NOT_OK(AddShuffleOp(shuffle_size, txt_op, &shuffle_op)); + *top = shuffle_op; + *bottom = txt_op; + } + return Status::OK(); } @@ -1208,7 +1326,8 @@ Status DEPipeline::ParsePadInfo(py::handle value, PadInfo *pad_info) { return Status::OK(); } -Status DEPipeline::ParseBuildVocabOp(const py::dict &args, std::shared_ptr *ptr) { +Status DEPipeline::ParseBuildVocabOp(const py::dict &args, std::shared_ptr *top, + std::shared_ptr *bottom) { std::shared_ptr builder = std::make_shared(); for (auto arg : args) { std::string key = py::str(arg.first); @@ -1235,18 +1354,23 @@ Status DEPipeline::ParseBuildVocabOp(const py::dict &args, std::shared_ptr op; RETURN_IF_NOT_OK(builder->Build(&op)); - *ptr = op; + *top = op; return Status::OK(); } -Status DEPipeline::ParseClueOp(const py::dict &args, std::shared_ptr *ptr) { +Status DEPipeline::ParseClueOp(const py::dict &args, std::shared_ptr *top, + std::shared_ptr *bottom) { + std::vector files_list; std::shared_ptr builder = std::make_shared(); if (!args["dataset_files"].is_none()) { - (void)builder->SetClueFilesList(ToStringVector(args["dataset_files"])); + files_list = ToStringVector(args["dataset_files"]); + (void)builder->SetClueFilesList(files_list); } else { RETURN_STATUS_UNEXPECTED("Error: dataset_files is missing"); } // Optional arguments + bool shuffle_required = false; + int64_t num_devices = 0; for (auto arg : args) { std::string key = py::str(arg.first); py::handle value = arg.second; @@ -1256,11 +1380,12 @@ Status DEPipeline::ParseClueOp(const py::dict &args, std::shared_ptr } else if (key == "shuffle_files") { (void)builder->SetShuffleFiles(ToBool(value)); } else if (key == "shuffle_global") { - (void)builder->SetShuffleGlobal(ToBool(value)); + shuffle_required = ToBool(value); } else if (key == "num_samples") { (void)builder->SetNumSamples(ToInt(value)); } else if (key == "num_shards") { - (void)builder->SetNumDevices(ToInt(value)); + num_devices = ToInt(value); + (void)builder->SetNumDevices(num_devices); } else if (key == "shard_id") { (void)builder->SetDeviceId(ToInt(value)); } else if (key == "cols_to_keyword") { @@ -1276,9 +1401,76 @@ Status DEPipeline::ParseClueOp(const py::dict &args, std::shared_ptr } } } - std::shared_ptr op; - RETURN_IF_NOT_OK(builder->Build(&op)); - *ptr = op; + + std::shared_ptr clue_op; + RETURN_IF_NOT_OK(builder->Build(&clue_op)); + RETURN_IF_NOT_OK(tree_->AssociateNode(clue_op)); + *top = clue_op; + + if (shuffle_required) { + std::shared_ptr shuffle_op = nullptr; + int64_t shuffle_size = 0; + int64_t num_rows = 0; + + // First, get the number of rows in the dataset and then compute the shuffle size + RETURN_IF_NOT_OK(ClueOp::CountAllFileRows(files_list, &num_rows)); + RETURN_IF_NOT_OK(ComputeShuffleSize(files_list.size(), num_devices, num_rows, 0, &shuffle_size)); + + // Add the shuffle op over top of this op and return the subtree (top/bottom) to caller + RETURN_IF_NOT_OK(AddShuffleOp(shuffle_size, clue_op, &shuffle_op)); + *top = shuffle_op; + *bottom = clue_op; + } + + return Status::OK(); +} + +// Helper function to inject a shuffle operator over top of the current operation being built. +Status DEPipeline::AddShuffleOp(int64_t shuffle_size, std::shared_ptr input_op, + std::shared_ptr *shuffle_op) { + std::shared_ptr new_shuffle_op = nullptr; + ShuffleOp::Builder shuffle_builder; + + (void)shuffle_builder.SetShuffleSize(shuffle_size); + RETURN_IF_NOT_OK(shuffle_builder.Build(&new_shuffle_op)); + RETURN_IF_NOT_OK(tree_->AssociateNode(new_shuffle_op)); + RETURN_IF_NOT_OK(new_shuffle_op->AddChild(input_op)); + // We have now created: + // + // ShuffleOp + // | + // input_op + // + *shuffle_op = new_shuffle_op; + + return Status::OK(); +} + +// Common code for computing a default shuffle size +Status DEPipeline::ComputeShuffleSize(int64_t num_files, int64_t num_devices, int64_t num_rows, int64_t total_rows, + int64_t *shuffle_size) { + const int64_t average_files_multiplier = 4; + const int64_t shuffle_max = 10000; + int64_t avg_rows_per_file = 0; + + // Adjust the num rows per shard if sharding was given + if (num_devices > 0) { + if (num_rows % num_devices == 0) { + num_rows = num_rows / num_devices; + } else { + num_rows = (num_rows / num_devices) + 1; + } + } + + // Cap based on total rows directive. Some ops do not have this and give value of 0. + if (total_rows > 0) { + num_rows = std::min(num_rows, total_rows); + } + + // get the average per file + avg_rows_per_file = num_rows / num_files; + + *shuffle_size = std::max(avg_rows_per_file * average_files_multiplier, shuffle_max); return Status::OK(); } } // namespace dataset diff --git a/mindspore/ccsrc/dataset/api/de_pipeline.h b/mindspore/ccsrc/dataset/api/de_pipeline.h index d6127d5d44..7cfc73307c 100644 --- a/mindspore/ccsrc/dataset/api/de_pipeline.h +++ b/mindspore/ccsrc/dataset/api/de_pipeline.h @@ -77,7 +77,7 @@ class DEPipeline { ~DEPipeline(); // Function to add a Node to the Execution Tree. - Status AddNodeToTree(const OpName &op_name, const py::dict &args, DsOpPtr *out); + Status AddNodeToTree(const OpName &op_name, const py::dict &args, py::dict *output); // Function to add a child and parent relationship. static Status AddChildToParentNode(const DsOpPtr &child_op, const DsOpPtr &parent_op); @@ -104,73 +104,74 @@ class DEPipeline { int GetRepeatCount() const; - Status ParseShuffleOp(const py::dict &args, std::shared_ptr *ptr); + Status ParseShuffleOp(const py::dict &args, std::shared_ptr *top, std::shared_ptr *bottom); - Status ParseMindRecordOp(const py::dict &args, std::shared_ptr *ptr); + Status ParseMindRecordOp(const py::dict &args, std::shared_ptr *top, std::shared_ptr *bottom); Status BuildMindrecordSamplerChain(const py::handle &handle, std::vector> *operators, int num_padded); - Status ParseMapOp(const py::dict &args, std::shared_ptr *ptr); + Status ParseMapOp(const py::dict &args, std::shared_ptr *top, std::shared_ptr *bottom); - Status ParseFilterOp(const py::dict &args, std::shared_ptr *ptr); + Status ParseFilterOp(const py::dict &args, std::shared_ptr *top, std::shared_ptr *bottom); - Status ParseRepeatOp(const py::dict &args, std::shared_ptr *ptr); + Status ParseRepeatOp(const py::dict &args, std::shared_ptr *top, std::shared_ptr *bottom); - Status ParseSkipOp(const py::dict &args, std::shared_ptr *ptr); + Status ParseSkipOp(const py::dict &args, std::shared_ptr *top, std::shared_ptr *bottom); - Status ParseBatchOp(const py::dict &args, std::shared_ptr *ptr); + Status ParseBatchOp(const py::dict &args, std::shared_ptr *top, std::shared_ptr *bottom); - Status ParseBucketBatchByLengthOp(const py::dict &args, std::shared_ptr *ptr); + Status ParseBucketBatchByLengthOp(const py::dict &args, std::shared_ptr *top, + std::shared_ptr *bottom); - Status ParseBarrierOp(const py::dict &args, std::shared_ptr *ptr); + Status ParseBarrierOp(const py::dict &args, std::shared_ptr *top, std::shared_ptr *bottom); - Status ParseGeneratorOp(const py::dict &args, std::shared_ptr *ptr); + Status ParseGeneratorOp(const py::dict &args, std::shared_ptr *top, std::shared_ptr *bottom); - Status ParseRenameOp(const py::dict &args, std::shared_ptr *ptr); + Status ParseRenameOp(const py::dict &args, std::shared_ptr *top, std::shared_ptr *bottom); - Status ParseTakeOp(const py::dict &args, std::shared_ptr *ptr); + Status ParseTakeOp(const py::dict &args, std::shared_ptr *top, std::shared_ptr *bottom); - Status ParseZipOp(const py::dict &args, std::shared_ptr *ptr); + Status ParseZipOp(const py::dict &args, std::shared_ptr *top, std::shared_ptr *bottom); - Status ParseConcatOp(const py::dict &args, std::shared_ptr *ptr); + Status ParseConcatOp(const py::dict &args, std::shared_ptr *top, std::shared_ptr *bottom); - Status ParseDeviceQueueOp(const py::dict &args, std::shared_ptr *ptr); + Status ParseDeviceQueueOp(const py::dict &args, std::shared_ptr *top, std::shared_ptr *bottom); - Status ParseTFReaderOp(const py::dict &args, std::shared_ptr *ptr); + Status ParseTFReaderOp(const py::dict &args, std::shared_ptr *top, std::shared_ptr *bottom); - Status ParseProjectOp(const py::dict &args, std::shared_ptr *ptr); + Status ParseProjectOp(const py::dict &args, std::shared_ptr *top, std::shared_ptr *bottom); - Status ParseImageFolderOp(const py::dict &args, std::shared_ptr *ptr); + Status ParseImageFolderOp(const py::dict &args, std::shared_ptr *top, std::shared_ptr *bottom); - Status ParseManifestOp(const py::dict &args, std::shared_ptr *ptr); + Status ParseManifestOp(const py::dict &args, std::shared_ptr *top, std::shared_ptr *bottom); - Status ParseVOCOp(const py::dict &args, std::shared_ptr *ptr); + Status ParseVOCOp(const py::dict &args, std::shared_ptr *top, std::shared_ptr *bottom); - Status ParseCocoOp(const py::dict &args, std::shared_ptr *ptr); + Status ParseCocoOp(const py::dict &args, std::shared_ptr *top, std::shared_ptr *bottom); - Status ParseCifar10Op(const py::dict &args, std::shared_ptr *ptr); + Status ParseCifar10Op(const py::dict &args, std::shared_ptr *top, std::shared_ptr *bottom); - Status ParseCifar100Op(const py::dict &args, std::shared_ptr *ptr); + Status ParseCifar100Op(const py::dict &args, std::shared_ptr *top, std::shared_ptr *bottom); - Status ParseRandomDataOp(const py::dict &args, std::shared_ptr *ptr); + Status ParseRandomDataOp(const py::dict &args, std::shared_ptr *top, std::shared_ptr *bottom); void PrintTree(); int32_t GetNumClasses() const; - Status ParseMnistOp(const py::dict &args, std::shared_ptr *ptr); + Status ParseMnistOp(const py::dict &args, std::shared_ptr *top, std::shared_ptr *bottom); Status SetBatchParameters(const py::dict &args); - Status ParseCelebAOp(const py::dict &args, std::shared_ptr *ptr); + Status ParseCelebAOp(const py::dict &args, std::shared_ptr *top, std::shared_ptr *bottom); - Status ParseTextFileOp(const py::dict &args, std::shared_ptr *ptr); + Status ParseTextFileOp(const py::dict &args, std::shared_ptr *top, std::shared_ptr *bottom); - Status ParseBuildVocabOp(const py::dict &args, std::shared_ptr *ptr); + Status ParseBuildVocabOp(const py::dict &args, std::shared_ptr *top, std::shared_ptr *bottom); - Status ParseClueOp(const py::dict &args, std::shared_ptr *ptr); + Status ParseClueOp(const py::dict &args, std::shared_ptr *top, std::shared_ptr *bottom); private: // Execution tree that links the dataset operators. @@ -180,6 +181,25 @@ class DEPipeline { static Status ParsePadInfo(py::handle value, PadInfo *pad_info); + /// \brief Helper function to inject a shuffle operator over top of the current operation being built. + /// \param[in] shuffle_size The size to use in the shuffle buffer + /// \param[in] input_op The operator to build shuffle on top of + /// \param[out] shuffle_op The top node of the created subtree (subtree contains two nodes). In this case it will be + /// the shuffle operator + /// \return Status return code + Status AddShuffleOp(int64_t shuffle_size, std::shared_ptr input_op, + std::shared_ptr *shuffle_op); + + /// \brief Helper function to compute the shuffle size + /// \param[in] num_files The number of files in the dataset + /// \param[in] num_devices The number of devices in the dataset + /// \param[in] num_rows The number of rows in the dataset + /// \param[in] total_rows An upper bound on the total rows in the dataset + /// \param[out] shuffle_size The resultant computed shuffle size + /// \return Status return code + Status ComputeShuffleSize(int64_t num_files, int64_t num_devices, int64_t num_rows, int64_t total_rows, + int64_t *shuffle_size); + int batch_size_; int repeat_num_; int num_rows_; diff --git a/mindspore/ccsrc/dataset/api/python_bindings.cc b/mindspore/ccsrc/dataset/api/python_bindings.cc index 51f2be49d5..ed3f993fb8 100644 --- a/mindspore/ccsrc/dataset/api/python_bindings.cc +++ b/mindspore/ccsrc/dataset/api/python_bindings.cc @@ -63,12 +63,14 @@ #include "dataset/kernels/image/random_horizontal_flip_bbox_op.h" #include "dataset/kernels/image/random_horizontal_flip_op.h" #include "dataset/kernels/image/random_resize_op.h" +#include "dataset/kernels/image/random_resize_with_bbox_op.h" #include "dataset/kernels/image/random_rotation_op.h" #include "dataset/kernels/image/random_vertical_flip_op.h" #include "dataset/kernels/image/random_vertical_flip_with_bbox_op.h" #include "dataset/kernels/image/rescale_op.h" #include "dataset/kernels/image/resize_bilinear_op.h" #include "dataset/kernels/image/resize_op.h" +#include "dataset/kernels/image/resize_with_bbox_op.h" #include "dataset/kernels/image/uniform_aug_op.h" #include "dataset/kernels/no_op.h" #include "dataset/text/kernels/jieba_tokenizer_op.h" @@ -116,9 +118,9 @@ void bindDEPipeline(py::module *m) { .def( "AddNodeToTree", [](DEPipeline &de, const OpName &op_name, const py::dict &args) { - DsOpPtr op; - THROW_IF_ERROR(de.AddNodeToTree(op_name, args, &op)); - return op; + py::dict out; + THROW_IF_ERROR(de.AddNodeToTree(op_name, args, &out)); + return out; }, py::return_value_policy::reference) .def_static("AddChildToParentNode", @@ -348,6 +350,18 @@ void bindTensorOps1(py::module *m) { .def(py::init(), py::arg("targetHeight"), py::arg("targetWidth") = ResizeOp::kDefWidth, py::arg("interpolation") = ResizeOp::kDefInterpolation); + (void)py::class_>( + *m, "ResizeWithBBoxOp", "Tensor operation to resize an image. Takes height, width and mode.") + .def(py::init(), py::arg("targetHeight"), + py::arg("targetWidth") = ResizeWithBBoxOp::kDefWidth, + py::arg("interpolation") = ResizeWithBBoxOp::kDefInterpolation); + + (void)py::class_>( + *m, "RandomResizeWithBBoxOp", + "Tensor operation to resize an image using a randomly selected interpolation. Takes height and width.") + .def(py::init(), py::arg("targetHeight"), + py::arg("targetWidth") = RandomResizeWithBBoxOp::kDefTargetWidth); + (void)py::class_>( *m, "UniformAugOp", "Tensor operation to apply random augmentation(s).") .def(py::init>, int32_t>(), py::arg("operations"), diff --git a/mindspore/ccsrc/dataset/core/config_manager.cc b/mindspore/ccsrc/dataset/core/config_manager.cc index a489b4a4ce..9291a8f832 100644 --- a/mindspore/ccsrc/dataset/core/config_manager.cc +++ b/mindspore/ccsrc/dataset/core/config_manager.cc @@ -41,6 +41,7 @@ Status ConfigManager::FromJson(const nlohmann::json &j) { set_worker_connector_size(j.value("workerConnectorSize", worker_connector_size_)); set_op_connector_size(j.value("opConnectorSize", op_connector_size_)); set_seed(j.value("seed", seed_)); + set_monitor_sampling_interval(j.value("monitorSamplingInterval", monitor_sampling_interval_)); return Status::OK(); } diff --git a/mindspore/ccsrc/dataset/core/data_type.cc b/mindspore/ccsrc/dataset/core/data_type.cc index 71a510d88f..bb10fae52f 100644 --- a/mindspore/ccsrc/dataset/core/data_type.cc +++ b/mindspore/ccsrc/dataset/core/data_type.cc @@ -18,7 +18,6 @@ #include "utils/log_adapter.h" #include "dataset/core/pybind_support.h" -#include "dataset/util/de_error.h" namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/dataset/core/tensor.cc b/mindspore/ccsrc/dataset/core/tensor.cc index abab8cf3f4..8de3425c5b 100644 --- a/mindspore/ccsrc/dataset/core/tensor.cc +++ b/mindspore/ccsrc/dataset/core/tensor.cc @@ -152,7 +152,7 @@ Tensor::Tensor(const std::vector &strings, const TensorShape &shape this->data_end_ = data_ + offset_arr[i]; - DS_ASSERT(num_bytes == 0); + MS_ASSERT(num_bytes == 0); if (shape.known()) Tensor::Reshape(shape); } Tensor::Tensor(const dataengine::BytesList &bytes_list, const TensorShape &shape) @@ -191,7 +191,7 @@ Tensor::Tensor(const dataengine::BytesList &bytes_list, const TensorShape &shape data_end_ = data_ + offset_arr[i]; - DS_ASSERT(num_bytes == 0); + MS_ASSERT(num_bytes == 0); if (shape.known()) Tensor::Reshape(shape); } Status Tensor::CreateTensor(std::shared_ptr *ptr, TensorImpl tensor_impl, const TensorShape &shape, @@ -420,7 +420,7 @@ bool Tensor::operator==(const Tensor &rhs) const { // Description: A function that print the value as specified by its index void Tensor::PrintItemAt(const std::vector &index, std::ostream &out) const { Status rc; - DS_ASSERT(data_); + MS_ASSERT(data_); switch (type_.value()) { CASE_PRINT_HEX(DataType::DE_BOOL, bool); diff --git a/mindspore/ccsrc/dataset/core/tensor.h b/mindspore/ccsrc/dataset/core/tensor.h index a3dbb391e5..9fed0bbc97 100644 --- a/mindspore/ccsrc/dataset/core/tensor.h +++ b/mindspore/ccsrc/dataset/core/tensor.h @@ -33,7 +33,6 @@ #include "dataset/core/data_type.h" #include "dataset/core/tensor_shape.h" #include "dataset/util/allocator.h" -#include "dataset/util/de_error.h" #include "dataset/util/status.h" #include "proto/example.pb.h" diff --git a/mindspore/ccsrc/dataset/core/tensor_shape.cc b/mindspore/ccsrc/dataset/core/tensor_shape.cc index 30afdf38bc..a0d6b9cd8d 100644 --- a/mindspore/ccsrc/dataset/core/tensor_shape.cc +++ b/mindspore/ccsrc/dataset/core/tensor_shape.cc @@ -22,7 +22,6 @@ #include "common/utils.h" #include "utils/log_adapter.h" #include "dataset/core/constants.h" -#include "dataset/util/de_error.h" namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/dataset/engine/connector.h b/mindspore/ccsrc/dataset/engine/connector.h index cdce592c1b..bd66172be5 100644 --- a/mindspore/ccsrc/dataset/engine/connector.h +++ b/mindspore/ccsrc/dataset/engine/connector.h @@ -97,13 +97,15 @@ class Connector { virtual Status Pop(int32_t worker_id, // The worker-id of the caller. See the requirement at the top of this file. T *result) noexcept { { - DS_ASSERT(worker_id < num_consumers_); + MS_ASSERT(worker_id < num_consumers_); std::unique_lock lk(m_); RETURN_IF_NOT_OK(cv_.Wait(&lk, [this, worker_id]() { return expect_consumer_ == worker_id; })); RETURN_IF_NOT_OK(queues_[pop_from_]->PopFront(result)); pop_from_ = (pop_from_ + 1) % num_producers_; + out_buffers_count_++; expect_consumer_ = (expect_consumer_ + 1) % num_consumers_; } + cv_.NotifyAll(); return Status::OK(); } @@ -114,19 +116,21 @@ class Connector { // @param worker_id The id of a worker thread calling this method. // @param el A const lvalue element to be passed/added/pushed. Status Push(int32_t worker_id, const T &el) noexcept { - DS_ASSERT(worker_id < static_cast(queues_.size())); - DS_ASSERT(queues_[worker_id] != nullptr); + MS_ASSERT(worker_id < static_cast(queues_.size())); + MS_ASSERT(queues_[worker_id] != nullptr); return (queues_[worker_id]->Add(el)); } + auto out_buffers_count() const { return out_buffers_count_.load(); } + // Add an element into the DbConnector without the overhead of synchronization. // It may block when the internal queue is full. // The element passed to this function will be forwarded into the internal queue. // @param worker_id The id of a worker thread calling this method. // @param el An element to be passed/added/pushed. virtual Status Push(int32_t worker_id, T &&el) noexcept { - DS_ASSERT(worker_id < static_cast(queues_.size())); - DS_ASSERT(queues_[worker_id] != nullptr); + MS_ASSERT(worker_id < static_cast(queues_.size())); + MS_ASSERT(queues_[worker_id] != nullptr); return (queues_[worker_id]->Add(std::forward(el))); } @@ -138,6 +142,7 @@ class Connector { } expect_consumer_ = 0; pop_from_ = 0; + out_buffers_count_ = 0; MS_LOG(DEBUG) << "Connector counters reset."; } @@ -198,6 +203,7 @@ class Connector { // Used in the Pop(), when a thread call pop() but it is not the expect_consumer_. std::mutex m_; CondVar cv_; + std::atomic out_buffers_count_ = 0; }; } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/data_schema.cc b/mindspore/ccsrc/dataset/engine/data_schema.cc index 9b86d18912..6c5f882bed 100644 --- a/mindspore/ccsrc/dataset/engine/data_schema.cc +++ b/mindspore/ccsrc/dataset/engine/data_schema.cc @@ -27,7 +27,6 @@ #include "dataset/util/status.h" #include "dataset/core/tensor_shape.h" #include "utils/log_adapter.h" -#include "dataset/util/de_error.h" namespace mindspore { namespace dataset { @@ -184,35 +183,7 @@ TensorShape ColDescriptor::shape() const { const char DataSchema::DEFAULT_DATA_SCHEMA_FILENAME[] = "datasetSchema.json"; // Constructor 1: Simple constructor that leaves things uninitialized. -DataSchema::DataSchema() : dataset_type_(DatasetType::kUnknown), num_rows_(0) {} - -DatasetType DataSchema::GetDatasetTYpeFromString(const std::string &type) const { - // Convert the string to a more easy to manage enum flavour of the buffer type. - if (type == "ARROW") { - return DatasetType::kArrow; - } else if (type == "TF") { - return DatasetType::kTf; - } else { - return DatasetType::kUnknown; - } -} - -Status DataSchema::LoadDatasetType(const std::string &schema_file_path) { - try { - std::ifstream in(schema_file_path); - nlohmann::json js; - in >> js; - // First, get the column for the type of dataset. - dataset_type_str_ = js.value("datasetType", ""); - dataset_type_ = GetDatasetTYpeFromString(dataset_type_str_); - dir_structure_ = js.value("directoryStructure", ""); - } - // Catch any exception and convert to Status return code - catch (const std::exception &err) { - RETURN_STATUS_UNEXPECTED("Schema file failed to load"); - } - return Status::OK(); -} +DataSchema::DataSchema() : num_rows_(0) {} // Internal helper function. Parses the json schema file in any order and produces a schema that // does not follow any particular order (json standard does not enforce any ordering protocol). @@ -400,8 +371,6 @@ Status DataSchema::LoadSchemaString(const std::string &schema_json_string, nlohmann::json js = nlohmann::json::parse(schema_json_string); RETURN_IF_NOT_OK(PreLoadExceptionCheck(js)); num_rows_ = js.value("numRows", 0); - dataset_type_str_ = js.value("datasetType", ""); - dataset_type_ = GetDatasetTYpeFromString(dataset_type_str_); nlohmann::json column_tree = js.at("columns"); if (column_tree.empty()) { RETURN_STATUS_UNEXPECTED("columns is null"); @@ -425,22 +394,16 @@ DataSchema::~DataSchema() = default; // Getter for the ColDescriptor by index const ColDescriptor &DataSchema::column(int32_t idx) const { - DS_ASSERT(idx < static_cast(col_descs_.size())); + MS_ASSERT(idx < static_cast(col_descs_.size())); return col_descs_[idx]; } // A print method typically used for debugging void DataSchema::Print(std::ostream &out) const { - out << "Dataset type string : ("; - if (dataset_type_str_.empty()) { - out << "none specified)\n"; - } else { - out << dataset_type_str_ << ")\n"; - } + out << "Dataset schema: ("; for (const auto &col_desc : col_descs_) { out << col_desc << "\n"; } - out << "Dataset type: " << static_cast(dataset_type_) << "\n"; } // Adds a column descriptor to the schema diff --git a/mindspore/ccsrc/dataset/engine/data_schema.h b/mindspore/ccsrc/dataset/engine/data_schema.h index 9debd2d466..ce61b8952d 100644 --- a/mindspore/ccsrc/dataset/engine/data_schema.h +++ b/mindspore/ccsrc/dataset/engine/data_schema.h @@ -30,196 +30,176 @@ namespace mindspore { namespace dataset { -// A simple class to provide meta info about a column. +/// \class ColDescriptor data_schema.h +/// \brief A simple class to provide meta info about a column. class ColDescriptor { public: - // Constructor 1: Simple constructor that leaves things uninitialized. + /// \brief Constructor 1: Simple constructor that leaves things uninitialized. ColDescriptor(); - // Constructor 2: Main constructor - // @param col_name - The name of the column - // @param col_type - The DE Datatype of the column - // @param tensor_impl - The (initial) type of tensor implementation for the column - // @param rank - The number of dimension of the data - // @param in_shape - option argument for input shape + /// \brief Constructor 2: Main constructor + /// \param[in] col_name - The name of the column + /// \param[in] col_type - The DE Datatype of the column + /// \param[in] tensor_impl - The (initial) type of tensor implementation for the column + /// \param[in] rank - The number of dimension of the data + /// \param[in] in_shape - option argument for input shape ColDescriptor(const std::string &col_name, DataType col_type, TensorImpl tensor_impl, int32_t rank, const TensorShape *in_shape = nullptr); - // Explicit copy constructor is required - // @param in_cd - the source ColDescriptor + /// \brief Explicit copy constructor is required + /// \param[in] in_cd - the source ColDescriptor ColDescriptor(const ColDescriptor &in_cd); - // Assignment overload - // @param in_cd - the source ColDescriptor + /// \brief Assignment overload + /// \param in_cd - the source ColDescriptor ColDescriptor &operator=(const ColDescriptor &in_cd); - // Destructor + /// \brief Destructor ~ColDescriptor(); - // A print method typically used for debugging - // @param out - The output stream to write output to + /// \brief A print method typically used for debugging + /// \param out - The output stream to write output to void Print(std::ostream &out) const; - // Given a number of elements, this function will compute what the actual Tensor shape would be. - // If there is no starting TensorShape in this column, or if there is a shape but it contains - // an unknown dimension, then the output shape returned shall resolve dimensions as needed. - // @param num_elements - The number of elements in the data for a Tensor - // @param out_shape - The materialized output Tensor shape - // @return Status - The error code return + /// \brief Given a number of elements, this function will compute what the actual Tensor shape would be. + /// If there is no starting TensorShape in this column, or if there is a shape but it contains + /// an unknown dimension, then the output shape returned shall resolve dimensions as needed. + /// \param[in] num_elements - The number of elements in the data for a Tensor + /// \param[inout] out_shape - The materialized output Tensor shape + /// \return Status - The error code return Status MaterializeTensorShape(int32_t num_elements, TensorShape *out_shape) const; - // << Stream output operator overload - // @notes This allows you to write the debug print info using stream operators - // @param out - reference to the output stream being overloaded - // @param cd - reference to the ColDescriptor to display - // @return - the output stream must be returned + /// \brief << Stream output operator overload + /// This allows you to write the debug print info using stream operators + /// \param[in] out - reference to the output stream being overloaded + /// \param[in] cd - reference to the ColDescriptor to display + /// \return - the output stream must be returned friend std::ostream &operator<<(std::ostream &out, const ColDescriptor &cd) { cd.Print(out); return out; } - // getter function - // @return The column's DataType + /// \brief getter function + /// \return The column's DataType DataType type() const { return type_; } - // getter function - // @return The column's rank + /// \brief getter function + /// \return The column's rank int32_t rank() const { return rank_; } - // getter function - // @return The column's name + /// \brief getter function + /// \return The column's name std::string name() const { return col_name_; } - // getter function - // @return The column's shape + /// \brief getter function + /// \return The column's shape TensorShape shape() const; - // getter function - // @return TF if the column has an assigned fixed shape. + /// \brief getter function + /// \return TF if the column has an assigned fixed shape. bool hasShape() const { return tensor_shape_ != nullptr; } - // getter function - // @return The column's tensor implementation type + /// \brief getter function + /// \return The column's tensor implementation type TensorImpl tensorImpl() const { return tensor_impl_; } private: DataType type_; // The columns type int32_t rank_; // The rank for this column (number of dimensions) - TensorImpl tensor_impl_; // The initial flavour of the tensor for this column. + TensorImpl tensor_impl_; // The initial flavour of the tensor for this column std::unique_ptr tensor_shape_; // The fixed shape (if given by user) std::string col_name_; // The name of the column }; -// A list of the columns. +/// \class DataSchema data_schema.h +/// \brief A list of the columns. class DataSchema { public: - // Constructor + /// \brief Constructor DataSchema(); - // Destructor + /// \brief Destructor ~DataSchema(); - // Populates the schema with a dataset type from a json file. It does not populate any of the - // column info. To populate everything, use loadSchema() afterwards. - // @param schema_file_path - Absolute path to the schema file to use for getting dataset type info. - Status LoadDatasetType(const std::string &schema_file_path); - - // Parses a schema json file and populates the columns and meta info. - // @param schema_file_path - the schema file that has the column's info to load - // @param columns_to_load - list of strings for columns to load. if empty, assumes all columns. - // @return Status - The error code return + /// \brief Parses a schema json file and populates the columns and meta info. + /// \param[in] schema_file_path - the schema file that has the column's info to load + /// \param[in] columns_to_load - list of strings for columns to load. if empty, assumes all columns. + /// \return Status - The error code return Status LoadSchemaFile(const std::string &schema_file_path, const std::vector &columns_to_load); - // Parses a schema JSON string and populates the columns and meta info. - // @param schema_json_string - the schema file that has the column's info to load - // @param columns_to_load - list of strings for columns to load. if empty, assumes all columns. - // @return Status - The error code return + /// \brief Parses a schema JSON string and populates the columns and meta info. + /// \param[in] schema_json_string - the schema file that has the column's info to load + /// \param[in] columns_to_load - list of strings for columns to load. if empty, assumes all columns. + /// \return Status - The error code return Status LoadSchemaString(const std::string &schema_json_string, const std::vector &columns_to_load); - // A print method typically used for debugging - // @param out - The output stream to write output to + /// \brief A print method typically used for debugging + /// \param[in] out - The output stream to write output to void Print(std::ostream &out) const; - // << Stream output operator overload - // @notes This allows you to write the debug print info using stream operators - // @param out - reference to the output stream being overloaded - // @param ds - reference to the DataSchema to display - // @return - the output stream must be returned + /// \brief << Stream output operator overload. This allows you to write the debug print info using stream operators + /// \param[in] out - reference to the output stream being overloaded + /// \param[in] ds - reference to the DataSchema to display + /// \return - the output stream must be returned friend std::ostream &operator<<(std::ostream &out, const DataSchema &ds) { ds.Print(out); return out; } - // Adds a column descriptor to the schema - // @param cd - The ColDescriptor to add - // @return Status - The error code return + /// \brief Adds a column descriptor to the schema + /// \param[in] cd - The ColDescriptor to add + /// \return Status - The error code return Status AddColumn(const ColDescriptor &cd); - // Setter - // @param in_type - The Dataset type to set into the schema - void set_dataset_type(DatasetType in_type) { dataset_type_ = in_type; } - - // getter - // @return The dataset type of the schema - DatasetType dataset_type() const { return dataset_type_; } - - // getter - // @return The reference to a ColDescriptor to get (const version) + /// \brief getter + /// \return The reference to a ColDescriptor to get (const version) const ColDescriptor &column(int32_t idx) const; - // getter - // @return The number of columns in the schema + /// \brief getter + /// \return The number of columns in the schema int32_t NumColumns() const { return col_descs_.size(); } bool Empty() const { return NumColumns() == 0; } - std::string dir_structure() const { return dir_structure_; } - - std::string dataset_type_str() const { return dataset_type_str_; } - + /// \brief getter + /// \return The number of rows read from schema int64_t num_rows() const { return num_rows_; } static const char DEFAULT_DATA_SCHEMA_FILENAME[]; - // Loops through all columns in the schema and returns a map with the column - // name to column index number. - // @param out_column_name_map - The output map of columns names to column index - // @return Status - The error code return + /// \brief Loops through all columns in the schema and returns a map with the column name to column index number. + /// \param[inout] out_column_name_map - The output map of columns names to column index + /// \return Status - The error code return Status GetColumnNameMap(std::unordered_map *out_column_name_map); private: - // Internal helper function. Parses the json schema file in any order and produces a schema that - // does not follow any particular order (json standard does not enforce any ordering protocol). - // This one produces a schema that contains all of the columns from the schema file. - // @param column_tree - The nlohmann tree from the json file to parse - // @return Status - The error code return + /// \brief Internal helper function. Parses the json schema file in any order and produces a schema that + /// does not follow any particular order (json standard does not enforce any ordering protocol). + /// This one produces a schema that contains all of the columns from the schema file. + /// \param[in] column_tree - The nlohmann tree from the json file to parse + /// \return Status - The error code return Status AnyOrderLoad(nlohmann::json column_tree); - // Internal helper function. For each input column name, perform a lookup to the json document to - // find the matching column. When the match is found, process that column to build the column - // descriptor and add to the schema in the order in which the input column names are given. - // @param column_tree - The nlohmann tree from the json file to parse - // @param columns_to_load - list of strings for the columns to add to the schema - // @return Status - The error code return + /// \brief Internal helper function. For each input column name, perform a lookup to the json document to + /// find the matching column. When the match is found, process that column to build the column + /// descriptor and add to the schema in the order in which the input column names are given. + /// \param[in] column_tree - The nlohmann tree from the json file to parse + /// \param[in] columns_to_load - list of strings for the columns to add to the schema + /// \return Status - The error code return Status ColumnOrderLoad(nlohmann::json column_tree, const std::vector &columns_to_load); - // Internal helper function. Given the json tree for a given column, load it into our schema. - // @param columnTree - The nlohmann child tree for a given column to load. - // @param col_name - The string name of the column for that subtree. - // @return Status - The error code return + /// \brief Internal helper function. Given the json tree for a given column, load it into our schema. + /// \param[in] columnTree - The nlohmann child tree for a given column to load. + /// \param[in] col_name - The string name of the column for that subtree. + /// \return Status - The error code return Status ColumnLoad(nlohmann::json column_child_tree, const std::string &col_name); - // Internal helper function. Performs sanity checks on the json file setup. - // @param js - The nlohmann tree for the schema file - // @return Status - The error code return + /// \brief Internal helper function. Performs sanity checks on the json file setup. + /// \param[in] js - The nlohmann tree for the schema file + /// \return Status - The error code return Status PreLoadExceptionCheck(const nlohmann::json &js); - DatasetType GetDatasetTYpeFromString(const std::string &type) const; - std::vector col_descs_; // Vector of column descriptors - std::string dataset_type_str_; // A string that represents the type of dataset - DatasetType dataset_type_; // The numeric form of the dataset type from enum - std::string dir_structure_; // Implicit or flatten int64_t num_rows_; }; } // namespace dataset diff --git a/mindspore/ccsrc/dataset/engine/dataset_iterator.cc b/mindspore/ccsrc/dataset/engine/dataset_iterator.cc index 7eb38785aa..be333741b1 100644 --- a/mindspore/ccsrc/dataset/engine/dataset_iterator.cc +++ b/mindspore/ccsrc/dataset/engine/dataset_iterator.cc @@ -27,7 +27,7 @@ namespace mindspore { namespace dataset { // Constructor of the IteratorBase -IteratorBase::IteratorBase() : curr_buffer_(nullptr), eof_handled_(false), first_row_(true) {} +IteratorBase::IteratorBase() : curr_buffer_(nullptr), eof_handled_(false) {} IteratorBase::~IteratorBase() = default; @@ -51,13 +51,10 @@ Status IteratorBase::GetNextAsMap(TensorMap *out_map) { // The column name mapping comes from the source operator that is producing the data into the iterator. // To avoid having to fetch this for every time, we'll take a local copy of the column name id mapping // and save in the iterator. We only have to do this once. All subsequent iterations use the same mapping. - // Note: This can only be done after the first row has been produced, as this guarantees the the child has - // it's column mapping set up. - if (first_row_) { + if (col_name_id_map_.empty()) { // Determine the column name map by calling the derived class method to retrieve the column // name map col_name_id_map_ = this->GetColumnNameMap(); - first_row_ = false; } // Populate the out map from the row and return it diff --git a/mindspore/ccsrc/dataset/engine/dataset_iterator.h b/mindspore/ccsrc/dataset/engine/dataset_iterator.h index ada2b0ffb6..4e40e77c74 100644 --- a/mindspore/ccsrc/dataset/engine/dataset_iterator.h +++ b/mindspore/ccsrc/dataset/engine/dataset_iterator.h @@ -72,7 +72,6 @@ class IteratorBase { protected: std::unique_ptr curr_buffer_; // holds the current buffer bool eof_handled_; // T/F if this op got an eof - bool first_row_; // internal tracking for first row case std::unordered_map col_name_id_map_; }; diff --git a/mindspore/ccsrc/dataset/engine/datasetops/barrier_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/barrier_op.cc index 5ab2df4ac4..6fc276a75e 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/barrier_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/barrier_op.cc @@ -144,9 +144,6 @@ Status BarrierOp::prepare(TensorQTable *const table) { table->push_back(std::move(new_row)); - // Assign the column name id map - RETURN_IF_NOT_OK(DatasetOp::AssignColMapFromChild()); - // the update code below shouldn't do anything bad if the column name already exists. return Status::OK(); } diff --git a/mindspore/ccsrc/dataset/engine/datasetops/batch_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/batch_op.cc index 60643c90ba..8bfa8c287c 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/batch_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/batch_op.cc @@ -76,7 +76,6 @@ Status BatchOp::operator()() { std::unique_ptr table = std::make_unique(); child_iterator_ = std::make_unique(this, 0, 0); RETURN_IF_NOT_OK(child_iterator_->FetchNextTensorRow(&new_row)); - RETURN_IF_NOT_OK(DatasetOp::AssignColMapFromChild()); // must come after the first fetch above int32_t cur_batch_size = 0; RETURN_IF_NOT_OK(GetBatchSize(&cur_batch_size, CBatchInfo(0, 0, 0))); while (child_iterator_->eof_handled() == false) { @@ -410,7 +409,7 @@ Status BatchOp::UnpackPadInfo(const PadInfo &pad_info, // Visitor accept method for NodePass Status BatchOp::Accept(NodePass *p, bool *modified) { // Downcast shared pointer then call visitor - return p->RunOnNode(std::static_pointer_cast(shared_from_this()), modified); + return p->RunOnNode(shared_from_base(), modified); } } // namespace dataset diff --git a/mindspore/ccsrc/dataset/engine/datasetops/bucket_batch_by_length_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/bucket_batch_by_length_op.cc index def2ea0fee..5e143b700f 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/bucket_batch_by_length_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/bucket_batch_by_length_op.cc @@ -115,7 +115,6 @@ Status BucketBatchByLengthOp::operator()() { TensorRow current_row; child_iterator_ = std::make_unique(this, 0, 0); RETURN_IF_NOT_OK(child_iterator_->FetchNextTensorRow(¤t_row)); - RETURN_IF_NOT_OK(AssignColMapFromChild()); while (!child_iterator_->eof_handled()) { while (!current_row.empty()) { int32_t element_length; diff --git a/mindspore/ccsrc/dataset/engine/datasetops/build_vocab_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/build_vocab_op.cc index f99804ec9b..ceb5058593 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/build_vocab_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/build_vocab_op.cc @@ -86,7 +86,6 @@ Status BuildVocabOp::operator()() { child_iterator_ = std::make_unique(this, 0, 0); TensorRow new_row; RETURN_IF_NOT_OK(child_iterator_->FetchNextTensorRow(&new_row)); - RETURN_IF_NOT_OK(AssignColMapFromChild()); if (!col_names_.empty()) { col_ids_.reserve(col_names_.size()); for (std::string col : col_names_) { diff --git a/mindspore/ccsrc/dataset/engine/datasetops/concat_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/concat_op.cc index 7162dc0b47..4bada31e7e 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/concat_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/concat_op.cc @@ -66,12 +66,6 @@ Status ConcatOp::operator()() { std::unique_ptr buf; RETURN_IF_NOT_OK(child_[0]->GetNextBuffer(&buf)); - // Obtain columns_name_id_map from child_[0] - column_name_id_map_ = child_[0]->column_name_id_map(); - if (column_name_id_map_.empty()) { - RETURN_STATUS_UNEXPECTED("Child column name map cannot be empty!"); - } - int eof_count = 0; while (eof_count != children_num_) { for (int i = 0; i < children_num_; i++) { @@ -115,17 +109,13 @@ Status ConcatOp::Verify(int32_t id, const std::unique_ptr &buf) { buf->GetRow(0, &new_row); if (id == 0) { - // Obtain the column name, data type and data rank in child[0] - column_name_id_ = child_[id]->column_name_id_map(); + // Obtain the data type and data rank in child[0] for (auto item : new_row) { data_type_.push_back(item->type()); data_rank_.push_back(item->Rank()); } } else { - // Compare the column name, data type and data rank with these in child[0] - if (child_[id]->column_name_id_map() != column_name_id_) { - RETURN_STATUS_UNEXPECTED("The column name or column order is not the same with previous dataset."); - } + // Compare the data type and data rank with these in child[0] int32_t index = 0; for (auto item : new_row) { if ((item->type() != data_type_[index]) || item->Rank() != data_rank_[index++]) { @@ -138,7 +128,27 @@ Status ConcatOp::Verify(int32_t id, const std::unique_ptr &buf) { Status ConcatOp::PrepareNodePostAction() { RETURN_IF_NOT_OK(PipelineOp::PrepareNodePostAction()); - tree_->AddToRepeatStack(shared_from_this()); + tree_->AddToEOEOpStack(shared_from_this()); + return Status::OK(); +} + +// We need to overwrite the super class ComputeColMap here because the number of children is more than 1. +Status ConcatOp::ComputeColMap() { + if (column_name_id_map_.empty()) { + // Obtain columns_name_id_map from child_[0] + column_name_id_map_ = child_[0]->column_name_id_map(); + if (column_name_id_map_.empty()) { + RETURN_STATUS_UNEXPECTED("Child column name map cannot be empty!"); + } + // Verify all children have the same column name map + for (int32_t i = 0; i < child_.size(); ++i) { + if (child_[i]->column_name_id_map() != column_name_id_map_) { + RETURN_STATUS_UNEXPECTED("The column name or column order is not the same with previous dataset."); + } + } + } else { + MS_LOG(WARNING) << "Column name map is already set!"; + } return Status::OK(); } } // namespace dataset diff --git a/mindspore/ccsrc/dataset/engine/datasetops/concat_op.h b/mindspore/ccsrc/dataset/engine/datasetops/concat_op.h index 0fb8ec8362..4bcfdbf6c6 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/concat_op.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/concat_op.h @@ -85,6 +85,10 @@ class ConcatOp : public PipelineOp { // @return Name of the current Op std::string Name() const override { return "ConcatOp"; } + // Private function for computing the assignment of the column name map. + // @return - Status + Status ComputeColMap() override; + private: Status Verify(int32_t id, const std::unique_ptr &buf); diff --git a/mindspore/ccsrc/dataset/engine/datasetops/dataset_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/dataset_op.cc index bf991ea7d9..3e31f6c017 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/dataset_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/dataset_op.cc @@ -18,29 +18,31 @@ #include #include #include +#include #include #include #include #include "dataset/engine/execution_tree.h" #include "dataset/engine/datasetops/device_queue_op.h" +#include "dataset/engine/datasetops/source/sampler/sampler.h" #include "dataset/engine/data_buffer.h" #include "dataset/engine/db_connector.h" #include "dataset/engine/opt/pass.h" - +#include "utils/system/crc32c.h" #include "utils/log_adapter.h" namespace mindspore { namespace dataset { // Constructor -DatasetOp::DatasetOp(int32_t op_connector_size) +DatasetOp::DatasetOp(int32_t op_connector_size, std::shared_ptr sampler) : oc_queue_size_(op_connector_size), + sampler_(sampler), operator_id_(kInvalidOperatorId), tree_(nullptr), state_(OpState::kDeOpIdle), op_ctrl_flags_(kDeOpNone), - out_connector_(nullptr), - first_fetch_(true) { + out_connector_(nullptr) { // The operator starts out with an invalid operator id. The only way to // get it out of invalid state is to assign the operator to an execution tree. } @@ -105,13 +107,58 @@ Status DatasetOp::InsertAsParent(std::shared_ptr to_add) { void DatasetOp::AddParent(DatasetOp *parent) { parent_.push_back(parent); } // Removes a parent operator from this operator -void DatasetOp::RemoveParent(DatasetOp *parent) { +void DatasetOp::RemoveParent(const DatasetOp *parent) { parent_.erase(std::remove(parent_.begin(), parent_.end(), parent), parent_.end()); } +// Removes this node from the tree and connects it's parent/child together +Status DatasetOp::Remove() { + if (parent_.size() > 1) { + std::string err_msg("No support for op removal if the operator has more than one parent"); + RETURN_STATUS_UNEXPECTED(err_msg); + } + if (child_.size() > 1) { + std::string err_msg("No support for op removal if the operator has more than one child"); + RETURN_STATUS_UNEXPECTED(err_msg); + } + + // Scenario's when removing node B: + // A -> B -> C + // A -> B + // B -> C + // + // If we remove B, then first take our child A and update it's parent to be C + // It's possible the parent is null if we are the root node being removed. + if (!child_.empty()) { + // If we have a parent, then assign chlid's parent to point to our parent. + if (!parent_.empty()) { + child_[0]->parent_[0] = parent_[0]; + } else { + // We don't have a parent, so we are the root node being removed. + // clear the parent list of our child so that it becomes the new root. + child_[0]->parent_.clear(); + tree_->AssignRoot(child_[0]); + } + } + + // Next, if we had a parent, then set it's child to be our child. + if (!parent_.empty()) { + // if we have a child, then set our parent to point to it + if (!child_.empty()) { + parent_[0]->child_[0] = child_[0]; + } else { + // We don't have a child, so clear the child list of the current + // parent because it will be empty once we are removed. + parent_[0]->child_.clear(); + } + } + + return Status::OK(); +} + // Getter function to get a shared pointer to our childAdds a operator to become our child. std::shared_ptr DatasetOp::child(int32_t child_index) const { - DS_ASSERT(child_index < static_cast(child_.size())); + MS_ASSERT(child_index < static_cast(child_.size())); // Return a shared pointer return child_[child_index]; } @@ -151,6 +198,9 @@ void DatasetOp::Print(std::ostream &out, bool show_all) const { } out << "\nConnector queue size : " << oc_queue_size_ << "\nOperator control flags : 0x" << std::hex << std::setw(8) << std::setfill('0') << op_ctrl_flags_ << std::dec << std::setfill(' '); + if (sampler_) { + sampler_->Print(out, show_all); + } } } @@ -223,11 +273,10 @@ Status DatasetOp::PrepareNodePreAction() { Status DatasetOp::PrepareNodePostAction() { // If this op does not have any children and it is in a repeat path of the tree... if (child_.empty() && BitTest(op_ctrl_flags_, kDeOpRepeated)) { - // push ourselves onto the tree repeat stack. Later, the repeat operator + // push ourselves onto the eoe operator stack. Later, a repeat/epoch ctrl operator // above us will consume them. - tree_->AddToRepeatStack(shared_from_this()); + tree_->AddToEOEOpStack(shared_from_this()); } - // Creating Connector object for each op. // The consumer of the root node is assumed to be one thread. // If multiple threads are consuming from the root node, they will get the ordered data in round robin fashion. @@ -240,6 +289,10 @@ Status DatasetOp::PrepareNodePostAction() { RETURN_IF_NOT_OK(out_connector_->Register(tree_->AllTasks())); } RETURN_IF_NOT_OK(this->RegisterWorkerConnectors()); + + // Generate the column name map for the current op. + RETURN_IF_NOT_OK(this->ComputeColMap()); + return Status::OK(); } @@ -262,38 +315,84 @@ std::string DatasetOp::ColumnNameMapAsString() const { return outStr; } -// A helper function for providing assignment of the column name map. -// This grabs the map from child 0 and assigns it into this op. -// Can only be used if number of children is 1. -Status DatasetOp::AssignColMapFromChild() { +// Computing the assignment of the column name map. +// This just inherits the column map from its first child, can only be used if the number of children is 1. +// Operations changing the column map must overwrite this function. +Status DatasetOp::ComputeColMap() { if (child_.size() > 1) { RETURN_STATUS_UNEXPECTED("Assigning column name map from child only works for single-child operators."); } - // Assign the correct column name map to this op by taking it from the input child. - // This must be done AFTER the first fetch, but only needs to be done once by the first worker to - // do the first fetch. - if (first_fetch_) { - // If there was a single worker, or this is being called from a master thread in a parallel op, - // then the mutex is not really needed here, although it's harmless. - std::unique_lock lock(column_name_map_mutex_); - // If the map has not been set up yet, then we are the first one in to set it up. The first_fetch_ (dirty read) - // bool allows us to avoid acquiring the lock if the map has already been set. + if (column_name_id_map_.empty()) { + column_name_id_map_ = child_[0]->column_name_id_map(); if (column_name_id_map_.empty()) { - column_name_id_map_ = child_[0]->column_name_id_map(); - first_fetch_ = false; - if (column_name_id_map_.empty()) { - RETURN_STATUS_UNEXPECTED("Child column name map cannot be empty!"); - } + RETURN_STATUS_UNEXPECTED("Child column name map cannot be empty!"); } - MS_LOG(DEBUG) << "Setting column map after first fetch:\n" << DatasetOp::ColumnNameMapAsString(); + MS_LOG(DEBUG) << "Setting column map:\n" << DatasetOp::ColumnNameMapAsString(); + } else { + MS_LOG(WARNING) << "Column name map is already set!"; } return Status::OK(); } +Status DatasetOp::PreAccept(NodePass *p, bool *modified) { + // DatasetOp is the base class of visitor target pre-visit. + // This method will only be called if its derived class does not implement one. + return p->PreRunOnNode(shared_from_this(), modified); +} + Status DatasetOp::Accept(NodePass *p, bool *modified) { // DatasetOp is the base class of visitor target. // This method will only be called if its derived class does not implement one. return p->RunOnNode(shared_from_this(), modified); } + +// A helper function with some common code that leaf nodes can use during +// prepare phase for checking if they need to assign a sampler to the cache. +Status DatasetOp::SaveSamplerForCache(bool random_access_op) { + // If we are a descendant under a cache op and we have a sampler, then save this sampler + // to a stack so that the cache can pick it up during it's processing above us. + if (sampler_) { + if (BitTest(tree_->PrepareFlags(), ExecutionTree::kDePrepCache)) { + // use move semantic to set our sampler_ to null after the move. This is okay because a sampler is + // useless to a random data op. It was only being used as a temporary holding until the cache can + // be created + tree_->AddToSamplerStack(sampler_); + MS_LOG(INFO) << "Preparing a leaf op: passing sampler up the tree for Cache handling."; + } else if (!random_access_op) { + // A sampler exists, but we are not in a caching tree and we are not a random access mappable leaf. + // This is an error because that type of leaf does not use sampling unless there's a cache to hook it into. + RETURN_STATUS_UNEXPECTED( + "Non-mappable leaf op has a sampler, but it only supports sampling if there is a cache after it in the tree"); + } + } + + if (!random_access_op) { + // Since we don't truly need the sampler for this non-mappable dataset and it's been saved for the cache + // we can remove it now from the base. + sampler_.reset(); + } + + return Status::OK(); +} +uint32_t DatasetOp::GenerateCRC(const std::shared_ptr &op) { + std::stringstream ss; + op->tree_->Print(ss, op); + std::string ss_str = ss.str(); + + // Filter out the Operator control flags field when generating the check sum + ss_str = std::regex_replace(ss_str, std::regex("Operator control flags.*\n"), ""); + + // Filter out the Device id field to allow cache sharing for a distributed run of the same pipeline + ss_str = std::regex_replace(ss_str, std::regex("Device id.*\n"), ""); + ss_str = std::regex_replace(ss_str, std::regex("device_id.*\n"), ""); + + // The Cache crc and Server cache id field is different when creating new cache_client and re-using the same + // cache_client later. So we filter out these two fields to allow cache sharing. + ss_str = std::regex_replace(ss_str, std::regex("Cache crc.*\n"), ""); + ss_str = std::regex_replace(ss_str, std::regex("Server cache id.*\n"), ""); + + uint32_t cache_crc = system::Crc32c::GetMaskCrc32cValue(ss_str.c_str(), ss_str.length()); + return cache_crc; +} } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/dataset_op.h b/mindspore/ccsrc/dataset/engine/datasetops/dataset_op.h index 973b5be962..ab5cb90357 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/dataset_op.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/dataset_op.h @@ -34,8 +34,10 @@ class DataBuffer; class NodePass; -// The base class DatasetOp is the main tree node. It is an abstract class, so -// the actual implementation of the operators will be derived from here. +class Sampler; + +/// \brief The base class DatasetOp is the main tree node. It is an abstract class, so +/// the actual implementation of the operators will be derived from here. class DatasetOp : public std::enable_shared_from_this { // Allow execution tree to access internal members friend class ExecutionTree; @@ -53,109 +55,114 @@ class DatasetOp : public std::enable_shared_from_this { // Flags that control operator runtime behaviours enum OpState { kDeOpRunning = 0, kDeOpIdle = 1, kDeOpTerminated }; - // Constructor - // @param op_connector_size - The size for the output connector of this operator. - explicit DatasetOp(int32_t op_connector_size); + /// Constructor + /// \param op_connector_size - The size for the output connector of this operator. + /// \param sampler - The sampler for the op + explicit DatasetOp(int32_t op_connector_size, std::shared_ptr sampler); - // Destructor + /// Destructor virtual ~DatasetOp() { tree_ = nullptr; } - // Adds a operator to become our child. - // @param child - shared pointer to the child to add. + /// Adds a operator to become our child. + /// \param child - shared pointer to the child to add. Status AddChild(std::shared_ptr child); - // Remove a operator from our children. - // @param child - shared pointer to the child to remove. + /// Remove a operator from our children. + /// \param child - shared pointer to the child to remove. Status RemoveChild(std::shared_ptr child); - // Getter function to get a shared pointer to our child - // @param child_index - An operator can have n children. Indicates choose which child to return. + /// \brief Removes this node from the tree and connects it's parent/child together. + /// \return Status eerror code returned + Status Remove(); + + /// \brief Getter function to get a shared pointer to our child + /// \param child_index - An operator can have n children. Indicates choose which child to return. std::shared_ptr child(int32_t child_index) const; - // Inserts a operator as the parent current op. - // Inserted op will become the sole parent of the current op. - // The existing parent of the current op will be transferred to the inserted op. + /// \brief Inserts a operator as the parent current op. + /// Inserted op will become the sole parent of the current op. + /// The existing parent of the current op will be transferred to the inserted op. Status InsertAsParent(std::shared_ptr to_add); - // Creates the connector within this operator - // @param num_producers - number of threads that write into this connector - // @param num_consumers - number of threads that read from this connector + /// \brief Creates the connector within this operator + /// \param num_producers - number of threads that write into this connector + /// \param num_consumers - number of threads that read from this connector void CreateConnector(int32_t num_producers, int32_t num_consumers); - // A print method typically used for debugging - // @param out - The output stream to write output to - // @param show_all - A bool to control if you want to show all info or just a summary + /// \brief A print method typically used for debugging + /// \param out - The output stream to write output to + /// \param show_all - A bool to control if you want to show all info or just a summary virtual void Print(std::ostream &out, bool show_all) const; - // << Stream output operator overload - // @notes This allows you to write the debug print info using stream operators - // @param out - reference to the output stream being overloaded - // @param dO - reference to the DatasetOp to display - // @return - the output stream must be returned + /// \brief << Stream output operator overload + /// \notes This allows you to write the debug print info using stream operators + /// \param out - reference to the output stream being overloaded + /// \param dO - reference to the DatasetOp to display + /// \return - the output stream must be returned friend std::ostream &operator<<(std::ostream &out, const DatasetOp &dO) { dO.Print(out, false); return out; } - // Class functor operator (). - // DatasetOps operate by launching a thread (see ExecutionTree). - // This pure virtual version makes the requirement that derived classes must provide a functor - // that will execute their main runtime loop code. - // @return Status - The error code return + /// \brief Class functor operator (). + /// DatasetOps operate by launching a thread (see ExecutionTree). + /// This pure virtual version makes the requirement that derived classes must provide a functor + /// that will execute their main runtime loop code. + /// \return Status - The error code return virtual Status operator()() = 0; - // Gets the next buffer from the given child - // @notes See GetNextInput for similar function that has built-in message handling - // @param p_buffer - The shared pointer for the fetched buffer to return (by reference) - // @param worker_id - The worker id - // @return Status - The error code return + /// \brief Gets the next buffer from the given child + /// \notes See GetNextInput for similar function that has built-in message handling + /// \param p_buffer - The shared pointer for the fetched buffer to return (by reference) + /// \param worker_id - The worker id + /// \return Status - The error code return virtual Status GetNextBuffer(std::unique_ptr *p_buffer, int32_t worker_id) { return GetNextBuffer(p_buffer, worker_id, false); } - // Gets the next buffer from the given child - // @notes See GetNextInput for similar function that has built-in message handling - // @param p_buffer - The shared pointer for the fetched buffer to return (by reference) - // @return Status - The error code return + /// \brief Gets the next buffer from the given child + /// \notes See GetNextInput for similar function that has built-in message handling + /// \param p_buffer - The shared pointer for the fetched buffer to return (by reference) + /// \return Status - The error code return virtual Status GetNextBuffer(std::unique_ptr *p_buffer) { return GetNextBuffer(p_buffer, 0, false); } - // Gets the next buffer from the given child - // @notes See GetNextInput for similar function that has built-in message handling - // @param p_buffer - The shared pointer for the fetched buffer to return (by reference) - // @param worker_id - The worker id - // @param retry_if_eoe Set this flag to true to allow calling pop() again after the first pop() returns EOE. - // @return Status - The error code return + /// \brief Gets the next buffer from the given child + /// \notes See GetNextInput for similar function that has built-in message handling + /// \param p_buffer - The shared pointer for the fetched buffer to return (by reference) + /// \param worker_id - The worker id + /// \param retry_if_eoe Set this flag to true to allow calling pop() again after the first pop() returns EOE. + /// \return Status - The error code return virtual Status GetNextBuffer(std::unique_ptr *p_buffer, int32_t worker_id, bool retry_if_eoe); - // Gets the next buffer from the given child . This function also has built-in eoe and eof - // message handling so that child classes don't have to manually code pass-through logic when - // those messages are received. - // @param p_buffer - The shared pointer for the fetched buffer to return (by reference) - // @param worker_id - The worker id - // @return Status - The error code return + /// \brief Gets the next buffer from the given child . This function also has built-in eoe and eof + /// message handling so that child classes don't have to manually code pass-through logic when + /// those messages are received. + /// \param p_buffer - The shared pointer for the fetched buffer to return (by reference) + /// \param worker_id - The worker id + /// \return Status - The error code return Status GetNextInput(std::unique_ptr *p_buffer, int32_t worker_id = 0, int32_t child_index = 0); - // Performs handling for when an eoe message is received. - // The base class implementation simply flows the eoe message to output. Derived classes - // may override if they need to perform special eoe handling. - // @param worker_id - The worker id - // @return Status - The error code return + /// \brief Performs handling for when an eoe message is received. + /// The base class implementation simply flows the eoe message to output. Derived classes + /// may override if they need to perform special eoe handling. + /// \param worker_id - The worker id + /// \return Status - The error code return virtual Status EoeReceived(int32_t worker_id); - // Performs handling for when an eof message is received. - // The base class implementation simply flows the eof message to output. Derived classes - // may override if they need to perform special eof handling. - // @param worker_id - The worker id - // @return Status - The error code return + /// \brief Performs handling for when an eof message is received. + /// The base class implementation simply flows the eof message to output. Derived classes + /// may override if they need to perform special eof handling. + /// \param worker_id - The worker id + /// \return Status - The error code return virtual Status EofReceived(int32_t worker_id); - // Derived classes may implement the reset function if the operator is stateful and needs - // specific reset handling that is not contained in this common code version of the reset - // @return Status - The error code return + /// \brief Derived classes may implement the reset function if the operator is stateful and needs + /// specific reset handling that is not contained in this common code version of the reset + /// \return Status - The error code return virtual Status Reset(); - // This calls the reset function on this subtree in pre-order - // @return Status - The error code return + /// \brief This calls the reset function on this subtree in pre-order + /// \return Status - The error code return virtual Status ResetSubtree() { RETURN_IF_NOT_OK(Reset()); for (const auto &c : child_) { @@ -164,64 +171,68 @@ class DatasetOp : public std::enable_shared_from_this { return Status::OK(); } - // During tree prepare phase, operators may have specific pre-operations to perform depending on - // their role. - // @notes Derived versions of this function should always call it's superclass version first - // before providing their own implementations. + /// \brief During tree prepare phase, operators may have specific pre-operations to perform depending on + /// their role. + /// \notes Derived versions of this function should always call it's superclass version first + /// before providing their own implementations. virtual Status PrepareNodePreAction(); - // During tree prepare phase, operators may have specific post-operations to perform depending on - // their role. - // @notes Derived versions of this function should always call it's superclass version first - // before providing their own implementations. + /// \brief During tree prepare phase, operators may have specific post-operations to perform depending on + /// their role. + /// \notes Derived versions of this function should always call it's superclass version first + /// before providing their own implementations. virtual Status PrepareNodePostAction(); - // Getter function - // @return The operator id + /// \brief Getter function + /// \return The operator id int32_t id() const { return operator_id_; } - // Getter function - // @return The prepare flags + /// \brief Getter function + /// \return The prepare flags virtual uint32_t PrepareFlags() const; - // Getter function - // @return The number of workers in this op + /// \brief Getter function + /// \return The number of workers in this op virtual int32_t num_workers() const = 0; - // Getter function - // @return The number of threads consuming from previous op. + /// \brief Getter function + /// \return The number of threads consuming from previous op. virtual int32_t num_consumers() const = 0; - // Getter function - // @return The number of threads producing to the output connector. + /// \brief Getter function + /// \return The number of threads producing to the output connector. virtual int32_t num_producers() const = 0; - // Getter function - // @return T/F if this is an inlined operator + /// \brief Getter function + /// \return T/F if this is an inlined operator bool inlined() const { return (oc_queue_size_ == 0); } - // Setter function - // @return Sets the control flags + /// \brief Setter function + /// \return Sets the control flags void set_control_flag(uint64_t flag) { BitSet(&op_ctrl_flags_, flag); } - // Register the internal worker connectors. No op unless it is a parallel op - // @return Status + /// \brief Setter function + /// \return Sets the control flags + void ClearControlFlag(uint64_t flag) { BitClear(&op_ctrl_flags_, flag); } + + /// \brief Register the internal worker connectors. No op unless it is a parallel op + /// \return Status virtual Status RegisterWorkerConnectors() { return Status::OK(); } - // Getter for the column name mapping - // @return The returned map + /// \brief Getter for the column name mapping + /// \return The returned map std::unordered_map column_name_id_map() const { return column_name_id_map_; } - // Checks if the column name map has been set up yet for this op - // @return - T/F if the operator has the map set up + /// \brief Checks if the column name map has been set up yet for this op + /// \return - T/F if the operator has the map set up bool HasColumnNameMap() const { return (column_name_id_map_.empty()); } - // gives a string output for the column map for handy debug printing - // @return - the column name map as a string + /// \brief gives a string output for the column map for handy debug printing + /// \return - the column name map as a string std::string ColumnNameMapAsString() const; - // Getter function - // @return connector size of current op + /// \brief Getter function + /// \return connector size of current op int32_t ConnectorSize() const { if (!inlined()) { return out_connector_->size(); @@ -230,8 +241,13 @@ class DatasetOp : public std::enable_shared_from_this { return ChildOpConnectorSize(); } - // Getter function - // @return connector size of current op + /// \brief Counting number of buffer sent out by a connector + int64_t ConnectorOutBufferCount() const { + return out_connector_ == nullptr ? int64_t(-1) : static_cast(out_connector_->out_buffers_count()); + } + + /// \brief Getter function + /// \return connector size of current op int32_t ConnectorCapacity() const { if (!inlined()) { return out_connector_->capacity(); @@ -240,51 +256,84 @@ class DatasetOp : public std::enable_shared_from_this { return ChildOpConnectorCapacity(); } - // Getter function - // @return connector size of child op + /// \brief Getter function + /// \return connector size of child op int32_t ChildOpConnectorSize(int32_t child_index = 0) const { return child_[child_index]->ConnectorSize(); } - // Getter function - // @return connector capacity of child op + /// \brief Getter function + /// \return connector capacity of child op int32_t ChildOpConnectorCapacity(int32_t child_index = 0) const { return child_[child_index]->ConnectorCapacity(); } - // Children Getter - // @return Vector of Children + /// \brief Children Getter + /// \return Vector of Children std::vector> Children() const { return child_; } - // Base method for NodePass visit. - // Subclass needs to override this if it requires special node visit access. - // Check "dataset/engine/opt/pass.h" for more details. - // @return Statue of the node visit + /// \brief Base method for NodePass pre-visit. A tree walk consists of walking down the tree and also walking back up + /// in a depth-first order. PreAccept is the node visit on the way down, whereas the regular Accept is the main + /// visit on the way back up the tree during a post-order traversal. Subclass needs to override this if it + /// requires special node visit access. Check "dataset/engine/opt/pass.h" for more details. + /// \param[in] p The node to visit + /// \param[out] modified Indicator if the node was modified + /// \return Status of the node visit + virtual Status PreAccept(NodePass *p, bool *modified); + + /// \brief Base method for NodePass visit. Subclass needs to override this if it requires special node visit access. + /// Check "dataset/engine/opt/pass.h" for more details. + /// \param[in] p The node to visit + /// \param[out] modified Indicator if the node was modified + /// \return Status of the node visit virtual Status Accept(NodePass *p, bool *modified); - // Op name getter - // @return Name of the current Op + /// Op name getter + /// \return Name of the current Op virtual std::string Name() const { return "DatasetOp"; } - // Execution Tree getter - // @return Pointer to the ExecutionTree the current op belongs to, no ownership + /// Execution Tree getter + /// \return Pointer to the ExecutionTree the current op belongs to, no ownership ExecutionTree *Tree() { return tree_; } + /// Getter for the sampler + /// \return Shared pointer to the sampler (may return nullptr) + std::shared_ptr sampler() { return sampler_; } + + /// Computes a CRC value for the operator + static uint32_t GenerateCRC(const std::shared_ptr &op); + + /// \brief A helper templated function for casting "this" pointer to shared_ptr + /// Similar to shared_from_this, except this one will give you the derived class as shared_ptr + /// \return A shared_ptr casted to the derived class + template + std::shared_ptr shared_from_base() { + return std::static_pointer_cast(shared_from_this()); + } + protected: - // Adds a parent operator to this operator - // @notes External callers do not have access to this function. - // @param parent - The parent node to add + /// Adds a parent operator to this operator + /// \notes External callers do not have access to this function. + /// \param parent - The parent node to add void AddParent(DatasetOp *parent); - // Removes a parent operator from this operator - // @notes External callers do not have access to this function. - // @param parent - The parent node to remove - void RemoveParent(DatasetOp *parent); + /// Removes a parent operator from this operator + /// \notes External callers do not have access to this function. + /// \param parent - The parent node to remove + void RemoveParent(const DatasetOp *parent); + + /// Compute the current op's column map using its child's column map. + /// Get called during the tree post-prepare phase in PrepareNodePostAction. + /// This base implementation just inherits the map from child 0, and can only be used if the number of children is 1. + /// Operations changing the column map it inherits from the child must overwrite this function. + /// \return - Status + virtual Status ComputeColMap(); - // A helper function for providing an assignment of the column name map. - // This grabs the map from child 0 and assigns it into this op. - // Can only be used if number of children is 1. - // @return - Status - Status AssignColMapFromChild(); + /// A helper function with some common code that leaf nodes can use during + /// pre/pare phase for checking if they need to assign a sampler to the cache. + /// \param random_access_op - indicate if this is a mappable random access leaf or not + /// \return - Status + Status SaveSamplerForCache(bool random_access_op); std::vector> child_; // Child nodes std::vector parent_; // Parent nodes. No ownership + std::shared_ptr sampler_; // Some leaf ops might have a sampler int32_t oc_queue_size_; // Capacity for each out_connector_ int32_t operator_id_; // Generated id for the node ExecutionTree *tree_; // Back pointer to our tree. @@ -292,18 +341,17 @@ class DatasetOp : public std::enable_shared_from_this { uint32_t op_ctrl_flags_; // Flags for the operator std::unique_ptr out_connector_; // Output Connector std::unordered_map column_name_id_map_; // Mapping between col index and col name - bool first_fetch_; // For use when setting column map std::mutex column_name_map_mutex_; // For protecting shared access to the column map private: - // Sets the operator id. - // @notes No public interface. Only the class itself, or it's friend the execution tree can set - // this - // @param op_id - the Id value to set into the operator + /// Sets the operator id. + /// \notes No public interface. Only the class itself, or it's friend the execution tree can set + /// this + /// \param op_id - the Id value to set into the operator void set_id(int32_t op_id) { operator_id_ = op_id; } - // Sets the tree into the op so that the operator has a back pointer to the tree. - // @param tree - the tree to assign to the op. + /// Sets the tree into the op so that the operator has a back pointer to the tree. + /// \param tree - the tree to assign to the op. void set_tree(ExecutionTree *tree) { tree_ = tree; } }; } // namespace dataset diff --git a/mindspore/ccsrc/dataset/engine/datasetops/device_queue_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/device_queue_op.cc index 84bad9db1a..0f1fefc0f0 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/device_queue_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/device_queue_op.cc @@ -313,7 +313,7 @@ void DeviceQueueOp::Print(std::ostream &out, bool show_all) const { // Visitor accept method for NodePass Status DeviceQueueOp::Accept(NodePass *p, bool *modified) { // Downcast shared pointer then call visitor - return p->RunOnNode(std::static_pointer_cast(shared_from_this()), modified); + return p->RunOnNode(shared_from_base(), modified); } } // namespace dataset diff --git a/mindspore/ccsrc/dataset/engine/datasetops/filter_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/filter_op.cc index 26b99080c8..81c93c6e1c 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/filter_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/filter_op.cc @@ -126,9 +126,6 @@ Status FilterOp::WorkerEntry(int32_t worker_id) { continue; } - // Now that the first fetch is in, use the helper function to assign the column name map to this op. - RETURN_IF_NOT_OK(DatasetOp::AssignColMapFromChild()); - RETURN_IF_NOT_OK(CheckColumns(in_buffer.get(), &in_columns_)); // if the databuffer was all filtered, it is marked as kFilterEmpty. @@ -264,7 +261,7 @@ Status FilterOp::InvokePredicateFunc(const TensorRow &input, bool *out_predicate // Visitor accept method for NodePass Status FilterOp::Accept(NodePass *p, bool *modified) { // Downcast shared pointer then call visitor - return p->RunOnNode(std::static_pointer_cast(shared_from_this()), modified); + return p->RunOnNode(shared_from_base(), modified); } } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/map_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/map_op.cc index 9918260201..05a1ac7925 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/map_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/map_op.cc @@ -54,20 +54,19 @@ Status MapOp::Builder::sanityCheck() const { Status MapOp::Builder::Build(std::shared_ptr *ptr) { RETURN_IF_NOT_OK(sanityCheck()); *ptr = std::make_shared(std::move(build_in_col_names_), std::move(build_out_col_names_), - std::move(build_tensor_funcs_), std::move(build_col_order_), build_num_workers_, - build_op_connector_size_, build_perf_mode_); + std::move(build_tensor_funcs_), build_num_workers_, build_op_connector_size_, + build_perf_mode_); return Status::OK(); } // Constructor of MapOp MapOp::MapOp(const std::vector &in_col_names, const std::vector &out_col_names, - std::vector> tensor_funcs, const std::vector &columns_order, - int32_t num_workers, int32_t op_connector_size, bool perf_mode) + std::vector> tensor_funcs, int32_t num_workers, int32_t op_connector_size, + bool perf_mode) : ParallelOp(num_workers, op_connector_size), tfuncs_(std::move(tensor_funcs)), in_columns_(in_col_names), out_columns_(out_col_names), - columns_order_(columns_order), perf_mode_(perf_mode) { // If caller didn't specify the out_col_names, assume they are same as the in_columns. if (out_columns_.empty() || out_columns_[0].empty()) { @@ -101,7 +100,7 @@ void MapOp::Print(std::ostream &out, bool show_all) const { } out << "\n TensorOps:"; for (size_t i = 0; i < tfuncs_.size(); i++) { - out << " " << tfuncs_[i]; + out << " " << *(tfuncs_[i].get()); } out << "\n\n"; } @@ -156,14 +155,15 @@ Status MapOp::WorkerEntry(int32_t worker_id) { // initializations that happen after the first fetch. RETURN_IF_NOT_OK(FetchNextBuffer(&in_buffer, worker_id)); - // Initialize details related to column selections and column map by calling WorkerEntryInit. - // WorkerEntryInit contains thread-safe lock to ensure that this init work is only performed once - // by the first worker to enter the codepath. All other threads will share the const info that - // gets set up here going forward. + // Sanity check the databuffer. // Special case: if there's more threads than buffers, some threads simply get the final control - // messages (eoe/eof), and so they will not perform the init work. + // messages (eoe/eof), and so they will not perform the check. if (!in_buffer->eoe() && !in_buffer->eof()) { - RETURN_IF_NOT_OK(WorkerEntryInit(in_buffer.get())); + int32_t num_rows = in_buffer->NumRows(); + int32_t num_cols = in_buffer->NumCols(); + if (num_rows == 0 || num_cols == 0) { + RETURN_STATUS_UNEXPECTED("MapOp is getting an empty DataBuffer."); + } } // Now that init work is done, drop into the main fetching loop. @@ -258,63 +258,18 @@ Status MapOp::WorkerCompute(DataBuffer *in_buffer, TensorQTable *new_tensor_tabl return Status::OK(); } -// initialize some internal data structure used by WorkerEntry() -Status MapOp::WorkerEntryInit(const DataBuffer *in_buf) { - int32_t num_rows = in_buf->NumRows(); - int32_t num_cols = in_buf->NumCols(); - if (num_rows == 0 || num_cols == 0) { - RETURN_STATUS_UNEXPECTED("MapOp is getting an empty DataBuffer."); +Status MapOp::ComputeColMap() { + // If the map has not been set up yet in the base class, then set it up + if (column_name_id_map_.empty()) { + std::unordered_map current_name_id_map = child_[0]->column_name_id_map(); + // Initialize private variables + RETURN_IF_NOT_OK(InitPrivateVariable(¤t_name_id_map)); + // Create the final column name to index mapping in the base class field + CreateFinalColMap(¤t_name_id_map); + MS_LOG(DEBUG) << "Column name map for map op set: " << this->ColumnNameMapAsString(); + } else { + MS_LOG(WARNING) << "Column name map is already set!"; } - - // We can't use AssignColMapFromChild() here since we need to modify the column map. We need to be threadsafe - // though for saving the final map in the op, so use the lock here. - if (first_fetch_) { - std::unique_lock lock(column_name_map_mutex_); - // If the map has not been set up yet in the base class, then we are the first one in to set it up - // (and we are under protection of the mutex lock) - if (column_name_id_map_.empty()) { - std::unordered_map current_name_id_map = child_[0]->column_name_id_map(); - - // If input_columns is empty(), The col at index-0 will be picked. - if (in_columns_.empty()) { - for (const auto &pair : current_name_id_map) { - if (pair.second == 0) { - MS_LOG(INFO) << "Input columns empty for map op, will apply to the first column in the current table."; - in_columns_.push_back(pair.first); - break; - } - } - - // If caller didn't specify the out_col_names, assume they are same as the input_columns. - // This was done in the constructor, but if input columns was empty to start we have to redo it here. - if (out_columns_.empty() || out_columns_[0].empty()) { - out_columns_ = in_columns_; - } - } - - // Before we continue, issue a sanity check to make sure the input columns from user and the incoming - // columns from child are correct - RETURN_IF_NOT_OK(this->ValidateInColumns(current_name_id_map)); - - // initialize keep_input_columns, true means to keep the column. - keep_input_columns_.resize(num_cols, true); - for (const auto &col_name : in_columns_) { - int32_t missed = current_name_id_map[col_name]; - keep_input_columns_[missed] = false; - } - - // initialize to_process_indices. - for (const auto &col_name : in_columns_) { - to_process_indices_.push_back(current_name_id_map[col_name]); - } - - // Create the final column name to index mapping in the base class field - CreateFinalColMap(¤t_name_id_map); - first_fetch_ = false; - } - } // mutex lock will release here - - MS_LOG(DEBUG) << "Column name map for map op set: " << this->ColumnNameMapAsString(); return Status::OK(); } @@ -330,6 +285,42 @@ Status MapOp::ValidateInColumns(const std::unordered_map & return Status::OK(); } +Status MapOp::InitPrivateVariable(std::unordered_map *col_name_id_map) { + // If input_columns is empty(), The col at index-0 will be picked. + if (in_columns_.empty()) { + for (const auto &pair : *col_name_id_map) { + if (pair.second == 0) { + MS_LOG(INFO) << "Input columns empty for map op, will apply to the first column in the current table."; + in_columns_.push_back(pair.first); + break; + } + } + + // If caller didn't specify the out_col_names, assume they are same as the input_columns. + // This was done in the constructor, but if input columns was empty to start we have to redo it here. + if (out_columns_.empty() || out_columns_[0].empty()) { + out_columns_ = in_columns_; + } + } + + // Before we continue, issue a sanity check to make sure the input columns from user and the incoming + // columns from child are correct + RETURN_IF_NOT_OK(this->ValidateInColumns(*col_name_id_map)); + + // initialize keep_input_columns, true means to keep the column. + keep_input_columns_.resize(col_name_id_map->size(), true); + for (const auto &col_name : in_columns_) { + int32_t missed = (*col_name_id_map)[col_name]; + keep_input_columns_[missed] = false; + } + + // initialize to_process_indices. + for (const auto &col_name : in_columns_) { + to_process_indices_.push_back((*col_name_id_map)[col_name]); + } + return Status::OK(); +} + // Create the final column name to index mapping and get indices of the columns this mapop does not use. void MapOp::CreateFinalColMap(std::unordered_map *col_name_id_map) { std::unordered_map final_col_name_id_map; @@ -376,7 +367,7 @@ void MapOp::CreateFinalColMap(std::unordered_map *col_name // Visitor accept method for NodePass Status MapOp::Accept(NodePass *p, bool *modified) { // Downcast shared pointer then call visitor - return p->RunOnNode(std::static_pointer_cast(shared_from_this()), modified); + return p->RunOnNode(shared_from_base(), modified); } } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/map_op.h b/mindspore/ccsrc/dataset/engine/datasetops/map_op.h index 4d7ffd1204..371d865196 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/map_op.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/map_op.h @@ -93,13 +93,6 @@ class MapOp : public ParallelOp { return *this; } - // Setter method. - // @return Builder setter method returns reference to the builder. - Builder &SetColOrder(const std::vector &col_order_) { - build_col_order_ = col_order_; - return *this; - } - // Setter method. // @return Builder setter method returns reference to the builder. Builder &SetNumWorkers(int32_t num_workers) { @@ -130,7 +123,6 @@ class MapOp : public ParallelOp { std::vector build_in_col_names_; std::vector build_out_col_names_; std::vector> build_tensor_funcs_; - std::vector build_col_order_; int32_t build_num_workers_; int32_t build_op_connector_size_; bool build_perf_mode_; // Default true. @@ -145,12 +137,11 @@ class MapOp : public ParallelOp { // @param in_col_names A list of input column names (should match the input/output \p tensorFuncs). // @param out_col_names A list of output column names (should match the input/output \p tensorFuncs). // @param tensor_funcs A list of TensorOp pointers for MapOp to apply to each data. - // @param columns_order names A full list of column names (should match the whole dataset view post \p tensorFuncs). // @param num_workers The number of worker threads. // @param op_connector_size The size of each queue in the connector. MapOp(const std::vector &in_col_names, const std::vector &out_col_names, - std::vector> tensor_funcs, const std::vector &columns_order, - int32_t num_workers, int32_t op_connector_size, bool perf_mode); + std::vector> tensor_funcs, int32_t num_workers, int32_t op_connector_size, + bool perf_mode); // Destructor ~MapOp() = default; @@ -190,10 +181,6 @@ class MapOp : public ParallelOp { // @return Name of the current Op std::string Name() const override { return "MapOp"; } - // Columns order getter - // @return The post map columns order - std::vector const &ColumnsOrder() const { return columns_order_; } - private: // Local queues where worker threads can pop from. // Popping directly from the Connector can block if the previous designated threads haven't pop. @@ -215,9 +202,6 @@ class MapOp : public ParallelOp { // Indices of the columns to process. std::vector to_process_indices_; - // Variable to store the column_order of all columns post tensorOps - std::vector columns_order_; - // Performance mode is when the main thread creates local queues, pulls databuffers from the previous // op's Connector and distributes them to the local queues. Workers pull from the local queues. // If this flag is false, each worker pulls directly from the Connector. This use less resources @@ -258,15 +242,18 @@ class MapOp : public ParallelOp { // @param col_name_id_map The column name to index mapping obtained from child operator void CreateFinalColMap(std::unordered_map *col_name_id_map); - // Private function that initialize some internal data structure used by WorkerEntry() - // @param in_buf A raw pointer to the DataBuffer. A raw pointer is fine because this function does not manage memory - // and is not shared with other threads. - Status WorkerEntryInit(const DataBuffer *in_buf); - // Validating if each of the input_columns exists in the DataBuffer. // @param - the column map to check // @return - status return code Status ValidateInColumns(const std::unordered_map &col_name_id_map); + + // Private function for computing the assignment of the column name map. + // @return - Status + Status ComputeColMap() override; + + // Private function for initializing private variables such as in_columns_, out_columns_. + // @return - Status + Status InitPrivateVariable(std::unordered_map *col_name_id_map); }; } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/parallel_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/parallel_op.cc index c0a8d95f15..244861a6c8 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/parallel_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/parallel_op.cc @@ -26,8 +26,8 @@ namespace mindspore { namespace dataset { // Constructor -ParallelOp::ParallelOp(int32_t num_workers, int32_t op_connector_size) - : DatasetOp(op_connector_size), +ParallelOp::ParallelOp(int32_t num_workers, int32_t op_connector_size, std::shared_ptr sampler) + : DatasetOp(op_connector_size, sampler), num_workers_(num_workers), num_producers_(num_workers), worker_connector_size_(1), diff --git a/mindspore/ccsrc/dataset/engine/datasetops/parallel_op.h b/mindspore/ccsrc/dataset/engine/datasetops/parallel_op.h index 142ec78360..f59d4bfc53 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/parallel_op.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/parallel_op.h @@ -38,7 +38,8 @@ class ParallelOp : public DatasetOp { // Constructor // @param num_workers // @param op_connector_size - size of the output connector for this operator - ParallelOp(int32_t num_workers, int32_t op_connector_size); + // @param sampler - The sampler for the op + ParallelOp(int32_t num_workers, int32_t op_connector_size, std::shared_ptr sampler = nullptr); // Destructor ~ParallelOp() = default; diff --git a/mindspore/ccsrc/dataset/engine/datasetops/pipeline_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/pipeline_op.cc index 46eded8ea1..1d017a4d3e 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/pipeline_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/pipeline_op.cc @@ -20,7 +20,8 @@ namespace mindspore { namespace dataset { // Constructor -PipelineOp::PipelineOp(int32_t op_connector_size) : DatasetOp(op_connector_size) {} +PipelineOp::PipelineOp(int32_t op_connector_size, std::shared_ptr sampler) + : DatasetOp(op_connector_size, sampler) {} // A print method typically used for debugging void PipelineOp::Print(std::ostream &out, bool show_all) const { diff --git a/mindspore/ccsrc/dataset/engine/datasetops/pipeline_op.h b/mindspore/ccsrc/dataset/engine/datasetops/pipeline_op.h index a14279032d..cb3c76813b 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/pipeline_op.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/pipeline_op.h @@ -32,7 +32,8 @@ class PipelineOp : public DatasetOp { // Constructor // @param op_connector_size - size of the output connector // @return Builder setter method returns reference to the builder. - explicit PipelineOp(int32_t op_connector_size); + // @param sampler - The sampler for the op + explicit PipelineOp(int32_t op_connector_size, std::shared_ptr sampler = nullptr); // Destructor ~PipelineOp() = default; diff --git a/mindspore/ccsrc/dataset/engine/datasetops/project_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/project_op.cc index 063c618aeb..5ce4056024 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/project_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/project_op.cc @@ -74,24 +74,6 @@ void ProjectOp::Print(std::ostream &out, bool show_all) const { Status ProjectOp::GetNextBuffer(std::unique_ptr *p_buffer, int32_t worker_id, bool retry_if_eoe) { RETURN_IF_NOT_OK(child_[0]->GetNextBuffer(p_buffer, worker_id, retry_if_eoe)); if (!((*p_buffer)->eoe()) && !((*p_buffer)->eof())) { - // Only for the first buffer fetched, get the column map of the incoming data and save it - // into our own column name map after making the appropriate mods - // We cannot use the super class AssignColMapFromChild here because we're making a modification of the - // map from the child map. - if (first_fetch_) { - std::unordered_map child_column_name_mapping = child_[0]->column_name_id_map(); - for (size_t i = 0; i < columns_to_project_.size(); i++) { - std::string ¤t_column = columns_to_project_[i]; - if (child_column_name_mapping.find(current_column) == child_column_name_mapping.end()) { - std::string err_msg = "ProjectOp: column " + current_column + " does not exist in child operator."; - RETURN_STATUS_UNEXPECTED(err_msg); - } - // Setup the new column name mapping for ourself (base class field) - column_name_id_map_[current_column] = i; - projected_column_indices_.push_back(child_column_name_mapping[current_column]); - } - first_fetch_ = false; // we only need to do this path once - } RETURN_IF_NOT_OK(Project(p_buffer)); } return Status::OK(); @@ -149,7 +131,29 @@ Status ProjectOp::EofReceived(int32_t worker_id) { return Status::OK(); } // Visitor accept method for NodePass Status ProjectOp::Accept(NodePass *p, bool *modified) { // Downcast shared pointer then call visitor - return p->RunOnNode(std::static_pointer_cast(shared_from_this()), modified); + return p->RunOnNode(shared_from_base(), modified); +} + +// Compute the column map and save it into our own column name map +// We cannot use the super class ComputeColMap here because we're making a modification of the +// map from the child map. +Status ProjectOp::ComputeColMap() { + if (column_name_id_map_.empty()) { + std::unordered_map child_column_name_mapping = child_[0]->column_name_id_map(); + for (size_t i = 0; i < columns_to_project_.size(); i++) { + std::string ¤t_column = columns_to_project_[i]; + if (child_column_name_mapping.find(current_column) == child_column_name_mapping.end()) { + std::string err_msg = "ProjectOp: column " + current_column + " does not exist in child operator."; + RETURN_STATUS_UNEXPECTED(err_msg); + } + // Setup the new column name mapping for ourself (base class field) + column_name_id_map_[current_column] = i; + projected_column_indices_.push_back(child_column_name_mapping[current_column]); + } + } else { + MS_LOG(WARNING) << "Column name map is already set!"; + } + return Status::OK(); } } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/project_op.h b/mindspore/ccsrc/dataset/engine/datasetops/project_op.h index ced0f9e5a9..628c1342ba 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/project_op.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/project_op.h @@ -116,6 +116,10 @@ class ProjectOp : public PipelineOp { std::vector projected_column_indices_; Status Project(std::unique_ptr *data_buffer); + + // Computing the assignment of the column name map. + // @return - Status + Status ComputeColMap() override; }; } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/rename_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/rename_op.cc index c026aac4fa..23cd29d295 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/rename_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/rename_op.cc @@ -69,12 +69,6 @@ Status RenameOp::operator()() { RETURN_STATUS_UNEXPECTED(err_msg); } - // First, populate the column map from the input child. - // This will not be the final map for output from this op. - RETURN_IF_NOT_OK(DatasetOp::AssignColMapFromChild()); - // core rename functionality only needs to happen once, to identify the new column names/indexes - RETURN_IF_NOT_OK(RenameColumns()); - while (curr_buffer->eof() == false) { while (curr_buffer->eoe() == false) { // push the renamed input buffer @@ -95,45 +89,52 @@ Status RenameOp::operator()() { return Status::OK(); } -// renames the columns -Status RenameOp::RenameColumns() { - // iterate over my index in input vector, find the corresponding position - std::unordered_map new_col_name_id_map = {}; - // parameter for input check - size_t found = 0; - - // iterate over all the pairs and if there is a name match with rename, rename the column and add it to new map - // by doing it this way we recreate a new ColNameIdMap and allow for switching - for (const auto &pair : column_name_id_map_) { - std::string name = pair.first; - int32_t id = pair.second; - // find name - std::vector::iterator it; - it = std::find(in_columns_.begin(), in_columns_.end(), name); - // for c input checks here we have to count the number of times we find the stuff in in_columns_ - // because we iterate over the mInputList n times - if (it != in_columns_.end()) { - // found - found += 1; - int index = std::distance(in_columns_.begin(), it); - MS_LOG(DEBUG) << "Rename operator index found " << index << " value " << id << "."; - - new_col_name_id_map[out_columns_[index]] = id; - } else { - // not found - MS_LOG(DEBUG) << "Rename operator index not found: " << id << " is the column id."; - new_col_name_id_map[name] = id; +// Rename core functionality to compute the new column name id map. +// We need to overwrite the super class ComputeColMap here because we're making a modification of the +// map from the child map. +Status RenameOp::ComputeColMap() { + if (column_name_id_map_.empty()) { + column_name_id_map_ = child_[0]->column_name_id_map(); + // iterate over my index in input vector, find the corresponding position + std::unordered_map new_col_name_id_map = {}; + // parameter for input check + size_t found = 0; + + // iterate over all the pairs and if there is a name match with rename, rename the column and add it to new map + // by doing it this way we recreate a new ColNameIdMap and allow for switching + for (const auto &pair : column_name_id_map_) { + std::string name = pair.first; + int32_t id = pair.second; + // find name + std::vector::iterator it; + it = std::find(in_columns_.begin(), in_columns_.end(), name); + // for c input checks here we have to count the number of times we find the stuff in in_columns_ + // because we iterate over the mInputList n times + if (it != in_columns_.end()) { + // found + found += 1; + int index = std::distance(in_columns_.begin(), it); + MS_LOG(DEBUG) << "Rename operator index found " << index << " value " << id << "."; + + new_col_name_id_map[out_columns_[index]] = id; + } else { + // not found + MS_LOG(DEBUG) << "Rename operator index not found: " << id << " is the column id."; + new_col_name_id_map[name] = id; + } + } + // only checks number of renamed columns have been found, this input check doesn't check everything + if (found != in_columns_.size()) { + MS_LOG(DEBUG) << "Rename operator column names found: " << found << " out of " << in_columns_.size() << "."; + std::string err_msg = "Renamed column doesn't exist in dataset"; + RETURN_STATUS_UNEXPECTED(err_msg); } - } - // only checks number of renamed columns have been found, this input check doesn't check everything - if (found != in_columns_.size()) { - MS_LOG(DEBUG) << "Rename operator column names found: " << found << " out of " << in_columns_.size() << "."; - std::string err_msg = "Renamed column doesn't exist in dataset"; - RETURN_STATUS_UNEXPECTED(err_msg); - } - // Now, overwrite our column map with the new renamed columns/id's - column_name_id_map_ = new_col_name_id_map; + // Now, overwrite our column map with the new renamed columns/id's + column_name_id_map_ = new_col_name_id_map; + } else { + MS_LOG(WARNING) << "Column name map is already set!"; + } return Status::OK(); } @@ -175,7 +176,7 @@ Status RenameOp::EoeReceived(int32_t) { // Visitor accept method for NodePass Status RenameOp::Accept(NodePass *p, bool *modified) { // Downcast shared pointer then call visitor - return p->RunOnNode(std::static_pointer_cast(shared_from_this()), modified); + return p->RunOnNode(shared_from_base(), modified); } } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/rename_op.h b/mindspore/ccsrc/dataset/engine/datasetops/rename_op.h index eaca20ccc8..e209c075d6 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/rename_op.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/rename_op.h @@ -122,7 +122,9 @@ class RenameOp : public PipelineOp { protected: // Rename core functionality - Status RenameColumns(); + // Computing the assignment of the new column name map. + // @return - Status + Status ComputeColMap() override; // Variable to store the input column names std::vector in_columns_; diff --git a/mindspore/ccsrc/dataset/engine/datasetops/repeat_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/repeat_op.cc index 524603fd94..4999dddd02 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/repeat_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/repeat_op.cc @@ -82,14 +82,14 @@ void RepeatOp::Print(std::ostream &out, bool show_all) const { Status RepeatOp::PrepareNodePostAction() { // Run any common code from super class first before adding our own specific logic RETURN_IF_NOT_OK(PipelineOp::PrepareNodePostAction()); - std::shared_ptr leaf_op = tree_->PopFromRepeatStack(); + std::shared_ptr leaf_op = tree_->PopFromEOEOpStack(); while (leaf_op != nullptr) { // Track the leaf operators that are under this repeat op. eoe_ops_.push_back(leaf_op); - leaf_op = tree_->PopFromRepeatStack(); + leaf_op = tree_->PopFromEOEOpStack(); } // Push ourselves to the stack in case one of our ascendants is repeat too. - tree_->AddToRepeatStack(shared_from_this()); + tree_->AddToEOEOpStack(shared_from_this()); return Status::OK(); } @@ -123,8 +123,6 @@ Status RepeatOp::GetNextBuffer(std::unique_ptr *p_buffer, int32_t wo if (buf->eof()) { RETURN_IF_NOT_OK(EofReceived(worker_id)); } - // Update the column name map if needed - RETURN_IF_NOT_OK(DatasetOp::AssignColMapFromChild()); *p_buffer = std::move(buf); return Status::OK(); } @@ -192,7 +190,7 @@ int32_t RepeatOp::num_producers() const { // Visitor accept method for NodePass Status RepeatOp::Accept(NodePass *p, bool *modified) { // Downcast shared pointer then call visitor - return p->RunOnNode(std::static_pointer_cast(shared_from_this()), modified); + return p->RunOnNode(shared_from_base(), modified); } } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/shuffle_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/shuffle_op.cc index 69f7e09986..f86fcc602b 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/shuffle_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/shuffle_op.cc @@ -266,9 +266,6 @@ Status ShuffleOp::InitShuffleBuffer() { RETURN_STATUS_UNEXPECTED("Unable to fetch a single row for shuffle buffer."); } - // Now that a first fetch is done, assign the column map for this operator - RETURN_IF_NOT_OK(DatasetOp::AssignColMapFromChild()); - // Now fill the rest of the shuffle buffer until we are unable to get the next row or we reached // the desired shuffle buffer size. while (!new_row.empty() && shuffle_buffer_->size() < static_cast(shuffle_size_ - 1)) { @@ -301,7 +298,7 @@ Status ShuffleOp::EoeReceived(int32_t worker_id) { // Visitor accept method for NodePass Status ShuffleOp::Accept(NodePass *p, bool *modified) { // Downcast shared pointer then call visitor - return p->RunOnNode(std::static_pointer_cast(shared_from_this()), modified); + return p->RunOnNode(shared_from_base(), modified); } } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/skip_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/skip_op.cc index 35fae8d091..f6b0fe689c 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/skip_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/skip_op.cc @@ -86,9 +86,6 @@ Status SkipOp::operator()() { std::unique_ptr curr_buffer; RETURN_IF_NOT_OK(GetNextInput(&curr_buffer)); - // After the first buffer fetch above we can do the one-time assign of the column name map - RETURN_IF_NOT_OK(DatasetOp::AssignColMapFromChild()); - while (curr_buffer->eof() == false) { // Reset count skip_count_ = 0; @@ -133,7 +130,7 @@ Status SkipOp::EofReceived(int32_t worker_id) { // Visitor accept method for NodePass Status SkipOp::Accept(NodePass *p, bool *modified) { // Downcast shared pointer then call visitor - return p->RunOnNode(std::static_pointer_cast(shared_from_this()), modified); + return p->RunOnNode(shared_from_base(), modified); } } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/celeba_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/celeba_op.cc index 4b32201d6d..c7a4269a39 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/celeba_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/celeba_op.cc @@ -70,20 +70,14 @@ Status CelebAOp::Builder::SanityCheck() { CelebAOp::CelebAOp(int32_t num_workers, int32_t rows_per_buffer, const std::string &dir, int32_t queue_size, bool decode, const std::string &dataset_type, const std::set &exts, std::unique_ptr schema, std::shared_ptr sampler) - : ParallelOp(num_workers, queue_size), + : ParallelOp(num_workers, queue_size, std::move(sampler)), rows_per_buffer_(rows_per_buffer), folder_path_(dir), decode_(decode), extensions_(exts), data_schema_(std::move(schema)), - sampler_(std::move(sampler)), num_rows_in_attr_file_(0), dataset_type_(dataset_type) { - // Set the column name map (base class field) - for (int32_t index = 0; index < data_schema_->NumColumns(); index++) { - column_name_id_map_[data_schema_->column(index).name()] = index; - } - attr_info_queue_ = std::make_unique>>(queue_size); io_block_queues_.Init(num_workers_, queue_size); } @@ -413,5 +407,17 @@ Status CelebAOp::Reset() { wp_.Set(); // wake up master thread after reset is done return Status::OK(); } + +Status CelebAOp::ComputeColMap() { + // Set the column name map (base class field) + if (column_name_id_map_.empty()) { + for (int32_t index = 0; index < data_schema_->NumColumns(); index++) { + column_name_id_map_[data_schema_->column(index).name()] = index; + } + } else { + MS_LOG(WARNING) << "Column name map is already set!"; + } + return Status::OK(); +} } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/celeba_op.h b/mindspore/ccsrc/dataset/engine/datasetops/source/celeba_op.h index f4b5d040ca..a6fa495a14 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/celeba_op.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/celeba_op.h @@ -212,12 +212,15 @@ class CelebAOp : public ParallelOp, RandomAccessOp { // @return Status - The error code return Status Reset() override; + // Private function for computing the assignment of the column name map. + // @return - Status + Status ComputeColMap() override; + int32_t rows_per_buffer_; std::string folder_path_; // directory of celeba folder bool decode_; std::set extensions_; // extensions allowed std::unique_ptr data_schema_; - std::shared_ptr sampler_; std::unique_ptr>> attr_info_queue_; int64_t num_rows_in_attr_file_; // rows number specified in attr file QueueList> io_block_queues_; diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/cifar_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/cifar_op.cc index ad87e394eb..8dd615a8c1 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/cifar_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/cifar_op.cc @@ -79,18 +79,13 @@ Status CifarOp::Builder::SanityCheck() { CifarOp::CifarOp(CifarType type, int32_t num_works, int32_t rows_per_buf, const std::string &file_dir, int32_t queue_size, std::unique_ptr data_schema, std::shared_ptr sampler) - : ParallelOp(num_works, queue_size), + : ParallelOp(num_works, queue_size, std::move(sampler)), cifar_type_(type), rows_per_buffer_(rows_per_buf), folder_path_(file_dir), data_schema_(std::move(data_schema)), - sampler_(std::move(sampler)), row_cnt_(0), buf_cnt_(0) { - // set the column name map (base class field) - for (uint32_t i = 0; i < data_schema_->NumColumns(); ++i) { - column_name_id_map_[data_schema_->column(i).name()] = i; - } constexpr uint64_t kUtilQueueSize = 512; cifar_raw_data_block_ = std::make_unique>>(kUtilQueueSize); io_block_queues_.Init(num_workers_, queue_size); @@ -454,5 +449,17 @@ Status CifarOp::CountTotalRows(const std::string &dir, bool isCIFAR10, int64_t * return Status::OK(); } } + +Status CifarOp::ComputeColMap() { + // set the column name map (base class field) + if (column_name_id_map_.empty()) { + for (uint32_t i = 0; i < data_schema_->NumColumns(); ++i) { + column_name_id_map_[data_schema_->column(i).name()] = i; + } + } else { + MS_LOG(WARNING) << "Column name map is already set!"; + } + return Status::OK(); +} } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/cifar_op.h b/mindspore/ccsrc/dataset/engine/datasetops/source/cifar_op.h index 62c20ac401..917b23db94 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/cifar_op.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/cifar_op.h @@ -208,11 +208,14 @@ class CifarOp : public ParallelOp, public RandomAccessOp { // @return Status - The error code return Status GetClassIds(std::map> *cls_ids) const override; + // Private function for computing the assignment of the column name map. + // @return - Status + Status ComputeColMap() override; + CifarType cifar_type_; int32_t rows_per_buffer_; std::string folder_path_; std::unique_ptr data_schema_; - std::shared_ptr sampler_; int64_t row_cnt_; int64_t buf_cnt_; diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/clue_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/clue_op.cc index e92ca0d26c..9fceb6f333 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/clue_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/clue_op.cc @@ -31,11 +31,7 @@ namespace mindspore { namespace dataset { ClueOp::Builder::Builder() - : builder_device_id_(0), - builder_num_devices_(1), - builder_num_samples_(0), - builder_shuffle_files_(false), - builder_shuffle_global_(false) { + : builder_device_id_(0), builder_num_devices_(1), builder_num_samples_(0), builder_shuffle_files_(false) { std::shared_ptr config_manager = GlobalContext::config_manager(); builder_num_workers_ = config_manager->num_parallel_workers(); builder_op_connector_size_ = config_manager->op_connector_size(); @@ -66,8 +62,8 @@ Status ClueOp::Builder::Build(std::shared_ptr *op) { std::shared_ptr clue_op = std::make_shared( builder_num_workers_, builder_rows_per_buffer_, builder_num_samples_, builder_worker_connector_size_, ck_map, - builder_clue_files_list_, builder_op_connector_size_, builder_shuffle_files_, builder_shuffle_global_, - builder_num_devices_, builder_device_id_); + builder_clue_files_list_, builder_op_connector_size_, builder_shuffle_files_, builder_num_devices_, + builder_device_id_); RETURN_IF_NOT_OK(clue_op->Init()); *op = std::move(clue_op); @@ -87,7 +83,7 @@ std::vector ClueOp::Builder::split(const std::string &s, char delim ClueOp::ClueOp(int32_t num_workers, int64_t rows_per_buffer, int64_t num_samples, int32_t worker_connector_size, ColKeyMap cols_to_keyword, std::vector clue_files_list, int32_t op_connector_size, - bool shuffle_files, bool shuffle_global, int32_t num_device, int32_t device_id) + bool shuffle_files, int32_t num_device, int32_t device_id) : ParallelOp(num_workers, op_connector_size), rows_per_buffer_(rows_per_buffer), num_rows_per_shard_(0), @@ -98,7 +94,6 @@ ClueOp::ClueOp(int32_t num_workers, int64_t rows_per_buffer, int64_t num_samples load_jagged_connector_(true), cols_to_keyword_(cols_to_keyword), shuffle_files_(shuffle_files), - shuffle_global_(shuffle_global), finished_reading_dataset_(false), num_devices_(num_device), device_id_(device_id), @@ -112,13 +107,6 @@ Status ClueOp::Init() { int32_t safe_queue_size = static_cast(std::ceil(clue_files_list_.size() / num_workers_) + 1); io_block_queues_.Init(num_workers_, safe_queue_size); - // Set the column name mapping (base class field) - int count = 0; - for (auto &p : cols_to_keyword_) { - column_name_id_map_[p.first] = count; - count++; - } - RETURN_IF_NOT_OK(ParallelOp::CreateWorkerConnector(worker_connector_size_)); jagged_buffer_connector_ = std::make_unique(num_workers_, 1, worker_connector_size_); @@ -549,5 +537,19 @@ Status ClueOp::CountAllFileRows(const std::vector &files, int64_t * } return Status::OK(); } + +Status ClueOp::ComputeColMap() { + // Set the column name mapping (base class field) + if (column_name_id_map_.empty()) { + int count = 0; + for (auto &p : cols_to_keyword_) { + column_name_id_map_[p.first] = count; + count++; + } + } else { + MS_LOG(WARNING) << "Column name map is already set!"; + } + return Status::OK(); +} } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/clue_op.h b/mindspore/ccsrc/dataset/engine/datasetops/source/clue_op.h index b6a797d3f4..487ed0d47f 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/clue_op.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/clue_op.h @@ -104,13 +104,6 @@ class ClueOp : public ParallelOp { return *this; } - // Setter method. - // @return Builder - setter method returns reference to the builder. - Builder &SetShuffleGlobal(bool shuffle_global) { - builder_shuffle_global_ = shuffle_global; - return *this; - } - // Setter method. // @return Builder - setter method returns reference to the builder. Builder &SetNumSamples(int64_t num_samples) { @@ -139,15 +132,13 @@ class ClueOp : public ParallelOp { int32_t builder_worker_connector_size_; std::vector builder_clue_files_list_; bool builder_shuffle_files_; - bool builder_shuffle_global_; std::map builder_cols_to_keyword_; }; // Constructor of ClueOp - // @param shuffle_global - whether or not to shuffle the entire dataset. ClueOp(int32_t num_workers, int64_t rows_per_buffer, int64_t num_samples, int32_t worker_connector_size, ColKeyMap cols_to_keyword, std::vector clue_files_list, int32_t op_connector_size, - bool shuffle_files, bool shuffle_global, int32_t num_devices, int32_t device_id); + bool shuffle_files, int32_t num_devices, int32_t device_id); // Default destructor ~ClueOp() = default; @@ -182,10 +173,6 @@ class ClueOp : public ParallelOp { // @return Vector of the input file names std::vector FileNames() { return clue_files_list_; } - // Global shuffle flag getter - // @return Bool - whether this Op requires global shuffle - bool RequireGlobalShuffle() { return shuffle_global_; } - private: // The entry point for when workers are launched. // @param worker_id - the id of the worker that is executing this function. @@ -263,9 +250,12 @@ class ClueOp : public ParallelOp { // @return Status - the error code returned. Status GetValue(const nlohmann::json &js, std::vector key_chain, std::shared_ptr *t); + // Private function for computing the assignment of the column name map. + // @return - Status + Status ComputeColMap() override; + int32_t device_id_; bool shuffle_files_; - bool shuffle_global_; bool finished_reading_dataset_; int32_t num_devices_; int64_t rows_per_buffer_; diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/coco_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/coco_op.cc index 8d352bbd6c..92f6794769 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/coco_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/coco_op.cc @@ -129,10 +129,6 @@ CocoOp::CocoOp(const TaskType &task_type, const std::string &image_folder_path, rows_per_buffer_(rows_per_buffer), sampler_(std::move(sampler)), data_schema_(std::move(data_schema)) { - // Set the column name map (base class field) - for (int32_t i = 0; i < data_schema_->NumColumns(); ++i) { - column_name_id_map_[data_schema_->column(i).name()] = i; - } io_block_queues_.Init(num_workers_, queue_size); } @@ -627,5 +623,17 @@ Status CocoOp::GetClassIndexing(const std::string &dir, const std::string &file, *output_class_indexing = op->label_index_; return Status::OK(); } + +Status CocoOp::ComputeColMap() { + // Set the column name map (base class field) + if (column_name_id_map_.empty()) { + for (int32_t i = 0; i < data_schema_->NumColumns(); ++i) { + column_name_id_map_[data_schema_->column(i).name()] = i; + } + } else { + MS_LOG(WARNING) << "Column name map is already set!"; + } + return Status::OK(); +} } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/coco_op.h b/mindspore/ccsrc/dataset/engine/datasetops/source/coco_op.h index f5abeed72e..3791853798 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/coco_op.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/coco_op.h @@ -306,6 +306,10 @@ class CocoOp : public ParallelOp, public RandomAccessOp { template Status SearchNodeInJson(nlohmann::json input_tree, std::string node_name, T *output_node); + // Private function for computing the assignment of the column name map. + // @return - Status + Status ComputeColMap() override; + bool decode_; int64_t row_cnt_; int64_t buf_cnt_; diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/generator_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/generator_op.cc index d316524c04..36c221fc16 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/generator_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/generator_op.cc @@ -94,12 +94,6 @@ void GeneratorOp::Dealloc() noexcept { Status GeneratorOp::Init() { // Reset BufferID buffer_id_ = 0; - // Setup column names map (base class field) - if (column_name_id_map_.empty()) { - for (int i = 0; i < column_names_.size(); ++i) { - column_name_id_map_[column_names_[i]] = i; - } - } Status ret; { // Acquire Python GIL @@ -255,7 +249,19 @@ Status GeneratorOp::Reset() { // Visitor accept method for NodePass Status GeneratorOp::Accept(NodePass *p, bool *modified) { // Downcast shared pointer then call visitor - return p->RunOnNode(std::static_pointer_cast(shared_from_this()), modified); + return p->RunOnNode(shared_from_base(), modified); +} + +Status GeneratorOp::ComputeColMap() { + // Setup column names map (base class field) + if (column_name_id_map_.empty()) { + for (int i = 0; i < column_names_.size(); ++i) { + column_name_id_map_[column_names_[i]] = i; + } + } else { + MS_LOG(WARNING) << "Column name map is already set!"; + } + return Status::OK(); } } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/generator_op.h b/mindspore/ccsrc/dataset/engine/datasetops/source/generator_op.h index 82b395d6de..98dd2d70a1 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/generator_op.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/generator_op.h @@ -150,6 +150,10 @@ class GeneratorOp : public PipelineOp { Status PyRowToTensorRow(py::object py_data, TensorRow *tensor_row); Status FillBuffer(TensorQTable *tt); + + // Private function for computing the assignment of the column name map. + // @return - Status + Status ComputeColMap() override; }; #pragma GCC visibility pop diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/image_folder_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/image_folder_op.cc index 5cdfa8bb76..837eae1e3c 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/image_folder_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/image_folder_op.cc @@ -65,7 +65,7 @@ ImageFolderOp::ImageFolderOp(int32_t num_wkrs, int32_t rows_per_buffer, std::str bool recursive, bool do_decode, const std::set &exts, const std::map &map, std::unique_ptr data_schema, std::shared_ptr sampler) - : ParallelOp(num_wkrs, queue_size), + : ParallelOp(num_wkrs, queue_size, std::move(sampler)), rows_per_buffer_(rows_per_buffer), folder_path_(file_dir), recursive_(recursive), @@ -73,15 +73,10 @@ ImageFolderOp::ImageFolderOp(int32_t num_wkrs, int32_t rows_per_buffer, std::str extensions_(exts), class_index_(map), data_schema_(std::move(data_schema)), - sampler_(std::move(sampler)), row_cnt_(0), buf_cnt_(0), sampler_ind_(0), dirname_offset_(0) { - // Set the column name map (base class field) - for (int32_t i = 0; i < data_schema_->NumColumns(); ++i) { - column_name_id_map_[data_schema_->column(i).name()] = i; - } folder_name_queue_ = std::make_unique>(num_wkrs * queue_size); image_name_queue_ = std::make_unique>(num_wkrs * queue_size); io_block_queues_.Init(num_workers_, queue_size); @@ -108,7 +103,7 @@ Status ImageFolderOp::PrescanMasterEntry(const std::string &filedir) { // following loop puts the 2 level of shuffles together into 1 vector for (size_t ind = 0; ind < v.size(); ++ind) { while (v[ind]->second.empty() == false) { - DS_ASSERT(!(v[ind]->first.empty())); // make sure that v[ind]->first.substr(1) is not out of bound + MS_ASSERT(!(v[ind]->first.empty())); // make sure that v[ind]->first.substr(1) is not out of bound v[ind]->second.front()->second = class_index_.empty() ? ind : class_index_[v[ind]->first.substr(1)]; image_label_pairs_.push_back(v[ind]->second.front()); v[ind]->second.pop(); @@ -416,7 +411,19 @@ Status ImageFolderOp::CountRowsAndClasses(const std::string &path, const std::se // Visitor accept method for NodePass Status ImageFolderOp::Accept(NodePass *p, bool *modified) { // Downcast shared pointer then call visitor - return p->RunOnNode(std::static_pointer_cast(shared_from_this()), modified); + return p->RunOnNode(shared_from_base(), modified); +} + +Status ImageFolderOp::ComputeColMap() { + // Set the column name map (base class field) + if (column_name_id_map_.empty()) { + for (int32_t i = 0; i < data_schema_->NumColumns(); ++i) { + column_name_id_map_[data_schema_->column(i).name()] = i; + } + } else { + MS_LOG(WARNING) << "Column name map is already set!"; + } + return Status::OK(); } } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/image_folder_op.h b/mindspore/ccsrc/dataset/engine/datasetops/source/image_folder_op.h index e1d578e034..6629fd6092 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/image_folder_op.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/image_folder_op.h @@ -248,6 +248,10 @@ class ImageFolderOp : public ParallelOp, public RandomAccessOp { // @return Status - The error code return Status Reset() override; + // Private function for computing the assignment of the column name map. + // @return - Status + Status ComputeColMap() override; + int32_t rows_per_buffer_; std::string folder_path_; // directory of image folder bool recursive_; @@ -255,7 +259,6 @@ class ImageFolderOp : public ParallelOp, public RandomAccessOp { std::set extensions_; // extensions allowed std::map class_index_; std::unique_ptr data_schema_; - std::shared_ptr sampler_; int64_t row_cnt_; int64_t buf_cnt_; int64_t sampler_ind_; diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/manifest_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/manifest_op.cc index 0762f36d5a..e65da8707b 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/manifest_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/manifest_op.cc @@ -64,7 +64,7 @@ Status ManifestOp::Builder::SanityCheck() { ManifestOp::ManifestOp(int32_t num_works, int32_t rows_per_buffer, std::string file, int32_t queue_size, bool decode, const std::map &class_index, std::unique_ptr data_schema, std::shared_ptr sampler, std::string usage) - : ParallelOp(num_works, queue_size), + : ParallelOp(num_works, queue_size, std::move(sampler)), rows_per_buffer_(rows_per_buffer), io_block_pushed_(0), row_cnt_(0), @@ -72,14 +72,9 @@ ManifestOp::ManifestOp(int32_t num_works, int32_t rows_per_buffer, std::string f data_schema_(std::move(data_schema)), file_(file), class_index_(class_index), - sampler_(std::move(sampler)), decode_(decode), usage_(usage), buf_cnt_(0) { - // Set the column name map (base class field) - for (int32_t i = 0; i < data_schema_->NumColumns(); ++i) { - column_name_id_map_[data_schema_->column(i).name()] = i; - } io_block_queues_.Init(num_workers_, queue_size); (void)std::transform(usage_.begin(), usage_.end(), usage_.begin(), ::tolower); } @@ -420,5 +415,17 @@ Status ManifestOp::GetClassIndexing(const std::string &file, const py::dict &dic return Status::OK(); } + +Status ManifestOp::ComputeColMap() { + // Set the column name map (base class field) + if (column_name_id_map_.empty()) { + for (int32_t i = 0; i < data_schema_->NumColumns(); ++i) { + column_name_id_map_[data_schema_->column(i).name()] = i; + } + } else { + MS_LOG(WARNING) << "Column name map is already set!"; + } + return Status::OK(); +} } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/manifest_op.h b/mindspore/ccsrc/dataset/engine/datasetops/source/manifest_op.h index edfdbb51ae..c180ea581d 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/manifest_op.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/manifest_op.h @@ -219,6 +219,10 @@ class ManifestOp : public ParallelOp, public RandomAccessOp { // @return Status - The error code return Status CountDatasetInfo(); + // Private function for computing the assignment of the column name map. + // @return - Status + Status ComputeColMap() override; + int32_t rows_per_buffer_; int64_t io_block_pushed_; int64_t row_cnt_; @@ -226,7 +230,6 @@ class ManifestOp : public ParallelOp, public RandomAccessOp { std::unique_ptr data_schema_; std::string file_; // file that store the information of images std::map class_index_; - std::shared_ptr sampler_; bool decode_; std::string usage_; int64_t buf_cnt_; diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/mindrecord_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/mindrecord_op.cc index 0f762386af..2b9d010ebb 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/mindrecord_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/mindrecord_op.cc @@ -196,10 +196,6 @@ Status MindRecordOp::Init() { data_schema_ = std::move(tmp_schema); } - for (int i = 0; i < static_cast(columns_to_load_.size()); i++) { - column_name_id_map_[columns_to_load_[i]] = i; - } - return Status::OK(); } @@ -500,7 +496,18 @@ Status MindRecordOp::CountTotalRows(const std::vector dataset_path, // Visitor accept method for NodePass Status MindRecordOp::Accept(NodePass *p, bool *modified) { // Downcast shared pointer then call visitor - return p->RunOnNode(std::static_pointer_cast(shared_from_this()), modified); + return p->RunOnNode(shared_from_base(), modified); +} + +Status MindRecordOp::ComputeColMap() { + if (column_name_id_map_.empty()) { + for (int i = 0; i < static_cast(columns_to_load_.size()); i++) { + column_name_id_map_[columns_to_load_[i]] = i; + } + } else { + MS_LOG(WARNING) << "Column name map is already set!"; + } + return Status::OK(); } } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/mindrecord_op.h b/mindspore/ccsrc/dataset/engine/datasetops/source/mindrecord_op.h index b704240aaa..af405a8f5b 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/mindrecord_op.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/mindrecord_op.h @@ -234,6 +234,10 @@ class MindRecordOp : public ParallelOp { Status FetchBlockBuffer(const int32_t &buffer_id); + // Private function for computing the assignment of the column name map. + // @return - Status + Status ComputeColMap() override; + int32_t rows_per_buffer_; // The number of requested rows per buffer. std::vector dataset_file_; // dataset files bool load_dataset_; // load dataset from single file or not diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/mnist_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/mnist_op.cc index eacd9daf75..e98f8ae8c1 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/mnist_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/mnist_op.cc @@ -66,17 +66,12 @@ Status MnistOp::Builder::SanityCheck() { MnistOp::MnistOp(int32_t num_workers, int32_t rows_per_buffer, std::string folder_path, int32_t queue_size, std::unique_ptr data_schema, std::shared_ptr sampler) - : ParallelOp(num_workers, queue_size), + : ParallelOp(num_workers, queue_size, std::move(sampler)), buf_cnt_(0), row_cnt_(0), folder_path_(folder_path), rows_per_buffer_(rows_per_buffer), - sampler_(std::move(sampler)), data_schema_(std::move(data_schema)) { - // set the column name map (base class field) - for (int32_t i = 0; i < data_schema_->NumColumns(); ++i) { - column_name_id_map_[data_schema_->column(i).name()] = i; - } io_block_queues_.Init(num_workers, queue_size); } @@ -432,5 +427,17 @@ Status MnistOp::CountTotalRows(const std::string &dir, int64_t *count) { return Status::OK(); } + +Status MnistOp::ComputeColMap() { + // set the column name map (base class field) + if (column_name_id_map_.empty()) { + for (int32_t i = 0; i < data_schema_->NumColumns(); ++i) { + column_name_id_map_[data_schema_->column(i).name()] = i; + } + } else { + MS_LOG(WARNING) << "Column name map is already set!"; + } + return Status::OK(); +} } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/mnist_op.h b/mindspore/ccsrc/dataset/engine/datasetops/source/mnist_op.h index 909ac22124..9bd6276a11 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/mnist_op.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/mnist_op.h @@ -226,12 +226,15 @@ class MnistOp : public ParallelOp, public RandomAccessOp { // @return Status - The error code return Status Reset() override; + // Private function for computing the assignment of the column name map. + // @return - Status + Status ComputeColMap() override; + int64_t buf_cnt_; int64_t row_cnt_; WaitPost wp_; std::string folder_path_; // directory of image folder int32_t rows_per_buffer_; - std::shared_ptr sampler_; std::unique_ptr data_schema_; std::vector image_label_pairs_; std::vector image_names_; diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/random_data_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/random_data_op.cc index 9e3d1140a7..3a865d8d69 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/random_data_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/random_data_op.cc @@ -21,6 +21,7 @@ #include "dataset/core/config_manager.h" #include "dataset/util/random.h" #include "dataset/util/wait_post.h" +#include "dataset/engine/datasetops/source/sampler/sequential_sampler.h" namespace mindspore { namespace dataset { @@ -30,7 +31,8 @@ RandomDataOp::Builder::Builder() builder_num_workers_(0), builder_op_connector_size_(0), builder_rows_per_buffer_(0), - builder_total_rows_(0) { + builder_total_rows_(0), + builder_sampler_(nullptr) { // Some arguments to the RandomDataOp have a default argument that is taken from the config. // The user may override these defaults by using the builder set methods. std::shared_ptr cfg = GlobalContext::config_manager(); @@ -43,8 +45,9 @@ RandomDataOp::Builder::Builder() Status RandomDataOp::Builder::Build(std::shared_ptr *out_op) { RETURN_IF_NOT_OK(SanityCheck()); - *out_op = std::make_shared(builder_num_workers_, builder_op_connector_size_, builder_rows_per_buffer_, - builder_total_rows_, std::move(builder_data_schema_)); + *out_op = + std::make_shared(builder_num_workers_, builder_op_connector_size_, builder_rows_per_buffer_, + builder_total_rows_, std::move(builder_data_schema_), std::move(builder_sampler_)); // If the user did not provide a schema, then we will ask the op to generate a pseudo-random // schema. @@ -53,9 +56,6 @@ Status RandomDataOp::Builder::Build(std::shared_ptr *out_op) { RETURN_IF_NOT_OK((*out_op)->GenerateSchema()); } - // Extract the column name mapping from the schema and save it in the class. - RETURN_IF_NOT_OK((*out_op)->data_schema_->GetColumnNameMap(&((*out_op)->column_name_id_map_))); - return Status::OK(); } @@ -69,8 +69,8 @@ Status RandomDataOp::Builder::SanityCheck() const { // Constructor for RandomDataOp RandomDataOp::RandomDataOp(int32_t num_workers, int32_t op_connector_size, int64_t rows_per_buffer, int64_t total_rows, - std::unique_ptr data_schema) - : ParallelOp(num_workers, op_connector_size), + std::unique_ptr data_schema, std::shared_ptr sampler) + : ParallelOp(num_workers, op_connector_size, std::move(sampler)), buffer_id_(0), rows_per_buffer_(rows_per_buffer), total_rows_(total_rows), @@ -127,7 +127,7 @@ Status RandomDataOp::GenerateSchema() { // For each column: // - choose a datatype // - generate a shape that randomly chooses the number of dimensions and the dimension values. - DataType::Type newType = static_cast(GenRandomInt(0, DataType::NUM_OF_TYPES - 2)); + DataType::Type newType = static_cast(GenRandomInt(1, DataType::NUM_OF_TYPES - 2)); int32_t rank = GenRandomInt(1, kMaxRank); std::vector dims; for (int32_t d = 0; d < rank; d++) { @@ -405,5 +405,25 @@ Status RandomDataOp::Reset() { return Status::OK(); } + +Status RandomDataOp::ComputeColMap() { + // Extract the column name mapping from the schema and save it in the class. + if (column_name_id_map_.empty()) { + RETURN_IF_NOT_OK(data_schema_->GetColumnNameMap(&(column_name_id_map_))); + } else { + MS_LOG(WARNING) << "Column name map is already set!"; + } + return Status::OK(); +} + +// During tree prepare phase, operators may have specific post-operations to perform depending on +// their role. +Status RandomDataOp::PrepareNodePostAction() { + // Run common code from super class before adding RandomDataOp specific handling + RETURN_IF_NOT_OK(ParallelOp::PrepareNodePostAction()); + // Specific handling for this op, we need to do cache op work to assign the sampler to the cache. + RETURN_IF_NOT_OK(DatasetOp::SaveSamplerForCache(false)); + return Status::OK(); +} } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/random_data_op.h b/mindspore/ccsrc/dataset/engine/datasetops/source/random_data_op.h index 48cfb0be51..b2af27dda3 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/random_data_op.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/random_data_op.h @@ -42,7 +42,7 @@ class RandomDataOp : public ParallelOp { // Some constants to provide limits to random generation. static constexpr int32_t kMaxNumColumns = 4; static constexpr int32_t kMaxRank = 4; - static constexpr int32_t kMaxDimValue = 2048; + static constexpr int32_t kMaxDimValue = 32; static constexpr int32_t kMaxTotalRows = 1024; // A nested builder class to aid in the construction of a RandomDataOp @@ -117,6 +117,14 @@ class RandomDataOp : public ParallelOp { return *this; } + // Setter method + // @param std::shared_ptr sampler + // @return Builder setter method returns reference to the builder. + Builder &SetSampler(std::shared_ptr sampler) { + builder_sampler_ = std::move(sampler); + return *this; + } + private: /** * Check if the required parameters are set by the builder. @@ -125,6 +133,7 @@ class RandomDataOp : public ParallelOp { Status SanityCheck() const; std::unique_ptr builder_data_schema_; + std::shared_ptr builder_sampler_; int32_t builder_num_workers_; int32_t builder_op_connector_size_; int64_t builder_rows_per_buffer_; @@ -139,10 +148,11 @@ class RandomDataOp : public ParallelOp { * @param rows_per_buffer - The number of rows in each DataBuffer * @param data_schema - A user-provided schema * @param total_rows - The total number of rows in the dataset + * @param sampler - allow a sampler. Only valid if a cache exists in ascendent tree nodes * @return Builder - The modified builder by reference */ RandomDataOp(int32_t num_workers, int32_t op_connector_size, int64_t rows_per_buffer, int64_t total_rows, - std::unique_ptr data_schema); + std::unique_ptr data_schema, std::shared_ptr sampler); /** * Destructor @@ -193,6 +203,12 @@ class RandomDataOp : public ParallelOp { // @return Name of the current Op std::string Name() const override { return "RandomDataOp"; } + // During tree prepare phase, operators may have specific post-operations to perform depending on + // their role. + // @notes Derived versions of this function should always call it's superclass version first + // before providing their own implementations. + Status PrepareNodePostAction() override; + private: /** * The entry point code for when workers are launched @@ -250,6 +266,10 @@ class RandomDataOp : public ParallelOp { return ++buffer_id_; } + // Private function for computing the assignment of the column name map. + // @return - Status + Status ComputeColMap() override; + int32_t buffer_id_; int64_t rows_per_buffer_; int64_t total_rows_; diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/distributed_sampler.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/distributed_sampler.cc index 226647df14..9f4a9cf55c 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/distributed_sampler.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/distributed_sampler.cc @@ -107,12 +107,11 @@ Status DistributedSampler::ResetSampler() { } void DistributedSampler::Print(std::ostream &out, bool show_all) const { - out << "(sampler): DistributedSampler\n"; + out << "\nSampler: DistributedSampler"; if (show_all) { - out << "seed_: " << seed_ << '\n'; - out << "device_id_: " << device_id_ << '\n'; - out << "num_devices_: " << num_devices_ << '\n'; - out << "shuffle_: " << shuffle_ << '\n'; + Sampler::Print(out, show_all); + out << "\nseed: " << seed_ << "\ndevice_id: " << device_id_ << "\nnum_devices: " << num_devices_ + << "\nshuffle: " << shuffle_; } } diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/pk_sampler.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/pk_sampler.cc index 92a880d599..cd2cadb9ff 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/pk_sampler.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/pk_sampler.cc @@ -113,5 +113,13 @@ Status PKSampler::HandshakeRandomAccessOp(const RandomAccessOp *op) { return Status::OK(); } +void PKSampler::Print(std::ostream &out, bool show_all) const { + out << "\nSampler: PKSampler"; + if (show_all) { + // Call the super class for displaying any common detailed info + Sampler::Print(out, show_all); + // Then add our own info if any + } +} } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/pk_sampler.h b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/pk_sampler.h index 7b1423326a..cde8a75b5b 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/pk_sampler.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/pk_sampler.h @@ -56,6 +56,11 @@ class PKSampler : public Sampler { // NOT YET FINISHED // @return - The error code return Status ResetSampler() override; + // Printer for debugging purposes. + // @param out - output stream to write to + // @param show_all - bool to show detailed vs summary + void Print(std::ostream &out, bool show_all) const override; + private: bool shuffle_; uint32_t seed_; diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/python_sampler.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/python_sampler.cc index af4aa20bb2..d204c55ce9 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/python_sampler.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/python_sampler.cc @@ -103,5 +103,14 @@ Status PythonSampler::ResetSampler() { return Status::OK(); } + +void PythonSampler::Print(std::ostream &out, bool show_all) const { + out << "\nSampler: PythonSampler"; + if (show_all) { + // Call the super class for displaying any common detailed info + Sampler::Print(out, show_all); + // Then add our own info if any + } +} } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/python_sampler.h b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/python_sampler.h index 49ff12878d..7d653b2087 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/python_sampler.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/python_sampler.h @@ -50,6 +50,11 @@ class PythonSampler : public Sampler { // @return - The error code return Status GetNextSample(std::unique_ptr *out_buffer) override; + // Printer for debugging purposes. + // @param out - output stream to write to + // @param show_all - bool to show detailed vs summary + void Print(std::ostream &out, bool show_all) const override; + private: bool need_to_reset_; // Whether Reset() should be called before calling GetNextBuffer() diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/random_sampler.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/random_sampler.cc index b3dfaad7f7..db0a96ea3a 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/random_sampler.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/random_sampler.cc @@ -113,13 +113,12 @@ Status RandomSampler::ResetSampler() { } void RandomSampler::Print(std::ostream &out, bool show_all) const { - out << "(sampler): RandomSampler\n"; - + out << "\nSampler: RandomSampler"; if (show_all) { - out << "num_samples_: " << num_samples_ << '\n'; - out << "next_id_: " << next_id_ << '\n'; + // Call the super class for displaying any common detailed info + Sampler::Print(out, show_all); + // Then add our own info if any } } - } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/sampler.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/sampler.cc index 3f737c167c..1584166dc3 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/sampler.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/sampler.cc @@ -80,22 +80,26 @@ Status Sampler::CreateSamplerTensor(std::shared_ptr *sample_ids, int64_t } void Sampler::Print(std::ostream &out, bool show_all) const { - out << "(sampler): base\n"; - + // Sampler printing is usually only called in the show_all mode. + // Derived classes will display the name, then call back to this base + // for common info. + // No-op in the summary mode. if (show_all) { - out << "num_rows_: " << num_rows_ << '\n'; - out << "num_samples_: " << num_samples_ << '\n'; + out << "\nnum_rows_: " << num_rows_ << "\nnum_samples_: " << num_samples_; } } Status Sampler::GetAllIdsThenReset(py::array *data) { std::unique_ptr db; std::shared_ptr sample_ids; + TensorRow sample_row; // A call to derived class to get sample ids wrapped inside a buffer RETURN_IF_NOT_OK(GetNextSample(&db)); // Get the only tensor inside the buffer that contains the actual SampleIds for the entire epoch - RETURN_IF_NOT_OK(db->GetTensor(&sample_ids, 0, 0)); + RETURN_IF_NOT_OK(db->GetRow(0, &sample_row)); + sample_ids = sample_row[0]; + // check this buffer is not a ctrl buffer CHECK_FAIL_RETURN_UNEXPECTED(db->buffer_flags() == DataBuffer::kDeBFlagNone, "ERROR ctrl buffer received"); { diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/sequential_sampler.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/sequential_sampler.cc index f0ff6a2c02..28598da55f 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/sequential_sampler.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/sequential_sampler.cc @@ -89,7 +89,14 @@ Status SequentialSampler::ResetSampler() { return Status::OK(); } -void SequentialSampler::Print(std::ostream &out, bool show_all) const { out << "(sampler): SequentialSampler\n"; } - +void SequentialSampler::Print(std::ostream &out, bool show_all) const { + out << "\nSampler: SequentialSampler"; + if (show_all) { + // Call the super class for displaying any common detailed info + Sampler::Print(out, show_all); + // Then add our own info + out << "\nStart index: " << start_index_; + } +} } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/sequential_sampler.h b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/sequential_sampler.h index 2cb7a9ff8d..06f084fb7a 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/sequential_sampler.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/sequential_sampler.h @@ -49,6 +49,9 @@ class SequentialSampler : public Sampler { // @return - The error code return Status GetNextSample(std::unique_ptr *out_buffer) override; + // Printer for debugging purposes. + // @param out - output stream to write to + // @param show_all - bool to show detailed vs summary void Print(std::ostream &out, bool show_all) const override; private: diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/subset_random_sampler.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/subset_random_sampler.cc index 54491889fc..08a623ed1b 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/subset_random_sampler.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/subset_random_sampler.cc @@ -119,5 +119,14 @@ Status SubsetRandomSampler::GetNextSample(std::unique_ptr *out_buffe return Status::OK(); } + +void SubsetRandomSampler::Print(std::ostream &out, bool show_all) const { + out << "\nSampler: SubsetRandomSampler"; + if (show_all) { + // Call the super class for displaying any common detailed info + Sampler::Print(out, show_all); + // Then add our own info if any + } +} } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/subset_random_sampler.h b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/subset_random_sampler.h index 980ffe578a..ffc7cb17bc 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/subset_random_sampler.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/subset_random_sampler.h @@ -51,6 +51,11 @@ class SubsetRandomSampler : public Sampler { // @note the sample ids (int64_t) will be placed in one Tensor and be placed into pBuffer. Status GetNextSample(std::unique_ptr *out_buffer) override; + // Printer for debugging purposes. + // @param out - output stream to write to + // @param show_all - bool to show detailed vs summary + void Print(std::ostream &out, bool show_all) const override; + private: // A list of indices (already randomized in constructor). std::vector indices_; diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/weighted_random_sampler.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/weighted_random_sampler.cc index 759af99352..6bf3d2d85e 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/weighted_random_sampler.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/weighted_random_sampler.cc @@ -156,5 +156,14 @@ Status WeightedRandomSampler::GetNextSample(std::unique_ptr *out_buf return Status::OK(); } + +void WeightedRandomSampler::Print(std::ostream &out, bool show_all) const { + out << "\nSampler: WeightedRandomSampler"; + if (show_all) { + // Call the super class for displaying any common detailed info + Sampler::Print(out, show_all); + // Then add our own info if any + } +} } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/weighted_random_sampler.h b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/weighted_random_sampler.h index 257501250d..1fbe29ed80 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/weighted_random_sampler.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/weighted_random_sampler.h @@ -53,6 +53,11 @@ class WeightedRandomSampler : public Sampler { // @note the sample ids (int64_t) will be placed in one Tensor and be placed into pBuffer. Status GetNextSample(std::unique_ptr *out_buffer) override; + // Printer for debugging purposes. + // @param out - output stream to write to + // @param show_all - bool to show detailed vs summary + void Print(std::ostream &out, bool show_all) const override; + private: // A list of weights for each sample. std::vector weights_; diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/text_file_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/text_file_op.cc index 26058cc8b8..818b5ab3f4 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/text_file_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/text_file_op.cc @@ -37,7 +37,7 @@ TextFileOp::Builder::Builder() builder_num_devices_(1), builder_total_rows_(0), builder_shuffle_files_(false), - builder_shuffle_global_(false) { + builder_sampler_(nullptr) { std::shared_ptr config_manager = GlobalContext::config_manager(); builder_num_workers_ = config_manager->num_parallel_workers(); builder_op_connector_size_ = config_manager->op_connector_size(); @@ -68,7 +68,7 @@ Status TextFileOp::Builder::Build(std::shared_ptr *op) { std::shared_ptr text_file_op = std::make_shared( builder_num_workers_, builder_rows_per_buffer_, builder_total_rows_, builder_worker_connector_size_, std::move(builder_schema_), builder_text_files_list_, builder_op_connector_size_, builder_shuffle_files_, - builder_shuffle_global_, builder_num_devices_, builder_device_id_); + builder_num_devices_, builder_device_id_, std::move(builder_sampler_)); RETURN_IF_NOT_OK(text_file_op->Init()); *op = std::move(text_file_op); @@ -77,16 +77,15 @@ Status TextFileOp::Builder::Build(std::shared_ptr *op) { TextFileOp::TextFileOp(int32_t num_workers, int64_t rows_per_buffer, int64_t total_rows, int32_t worker_connector_size, std::unique_ptr schema, std::vector text_files_list, - int32_t op_connector_size, bool shuffle_files, bool shuffle_global, int32_t num_device, - int32_t device_id) - : ParallelOp(num_workers, op_connector_size), + int32_t op_connector_size, bool shuffle_files, int32_t num_device, int32_t device_id, + std::shared_ptr sampler) + : ParallelOp(num_workers, op_connector_size, std::move(sampler)), device_id_(device_id), num_devices_(num_device), rows_per_buffer_(rows_per_buffer), total_rows_(total_rows), text_files_list_(std::move(text_files_list)), shuffle_files_(shuffle_files), - shuffle_global_(shuffle_global), data_schema_(std::move(schema)), all_num_rows_(0), num_rows_per_shard_(0), @@ -127,11 +126,6 @@ Status TextFileOp::Init() { int32_t safe_queue_size = static_cast(std::ceil(text_files_list_.size() / num_workers_) + 1); io_block_queues_.Init(num_workers_, safe_queue_size); - // Set the column name mapping (base class field) - for (int32_t i = 0; i < data_schema_->NumColumns(); ++i) { - column_name_id_map_[data_schema_->column(i).name()] = i; - } - RETURN_IF_NOT_OK(ParallelOp::CreateWorkerConnector(worker_connector_size_)); jagged_buffer_connector_ = std::make_unique(num_workers_, 1, worker_connector_size_); @@ -488,5 +482,17 @@ Status TextFileOp::CountAllFileRows(const std::vector &files, int64 } return Status::OK(); } + +Status TextFileOp::ComputeColMap() { + // Set the column name mapping (base class field) + if (column_name_id_map_.empty()) { + for (int32_t i = 0; i < data_schema_->NumColumns(); ++i) { + column_name_id_map_[data_schema_->column(i).name()] = i; + } + } else { + MS_LOG(WARNING) << "Column name map is already set!"; + } + return Status::OK(); +} } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/text_file_op.h b/mindspore/ccsrc/dataset/engine/datasetops/source/text_file_op.h index dd258d914e..5b787d4dad 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/text_file_op.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/text_file_op.h @@ -20,6 +20,7 @@ #include #include #include +#include #include #include "dataset/util/status.h" @@ -107,15 +108,16 @@ class TextFileOp : public ParallelOp { // Setter method. // @return Builder - setter method returns reference to the builder. - Builder &SetShuffleGlobal(bool shuffle_global) { - builder_shuffle_global_ = shuffle_global; + Builder &SetTotalRows(int64_t total_rows) { + builder_total_rows_ = total_rows; return *this; } - // Setter method. - // @return Builder - setter method returns reference to the builder. - Builder &SetTotalRows(int64_t total_rows) { - builder_total_rows_ = total_rows; + // Setter method + // @param std::shared_ptr sampler + // @return Builder setter method returns reference to the builder. + Builder &SetSampler(std::shared_ptr sampler) { + builder_sampler_ = std::move(sampler); return *this; } @@ -129,8 +131,8 @@ class TextFileOp : public ParallelOp { int32_t builder_worker_connector_size_; std::vector builder_text_files_list_; bool builder_shuffle_files_; - bool builder_shuffle_global_; std::unique_ptr builder_schema_; + std::shared_ptr builder_sampler_; }; // Constructor of TextFileOp @@ -143,11 +145,11 @@ class TextFileOp : public ParallelOp { // @param op_connector_size - size of each queue in the connector that the child operator pulls from. // @param columns_to_load - the names of the columns to load data from. // @param shuffle_files - whether or not to shuffle the files before reading data. - // @param shuffle_global - whether or not to shuffle the entire dataset. // @param equal_rows_per_shard - whether or not to get equal rows for each process. + // @param sampler - allow a sampler. Only valid if a cache exists in ascendent tree nodes TextFileOp(int32_t num_workers, int64_t rows_per_buffer, int64_t total_rows, int32_t worker_connector_size, std::unique_ptr, std::vector text_files_list, int32_t op_connector_size, - bool shuffle_files, bool shuffle_global, int32_t num_devices, int32_t device_id); + bool shuffle_files, int32_t num_devices, int32_t device_id, std::shared_ptr sampler); // Default destructor ~TextFileOp() = default; @@ -186,10 +188,6 @@ class TextFileOp : public ParallelOp { // @return Vector of the input file names std::vector FileNames() { return text_files_list_; } - // Global shuffle flag getter - // @return Bool - whether this Op requires global shuffle - bool RequireGlobalShuffle() { return shuffle_global_; } - private: // The entry point for when workers are launched. // @param worker_id - the id of the worker that is executing this function. @@ -264,13 +262,16 @@ class TextFileOp : public ParallelOp { // @return Status - the error code returned. Status PostEndOfEpoch(int32_t queue_index); + // Private function for computing the assignment of the column name map. + // @return - Status + Status ComputeColMap() override; + int32_t device_id_; int32_t num_devices_; int64_t rows_per_buffer_; int64_t total_rows_; std::vector text_files_list_; bool shuffle_files_; - bool shuffle_global_; std::unique_ptr data_schema_; int64_t all_num_rows_; int64_t num_rows_per_shard_; diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.cc index 23dce8dc10..48f13ff766 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.cc @@ -48,14 +48,17 @@ namespace mindspore { namespace dataset { TFReaderOp::Builder::Builder() - : builder_device_id_(0), builder_num_devices_(1), builder_total_rows_(0), builder_equal_rows_per_shard_(false) { + : builder_device_id_(0), + builder_num_devices_(1), + builder_total_rows_(0), + builder_equal_rows_per_shard_(false), + builder_sampler_(nullptr) { std::shared_ptr config_manager = GlobalContext::config_manager(); builder_num_workers_ = config_manager->num_parallel_workers(); builder_worker_connector_size_ = config_manager->worker_connector_size(); builder_op_connector_size_ = config_manager->op_connector_size(); builder_rows_per_buffer_ = config_manager->rows_per_buffer(); builder_shuffle_files_ = false; - builder_shuffle_global_ = false; builder_data_schema_ = std::make_unique(); } @@ -88,11 +91,6 @@ Status TFReaderOp::Builder::ValidateInputs() const { err_msg += "Number of parallel workers is smaller or equal to 0\n"; } - if (!builder_equal_rows_per_shard_ && - builder_dataset_files_list_.size() < static_cast(builder_num_devices_)) { - err_msg += "Not enough tfrecord files provided\n"; - } - if (builder_device_id_ >= builder_num_devices_ || builder_num_devices_ < 1) { err_msg += "Wrong sharding configs\n"; } @@ -126,8 +124,8 @@ Status TFReaderOp::Builder::Build(std::shared_ptr *out_tf_reader_op) std::shared_ptr new_tf_reader_op = std::make_shared( builder_num_workers_, builder_worker_connector_size_, builder_rows_per_buffer_, builder_total_rows_, builder_dataset_files_list_, std::move(builder_data_schema_), builder_op_connector_size_, builder_columns_to_load_, - builder_shuffle_files_, builder_shuffle_global_, builder_num_devices_, builder_device_id_, - builder_equal_rows_per_shard_); + builder_shuffle_files_, builder_num_devices_, builder_device_id_, builder_equal_rows_per_shard_, + std::move(builder_sampler_)); RETURN_IF_NOT_OK(new_tf_reader_op->Init()); *out_tf_reader_op = std::move(new_tf_reader_op); @@ -137,9 +135,9 @@ Status TFReaderOp::Builder::Build(std::shared_ptr *out_tf_reader_op) TFReaderOp::TFReaderOp(int32_t num_workers, int32_t worker_connector_size, int64_t rows_per_buffer, int64_t total_num_rows, std::vector dataset_files_list, std::unique_ptr data_schema, int32_t op_connector_size, - std::vector columns_to_load, bool shuffle_files, bool shuffle_global, - int32_t num_device, int32_t device_id, bool equal_rows_per_shard) - : ParallelOp(num_workers, op_connector_size), + std::vector columns_to_load, bool shuffle_files, int32_t num_device, + int32_t device_id, bool equal_rows_per_shard, std::shared_ptr sampler) + : ParallelOp(num_workers, op_connector_size, std::move(sampler)), device_id_(device_id), num_devices_(num_device), rows_per_buffer_(rows_per_buffer), @@ -148,7 +146,6 @@ TFReaderOp::TFReaderOp(int32_t num_workers, int32_t worker_connector_size, int64 columns_to_load_(std::move(columns_to_load)), finished_reading_dataset_(false), shuffle_files_(shuffle_files), - shuffle_global_(shuffle_global), data_schema_(std::move(data_schema)), filename_index_(std::make_unique()), load_io_block_queue_(true), @@ -174,7 +171,6 @@ void TFReaderOp::Print(std::ostream &out, bool show_all) const { // Then show any custom derived-internal stuff out << "\nRows per buffer: " << rows_per_buffer_ << "\nTotal rows: " << total_rows_ << "\nDevice id: " << device_id_ << "\nNumber of devices: " << num_devices_ << "\nShuffle files: " << ((shuffle_files_) ? "yes" : "no") - << "\nShuffle global: " << ((shuffle_global_) ? "yes" : "no") << "\nDataset files list: Size: " << dataset_files_list_.size() << "\n"; for (int i = 0; i < dataset_files_list_.size(); ++i) { out << " " << dataset_files_list_[i]; @@ -195,11 +191,6 @@ Status TFReaderOp::Init() { RETURN_IF_NOT_OK(CreateSchema(dataset_files_list_[0], columns_to_load_)); } - // Construct the column name map for this operator (base class field) - for (int32_t i = 0; i < data_schema_->NumColumns(); ++i) { - column_name_id_map_[data_schema_->column(i).name()] = i; - } - if (total_rows_ == 0) { total_rows_ = data_schema_->num_rows(); } @@ -1013,7 +1004,54 @@ int64_t TFReaderOp::CountTotalRowsSectioned(const std::vector &file // Visitor accept method for NodePass Status TFReaderOp::Accept(NodePass *p, bool *modified) { // Downcast shared pointer then call visitor - return p->RunOnNode(std::static_pointer_cast(shared_from_this()), modified); + return p->RunOnNode(shared_from_base(), modified); +} + +Status TFReaderOp::ComputeColMap() { + // Construct the column name map for this operator (base class field) + if (column_name_id_map_.empty()) { + for (int32_t i = 0; i < data_schema_->NumColumns(); ++i) { + column_name_id_map_[data_schema_->column(i).name()] = i; + } + } else { + MS_LOG(WARNING) << "Column name map is already set!"; + } + return Status::OK(); +} + +// During tree prepare phase, operators may have specific post-operations to perform depending on +// their role. +Status TFReaderOp::PrepareNodePostAction() { + // Run common code from super class before adding TFReaderOp specific handling + RETURN_IF_NOT_OK(ParallelOp::PrepareNodePostAction()); + + // Specific handling for this op, we need to do cache op work so assign the sampler to the cache + // TF is a special case because it can support file-based sharding/shuffling, or, if there + // is a cache, then it can also do row-based sampler using the sampler on the cache. + // Thus, pass true for random access op flag when saving the sampler. This is a special case, + // since usually a non-mappable dataset would pass false here. + RETURN_IF_NOT_OK(DatasetOp::SaveSamplerForCache(true)); + + // Now that the sampler has been saved for the cache, we need to adjust the TFReaderOp to turn it into + // a simpler producer of all data (no shuffling or sharding or anything) + if (BitTest(tree_->PrepareFlags(), ExecutionTree::kDePrepCache)) { + device_id_ = 0; + num_devices_ = 1; + total_rows_ = 0; + shuffle_files_ = false; + equal_rows_per_shard_ = false; + sampler_.reset(); // Normally SaveSampler code did this for us, but we passed in true above (See comment) + } else { + // This sanity check had been delayed until now in the prepare loop. + // If we are not in a cache path, then we can validate the the file-based sharding config. + // If we are in a cache path, there is no file-based sharding so the check is not correct in that + // situation. + if (!equal_rows_per_shard_ && dataset_files_list_.size() < static_cast(num_devices_)) { + RETURN_STATUS_UNEXPECTED("Not enough tfrecord files provided\n"); + } + } + + return Status::OK(); } } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.h b/mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.h index 9c92d6d4be..9226c4c6c5 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.h @@ -148,20 +148,22 @@ class TFReaderOp : public ParallelOp { // Setter method. // @return Builder - setter method returns reference to the builder. - Builder &SetShuffleGlobal(bool shuffle_global) { - builder_shuffle_global_ = shuffle_global; + Builder &SetShardEqualRows(bool shard_equal_rows) { + builder_equal_rows_per_shard_ = shard_equal_rows; return *this; } - // Setter method. - // @return Builder - setter method returns reference to the builder. - Builder &SetShardEqualRows(bool shard_equal_rows) { - builder_equal_rows_per_shard_ = shard_equal_rows; + // Setter method + // @param std::shared_ptr sampler + // @return Builder setter method returns reference to the builder. + Builder &SetSampler(std::shared_ptr sampler) { + builder_sampler_ = std::move(sampler); return *this; } private: std::unique_ptr builder_data_schema_; + std::shared_ptr builder_sampler_; int32_t builder_device_id_; int32_t builder_num_devices_; int32_t builder_num_workers_; @@ -172,7 +174,6 @@ class TFReaderOp : public ParallelOp { std::vector builder_dataset_files_list_; std::vector builder_columns_to_load_; bool builder_shuffle_files_; - bool builder_shuffle_global_; bool builder_equal_rows_per_shard_; }; @@ -187,12 +188,12 @@ class TFReaderOp : public ParallelOp { // @param op_connector_size - size of each queue in the connector that the child operator pulls from. // @param columns_to_load - the names of the columns to load data from. // @param shuffle_files - whether or not to shuffle the files before reading data. - // @param shuffle_global - whether or not to shuffle the entire dataset. // @param equal_rows_per_shard - whether or not to get equal rows for each process. + // @param sampler - allow a sampler. Only valid if a cache exists in ascendent tree nodes TFReaderOp(int32_t num_workers, int32_t worker_connector_size, int64_t rows_per_buffer, int64_t total_num_rows, std::vector dataset_files_list, std::unique_ptr data_schema, int32_t op_connector_size, std::vector columns_to_load, bool shuffle_files, - bool shuffle_global, int32_t num_devices, int32_t device_id, bool equal_rows_per_shard); + int32_t num_devices, int32_t device_id, bool equal_rows_per_shard, std::shared_ptr sampler); // Default destructor ~TFReaderOp() = default; @@ -245,9 +246,11 @@ class TFReaderOp : public ParallelOp { // @return Vector of the input file names std::vector FileNames() { return dataset_files_list_; } - // Global shuffle flag getter - // @return Bool - whether this Op requires global shuffle - bool RequireGlobalShuffle() { return shuffle_global_; } + // During tree prepare phase, operators may have specific post-operations to perform depending on + // their role. + // @notes Derived versions of this function should always call it's superclass version first + // before providing their own implementations. + Status PrepareNodePostAction() override; private: // The entry point for when workers are launched. @@ -381,6 +384,10 @@ class TFReaderOp : public ParallelOp { // @return Status - the error code returned. Status CalculateNumRowsPerShard(); + // Private function for computing the assignment of the column name map. + // @return - Status + Status ComputeColMap() override; + int32_t device_id_; int32_t num_devices_; int64_t rows_per_buffer_; @@ -389,7 +396,6 @@ class TFReaderOp : public ParallelOp { std::vector columns_to_load_; bool finished_reading_dataset_; bool shuffle_files_; - bool shuffle_global_; std::unique_ptr data_schema_; std::unique_ptr filename_index_; bool load_io_block_queue_; diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/voc_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/voc_op.cc index d3c7ff397f..958aa65b06 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/voc_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/voc_op.cc @@ -88,7 +88,7 @@ Status VOCOp::Builder::SanityCheck() { VOCOp::VOCOp(const TaskType &task_type, const std::string &task_mode, const std::string &folder_path, const std::map &class_index, int32_t num_workers, int32_t rows_per_buffer, int32_t queue_size, bool decode, std::unique_ptr data_schema, std::shared_ptr sampler) - : ParallelOp(num_workers, queue_size), + : ParallelOp(num_workers, queue_size, std::move(sampler)), decode_(decode), row_cnt_(0), buf_cnt_(0), @@ -97,12 +97,7 @@ VOCOp::VOCOp(const TaskType &task_type, const std::string &task_mode, const std: folder_path_(folder_path), class_index_(class_index), rows_per_buffer_(rows_per_buffer), - sampler_(std::move(sampler)), data_schema_(std::move(data_schema)) { - // Set the column name map (base class field) - for (int32_t i = 0; i < data_schema_->NumColumns(); ++i) { - column_name_id_map_[data_schema_->column(i).name()] = i; - } io_block_queues_.Init(num_workers_, queue_size); } @@ -454,5 +449,17 @@ Status VOCOp::GetClassIndexing(const std::string &dir, const std::string &task_t return Status::OK(); } + +Status VOCOp::ComputeColMap() { + // Set the column name map (base class field) + if (column_name_id_map_.empty()) { + for (int32_t i = 0; i < data_schema_->NumColumns(); ++i) { + column_name_id_map_[data_schema_->column(i).name()] = i; + } + } else { + MS_LOG(WARNING) << "Column name map is already set!"; + } + return Status::OK(); +} } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/voc_op.h b/mindspore/ccsrc/dataset/engine/datasetops/source/voc_op.h index bce82a43c9..89875341ca 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/voc_op.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/voc_op.h @@ -263,6 +263,10 @@ class VOCOp : public ParallelOp, public RandomAccessOp { // @return Status - The error code return Status Reset() override; + // Private function for computing the assignment of the column name map. + // @return - Status + Status ComputeColMap() override; + bool decode_; int64_t row_cnt_; int64_t buf_cnt_; @@ -270,7 +274,6 @@ class VOCOp : public ParallelOp, public RandomAccessOp { TaskType task_type_; std::string task_mode_; int32_t rows_per_buffer_; - std::shared_ptr sampler_; std::unique_ptr data_schema_; WaitPost wp_; diff --git a/mindspore/ccsrc/dataset/engine/datasetops/take_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/take_op.cc index f9ff49a5b7..8bc449cdc9 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/take_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/take_op.cc @@ -73,7 +73,6 @@ Status TakeOp::operator()() { TaskManager::FindMe()->Post(); std::unique_ptr buf; RETURN_IF_NOT_OK(child_[0]->GetNextBuffer(&buf)); - RETURN_IF_NOT_OK(DatasetOp::AssignColMapFromChild()); while (buf->eof() == false) { if (take_count_ == max_takes_) { @@ -130,14 +129,14 @@ Status TakeOp::FillBuffer(std::unique_ptr *buffer, std::unique_ptrAddToRepeatStack(shared_from_this()); + tree_->AddToEOEOpStack(shared_from_this()); return Status::OK(); } // Visitor accept method for NodePass Status TakeOp::Accept(NodePass *p, bool *modified) { // Downcast shared pointer then call visitor - return p->RunOnNode(std::static_pointer_cast(shared_from_this()), modified); + return p->RunOnNode(shared_from_base(), modified); } } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/zip_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/zip_op.cc index bb438552f3..70bce16a89 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/zip_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/zip_op.cc @@ -139,24 +139,6 @@ Status ZipOp::prepare(TensorQTable *const table) { // Pack this first row into our tensor table table->push_back(std::move(new_row)); - // At this point we have at least 1 row produced, so all child iterators have their column names such that we - // can produce our column name map now. - column_name_id_map_ = {}; - for (int32_t i = 0; i < children_num_; ++i) { - // Initializing col_name_id_map_ from the first data buffer. - const std::unordered_map col_name_id_map = child_iterators_[i]->GetColumnNameMap(); - int32_t colsCurrent = column_name_id_map_.size(); - // the update code below shouldn't do anything bad if the column name already exists. - for (const auto &pair : col_name_id_map) { - std::string name = pair.first; - int32_t old_id = pair.second; - // check if name already exists in column name descriptor - if (column_name_id_map_.count(name) == 1) { - RETURN_STATUS_UNEXPECTED("key already exists when zipping datasets"); - } - column_name_id_map_[name] = old_id + colsCurrent; - } - } return Status::OK(); } @@ -255,7 +237,32 @@ Status ZipOp::EoeReceived(int32_t) { // Visitor accept method for NodePass Status ZipOp::Accept(NodePass *p, bool *modified) { // Downcast shared pointer then call visitor - return p->RunOnNode(std::static_pointer_cast(shared_from_this()), modified); + return p->RunOnNode(shared_from_base(), modified); +} + +Status ZipOp::ComputeColMap() { + if (column_name_id_map_.empty()) { + column_name_id_map_ = {}; + for (int32_t i = 0; i < child_.size(); ++i) { + // Initializing col_name_id_map from the child. + const std::unordered_map col_name_id_map = child_[i]->column_name_id_map(); + int32_t colsCurrent = column_name_id_map_.size(); + // the update code below shouldn't do anything bad if the column name already exists. + for (const auto &pair : col_name_id_map) { + std::string name = pair.first; + int32_t old_id = pair.second; + // check if name already exists in column name descriptor + if (column_name_id_map_.count(name) == 1) { + RETURN_STATUS_UNEXPECTED("key already exists when zipping datasets"); + } + column_name_id_map_[name] = old_id + colsCurrent; + } + } + MS_LOG(DEBUG) << "Setting column map:\n" << this->ColumnNameMapAsString(); + } else { + MS_LOG(WARNING) << "Column name map is already set!"; + } + return Status::OK(); } } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/zip_op.h b/mindspore/ccsrc/dataset/engine/datasetops/zip_op.h index 08b93c18b5..fad3c22eaa 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/zip_op.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/zip_op.h @@ -141,6 +141,10 @@ class ZipOp : public PipelineOp { // 1, a, T Status getNextTensorRow(TensorRow *const new_zip_row); + // Computing the assignment of the column name map. + // @return - Status + Status ComputeColMap() override; + int32_t children_num_; int32_t rows_per_buffer_; int32_t buffer_id_; diff --git a/mindspore/ccsrc/dataset/engine/db_connector.h b/mindspore/ccsrc/dataset/engine/db_connector.h index b1fdd14ab6..54909f51ba 100644 --- a/mindspore/ccsrc/dataset/engine/db_connector.h +++ b/mindspore/ccsrc/dataset/engine/db_connector.h @@ -83,6 +83,7 @@ class DbConnector : public Connector> { expect_consumer_ = (expect_consumer_ + 1) % num_consumers_; } } + out_buffers_count_++; cv_.NotifyAll(); return Status::OK(); } diff --git a/mindspore/ccsrc/dataset/engine/execution_tree.cc b/mindspore/ccsrc/dataset/engine/execution_tree.cc index 5c921bba84..385722e257 100644 --- a/mindspore/ccsrc/dataset/engine/execution_tree.cc +++ b/mindspore/ccsrc/dataset/engine/execution_tree.cc @@ -19,8 +19,8 @@ #include "dataset/engine/datasetops/dataset_op.h" #include "dataset/engine/datasetops/shuffle_op.h" #include "dataset/util/task_manager.h" -#include "dataset/engine/opt/pre/map_column_reorder.h" -#include "dataset/engine/opt/pre/global_shuffle.h" +#include "dataset/engine/opt/pass.h" +#include "dataset/engine/opt/pre/removal_pass.h" #include "dataset/engine/perf/profiling.h" #include "dataset/engine/perf/monitor.h" @@ -42,6 +42,10 @@ ExecutionTree::~ExecutionTree() { (void)tg_->ServiceStop(); } // provides it with a link to the tree. A node cannot form any relationships (parent/child) with // other nodes unless they are associated with the same tree. Status ExecutionTree::AssociateNode(const std::shared_ptr &op) { + // If we are already a part of the tree, no-op + if (op->tree_ == this) { + return Status::OK(); + } if (tree_state_ != kDeTStateInit && tree_state_ != kDeTStateBuilding) { std::string err_msg = "Invalid tree state for adding a node. Current state: " + std::to_string(static_cast(tree_state_)) + @@ -85,13 +89,13 @@ Status ExecutionTree::AssignRoot(const std::shared_ptr &op) { } // A print method typically used for debugging -void ExecutionTree::Print(std::ostream &out) const { +void ExecutionTree::Print(std::ostream &out, const std::shared_ptr &op) const { out << "Execution tree summary:\n" << "-----------------------\n"; - this->PrintNode(out, root_, "", true, false); + this->PrintNode(out, op == nullptr ? root_ : op, "", true, false); out << "\nExecution tree operator details:\n" << "--------------------------------\n"; - this->PrintNode(out, root_, "", true, true); + this->PrintNode(out, op == nullptr ? root_ : op, "", true, true); } // A helper functions for doing the recursive printing @@ -209,10 +213,10 @@ Status ExecutionTree::Prepare() { Status ExecutionTree::PrepareTreePreAction() { bool modified = false; - std::vector pre_actions; + std::vector> pre_actions; // Construct pre actions - pre_actions.push_back(new MapColumnReorder()); - pre_actions.push_back(new GlobalShufflePass()); + MS_LOG(INFO) << "Running pre pass"; + pre_actions.push_back(std::make_unique(RemovalPass())); // Apply pre action passes for (auto &pass : pre_actions) { RETURN_IF_NOT_OK(pass->Run(this, &modified)); @@ -267,27 +271,40 @@ Status ExecutionTree::PrepareNode(const std::shared_ptr &dataset_op) RETURN_IF_NOT_OK(this->PrepareNode(i)); } - // Then clear the flags from this op now that we have prepared it. - BitClear(&prepare_flags_, op_prep_flags); - // No more children, now we execute any prepare actions before going back up the // the tree on recursive function RETURN_IF_NOT_OK(dataset_op->PrepareNodePostAction()); + // Then clear the flags from this op now that we have prepared it. + BitClear(&prepare_flags_, op_prep_flags); + return Status::OK(); } -// Adds an operator to the repeat stack during prepare phase. -void ExecutionTree::AddToRepeatStack(std::shared_ptr dataset_op) { repeat_stack_.push(dataset_op); } +// Adds an operator to the eoe operator stack during prepare phase. +void ExecutionTree::AddToEOEOpStack(std::shared_ptr dataset_op) { eoe_stack_.push(dataset_op); } -// Pops an operator from the repeat stack during prepare phase. -std::shared_ptr ExecutionTree::PopFromRepeatStack() { +// Pops an operator from the eoe operator stack during prepare phase. +std::shared_ptr ExecutionTree::PopFromEOEOpStack() { std::shared_ptr top_op = nullptr; - if (!repeat_stack_.empty()) { - top_op = repeat_stack_.top(); - repeat_stack_.pop(); + if (!eoe_stack_.empty()) { + top_op = eoe_stack_.top(); + eoe_stack_.pop(); } return top_op; } + +// Adds a sampler to the sampler stack during prepare phase. +void ExecutionTree::AddToSamplerStack(std::shared_ptr sampler) { sampler_stack_.push(sampler); } + +// Pops an operator from the sampler stack during prepare phase. +std::shared_ptr ExecutionTree::PopFromSamplerStack() { + std::shared_ptr top_sampler = nullptr; + if (!sampler_stack_.empty()) { + top_sampler = sampler_stack_.top(); + sampler_stack_.pop(); + } + return top_sampler; +} } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/execution_tree.h b/mindspore/ccsrc/dataset/engine/execution_tree.h index e1c5e8ff54..5ebfa539ad 100644 --- a/mindspore/ccsrc/dataset/engine/execution_tree.h +++ b/mindspore/ccsrc/dataset/engine/execution_tree.h @@ -37,7 +37,8 @@ class ExecutionTree { // Prepare flags used during tree prepare phase enum PrepareFlags { kDePrepNone = 0, - kDePrepRepeat = 1 // Processing a repeat operation + kDePrepRepeat = 1, // Processing a repeat operation + kDePrepCache = 2 // Processing a cache operation }; // State flags for the lifecycle of the tree @@ -88,8 +89,10 @@ class ExecutionTree { bool operator!=(const Iterator &rhs) { return nodes_[ind_] != rhs.nodes_[rhs.ind_]; } + int32_t NumNodes() { return nodes_.size(); } + private: - int ind_; // the cur node our Iterator points to + int32_t ind_; // the cur node our Iterator points to std::vector> nodes_; // store the nodes in post order void PostOrderTraverse(const std::shared_ptr &); }; @@ -116,9 +119,9 @@ class ExecutionTree { // @return Status - The error code return Status Launch(); - // A print method typically used for debugging - // @param out - The output stream to write output to - void Print(std::ostream &out) const; + /// A print method typically used for debugging + /// \param out - The output stream to write output to + void Print(std::ostream &out, const std::shared_ptr &op = nullptr) const; // Returns an iterator positioned at the start // @return Iterator - The iterator @@ -197,14 +200,23 @@ class ExecutionTree { // @return Status - The error code return Status PrepareNode(const std::shared_ptr &dataset_op); - // Adds an operator to the repeat stack during prepare phase. - // @param op - The dataset op to work add to repeat stack - // @return Status - The error code return - void AddToRepeatStack(std::shared_ptr dataset_op); + /// Adds an operator to the eoe operator stack during prepare phase. + /// \param op - The dataset op to work add to eoe stack + /// \return Status - The error code return + void AddToEOEOpStack(std::shared_ptr dataset_op); + + /// Pops an operator from the eoe operator stack during prepare phase. + /// \return shared_ptr to the popped operator + std::shared_ptr PopFromEOEOpStack(); + + /// Adds a sampler to the sampler stack during prepare phase. + /// \param samplerop - The dataset op to work add to eoe stack + /// \return Status - The error code return + void AddToSamplerStack(std::shared_ptr sampler); - // Pops an operator from the repeat stack during prepare phase. - // @return shared_ptr to the popped operator - std::shared_ptr PopFromRepeatStack(); + /// Pops an operator from the sampler stack during prepare phase. + /// \return shared_ptr to the popped operator + std::shared_ptr PopFromSamplerStack(); // Return the pointer to the TaskGroup // @return raw pointer to the TaskGroup @@ -234,9 +246,10 @@ class ExecutionTree { int32_t id_count_; // Counter for generating operator id's uint32_t prepare_flags_; // Flags used during tree prepare TreeState tree_state_; // Tracking the current tree state - std::stack> repeat_stack_; // A stack used during prepare phase std::unique_ptr perf_monitor_; // Performance Monitor std::unique_ptr profiling_manager_; // Profiling manager + std::stack> eoe_stack_; // A stack used during prepare phase + std::stack> sampler_stack_; // A stack used during prepare phase }; } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/gnn/graph.cc b/mindspore/ccsrc/dataset/engine/gnn/graph.cc index 1017657397..a143bd4e38 100644 --- a/mindspore/ccsrc/dataset/engine/gnn/graph.cc +++ b/mindspore/ccsrc/dataset/engine/gnn/graph.cc @@ -149,14 +149,37 @@ Status Graph::GetAllNeighbors(const std::vector &node_list, NodeType return Status::OK(); } +Status Graph::CheckSamplesNum(NodeIdType samples_num) { + NodeIdType all_nodes_number = + std::accumulate(node_type_map_.begin(), node_type_map_.end(), 0, + [](NodeIdType t1, const auto &t2) -> NodeIdType { return t1 + t2.second.size(); }); + if ((samples_num < 1) || (samples_num > all_nodes_number)) { + std::string err_msg = "Wrong samples number, should be between 1 and " + std::to_string(all_nodes_number) + + ", got " + std::to_string(samples_num); + RETURN_STATUS_UNEXPECTED(err_msg); + } + return Status::OK(); +} + Status Graph::GetSampledNeighbors(const std::vector &node_list, const std::vector &neighbor_nums, const std::vector &neighbor_types, std::shared_ptr *out) { CHECK_FAIL_RETURN_UNEXPECTED(!node_list.empty(), "Input node_list is empty."); CHECK_FAIL_RETURN_UNEXPECTED(neighbor_nums.size() == neighbor_types.size(), "The sizes of neighbor_nums and neighbor_types are inconsistent."); + for (const auto &num : neighbor_nums) { + RETURN_IF_NOT_OK(CheckSamplesNum(num)); + } + for (const auto &type : neighbor_types) { + if (node_type_map_.find(type) == node_type_map_.end()) { + std::string err_msg = "Invalid neighbor type:" + std::to_string(type); + RETURN_STATUS_UNEXPECTED(err_msg); + } + } std::vector> neighbors_vec(node_list.size()); for (size_t node_idx = 0; node_idx < node_list.size(); ++node_idx) { + std::shared_ptr input_node; + RETURN_IF_NOT_OK(GetNodeByNodeId(node_list[node_idx], &input_node)); neighbors_vec[node_idx].emplace_back(node_list[node_idx]); std::vector input_list = {node_list[node_idx]}; for (size_t i = 0; i < neighbor_nums.size(); ++i) { @@ -204,6 +227,12 @@ Status Graph::NegativeSample(const std::vector &data, const std::uno Status Graph::GetNegSampledNeighbors(const std::vector &node_list, NodeIdType samples_num, NodeType neg_neighbor_type, std::shared_ptr *out) { CHECK_FAIL_RETURN_UNEXPECTED(!node_list.empty(), "Input node_list is empty."); + RETURN_IF_NOT_OK(CheckSamplesNum(samples_num)); + if (node_type_map_.find(neg_neighbor_type) == node_type_map_.end()) { + std::string err_msg = "Invalid neighbor type:" + std::to_string(neg_neighbor_type); + RETURN_STATUS_UNEXPECTED(err_msg); + } + std::vector> neighbors_vec; neighbors_vec.resize(node_list.size()); for (size_t node_idx = 0; node_idx < node_list.size(); ++node_idx) { diff --git a/mindspore/ccsrc/dataset/engine/gnn/graph.h b/mindspore/ccsrc/dataset/engine/gnn/graph.h index ea10363053..344a6c6bf2 100644 --- a/mindspore/ccsrc/dataset/engine/gnn/graph.h +++ b/mindspore/ccsrc/dataset/engine/gnn/graph.h @@ -226,6 +226,8 @@ class Graph { Status NegativeSample(const std::vector &input_data, const std::unordered_set &exclude_data, int32_t samples_num, std::vector *out_samples); + Status CheckSamplesNum(NodeIdType samples_num); + std::string dataset_file_; int32_t num_workers_; // The number of worker threads std::mt19937 rnd_; diff --git a/mindspore/ccsrc/dataset/engine/jagged_connector.h b/mindspore/ccsrc/dataset/engine/jagged_connector.h index 5f9f255f2b..2058c542a8 100644 --- a/mindspore/ccsrc/dataset/engine/jagged_connector.h +++ b/mindspore/ccsrc/dataset/engine/jagged_connector.h @@ -44,7 +44,7 @@ class JaggedConnector : public Connector> { Status Pop(int32_t worker_id, std::unique_ptr *result) noexcept override { { - DS_ASSERT(worker_id < num_consumers_); + MS_ASSERT(worker_id < num_consumers_); std::unique_lock lock(m_); RETURN_IF_NOT_OK(cv_.Wait(&lock, [this, worker_id]() { return expect_consumer_ == worker_id; })); if (is_queue_finished_[pop_from_]) { diff --git a/mindspore/ccsrc/dataset/engine/opt/CMakeLists.txt b/mindspore/ccsrc/dataset/engine/opt/CMakeLists.txt index 170cbb55e5..080d968cfc 100644 --- a/mindspore/ccsrc/dataset/engine/opt/CMakeLists.txt +++ b/mindspore/ccsrc/dataset/engine/opt/CMakeLists.txt @@ -1,8 +1,8 @@ file(GLOB_RECURSE _CURRENT_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc") set_property(SOURCE ${_CURRENT_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_MD) add_library(engine-opt OBJECT - pass.cc - pre/map_column_reorder.cc - pre/global_shuffle.cc - util/printer_pass.cc - ) \ No newline at end of file + pass.cc + pre/removal_nodes.cc + pre/removal_pass.cc + util/printer_pass.cc + ) diff --git a/mindspore/ccsrc/dataset/engine/opt/pass.cc b/mindspore/ccsrc/dataset/engine/opt/pass.cc index a032d46cba..27769f056b 100644 --- a/mindspore/ccsrc/dataset/engine/opt/pass.cc +++ b/mindspore/ccsrc/dataset/engine/opt/pass.cc @@ -61,6 +61,7 @@ Status NodePass::Run(ExecutionTree *tree, bool *modified) { // Helper function to perform DFS visit Status NodePass::DFSNodeVisit(std::shared_ptr node, bool *modified) { + RETURN_IF_NOT_OK(node->PreAccept(this, modified)); for (const auto &c : node->Children()) { RETURN_IF_NOT_OK(this->DFSNodeVisit(c, modified)); } @@ -159,6 +160,5 @@ Status NodePass::RunOnNode(std::shared_ptr node, bool *modified) // Fallback to base class visitor by default return RunOnNode(std::static_pointer_cast(node), modified); } - } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/opt/pass.h b/mindspore/ccsrc/dataset/engine/opt/pass.h index 39682b22f7..129c2fab37 100644 --- a/mindspore/ccsrc/dataset/engine/opt/pass.h +++ b/mindspore/ccsrc/dataset/engine/opt/pass.h @@ -66,14 +66,16 @@ class Pass : public std::enable_shared_from_this { // TreePass is a basic Pass class which performs transformation on ExecutionTree directly. class TreePass : public Pass { public: - // Run the transformation pass against the execution tree. - // @param tree - Pointer to the execution tree to be transformed. - // @param modified - Pointer to the modified flag, + /// \brief Run the transformation pass against the execution tree. + /// \param[inout] tree Pointer to the execution tree to be transformed. + /// \param[inout] modified Indicate if the tree was modified Status Run(ExecutionTree *tree, bool *modified) final; - // Derived classes may implement the runOnTree function to implement tree transformation. - // "modified" flag needs to be set to true if tree is modified during the pass execution. - // @return Status - The error code return + /// \brief Derived classes may implement the runOnTree function to implement tree transformation. + /// "modified" flag needs to be set to true if tree is modified during the pass execution. + /// \param[inout] tree The tree to operate on. + /// \param[inout] Indicate of the tree was modified. + /// \return Status The error code return virtual Status RunOnTree(ExecutionTree *tree, bool *modified) { return Status::OK(); } }; @@ -90,14 +92,23 @@ class NodePass : public Pass { ~NodePass() = default; - // Run the transformation pass against the execution tree. - // @param tree - Pointer to the execution tree to be transformed. - // @param modified - Pointer to the modified flag, + /// \brief Run the transformation pass against the execution tree + /// \param[inout] tree Pointer to the execution tree to be transformed + /// \param[inout] modified Indicator if the tree was changed Status Run(ExecutionTree *tree, bool *modified) final; - // Derived classes may implement the runOnNode function to implement node level tree transformation. - // "modified" flag needs to be set to true if tree is modified during the pass execution. - // @return Status - The error code return + /// \brief Derived classes may implement the PreRunOnNode function to implement any initial visit work on the way down + /// a tree traversal. "modified" flag needs to be set to true if tree is modified during the pass execution + /// \param[in] node The node being visited + /// \param[out] modified Indicator if the node was changed at all + /// \return Status The error code return + virtual Status PreRunOnNode(std::shared_ptr node, bool *modified) { return Status::OK(); } + + /// \brief Derived classes may implement the RunOnNode function to implement node level tree transformation + /// "modified" flag needs to be set to true if tree is modified during the pass execution + /// \param[in] node The node being visited + /// \param[out] modified Indicator if the node was changed at all. + /// \return Status The error code return virtual Status RunOnNode(std::shared_ptr node, bool *modified) { return Status::OK(); } // Visit methods to be overridden. diff --git a/mindspore/ccsrc/dataset/engine/opt/pre/global_shuffle.cc b/mindspore/ccsrc/dataset/engine/opt/pre/global_shuffle.cc deleted file mode 100644 index 2adf734a6c..0000000000 --- a/mindspore/ccsrc/dataset/engine/opt/pre/global_shuffle.cc +++ /dev/null @@ -1,98 +0,0 @@ -/** - * Copyright 2020 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include "dataset/engine/opt/pre/global_shuffle.h" -#include "dataset/engine/execution_tree.h" -#include "dataset/engine/datasetops/shuffle_op.h" -#include "dataset/engine/datasetops/source/tf_reader_op.h" -#include "dataset/engine/datasetops/source/text_file_op.h" -#include "dataset/engine/datasetops/source/clue_op.h" - -namespace mindspore { -namespace dataset { - -Status GlobalShufflePass::RunOnTree(ExecutionTree *tree, bool *modified) { - std::vector> tf_readers; - std::vector> text_files; - std::vector> clues; - - // Pass 1, search for all sources which requires global shuffle - for (auto &op : *tree) { - if (auto ptr = std::dynamic_pointer_cast(op.shared_from_this())) { - if (ptr->RequireGlobalShuffle()) { - tf_readers.push_back(ptr); - continue; - } - } - if (auto ptr = std::dynamic_pointer_cast(op.shared_from_this())) { - if (ptr->RequireGlobalShuffle()) { - text_files.push_back(ptr); - continue; - } - } - if (auto ptr = std::dynamic_pointer_cast(op.shared_from_this())) { - if (ptr->RequireGlobalShuffle()) { - clues.push_back(ptr); - continue; - } - } - } - - // Pass 2, insert shuffle nodes - // The following blocks can be implemented with template if we unify the CountTotalRows across all source nodes . - for (auto node : tf_readers) { - std::shared_ptr builder = std::make_shared(); - int64_t total_rows = 0; - TFReaderOp::CountTotalRows(&total_rows, node->FileNames(), 8, true); - int32_t avg_file_size = total_rows / (node->FileNames().size()); - builder->SetShuffleSize(std::max(avg_file_size * 4, 10000)); - std::shared_ptr op; - RETURN_IF_NOT_OK(builder->Build(&op)); - RETURN_IF_NOT_OK(tree->AssociateNode(op)); - RETURN_IF_NOT_OK(node->InsertAsParent(op)); - } - - for (auto node : text_files) { - std::shared_ptr builder = std::make_shared(); - int64_t total_rows = 0; - TextFileOp::CountAllFileRows(node->FileNames(), &total_rows); - int32_t avg_file_size = total_rows / (node->FileNames().size()); - builder->SetShuffleSize(std::max(avg_file_size * 4, 10000)); - std::shared_ptr op; - RETURN_IF_NOT_OK(builder->Build(&op)); - RETURN_IF_NOT_OK(tree->AssociateNode(op)); - RETURN_IF_NOT_OK(node->InsertAsParent(op)); - } - - for (auto node : clues) { - std::shared_ptr builder = std::make_shared(); - int64_t total_rows = 0; - ClueOp::CountAllFileRows(node->FileNames(), &total_rows); - int32_t avg_file_size = total_rows / (node->FileNames().size()); - builder->SetShuffleSize(std::max(avg_file_size * 4, 10000)); - std::shared_ptr op; - RETURN_IF_NOT_OK(builder->Build(&op)); - RETURN_IF_NOT_OK(tree->AssociateNode(op)); - RETURN_IF_NOT_OK(node->InsertAsParent(op)); - } - - return Status::OK(); -} - -} // namespace dataset -} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/opt/pre/global_shuffle.h b/mindspore/ccsrc/dataset/engine/opt/pre/global_shuffle.h deleted file mode 100644 index 6865ac9391..0000000000 --- a/mindspore/ccsrc/dataset/engine/opt/pre/global_shuffle.h +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Copyright 2020 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef DATASET_ENGINE_OPT_PASS_PRE_GLOBALSHUFFLE_H -#define DATASET_ENGINE_OPT_PASS_PRE_GLOBALSHUFFLE_H - -#include -#include "dataset/engine/opt/pass.h" - -namespace mindspore { -namespace dataset { -// Global Shuffle Pass will insert ShuffleOp when the leaf nodes requires global shuffle. -// Example: -// Input Tree: TFReader(GLOBAL_SHUFFLE) -> Batch -// Output Tree: TFReader -> Shuffle -> Batch -class GlobalShufflePass : public TreePass { - Status RunOnTree(ExecutionTree *tree, bool *modified) override; -}; -} // namespace dataset -} // namespace mindspore - -#endif // DATASET_ENGINE_OPT_PASS_PRE_GLOBALSHUFFLE_H diff --git a/mindspore/ccsrc/dataset/engine/opt/pre/map_column_reorder.cc b/mindspore/ccsrc/dataset/engine/opt/pre/map_column_reorder.cc deleted file mode 100644 index a3dbbfcc54..0000000000 --- a/mindspore/ccsrc/dataset/engine/opt/pre/map_column_reorder.cc +++ /dev/null @@ -1,51 +0,0 @@ -/** - * Copyright 2020 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include "dataset/engine/opt/pre/map_column_reorder.h" -#include "dataset/engine/execution_tree.h" -#include "dataset/engine/datasetops/map_op.h" -#include "dataset/engine/datasetops/project_op.h" - -namespace mindspore { -namespace dataset { - -Status MapColumnReorder::RunOnTree(ExecutionTree *tree, bool *modified) { - std::vector> to_process; - - // Pass 1, search for all MapOp with column orders - for (auto &op : *tree) { - if (auto mapOp = std::dynamic_pointer_cast(op.shared_from_this())) { - if (mapOp->ColumnsOrder().size() != 0) { - to_process.push_back(mapOp); - } - } - } - - // Pass 2, insert nodes for all MapOp - for (auto node : to_process) { - std::shared_ptr builder = std::make_shared(node->ColumnsOrder()); - std::shared_ptr op; - RETURN_IF_NOT_OK(builder->Build(&op)); - RETURN_IF_NOT_OK(tree->AssociateNode(op)); - RETURN_IF_NOT_OK(node->InsertAsParent(op)); - } - return Status::OK(); -} - -} // namespace dataset -} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/opt/pre/removal_nodes.cc b/mindspore/ccsrc/dataset/engine/opt/pre/removal_nodes.cc new file mode 100644 index 0000000000..831a2a76ba --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/opt/pre/removal_nodes.cc @@ -0,0 +1,42 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include "dataset/engine/opt/pre/removal_nodes.h" +#include "dataset/engine/opt/pre/removal_pass.h" +#include "dataset/engine/datasetops/shuffle_op.h" + +namespace mindspore { +namespace dataset { + +RemovalNodes::RemovalNodes(RemovalPass *removal_pass) : removal_pass_(removal_pass), is_caching_(false) {} + +// Perform ShuffleOp removal check. +Status RemovalNodes::RunOnNode(std::shared_ptr node, bool *modified) { + *modified = false; + // If we are in a cache descendant tree, then this shuffle op needs to be removed + if (is_caching_) { + MS_LOG(DEBUG) << "ShuffleOp identified for removal (CacheOp is in ascendant tree)"; + if (removal_pass_) { + removal_pass_->AddToRemovalList(std::static_pointer_cast(node)); + } else { + return Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, "Back reference to removal pass is missing!"); + } + } + return Status::OK(); +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/opt/pre/removal_nodes.h b/mindspore/ccsrc/dataset/engine/opt/pre/removal_nodes.h new file mode 100644 index 0000000000..11ef37d80c --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/opt/pre/removal_nodes.h @@ -0,0 +1,51 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef DATASET_ENGINE_OPT_PASS_PRE_REMOVAL_NODES_H_ +#define DATASET_ENGINE_OPT_PASS_PRE_REMOVAL_NODES_H_ + +#include +#include "dataset/engine/opt/pass.h" + +namespace mindspore { +namespace dataset { + +class RemovalPass; + +/// \class RemovalNodes removal_nodes.h +/// \brief This is a NodePass who's job is to identify which nodes should be removed. +/// It works in conjunction with the removal_pass. +class RemovalNodes : public NodePass { + public: + /// \brief Constructor + /// \param[in] removal_pass Raw pointer back to controlling tree pass + explicit RemovalNodes(RemovalPass *removal_pass); + + /// \brief Perform ShuffleOp removal check + /// \param[in] node The node being visited + /// \param[inout] modified Indicator if the node was changed at all + /// \return Status The error code return + Status RunOnNode(std::shared_ptr node, bool *modified) override; + + private: + bool is_caching_; + RemovalPass *removal_pass_; // Back pointer to the owning removal pass +}; + +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_ENGINE_OPT_PASS_PRE_REMOVAL_NODES_ diff --git a/mindspore/ccsrc/dataset/engine/opt/pre/removal_pass.cc b/mindspore/ccsrc/dataset/engine/opt/pre/removal_pass.cc new file mode 100644 index 0000000000..31ec31234f --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/opt/pre/removal_pass.cc @@ -0,0 +1,45 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include "dataset/engine/opt/pre/removal_nodes.h" +#include "dataset/engine/opt/pre/removal_pass.h" +#include "dataset/engine/execution_tree.h" + +namespace mindspore { +namespace dataset { + +// constructor +RemovalPass::RemovalPass() {} + +// Runs a removal_nodes pass first to find out which nodes to remove, then removes them. +Status RemovalPass::RunOnTree(ExecutionTree *tree, bool *modified) { + // Create the removal node pass which can identify which nodes need to be removed. + std::unique_ptr removal_nodes = std::make_unique(this); + RETURN_IF_NOT_OK(removal_nodes->Run(tree, modified)); + + // Then, execute the removal of any nodes that were set up for removal + for (auto node : removal_nodes_) { + node->Remove(); + } + return Status::OK(); +} + +// Adds an operator to the list of operators to be removed +void RemovalPass::AddToRemovalList(std::shared_ptr dataset_op) { removal_nodes_.push_back(dataset_op); } +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/opt/pre/removal_pass.h b/mindspore/ccsrc/dataset/engine/opt/pre/removal_pass.h new file mode 100644 index 0000000000..6523ca69b2 --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/opt/pre/removal_pass.h @@ -0,0 +1,53 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef DATASET_ENGINE_OPT_PASS_PRE_REMOVAL_PASS_H_ +#define DATASET_ENGINE_OPT_PASS_PRE_REMOVAL_PASS_H_ + +#include +#include +#include "dataset/engine/opt/pass.h" + +namespace mindspore { +namespace dataset { + +class DatasetOp; + +/// \class RemovalPass removal_pass.h +/// \brief This is a tree pass that will remove nodes. It uses removal_nodes to first identify which +/// nodes should be removed, and then removes them. +class RemovalPass : public TreePass { + public: + /// \brief Constructor + RemovalPass(); + + /// \brief Runs a removal_nodes pass first to find out which nodes to remove, then removes them. + /// \param[inout] tree The tree to operate on. + /// \param[inout] Indicate of the tree was modified. + /// \return Status The error code return + Status RunOnTree(ExecutionTree *tree, bool *modified) override; + + /// \brief Adds an operator to the list of operators to be removed + /// \param[in] dataset_op The operator to add to the removal list + void AddToRemovalList(std::shared_ptr dataset_op); + + private: + std::vector> removal_nodes_; +}; +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_ENGINE_OPT_PASS_PRE_REMOVAL_PASS_H_ diff --git a/mindspore/ccsrc/dataset/engine/perf/CMakeLists.txt b/mindspore/ccsrc/dataset/engine/perf/CMakeLists.txt index 0b67469d2d..e611add983 100644 --- a/mindspore/ccsrc/dataset/engine/perf/CMakeLists.txt +++ b/mindspore/ccsrc/dataset/engine/perf/CMakeLists.txt @@ -3,4 +3,6 @@ add_library(engine-perf OBJECT monitor.cc device_queue_tracing.cc connector_size.cc - dataset_iterator_tracing.cc) + dataset_iterator_tracing.cc + connector_throughput.cc + ) diff --git a/mindspore/ccsrc/dataset/engine/perf/connector_size.cc b/mindspore/ccsrc/dataset/engine/perf/connector_size.cc index 862ec51c49..0bd2754075 100644 --- a/mindspore/ccsrc/dataset/engine/perf/connector_size.cc +++ b/mindspore/ccsrc/dataset/engine/perf/connector_size.cc @@ -14,7 +14,6 @@ * limitations under the License. */ #include "dataset/engine/perf/connector_size.h" - #include #include #include diff --git a/mindspore/ccsrc/dataset/engine/perf/connector_size.h b/mindspore/ccsrc/dataset/engine/perf/connector_size.h index 6840ffe244..2584289fb4 100644 --- a/mindspore/ccsrc/dataset/engine/perf/connector_size.h +++ b/mindspore/ccsrc/dataset/engine/perf/connector_size.h @@ -13,8 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef MINDSPORE_QUEUE_DEPTH_H -#define MINDSPORE_QUEUE_DEPTH_H +#ifndef DATASET_CONNECTOR_SIZE_H +#define DATASET_CONNECTOR_SIZE_H #include #include @@ -50,7 +50,7 @@ class ConnectorSize : public Sampling { // This function samples the connector size of every nodes within the ExecutionTree Status Sample() override; - std::string Name() const override { return kDeviceQueueTracingName; }; + std::string Name() const override { return kConnectorSizeSamplingName; } // Save sampling data to file // @return Status - The error code return @@ -65,6 +65,8 @@ class ConnectorSize : public Sampling { ExecutionTree *tree_ = nullptr; // ExecutionTree pointer ConnectorSizeSampleTable sample_table_; // Dataset structure to store all samples of connector size sampling }; + } // namespace dataset } // namespace mindspore -#endif // MINDSPORE_QUEUE_DEPTH_H + +#endif // DATASET_CONNECTOR_SIZE_H diff --git a/mindspore/ccsrc/dataset/engine/perf/connector_throughput.cc b/mindspore/ccsrc/dataset/engine/perf/connector_throughput.cc new file mode 100644 index 0000000000..4fd59de390 --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/perf/connector_throughput.cc @@ -0,0 +1,109 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include "dataset/engine/perf/connector_throughput.h" +#include "dataset/engine/execution_tree.h" +#include "dataset/util/path.h" + +namespace mindspore { +namespace dataset { + +// temporary helper +int ConnectorThroughput::InitNodes() { + auto it = (*tree_).begin(); + return it.NumNodes(); +} +// Sample action +Status ConnectorThroughput::Sample() { + std::vector out_buffer_count_row(n_nodes_); + std::vector throughput_row(n_nodes_); + TimePoint cur_time; // initialised inside the loop, used outside the loop to update prev sample time. + auto col = 0; + for (const auto &node : *tree_) { + auto cur_out_buffer_count = node.ConnectorOutBufferCount(); + out_buffer_count_row[col] = cur_out_buffer_count; + auto sz = timestamps_.size(); + cur_time = std::chrono::steady_clock::now(); + auto _dt = std::chrono::duration_cast(timestamps_[0][sz - 1] - timestamps_[0][sz - 2]); + auto dt = std::chrono::duration(_dt).count(); + auto prev_out_buffer_count = out_buffer_count_table_[col][out_buffer_count_table_.size() - 1]; + if (dt != 0) { + auto thr = (cur_out_buffer_count - prev_out_buffer_count) / (1000 * dt); + throughput_row[col] = thr; + } else { + throughput_row[col] = -1; + } + col++; + } + std::vector v = {cur_time}; // temporary fix + timestamps_.AddSample(v); + // Push new row of sample + out_buffer_count_table_.AddSample(out_buffer_count_row); + throughput_.AddSample(throughput_row); + return Status::OK(); +} + +json ConnectorThroughput::ParseOpInfo(const DatasetOp &node, const std::vector &thr) { + auto children = node.Children(); + std::vector children_id; + std::transform(children.begin(), children.end(), std::back_inserter(children_id), + [](std::shared_ptr op) -> int32_t { return op->id(); }); + json json_node; + json_node["op_id"] = node.id(); + json_node["op_type"] = node.Name(); + json_node["num_workers"] = node.num_workers(); + json metrics; + metrics["output_queue"] = {{"throughput", thr}}; + + json_node["metrics"] = metrics; + if (!children_id.empty()) { + json_node["children"] = children_id; + } + + return json_node; +} + +// Save profiling data to file +Status ConnectorThroughput::SaveToFile() { + std::ofstream os(file_path_); + json output; + output["sampling_interval"] = 10; + // Traverse the ExecutionTree for JSON node generation + int col = 0; + for (auto &node : *tree_) { + std::vector throughput; + for (auto i = 0; i < throughput_.size(); i++) { + throughput.push_back(throughput_[col][i]); + } + json json_node = ParseOpInfo(node, throughput); + output["op_info"].push_back(json_node); + col++; + } + os << output; + return Status::OK(); +} +Status ConnectorThroughput::Init(const std::string &dir_path, const std::string &device_id) { + file_path_ = (Path(dir_path) / Path("pipeline_profiling_" + Name() + "_" + device_id + ".json")).toString(); + return Status::OK(); +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/perf/connector_throughput.h b/mindspore/ccsrc/dataset/engine/perf/connector_throughput.h new file mode 100644 index 0000000000..e873eb8315 --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/perf/connector_throughput.h @@ -0,0 +1,100 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef DATASET_CONNECTOR_THROUGHPUT_H +#define DATASET_CONNECTOR_THROUGHPUT_H + +#include +#include +#include +#include +#include +#include "dataset/engine/perf/profiling.h" +#include "dataset/engine/perf/perf_data.h" +#include "dataset/engine/perf/cyclic_array.h" +#include "dataset/engine/datasetops/dataset_op.h" + +using json = nlohmann::json; +namespace mindspore { +namespace dataset { +class ExecutionTree; + +// Connector throughput samples the output connector size of each op in the pipeline. +// For the description of the data structure see perf_buffer.h +// It support JSON serialization for external usage. +class ConnectorThroughput : public Sampling { + using OutBufferCount = PerfData>; + using Throughput = PerfData>; + using TimePoint = std::chrono::time_point; + using TimeStamps = PerfData>; + + public: + explicit ConnectorThroughput(ExecutionTree *tree, int64_t max_rows = 1000000) + : tree_(tree), + max_rows_(max_rows), + n_nodes_(InitNodes()), + out_buffer_count_table_(OutBufferCount(max_rows_, n_nodes_)), + throughput_(Throughput(max_rows_, n_nodes_)), + timestamps_(TimeStamps(max_rows_, 1)) { + timestamps_.AddSample(std::vector(1)); + out_buffer_count_table_.AddSample(std::vector(n_nodes_)); + } + // Driver function for connector size sampling. + // This function samples the connector size of every nodes within the ExecutionTree + Status Sample() override; + + /* Status TestPrint() override { + std::ofstream os("performance_monitor.txt"); + if (throughput_.size() == 0) { + os << "data is empty" << std::endl; + return Status::OK(); + } + for (int i = 0; i < throughput_.size(); i++) { + for (int j = 0; j < n_nodes_; j++) { + os << throughput_[j][i] << " "; + } + os << std::endl; + } + return Status::OK(); + };*/ + + // Traverse the tree nodes and count them + int InitNodes(); + + std::string Name() const override { return name_; }; + + // Save sampling data to file + // @return Status - The error code return + Status SaveToFile() override; + + Status Init(const std::string &dir_path, const std::string &device_id); + + json ParseOpInfo(const DatasetOp &node, const std::vector &thr); + + private: + ExecutionTree *tree_ = nullptr; // ExecutionTree pointer + int64_t max_rows_; + int32_t n_nodes_; + OutBufferCount out_buffer_count_table_; + Throughput throughput_; + TimeStamps timestamps_; + std::string name_ = kConnectorThroughputSamplingName; +}; + +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_CONNECTOR_THROUGHPUT_H diff --git a/mindspore/ccsrc/dataset/engine/perf/cyclic_array.h b/mindspore/ccsrc/dataset/engine/perf/cyclic_array.h new file mode 100644 index 0000000000..fa60b401c5 --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/perf/cyclic_array.h @@ -0,0 +1,197 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef DATASET_CYCLIC_ARRAY_H +#define DATASET_CYCLIC_ARRAY_H + +#include +#include +#include +#include +#include "dataset/core/constants.h" + +namespace mindspore { +namespace dataset { + +/// \class CyclicArray "include/cyclic_array.h +/// \brief This is a container with a contiguous memory layout that pnly keeps N last entries, +/// when the number of entries exceeds the capacity +/// Must be preallocated +template +class CyclicArray { + public: + using value_type = T; + class Iterator { + // Add operator[] and make fully compliant with random access iterator + // and add a const iterator + // add resize(), empty() + public: + using iterator_category = std::random_access_iterator_tag; + using value_type = CyclicArray::value_type; + using difference_type = std::ptrdiff_t; + using pointer = CyclicArray::value_type *; + using reference = CyclicArray::value_type &; + + Iterator() = default; + + Iterator(dsize_t idx, pointer ptr, dsize_t capacity, dsize_t head) + : cur_idx_(idx), ptr_(ptr), capacity_(capacity), head_(head) {} + + Iterator(const Iterator &rhs) = default; + + ~Iterator() = default; + + Iterator &operator++() { + cur_idx_ = (cur_idx_ + 1) % (capacity_ + 1); + return *this; + } + + Iterator operator++(int) { + Iterator tmp(*this); + cur_idx_ = (cur_idx_ + 1) % (capacity_ + 1); + return tmp; + } + + Iterator &operator--() { + cur_idx_ = (cur_idx_ + capacity_) % (capacity_ + 1); + return *this; + } + + Iterator operator--(int) { + Iterator tmp(*this); + cur_idx_ = (cur_idx_ + capacity_) % (capacity_ + 1); + return tmp; + } + + Iterator operator+(dsize_t x) { return Iterator((cur_idx_ + x) % (capacity_ + 1), ptr_, capacity_, head_); } + + Iterator operator-(dsize_t x) { + return Iterator((cur_idx_ + (capacity_ + 1 - x)) % (capacity_ + 1), ptr_, capacity_, head_); + } + + bool operator<(const Iterator &rhs) { + return (head_ + cur_idx_) % (capacity_ + 1) < (rhs.head_ + rhs.cur_idx_) % (capacity_ + 1); + } + + bool operator>(const Iterator &rhs) { + return (head_ + cur_idx_) % (capacity_ + 1) > (rhs.head_ + rhs.cur_idx_) % (capacity_ + 1); + } + + bool operator>=(const Iterator &rhs) { + return (head_ + cur_idx_) % (capacity_ + 1) >= (rhs.head_ + rhs.cur_idx_) % (capacity_ + 1); + } + + bool operator<=(const Iterator &rhs) { + return (head_ + cur_idx_) % (capacity_ + 1) <= (rhs.head_ + rhs.cur_idx_) % (capacity_ + 1); + } + + difference_type operator-(const Iterator &rhs) { + return (cur_idx_ - rhs.cur_idx_ + capacity_ + 1) % (capacity_ + 1); + } + + reference operator*() { return ptr_[cur_idx_]; } + + pointer operator->() { return &(ptr_[cur_idx_]); } + + bool operator==(const Iterator &rhs) { return cur_idx_ == rhs.cur_idx_; } + + bool operator!=(const Iterator &rhs) { return cur_idx_ != rhs.cur_idx_; } + + private: + dsize_t cur_idx_; + pointer ptr_; + dsize_t capacity_; + dsize_t head_; + }; + + /// \brief Default constructor + CyclicArray() : buf_(nullptr), head_(0), tail_(0), size_(0), capacity_(0) {} + + /// \brief Constructor + /// \param[in] capacity + explicit CyclicArray(dsize_t capacity) + : buf_(std::make_unique(capacity + 1)), head_(0), tail_(0), size_(0), capacity_(capacity) {} + + CyclicArray(const CyclicArray &rhs) + : buf_(std::make_unique(rhs.capacity_ + 1)), + head_(rhs.head_), + tail_(rhs.tail_), + size_(rhs.size_), + capacity_(rhs.capacity_) { + std::copy(rhs.begin(), rhs.end(), begin()); + } + + CyclicArray(CyclicArray &&rhs) = default; + + ~CyclicArray() = default; + + /// \brief Iterator begin() + Iterator begin() { return Iterator(head_, buf_.get(), capacity_, head_); } + + /// \brief Iterator end() + Iterator end() { return Iterator(tail_, buf_.get(), capacity_, head_); } + + // not really const. + Iterator begin() const { return Iterator(head_, buf_.get(), capacity_, head_); } + + Iterator end() const { return Iterator(tail_, buf_.get(), capacity_, head_); } + + /// \brief clear the array. Does not deallocate memory, capacity remains the same + void clear() { + head_ = 0; + tail_ = 0; + size_ = 0; + } + + /// \brief returns current size + dsize_t size() { return size_; } + + /// \brief returns capacity + dsize_t capacity() { return capacity_; } + + /// \brief pushes a value + /// \param[in] val value + void push_back(T val) { + buf_[tail_] = val; + if (size_ >= capacity_) { + (tail_ != capacity_) ? tail_++ : tail_ = 0; + (head_ != capacity_) ? head_++ : head_ = 0; + } else { + tail_++; + size_++; + } + } + + /// \brief returns const reference to an element of the array + /// \param[in] idx index of the element + /// \param[out] const T& reference to an element of the array + const T &operator[](dsize_t idx) const { return buf_[(head_ + idx) % (capacity_ + 1)]; } + + /// \brief returns non-const reference to an element of the array + /// \param[in] idx index of the element + /// \param[out] T& reference to an element of the array + T &operator[](dsize_t idx) { return buf_[(head_ + idx) % (capacity_ + 1)]; } + + private: + std::unique_ptr buf_; + dsize_t head_; + dsize_t tail_; + dsize_t size_; + dsize_t capacity_; +}; +} // namespace dataset +} // namespace mindspore +#endif // DATASET_CYCLIC_ARRAY_H diff --git a/mindspore/ccsrc/dataset/engine/perf/dataset_iterator_tracing.h b/mindspore/ccsrc/dataset/engine/perf/dataset_iterator_tracing.h index 00264939fc..129863c6d1 100644 --- a/mindspore/ccsrc/dataset/engine/perf/dataset_iterator_tracing.h +++ b/mindspore/ccsrc/dataset/engine/perf/dataset_iterator_tracing.h @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + #ifndef MINDSPORE_DATASET_ITERATOR_TRACING_H #define MINDSPORE_DATASET_ITERATOR_TRACING_H diff --git a/mindspore/ccsrc/dataset/engine/perf/device_queue_tracing.h b/mindspore/ccsrc/dataset/engine/perf/device_queue_tracing.h index f7c6da3a04..13ef7121c1 100644 --- a/mindspore/ccsrc/dataset/engine/perf/device_queue_tracing.h +++ b/mindspore/ccsrc/dataset/engine/perf/device_queue_tracing.h @@ -35,7 +35,7 @@ class DeviceQueueTracing : public Tracing { // @return Status - The error code return Status Record(const int32_t type, const int32_t extra_info, const int32_t batch_num, const int32_t value); - std::string Name() const override { return "Device Queue Tracing"; }; + std::string Name() const override { return kDeviceQueueTracingName; }; // Save tracing data to file // @return Status - The error code return diff --git a/mindspore/ccsrc/dataset/engine/perf/monitor.cc b/mindspore/ccsrc/dataset/engine/perf/monitor.cc index c9dce004b5..8a0d682b81 100644 --- a/mindspore/ccsrc/dataset/engine/perf/monitor.cc +++ b/mindspore/ccsrc/dataset/engine/perf/monitor.cc @@ -28,7 +28,6 @@ Monitor::Monitor(ExecutionTree *tree) : tree_(tree) { max_samples_ = 0; cur_row_ = 0; } - Status Monitor::operator()() { // Register this thread with TaskManager to receive proper interrupt signal. TaskManager::FindMe()->Post(); diff --git a/mindspore/ccsrc/dataset/engine/perf/monitor.h b/mindspore/ccsrc/dataset/engine/perf/monitor.h index 2a482a6ad7..8b4245db8e 100644 --- a/mindspore/ccsrc/dataset/engine/perf/monitor.h +++ b/mindspore/ccsrc/dataset/engine/perf/monitor.h @@ -29,6 +29,7 @@ class ExecutionTree; class Monitor { public: // Monitor object constructor + explicit Monitor(ExecutionTree *tree); Monitor() = default; diff --git a/mindspore/ccsrc/dataset/engine/perf/perf_data.h b/mindspore/ccsrc/dataset/engine/perf/perf_data.h new file mode 100644 index 0000000000..a201d705ea --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/perf/perf_data.h @@ -0,0 +1,88 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef DATASET_PERF_DATA_H +#define DATASET_PERF_DATA_H + +#include +#include "dataset/core/constants.h" + +namespace mindspore { +namespace dataset { + +// PerfData is a convenience class to record and store the data produced by Monitor +// and represents a 2D column major table with every column storing samples +// for an operator. The number of rows equals to the number of samples, +// the number of columns equals to the number of operators. +// The capacity is determined on construction and cannot be changed. +// ColumnType can be std::vector or CyclicArray. In case of the latter data can be added +// indefinitely without the risk of overflowing otherwise the capacity must not be exceeded. +// Given PerfData pd(n_rows, n_cols) an element in the column i and row j can be accessed as +// pd[i][j] + +template +class PerfData { + public: + PerfData() = default; + ~PerfData() = default; + PerfData(dsize_t max_rows, dsize_t n_cols) : counter_(0), max_rows_(max_rows), n_cols_(n_cols) { + for (auto i = 0; i < n_cols_; i++) { + data_.push_back(ColumnType(max_rows_)); + } + } + PerfData(const PerfData &rhs) = default; + PerfData(PerfData &&rhs) = default; + + // Adds a row of data + // T must be any container working with range based loops + template + void AddSample(const T &row) { + auto i = 0; + for (const auto &e : row) { + data_[i++].push_back(e); + } + counter_++; + } + + // Fetches a row of data by copy + template + auto Row(dsize_t idx) { + std::vector row(n_cols_); + for (auto i = 0; i < n_cols_; i++) { + row[i] = data_[i][idx]; + } + return row; + } + + // returns a column of data + ColumnType &operator[](size_t idx) { return data_[idx]; } + + const ColumnType &operator[](size_t idx) const { return data_[idx]; } + + dsize_t size() { return counter_ < max_rows_ ? counter_ : max_rows_; } + + dsize_t capacity() { return max_rows_; } + + private: + std::vector data_; + dsize_t counter_; + dsize_t max_rows_; + int n_cols_; +}; + +} // namespace dataset +} // namespace mindspore +#endif // DATASET_PERF_DATA_H diff --git a/mindspore/ccsrc/dataset/engine/perf/profiling.cc b/mindspore/ccsrc/dataset/engine/perf/profiling.cc index 4786b8dd69..66f27c46ba 100644 --- a/mindspore/ccsrc/dataset/engine/perf/profiling.cc +++ b/mindspore/ccsrc/dataset/engine/perf/profiling.cc @@ -14,7 +14,6 @@ * limitations under the License. */ #include "dataset/engine/perf/profiling.h" - #include #include #include @@ -23,6 +22,7 @@ #include "dataset/engine/perf/monitor.h" #include "dataset/engine/perf/device_queue_tracing.h" #include "dataset/engine/perf/connector_size.h" +#include "dataset/engine/perf/connector_throughput.h" #include "dataset/engine/perf/dataset_iterator_tracing.h" #include "utils/log_adapter.h" @@ -72,9 +72,11 @@ Status ProfilingManager::Initialize() { std::shared_ptr dataset_iterator_tracing = std::make_shared(); RETURN_IF_NOT_OK(RegisterTracingNode(dataset_iterator_tracing)); - std::shared_ptr monitor_sampling = std::make_shared(tree_); - RETURN_IF_NOT_OK(RegisterSamplingNode(monitor_sampling)); + std::shared_ptr connector_size_sampling = std::make_shared(tree_); + RETURN_IF_NOT_OK(RegisterSamplingNode(connector_size_sampling)); + std::shared_ptr connector_thr_sampling = std::make_shared(tree_); + RETURN_IF_NOT_OK(RegisterSamplingNode(connector_thr_sampling)); return Status::OK(); } @@ -140,14 +142,15 @@ Status ProfilingManager::SaveProfilingData() { RETURN_IF_NOT_OK(node.second->SaveToFile()); } MS_LOG(INFO) << "Save profiling data end."; - return Status::OK(); } -double ProfilingTime::GetCurMilliSecond() { - struct timeval tv = {0, 0}; - (void)gettimeofday(&tv, nullptr); - return tv.tv_sec * 1000 + tv.tv_usec / 1000; +int64_t ProfilingTime::GetCurMilliSecond() { + // because cpplint does not allow using namespace + using std::chrono::duration_cast; + using std::chrono::milliseconds; + using std::chrono::steady_clock; + return duration_cast(steady_clock::now().time_since_epoch()).count(); } } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/perf/profiling.h b/mindspore/ccsrc/dataset/engine/perf/profiling.h index d0ea91d566..e38c2d5e54 100644 --- a/mindspore/ccsrc/dataset/engine/perf/profiling.h +++ b/mindspore/ccsrc/dataset/engine/perf/profiling.h @@ -20,6 +20,7 @@ #include #include #include +#include #include "dataset/util/status.h" namespace mindspore { @@ -28,9 +29,10 @@ namespace dataset { class Monitor; class ExecutionTree; -const char kDeviceQueueTracingName[] = "Device Queue Tracing"; -const char kDatasetIteratorTracingName[] = "Dataset Iterator Tracing"; -const char kConnectorSizeSamplingName[] = "Connector Size Sampling"; +const char kDeviceQueueTracingName[] = "Device_Queue_Tracing"; +const char kDatasetIteratorTracingName[] = "Dataset_Iterator_Tracing"; +const char kConnectorSizeSamplingName[] = "Connector_Size_Sampling"; +const char kConnectorThroughputSamplingName[] = "Connector_Throughput_Sampling"; // Profiling is a class of basic unit of profiling action // This base class encapsulate the serialization output logic @@ -59,6 +61,8 @@ class Sampling : public Profiling { public: // Sampling action function. This function will be invoked by performance monitor thread. virtual Status Sample() = 0; + // virtual Status TestPrint() = 0; + virtual ~Sampling() = default; }; // Tracing is class of profiling which record samples upon request. @@ -132,7 +136,7 @@ enum ProfilingTimeSubType { class ProfilingTime { public: - static double GetCurMilliSecond(); + static int64_t GetCurMilliSecond(); }; } // namespace dataset diff --git a/mindspore/ccsrc/dataset/kernels/image/CMakeLists.txt b/mindspore/ccsrc/dataset/kernels/image/CMakeLists.txt index 3d88d9989c..fef698912c 100644 --- a/mindspore/ccsrc/dataset/kernels/image/CMakeLists.txt +++ b/mindspore/ccsrc/dataset/kernels/image/CMakeLists.txt @@ -25,4 +25,6 @@ add_library(kernels-image OBJECT resize_bilinear_op.cc resize_op.cc uniform_aug_op.cc + resize_with_bbox_op.cc + random_resize_with_bbox_op.cc ) diff --git a/mindspore/ccsrc/dataset/kernels/image/image_utils.cc b/mindspore/ccsrc/dataset/kernels/image/image_utils.cc index bb88f991a4..ded9a8db11 100644 --- a/mindspore/ccsrc/dataset/kernels/image/image_utils.cc +++ b/mindspore/ccsrc/dataset/kernels/image/image_utils.cc @@ -108,7 +108,8 @@ Status Resize(const std::shared_ptr &input, std::shared_ptr *out } try { TensorShape shape{output_height, output_width}; - if (input_cv->Rank() == 3) shape = shape.AppendDim(input_cv->shape()[2]); + int num_channels = input_cv->shape()[2]; + if (input_cv->Rank() == 3) shape = shape.AppendDim(num_channels); std::shared_ptr output_cv = std::make_shared(shape, input_cv->type()); RETURN_UNEXPECTED_IF_NULL(output_cv); auto cv_mode = GetCVInterpolationMode(mode); @@ -351,7 +352,8 @@ Status Crop(const std::shared_ptr &input, std::shared_ptr *outpu } try { TensorShape shape{h, w}; - if (input_cv->Rank() == 3) shape = shape.AppendDim(input_cv->shape()[2]); + int num_channels = input_cv->shape()[2]; + if (input_cv->Rank() == 3) shape = shape.AppendDim(num_channels); std::shared_ptr output_cv = std::make_shared(shape, input_cv->type()); RETURN_UNEXPECTED_IF_NULL(output_cv); cv::Rect roi(x, y, w, h); @@ -374,15 +376,15 @@ Status HwcToChw(std::shared_ptr input, std::shared_ptr *output) *output = input; return Status::OK(); } + int num_channels = input_cv->shape()[2]; if (input_cv->shape().Size() < 2 || input_cv->shape().Size() > 3 || - (input_cv->shape().Size() == 3 && input_cv->shape()[2] != 3 && input_cv->shape()[2] != 1)) { + (input_cv->shape().Size() == 3 && num_channels != 3 && num_channels != 1)) { RETURN_STATUS_UNEXPECTED("The shape is incorrect: number of channels does not equal 3 nor 1"); } cv::Mat output_img; int height = input_cv->shape()[0]; int width = input_cv->shape()[1]; - int num_channels = input_cv->shape()[2]; auto output_cv = std::make_unique(TensorShape{num_channels, height, width}, input_cv->type()); for (int i = 0; i < num_channels; ++i) { @@ -400,7 +402,8 @@ Status HwcToChw(std::shared_ptr input, std::shared_ptr *output) Status SwapRedAndBlue(std::shared_ptr input, std::shared_ptr *output) { try { std::shared_ptr input_cv = CVTensor::AsCVTensor(std::move(input)); - if (input_cv->shape().Size() != 3 || input_cv->shape()[2] != 3) { + int num_channels = input_cv->shape()[2]; + if (input_cv->shape().Size() != 3 || num_channels != 3) { RETURN_STATUS_UNEXPECTED("The shape is incorrect: number of channels does not equal 3"); } auto output_cv = std::make_shared(input_cv->shape(), input_cv->type()); @@ -435,7 +438,8 @@ Status CropAndResize(const std::shared_ptr &input, std::shared_ptrmat(); TensorShape shape{target_height, target_width}; - if (input_cv->Rank() == 3) shape = shape.AppendDim(input_cv->shape()[2]); + int num_channels = input_cv->shape()[2]; + if (input_cv->Rank() == 3) shape = shape.AppendDim(num_channels); std::shared_ptr cvt_out = std::make_shared(shape, input_cv->type()); RETURN_UNEXPECTED_IF_NULL(cvt_out); cv::resize(cv_in(roi), cvt_out->mat(), cv::Size(target_width, target_height), 0, 0, cv_mode); @@ -540,7 +544,8 @@ Status AdjustBrightness(const std::shared_ptr &input, std::shared_ptrmat().data) { RETURN_STATUS_UNEXPECTED("Could not convert to CV Tensor"); } - if (input_cv->Rank() != 3 || input_cv->shape()[2] != 3) { + int num_channels = input_cv->shape()[2]; + if (input_cv->Rank() != 3 || num_channels != 3) { RETURN_STATUS_UNEXPECTED("The shape is incorrect: number of channels does not equal 3"); } auto output_cv = std::make_shared(input_cv->shape(), input_cv->type()); @@ -560,7 +565,8 @@ Status AdjustContrast(const std::shared_ptr &input, std::shared_ptrmat().data) { RETURN_STATUS_UNEXPECTED("Could not convert to CV Tensor"); } - if (input_cv->Rank() != 3 || input_cv->shape()[2] != 3) { + int num_channels = input_cv->shape()[2]; + if (input_cv->Rank() != 3 || num_channels != 3) { RETURN_STATUS_UNEXPECTED("The shape is incorrect: number of channels does not equal 3"); } cv::Mat gray, output_img; @@ -586,7 +592,8 @@ Status AdjustSaturation(const std::shared_ptr &input, std::shared_ptrmat().data) { RETURN_STATUS_UNEXPECTED("Could not convert to CV Tensor"); } - if (input_cv->Rank() != 3 || input_cv->shape()[2] != 3) { + int num_channels = input_cv->shape()[2]; + if (input_cv->Rank() != 3 || num_channels != 3) { RETURN_STATUS_UNEXPECTED("The shape is incorrect: number of channels does not equal 3"); } auto output_cv = std::make_shared(input_cv->shape(), input_cv->type()); @@ -614,7 +621,8 @@ Status AdjustHue(const std::shared_ptr &input, std::shared_ptr * if (!input_cv->mat().data) { RETURN_STATUS_UNEXPECTED("Could not convert to CV Tensor"); } - if (input_cv->Rank() != 3 || input_cv->shape()[2] != 3) { + int num_channels = input_cv->shape()[2]; + if (input_cv->Rank() != 3 || num_channels != 3) { RETURN_STATUS_UNEXPECTED("The shape is incorrect: number of channels does not equal 3"); } auto output_cv = std::make_shared(input_cv->shape(), input_cv->type()); @@ -643,7 +651,8 @@ Status Erase(const std::shared_ptr &input, std::shared_ptr *outp uint8_t fill_g, uint8_t fill_b) { try { std::shared_ptr input_cv = CVTensor::AsCVTensor(input); - if (input_cv->mat().data == nullptr || input_cv->Rank() != 3 || input_cv->shape()[2] != 3) { + int num_channels = input_cv->shape()[2]; + if (input_cv->mat().data == nullptr || input_cv->Rank() != 3 || num_channels != 3) { RETURN_STATUS_UNEXPECTED("bad CV Tensor input for erase"); } cv::Mat input_img = input_cv->mat(); @@ -717,7 +726,8 @@ Status Pad(const std::shared_ptr &input, std::shared_ptr *output std::shared_ptr output_cv = std::make_shared(out_image); RETURN_UNEXPECTED_IF_NULL(output_cv); // pad the dimension if shape information is only 2 dimensional, this is grayscale - if (input_cv->Rank() == 3 && input_cv->shape()[2] == 1 && output_cv->Rank() == 2) output_cv->ExpandDim(2); + int num_channels = input_cv->shape()[2]; + if (input_cv->Rank() == 3 && num_channels == 1 && output_cv->Rank() == 2) output_cv->ExpandDim(2); *output = std::static_pointer_cast(output_cv); return Status::OK(); @@ -782,7 +792,7 @@ Status UpdateBBoxesForCrop(std::shared_ptr *bboxList, size_t *bboxCount, return Status::OK(); } -Status PadBBoxes(std::shared_ptr *bboxList, const size_t &bboxCount, int32_t pad_top, int32_t pad_left) { +Status PadBBoxes(const std::shared_ptr *bboxList, const size_t &bboxCount, int32_t pad_top, int32_t pad_left) { for (int i = 0; i < bboxCount; i++) { uint32_t xMin, yMin; RETURN_IF_NOT_OK((*bboxList)->GetUnsignedIntAt(&xMin, {i, 0})); diff --git a/mindspore/ccsrc/dataset/kernels/image/image_utils.h b/mindspore/ccsrc/dataset/kernels/image/image_utils.h index 231ee77de0..57ffce6a12 100644 --- a/mindspore/ccsrc/dataset/kernels/image/image_utils.h +++ b/mindspore/ccsrc/dataset/kernels/image/image_utils.h @@ -244,7 +244,7 @@ Status UpdateBBoxesForCrop(std::shared_ptr *bboxList, size_t *bboxCount, // @param bboxCount: total Number of bounding boxes - required within caller function to run update loop // @param pad_top: Total amount of padding applied to image top // @param pad_left: Total amount of padding applied to image left side -Status PadBBoxes(std::shared_ptr *bboxList, const size_t &bboxCount, int32_t pad_top, int32_t pad_left); +Status PadBBoxes(const std::shared_ptr *bboxList, const size_t &bboxCount, int32_t pad_top, int32_t pad_left); // Updates bounding boxes for an Image Resize Operation - Takes in set of valid BBoxes // For e.g those that remain after a crop diff --git a/mindspore/ccsrc/dataset/kernels/image/random_crop_and_resize_with_bbox_op.cc b/mindspore/ccsrc/dataset/kernels/image/random_crop_and_resize_with_bbox_op.cc index b820779ed1..fbaf2c9326 100644 --- a/mindspore/ccsrc/dataset/kernels/image/random_crop_and_resize_with_bbox_op.cc +++ b/mindspore/ccsrc/dataset/kernels/image/random_crop_and_resize_with_bbox_op.cc @@ -30,8 +30,7 @@ Status RandomCropAndResizeWithBBoxOp::Compute(const TensorRow &input, TensorRow BOUNDING_BOX_CHECK(input); CHECK_FAIL_RETURN_UNEXPECTED(input[0]->shape().Size() >= 2, "The shape of input is abnormal"); - (*output).push_back(nullptr); // init memory for return vector - (*output).push_back(nullptr); + output->resize(2); (*output)[1] = std::move(input[1]); // move boxes over to output size_t bboxCount = input[1]->shape()[0]; // number of rows in bbox tensor diff --git a/mindspore/ccsrc/dataset/kernels/image/random_crop_with_bbox_op.cc b/mindspore/ccsrc/dataset/kernels/image/random_crop_with_bbox_op.cc index 2be37f1da3..c873307afd 100644 --- a/mindspore/ccsrc/dataset/kernels/image/random_crop_with_bbox_op.cc +++ b/mindspore/ccsrc/dataset/kernels/image/random_crop_with_bbox_op.cc @@ -36,8 +36,7 @@ Status RandomCropWithBBoxOp::Compute(const TensorRow &input, TensorRow *output) int32_t padded_image_h; int32_t padded_image_w; - (*output).push_back(nullptr); - (*output).push_back(nullptr); + output->resize(2); (*output)[1] = std::move(input[1]); // since some boxes may be removed bool crop_further = true; // Whether further cropping will be required or not, true unless required size matches diff --git a/mindspore/ccsrc/dataset/engine/opt/pre/map_column_reorder.h b/mindspore/ccsrc/dataset/kernels/image/random_resize_with_bbox_op.cc similarity index 54% rename from mindspore/ccsrc/dataset/engine/opt/pre/map_column_reorder.h rename to mindspore/ccsrc/dataset/kernels/image/random_resize_with_bbox_op.cc index 84274db3d5..de69c02e39 100644 --- a/mindspore/ccsrc/dataset/engine/opt/pre/map_column_reorder.h +++ b/mindspore/ccsrc/dataset/kernels/image/random_resize_with_bbox_op.cc @@ -14,22 +14,20 @@ * limitations under the License. */ -#ifndef DATASET_ENGINE_OPT_PASS_PRE_MAPCOLREORDER_H -#define DATASET_ENGINE_OPT_PASS_PRE_MAPCOLREORDER_H - -#include -#include "dataset/engine/opt/pass.h" +#include "dataset/kernels/image/random_resize_with_bbox_op.h" +#include "dataset/kernels/image/resize_with_bbox_op.h" +#include "dataset/util/status.h" namespace mindspore { namespace dataset { -// Map Column Recorder Pass will insert ProjectOp when MapOp requires a full output columns reorder. -// Example: -// Input Tree: TFReader -> MapOp(with col_order) -> Batch -// Output Tree: TFReader -> MapOp -> ProjectOp(col_order) -> Batch -class MapColumnReorder : public TreePass { - Status RunOnTree(ExecutionTree *tree, bool *modified) override; -}; +const int32_t RandomResizeWithBBoxOp::kDefTargetWidth = 0; + +Status RandomResizeWithBBoxOp::Compute(const TensorRow &input, TensorRow *output) { + // Randomly selects from the following four interpolation methods + // 0-bilinear, 1-nearest_neighbor, 2-bicubic, 3-area + interpolation_ = static_cast(distribution_(random_generator_)); + RETURN_IF_NOT_OK(ResizeWithBBoxOp::Compute(input, output)); + return Status::OK(); +} } // namespace dataset } // namespace mindspore - -#endif // DATASET_ENGINE_OPT_PASS_PRE_MAPCOLREORDER_H diff --git a/mindspore/ccsrc/dataset/kernels/image/random_resize_with_bbox_op.h b/mindspore/ccsrc/dataset/kernels/image/random_resize_with_bbox_op.h new file mode 100644 index 0000000000..4a7614525f --- /dev/null +++ b/mindspore/ccsrc/dataset/kernels/image/random_resize_with_bbox_op.h @@ -0,0 +1,56 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef DATASET_KERNELS_IMAGE_RANDOM_RESIZE_WITH_BBOX_OP_H +#define DATASET_KERNELS_IMAGE_RANDOM_RESIZE_WITH_BBOX_OP_H + +#include +#include + +#include "dataset/core/tensor.h" +#include "dataset/kernels/image/resize_op.h" +#include "dataset/kernels/image/resize_with_bbox_op.h" +#include "dataset/kernels/tensor_op.h" +#include "dataset/util/random.h" +#include "dataset/util/status.h" + +namespace mindspore { +namespace dataset { +class RandomResizeWithBBoxOp : public ResizeWithBBoxOp { + public: + // Default values, also used by python_bindings.cc + static const int32_t kDefTargetWidth; + explicit RandomResizeWithBBoxOp(int32_t size_1, int32_t size_2 = kDefTargetWidth) : ResizeWithBBoxOp(size_1, size_2) { + random_generator_.seed(GetSeed()); + } + + ~RandomResizeWithBBoxOp() = default; + + // Description: A function that prints info about the node + void Print(std::ostream &out) const override { + out << "RandomResizeWithBBoxOp: " << ResizeWithBBoxOp::size1_ << " " << ResizeWithBBoxOp::size2_; + } + + Status Compute(const TensorRow &input, TensorRow *output) override; + + private: + std::mt19937 random_generator_; + std::uniform_int_distribution distribution_{0, 3}; +}; +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_KERNELS_IMAGE_RANDOM_RESIZE_WITH_BBOX_OP_H diff --git a/mindspore/ccsrc/dataset/kernels/image/random_vertical_flip_with_bbox_op.cc b/mindspore/ccsrc/dataset/kernels/image/random_vertical_flip_with_bbox_op.cc index d88c009559..ffea851eac 100644 --- a/mindspore/ccsrc/dataset/kernels/image/random_vertical_flip_with_bbox_op.cc +++ b/mindspore/ccsrc/dataset/kernels/image/random_vertical_flip_with_bbox_op.cc @@ -41,12 +41,11 @@ Status RandomVerticalFlipWithBBoxOp::Compute(const TensorRow &input, TensorRow * RETURN_IF_NOT_OK(input[1]->GetUnsignedIntAt(&boxHeight, {i, 3})); // get height of bbox // subtract (curCorner + height) from (max) for new Corner position - newBoxCorner_y = (imHeight - 1) - (boxCorner_y + boxHeight); + newBoxCorner_y = (imHeight - 1) - ((boxCorner_y + boxHeight) - 1); RETURN_IF_NOT_OK(input[1]->SetItemAt({i, 1}, newBoxCorner_y)); } - (*output).push_back(nullptr); - (*output).push_back(nullptr); + output->resize(2); (*output)[1] = std::move(input[1]); return VerticalFlip(input[0], &(*output)[0]); diff --git a/mindspore/ccsrc/dataset/kernels/image/resize_with_bbox_op.cc b/mindspore/ccsrc/dataset/kernels/image/resize_with_bbox_op.cc new file mode 100644 index 0000000000..8a633d5678 --- /dev/null +++ b/mindspore/ccsrc/dataset/kernels/image/resize_with_bbox_op.cc @@ -0,0 +1,53 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "dataset/kernels/image/resize_with_bbox_op.h" +#include +#include +#include "dataset/kernels/image/resize_op.h" +#include "dataset/kernels/image/image_utils.h" +#include "dataset/core/cv_tensor.h" +#include "dataset/core/pybind_support.h" +#include "dataset/core/tensor.h" +#include "dataset/kernels/tensor_op.h" +#include "dataset/util/status.h" + +namespace mindspore { +namespace dataset { + +Status ResizeWithBBoxOp::Compute(const TensorRow &input, TensorRow *output) { + IO_CHECK_VECTOR(input, output); + BOUNDING_BOX_CHECK(input); + + int32_t input_h = input[0]->shape()[0]; + int32_t input_w = input[0]->shape()[1]; + + output->resize(2); + (*output)[1] = std::move(input[1]); // move boxes over to output + + std::shared_ptr input_cv = CVTensor::AsCVTensor(std::move(input[0])); + + RETURN_IF_NOT_OK(ResizeOp::Compute(std::static_pointer_cast(input_cv), &(*output)[0])); + + int32_t output_h = (*output)[0]->shape()[0]; // output height if ResizeWithBBox + int32_t output_w = (*output)[0]->shape()[1]; // output width if ResizeWithBBox + + size_t bboxCount = input[1]->shape()[0]; // number of rows in bbox tensor + RETURN_IF_NOT_OK(UpdateBBoxesForResize((*output)[1], bboxCount, output_w, output_h, input_w, input_h)); + return Status::OK(); +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/kernels/image/resize_with_bbox_op.h b/mindspore/ccsrc/dataset/kernels/image/resize_with_bbox_op.h new file mode 100644 index 0000000000..17bdd01ef1 --- /dev/null +++ b/mindspore/ccsrc/dataset/kernels/image/resize_with_bbox_op.h @@ -0,0 +1,43 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_KERNELS_IMAGE_RESIZE_WITH_BBOX_OP_H +#define DATASET_KERNELS_IMAGE_RESIZE_WITH_BBOX_OP_H + +#include "dataset/core/tensor.h" +#include "dataset/kernels/image/image_utils.h" +#include "dataset/kernels/tensor_op.h" +#include "dataset/util/status.h" +#include "dataset/kernels/image/resize_op.h" + +namespace mindspore { +namespace dataset { +class ResizeWithBBoxOp : public ResizeOp { + public: + // Constructor for ResizeWithBBoxOp, with default value and passing to base class constructor + explicit ResizeWithBBoxOp(int32_t size_1, int32_t size_2 = kDefWidth, + InterpolationMode mInterpolation = kDefInterpolation) + : ResizeOp(size_1, size_2, mInterpolation) {} + + ~ResizeWithBBoxOp() override = default; + + void Print(std::ostream &out) const override { out << "ResizeWithBBoxOp: " << size1_ << " " << size2_; } + + Status Compute(const TensorRow &input, TensorRow *output) override; +}; +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_KERNELS_IMAGE_RESIZE_WITH_BBOX_OP_H diff --git a/mindspore/ccsrc/dataset/kernels/tensor_op.h b/mindspore/ccsrc/dataset/kernels/tensor_op.h index 293d4a4f99..9aae50d6b0 100644 --- a/mindspore/ccsrc/dataset/kernels/tensor_op.h +++ b/mindspore/ccsrc/dataset/kernels/tensor_op.h @@ -45,14 +45,18 @@ #define BOUNDING_BOX_CHECK(input) \ do { \ + if (input.size() != 2) { \ + return Status(StatusCode::kBoundingBoxInvalidShape, __LINE__, __FILE__, \ + "Requires Image and Bounding Boxes, likely missed bounding boxes."); \ + } \ if (input[1]->shape().Size() < 2) { \ return Status(StatusCode::kBoundingBoxInvalidShape, __LINE__, __FILE__, \ - "Bounding boxes shape should have at least two dims"); \ + "Bounding boxes shape should have at least two dimensions."); \ } \ uint32_t num_of_features = input[1]->shape()[1]; \ if (num_of_features < 4) { \ return Status(StatusCode::kBoundingBoxInvalidShape, __LINE__, __FILE__, \ - "Bounding boxes should be have at least 4 features"); \ + "Bounding boxes should be have at least 4 features."); \ } \ uint32_t num_of_boxes = input[1]->shape()[0]; \ uint32_t img_h = input[0]->shape()[0]; \ diff --git a/mindspore/ccsrc/dataset/text/kernels/basic_tokenizer_op.cc b/mindspore/ccsrc/dataset/text/kernels/basic_tokenizer_op.cc index 1128990b44..3512a4b2d7 100644 --- a/mindspore/ccsrc/dataset/text/kernels/basic_tokenizer_op.cc +++ b/mindspore/ccsrc/dataset/text/kernels/basic_tokenizer_op.cc @@ -15,11 +15,16 @@ */ #include "dataset/text/kernels/basic_tokenizer_op.h" #include +#include #include #include #include #include +#include "unicode/errorcode.h" +#include "unicode/normalizer2.h" +#include "unicode/utypes.h" + namespace mindspore { namespace dataset { const bool BasicTokenizerOp::kDefLowerCase = false; @@ -40,8 +45,8 @@ const char BasicTokenizerOp::kCommonPattern[] = "|[\\x{2B820}-\\x{2CEAF}]" "|[\\x{F900}-\\x{FAFF}]" "|[\\x{2F800}-\\x{2FA1F}]"; -const char BasicTokenizerOp::kUnusedPattern[] = "\\[CLS\\]|\\[SEP\\]|\\[UNK\\]|\\[PAD\\]|\\[MASK\\]|"; - +const char BasicTokenizerOp::kUnusedPattern[] = "\\[CLS\\]|\\[SEP\\]|\\[UNK\\]|\\[PAD\\]|\\[MASK\\]|\\[unused\\d+\\]|"; +const std::unordered_set BasicTokenizerOp::kUnusedWords{"[CLS]", "[SEP]", "[UNK]", "[PAD]", "[MASK]"}; BasicTokenizerOp::BasicTokenizerOp(bool lower_case, bool keep_whitespace, NormalizeForm normalization_form, bool preserve_unused_token) : lower_case_(lower_case), @@ -67,6 +72,69 @@ BasicTokenizerOp::BasicTokenizerOp(bool lower_case, bool keep_whitespace, Normal regex_tokenizer_ = std::make_unique(delim_pattern, keep_delim_pattern); } +Status BasicTokenizerOp::CaseFoldWithoutUnusedWords(const std::string_view &text, + const std::unordered_set &unused_words, + std::string *outupt) { + icu::ErrorCode error; + const icu::Normalizer2 *nfkc_case_fold = icu::Normalizer2::getNFKCCasefoldInstance(error); + CHECK_FAIL_RETURN_UNEXPECTED(error.isSuccess(), "getNFKCCasefoldInstance failed."); + outupt->clear(); + + // 1. get start and end offsets of not case fold strs + std::queue> offsets; // offsets of not used words + int start = -1; + int len = 0; + for (int i = 0; i < text.length(); i++) { + if (text[i] == '[') { + start = i; + ++len; + } else if (text[i] == ']' && start >= 0) { + ++len; + std::string word(text.substr(start, len)); + if (unused_words.find(word) != unused_words.end()) { + offsets.push(std::make_pair(start, start + len - 1)); + } + start = -1; + len = 0; + } else if (start >= 0) { + ++len; + } + } + + // 2. Do not apply case fold on `unused_words` + start = 0; + for (int i = 0; i < text.length();) { + std::string_view process_text; + std::string preserve_token; + if (offsets.empty()) { + i = text.length(); + process_text = text.substr(start, i - start); + } else { + preserve_token = text.substr(offsets.front().first, offsets.front().second - offsets.front().first + 1); + process_text = text.substr(start, offsets.front().first - start); + i = offsets.front().second + 1; + offsets.pop(); + } + std::string temp; + icu::StringByteSink sink(&temp); + nfkc_case_fold->normalizeUTF8(0, icu::StringPiece(process_text.data(), process_text.size()), sink, nullptr, error); + *outupt += temp + preserve_token; + } + return Status::OK(); +} + +Status BasicTokenizerOp::CaseFoldWithoutUnusedWords(const std::shared_ptr &input, + std::shared_ptr *output) { + IO_CHECK(input, output); + std::vector strs(input->Size()); + int i = 0; + for (auto iter = input->begin(); iter != input->end(); iter++) { + RETURN_IF_NOT_OK(CaseFoldWithoutUnusedWords(*iter, kUnusedWords, &strs[i++])); + } + *output = std::make_shared(std::move(strs), input->shape()); + return Status::OK(); +} + Status BasicTokenizerOp::Compute(const std::shared_ptr &input, std::shared_ptr *output) { IO_CHECK(input, output); if (input->Rank() != 0 || input->type() != DataType::DE_STRING) { @@ -75,8 +143,13 @@ Status BasicTokenizerOp::Compute(const std::shared_ptr &input, std::shar std::shared_ptr cur_input; std::shared_ptr processed_tensor; if (lower_case_) { - // to lower case - RETURN_IF_NOT_OK(case_fold_->Compute(input, &processed_tensor)); + if (!preserve_unused_token_) { + // to lower case + RETURN_IF_NOT_OK(case_fold_->Compute(input, &processed_tensor)); + } else { + // to lower case except words in kUnusedWords + RETURN_IF_NOT_OK(CaseFoldWithoutUnusedWords(input, &processed_tensor)); + } cur_input = processed_tensor; // strip accent characters RETURN_IF_NOT_OK(nfd_normalize_->Compute(cur_input, &processed_tensor)); diff --git a/mindspore/ccsrc/dataset/text/kernels/basic_tokenizer_op.h b/mindspore/ccsrc/dataset/text/kernels/basic_tokenizer_op.h index a37e841573..01827a0ba4 100644 --- a/mindspore/ccsrc/dataset/text/kernels/basic_tokenizer_op.h +++ b/mindspore/ccsrc/dataset/text/kernels/basic_tokenizer_op.h @@ -17,6 +17,7 @@ #define DATASET_TEXT_KERNELS_BASIC_TOKENIZER_OP_H_ #include #include +#include #include "dataset/core/tensor.h" #include "dataset/kernels/tensor_op.h" @@ -45,9 +46,15 @@ class BasicTokenizerOp : public TensorOp { Status Compute(const std::shared_ptr &input, std::shared_ptr *output) override; + protected: + Status CaseFoldWithoutUnusedWords(const std::string_view &text, const std::unordered_set &unused_words, + std::string *outupt); + Status CaseFoldWithoutUnusedWords(const std::shared_ptr &input, std::shared_ptr *output); + private: static const char kCommonPattern[]; static const char kUnusedPattern[]; + static const std::unordered_set kUnusedWords; bool lower_case_; bool keep_whitespace_; NormalizeForm normalization_form_; diff --git a/mindspore/ccsrc/dataset/text/kernels/wordpiece_tokenizer_op.cc b/mindspore/ccsrc/dataset/text/kernels/wordpiece_tokenizer_op.cc index 48092d89cd..e488c527cd 100644 --- a/mindspore/ccsrc/dataset/text/kernels/wordpiece_tokenizer_op.cc +++ b/mindspore/ccsrc/dataset/text/kernels/wordpiece_tokenizer_op.cc @@ -81,9 +81,9 @@ Status WordpieceTokenizerOp::GetTokens(const std::string &input_token, std::vect if (!DecodeRunesInString(input_token.data(), input_token.size(), runes)) { RETURN_STATUS_UNEXPECTED("Decode utf8 string failed."); } - int end; + int end = 0; for (int start = 0; start < input_token.size();) { - bool found; + bool found = false; RETURN_IF_NOT_OK(LookupWord(input_token, runes, start, &found, &end)); if (found) { RETURN_IF_NOT_OK(AddSubword(input_token, start, end, out_tokens)); diff --git a/mindspore/ccsrc/dataset/util/CMakeLists.txt b/mindspore/ccsrc/dataset/util/CMakeLists.txt index b36d612435..96489add07 100644 --- a/mindspore/ccsrc/dataset/util/CMakeLists.txt +++ b/mindspore/ccsrc/dataset/util/CMakeLists.txt @@ -2,6 +2,8 @@ file(GLOB_RECURSE _CURRENT_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc" set_property(SOURCE ${_CURRENT_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_MD) add_library(utils OBJECT arena.cc + buddy.cc + cache_pool.cc circular_pool.cc memory_pool.cc cond_var.cc @@ -11,7 +13,11 @@ add_library(utils OBJECT service.cc services.cc lock.cc + semaphore.cc status.cc + storage_container.cc + storage_manager.cc + slice.cc path.cc wait_post.cc sig_handler.cc) diff --git a/mindspore/ccsrc/dataset/util/allocator.h b/mindspore/ccsrc/dataset/util/allocator.h index ba6c7786df..50a9cadbe3 100644 --- a/mindspore/ccsrc/dataset/util/allocator.h +++ b/mindspore/ccsrc/dataset/util/allocator.h @@ -17,8 +17,10 @@ #define DATASET_UTIL_ALLOCATOR_H_ #include +#include #include #include +#include #include "dataset/util/memory_pool.h" namespace mindspore { @@ -84,6 +86,91 @@ class Allocator { private: std::shared_ptr pool_; }; +/// \brief It is a wrapper of unique_ptr with a custom allocator and acts like std::lock_guard such that the memory will +/// be released when the object goes out of scope \tparam T The type of object to be allocated \tparam C Allocator. +/// Default to std::allocator +template > +class MemGuard { + public: + using allocator = C; + MemGuard() : n_(0) {} + explicit MemGuard(allocator a) : n_(0), alloc_(a) {} + // There is no copy constructor nor assignment operator because the memory is solely owned by this object. + MemGuard(const MemGuard &) = delete; + MemGuard &operator=(const MemGuard &) = delete; + // On the other hand, We can support move constructor + MemGuard(MemGuard &&lhs) noexcept : alloc_(std::move(lhs.alloc_)), ptr_(std::move(lhs.ptr_)), n_(lhs.n_) {} + MemGuard &operator=(MemGuard &&lhs) noexcept { + if (this != &lhs) { + this->deallocate(); + n_ = lhs.n_; + alloc_ = std::move(lhs.alloc_); + ptr_ = std::move(lhs.ptr_); + } + return *this; + } + /// \brief Explicitly deallocate the memory if allocated + void deallocate() { + if (ptr_) { + auto *p = ptr_.release(); + if (!std::is_arithmetic::value && std::is_destructible::value) { + for (auto i = 0; i < n_; ++i) { + p[i].~T(); + } + } + alloc_.deallocate(p, n_); + n_ = 0; + } + } + /// \brief Allocate memory (with emplace feature). Previous one will be released. If size is 0, no new memory is + /// allocated. + /// \param n Number of objects of type T to be allocated + /// \tparam Args Extra arguments pass to the constructor of T + template + Status allocate(size_t n, Args &&... args) noexcept { + try { + deallocate(); + if (n > 0) { + T *data = alloc_.allocate(n); + if (!std::is_arithmetic::value) { + for (auto i = 0; i < n; i++) { + std::allocator_traits::construct(alloc_, &(data[i]), std::forward(args)...); + } + } + ptr_ = std::unique_ptr(data); + n_ = n; + } + } catch (const std::bad_alloc &e) { + return Status(StatusCode::kOutOfMemory); + } catch (std::exception &e) { + RETURN_STATUS_UNEXPECTED(e.what()); + } + return Status::OK(); + } + ~MemGuard() noexcept { deallocate(); } + /// \brief Getter function + /// \return The pointer to the memory allocated + T *GetPointer() const { return ptr_.get(); } + /// \brief Getter function + /// \return The pointer to the memory allocated + T *GetMutablePointer() { return ptr_.get(); } + /// \brief Overload [] operator to access a particular element + /// \param x index to the element. Must be less than number of element allocated. + /// \return pointer to the x-th element + T *operator[](size_t x) { return GetMutablePointer() + x; } + /// \brief Overload [] operator to access a particular element + /// \param x index to the element. Must be less than number of element allocated. + /// \return pointer to the x-th element + T *operator[](size_t x) const { return GetPointer() + x; } + /// \brief Return how many bytes are allocated in total + /// \return Number of bytes allocated in total + size_t GetSizeInBytes() const { return n_ * sizeof(T); } + + private: + allocator alloc_; + std::unique_ptr> ptr_; + size_t n_; +}; } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/util/arena.cc b/mindspore/ccsrc/dataset/util/arena.cc index e9dec9d86e..af4f522678 100644 --- a/mindspore/ccsrc/dataset/util/arena.cc +++ b/mindspore/ccsrc/dataset/util/arena.cc @@ -17,7 +17,6 @@ #include #include #include "dataset/util/system_pool.h" -#include "dataset/util/de_error.h" #include "./securec.h" #include "utils/log_adapter.h" @@ -79,7 +78,7 @@ void Arena::Deallocate(void *p) { auto *q = get_base_addr(p); MemHdr hdr(0, 0); MemHdr::getHdr(q, &hdr); - DS_ASSERT(hdr.sig == 0xDEADBEEF); + MS_ASSERT(hdr.sig == 0xDEADBEEF); // We are going to insert a free block back to the treap. But first, check if we can combine // with the free blocks before and after to form a bigger block. std::unique_lock lck(mux_); @@ -103,8 +102,8 @@ void Arena::Deallocate(void *p) { } Status Arena::Reallocate(void **pp, size_t old_sz, size_t new_sz) { - DS_ASSERT(pp); - DS_ASSERT(*pp); + MS_ASSERT(pp); + MS_ASSERT(*pp); uint64_t actual_size = static_cast(new_sz) + ARENA_WALL_OVERHEAD_SZ; if (actual_size > this->get_max_size()) { RETURN_STATUS_UNEXPECTED("Request size too big : " + std::to_string(new_sz)); @@ -114,7 +113,7 @@ Status Arena::Reallocate(void **pp, size_t old_sz, size_t new_sz) { auto *oldHdr = get_base_addr(oldAddr); MemHdr hdr(0, 0); MemHdr::getHdr(oldHdr, &hdr); - DS_ASSERT(hdr.sig == 0xDEADBEEF); + MS_ASSERT(hdr.sig == 0xDEADBEEF); std::unique_lock lck(mux_); if (hdr.blk_size > req_blk) { // Refresh the header with the new smaller size. @@ -239,8 +238,8 @@ bool Arena::BlockEnlarge(uint64_t *addr, uint64_t old_sz, uint64_t new_sz) { } Status Arena::FreeAndAlloc(void **pp, size_t old_sz, size_t new_sz) { - DS_ASSERT(pp); - DS_ASSERT(*pp); + MS_ASSERT(pp); + MS_ASSERT(*pp); void *p = nullptr; void *q = *pp; RETURN_IF_NOT_OK(Allocate(new_sz, &p)); diff --git a/mindspore/ccsrc/dataset/util/auto_index.h b/mindspore/ccsrc/dataset/util/auto_index.h index 11a2e90b00..5c43ecfd80 100644 --- a/mindspore/ccsrc/dataset/util/auto_index.h +++ b/mindspore/ccsrc/dataset/util/auto_index.h @@ -91,7 +91,7 @@ class AutoIndexObj : public BPlusTree { } private: - static constexpr key_type kMinKey = 1; + static constexpr key_type kMinKey = 0; std::atomic inx_; }; } // namespace dataset diff --git a/mindspore/ccsrc/dataset/util/btree_impl.tpp b/mindspore/ccsrc/dataset/util/btree_impl.tpp index 8148a8d12c..fc3b05e3a1 100644 --- a/mindspore/ccsrc/dataset/util/btree_impl.tpp +++ b/mindspore/ccsrc/dataset/util/btree_impl.tpp @@ -61,8 +61,8 @@ typename BPlusTree::IndexRc BPlusTree::InnerNode:: template typename BPlusTree::IndexRc BPlusTree::InnerNode::Split( BPlusTree::InnerNode *to, key_type *split_key) { - DS_ASSERT(to); - DS_ASSERT(to->slotuse_ == 0); + MS_ASSERT(to); + MS_ASSERT(to->slotuse_ == 0); // It is simpler to sort first, then split. Other alternative is to move key by key to the // new node. Also we need to deal with the 'holes' after a key is moved. RETURN_IF_BAD_RC(this->Sort()); @@ -153,8 +153,8 @@ typename BPlusTree::IndexRc BPlusTree::LeafNode::S template typename BPlusTree::IndexRc BPlusTree::LeafNode::Split( BPlusTree::LeafNode *to) { - DS_ASSERT(to); - DS_ASSERT(to->slotuse_ == 0); + MS_ASSERT(to); + MS_ASSERT(to->slotuse_ == 0); // It is simpler to sort first, then split. Other alternative is to move key by key to the // new node. Also we need to deal with the 'holes' after a key is moved. RETURN_IF_BAD_RC(this->Sort()); diff --git a/mindspore/ccsrc/dataset/util/btree_iterator.tpp b/mindspore/ccsrc/dataset/util/btree_iterator.tpp index 91ba2acd7a..cffef3fa7a 100644 --- a/mindspore/ccsrc/dataset/util/btree_iterator.tpp +++ b/mindspore/ccsrc/dataset/util/btree_iterator.tpp @@ -15,7 +15,6 @@ #ifndef DATASET_UTIL_BTREE_ITERATOR_H_ #define DATASET_UTIL_BTREE_ITERATOR_H_ -#include "dataset/util/de_error.h" #include "utils/log_adapter.h" #include "btree.h" diff --git a/mindspore/ccsrc/dataset/util/buddy.cc b/mindspore/ccsrc/dataset/util/buddy.cc new file mode 100644 index 0000000000..540fa993d6 --- /dev/null +++ b/mindspore/ccsrc/dataset/util/buddy.cc @@ -0,0 +1,388 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/util/buddy.h" +#include +#include +#include "dataset/util/memory_pool.h" +#include "dataset/util/system_pool.h" +#include "utils/log_adapter.h" +#include "./securec.h" + +inline uint64_t BitLeftShift(uint64_t v, uint64_t n) { return (v << n); } + +inline uint64_t BitRightShift(uint64_t v, uint64_t n) { return (v >> n); } + +inline uint64_t BitOr(uint64_t rhs, uint64_t lhs) { return rhs | lhs; } + +inline uint64_t BitEx(uint64_t rhs, uint64_t lhs) { return rhs ^ lhs; } + +inline uint64_t BitAnd(uint64_t rhs, uint64_t lhs) { return rhs & lhs; } + +namespace mindspore { +namespace dataset { +Status BuddySpace::Init() { + if (log_min_ < 0) { + return Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, + "log_min must be positive : " + std::to_string(log_min_)); + } + if (num_lvl_ < 3 || num_lvl_ > 18) { + return Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, + "num_lvl must be between 3 and 18 : " + std::to_string(num_lvl_)); + } + min_ = BitLeftShift(1, log_min_); + max_ = BitLeftShift(1, log_min_ + num_lvl_ - 1); + size_t offset_1 = sizeof(rel_addr_t) * num_lvl_; + size_t offset_2 = sizeof(int) * num_lvl_ + offset_1; + size_t offset_3 = sizeof(char) * BitLeftShift(1, num_lvl_ - 3) + offset_2; + RETURN_IF_NOT_OK(DeMalloc(offset_3, &ptr_, true)); + hint_ = reinterpret_cast(ptr_); + count_ = reinterpret_cast((reinterpret_cast(ptr_) + offset_1)); + map_ = reinterpret_cast(ptr_) + offset_2; + count_[num_lvl_ - 1] = 1; + map_[0] = BitOr(MORE_BIT, num_lvl_ - 3); + return Status::OK(); +} + +Status BuddySpace::Alloc(const uint64_t sz, BSpaceDescriptor *desc, addr_t *p) noexcept { + std::lock_guard lock(mutex_); + addr_t addr = AllocNoLock(sz, desc); + if (addr != NOSPACE) { + *p = addr; + return Status::OK(); + } else { + return Status(StatusCode::kNoSpace, "BuddySpace full. Not an error. Please ignore."); + } +} + +addr_t BuddySpace::AllocNoLock(const uint64_t sz, BSpaceDescriptor *desc) noexcept { + MS_ASSERT(sz <= max_); + uint32_t reqSize = SizeToBlock(sz); + rel_addr_t rel_addr = AllocBuddySeg(reqSize); + if (rel_addr != static_cast(NOSPACE)) { + (void)memset_s(desc, sizeof(BSpaceDescriptor), 0, sizeof(BSpaceDescriptor)); + desc->sig = static_cast(0xDEADBEEF); + desc->addr = rel_addr; + desc->req_size = reqSize; + desc->blk_size = NextPowerOf2(reqSize); + return static_cast(rel_addr * min_); + } else { + return NOSPACE; + } +} + +void BuddySpace::FreeNoLock(const BSpaceDescriptor *desc) { + MS_ASSERT(desc->sig == 0XDEADBEEF); + rel_addr_t rel_addr = desc->addr; + size_t blk_size = desc->blk_size; + size_t req_size = desc->req_size; + FreeBuddySeg(rel_addr, blk_size, req_size); +} + +void BuddySpace::Free(const BSpaceDescriptor *desc) { + std::lock_guard lock(mutex_); + return FreeNoLock(desc); +} + +std::ostream &operator<<(std::ostream &os, const BuddySpace &s) { + os << "1 unit = " << s.GetMinSize() << "\n" + << "Size of buddy space = " << s.GetMaxSize() << "\n" + << "Number of levels = " << s.num_lvl_ << "\n\n" + << "Percent free = " << s.PercentFree() << "\n" + << "Dumping count array : " + << "\n"; + for (int i = 0; i < s.num_lvl_; i++) { + os << "[" << i << "] = " << s.count_[i] << " "; + if (((i + 1) % 4) == 0) { + os << "\n"; + } + } + os << "\n"; + os << "Dumping allocation info:" + << "\n"; + auto max_addr = static_cast(BitLeftShift(1, s.num_lvl_ - 1)); + rel_addr_t addr = 0; + while (addr < max_addr) { + size_t sz = 0; + BuddySpace::STATE st; + s.GetBuddySegState(addr, &sz, &st); + os << "Address : " << std::left << std::setw(8) << addr << " Size : " << std::setw(8) << sz << " State : " + << ((st == BuddySpace::STATE::kAlloc) ? "ALLOC" : ((st == BuddySpace::STATE::kFree) ? "FREE" : "Unkonwn")) + << "\n"; + addr += sz; + } + return os; +} + +void BuddySpace::GetBuddySegState(const rel_addr_t rel_addr, size_t *rel_sz, STATE *st) const { + char byte; + int pos; + int offset; + uint64_t val = 0; + int shift; + pos = BitRightShift(rel_addr, 2); + offset = rel_addr % 4; + shift = offset * 2; + byte = map_[pos]; + switch (offset) { + case 0: + val = byte; + break; + case 1: + case 3: + if (offset == 1) { + val = BitLeftShift(BitAnd(byte, 0x30), shift); + } else { + val = BitLeftShift(BitAnd(byte, 0x03), shift); + } + break; + case 2: + val = BitLeftShift(BitAnd(byte, 0x0F), shift); + break; + } + if (BitAnd(val, ONE_BIT)) { + *rel_sz = 1; + } else if (BitAnd(val, TWO_BIT)) { + *rel_sz = 2; + } else if (BitAnd(val, MORE_BIT)) { + log_t lg = BitAnd(val, 0x0F); + *rel_sz = BitLeftShift(1, lg + 2); + } else { + *st = STATE::kEmpty; + return; + } + *st = BitAnd(val, ALLOC_BIT) ? STATE::kAlloc : STATE::kFree; +} + +void BuddySpace::SetBuddySegState(rel_addr_t rel_addr, size_t rel_sz, STATE st) { + int clr; + int mask; + int pos; + int offset; + int val = 0; + int shift; + auto log_sz = static_cast(Log2(rel_sz)); + pos = BitRightShift(rel_addr, 2); + offset = rel_addr % 4; + shift = offset * 2; + if (rel_sz == 1) { + val = ONE_BIT; + mask = 0xC0; + } else if (rel_sz == 2) { + val = TWO_BIT; + mask = 0xF0; + } else { + val = BitOr(log_sz - 2, MORE_BIT); + mask = 0xFF; + } + if (st == STATE::kAlloc) { + val = BitOr(val, ALLOC_BIT); + } else if (st == STATE::kFree) { + val = BitAnd(val, ~(static_cast(ALLOC_BIT))); + } else if (st == STATE::kEmpty) { + val = 0; + } + clr = static_cast(~(BitRightShift(mask, shift))); + map_[pos] = static_cast(BitAnd(map_[pos], clr)); + map_[pos] = static_cast(BitOr(map_[pos], BitRightShift(val, shift))); + if (st == STATE::kAlloc) { + count_[log_sz]--; + } else if (st == STATE::kFree) { + count_[log_sz]++; + if (rel_addr < hint_[log_sz]) { + hint_[log_sz] = rel_addr; + } + } +} + +void BuddySpace::JoinBuddySeg(rel_addr_t addr, size_t blk_sz) { + while (blk_sz < BitLeftShift(1, num_lvl_)) { + rel_addr_t buddy = BitEx(addr, blk_sz); + size_t sz = 0; + STATE st; + GetBuddySegState(buddy, &sz, &st); + if (st == STATE::kFree && sz == blk_sz) { + auto log_sz = static_cast(Log2(blk_sz)); + rel_addr_t left = (buddy < addr) ? buddy : addr; + rel_addr_t right = left + blk_sz; + MS_ASSERT(count_[log_sz] >= 2); + count_[log_sz] -= 2; + SetBuddySegState(right, blk_sz, STATE::kEmpty); + SetBuddySegState(left, BitLeftShift(blk_sz, 1), STATE::kFree); + for (int i = 0; i < log_sz; i++) { + if (hint_[i] == right) { + hint_[i] = left; + } + } + addr = left; + blk_sz <<= 1u; + } else { + break; + } + } +} + +void BuddySpace::TrimBuddySeg(rel_addr_t addr, size_t blk_sz, size_t ask_sz) { + MS_ASSERT(ask_sz < blk_sz); + uint32_t inx = Log2(blk_sz); + size_t remaining_sz = ask_sz; + for (int i = inx; i > 0; i--) { + size_t b_size = BitLeftShift(1, i); + size_t half_sz = BitRightShift(b_size, 1); + count_[i]--; + SetBuddySegState(addr, half_sz, STATE::kFree); + SetBuddySegState(addr + half_sz, half_sz, STATE::kFree); + if (remaining_sz >= half_sz) { + SetBuddySegState(addr, half_sz, STATE::kAlloc); + remaining_sz -= half_sz; + if (remaining_sz == 0) { + break; + } + addr += half_sz; + } + } +} + +void BuddySpace::UnTrimBuddySeg(rel_addr_t addr, size_t blk_sz, size_t ask_sz) { + MS_ASSERT(ask_sz < blk_sz); + uint32_t inx = Log2(blk_sz); + size_t remaining_sz = ask_sz; + for (int i = inx; i > 0; i--) { + size_t b_size = BitLeftShift(1, i); + size_t half_sz = BitRightShift(b_size, 1); + if (remaining_sz >= half_sz) { +#ifdef DEBUG + { + size_t sz = 0; + STATE st; + GetBuddySegState(addr, &sz, &st); + MS_ASSERT(sz == half_sz && st == STATE::kAlloc); + } +#endif + SetBuddySegState(addr, half_sz, STATE::kFree); + remaining_sz -= half_sz; + if (remaining_sz == 0) { + JoinBuddySeg(addr, half_sz); + break; + } + addr += half_sz; + } + } +} + +rel_addr_t BuddySpace::AllocBuddySeg(uint32_t req_size) noexcept { + uint32_t blk_size = NextPowerOf2(req_size); + int start_inx = static_cast(Log2(blk_size)); + bool found = false; + rel_addr_t ask_addr = 0; + auto max_addr = static_cast(BitLeftShift(1, num_lvl_ - 1)); + STATE st; + size_t sz = 0; + for (int i = start_inx; !found && i < num_lvl_; i++) { + MS_ASSERT(count_[i] >= 0); + if (count_[i] == 0) { + continue; + } + auto blk_sz = static_cast(BitLeftShift(1, i)); + ask_addr = hint_[i]; + while (ask_addr < max_addr && !found) { + GetBuddySegState(ask_addr, &sz, &st); + if (st == STATE::kFree && sz == blk_sz) { + found = true; + } else { + MS_ASSERT(st != STATE::kEmpty); + ask_addr += ((sz > blk_sz) ? sz : blk_sz); + } + } + } + if (found) { + if (sz > req_size) { + TrimBuddySeg(ask_addr, sz, req_size); + } else { + SetBuddySegState(ask_addr, sz, STATE::kAlloc); + hint_[start_inx] = ask_addr; + } + return ask_addr; + } else { + return static_cast(NOSPACE); + } +} + +void BuddySpace::FreeBuddySeg(rel_addr_t addr, size_t blk_size, size_t req_size) { + if (req_size == blk_size) { +#ifdef DEBUG + { + size_t sz = 0; + STATE st; + GetBuddySegState(addr, &sz, &st); + } +#endif + SetBuddySegState(addr, blk_size, STATE::kFree); + JoinBuddySeg(addr, blk_size); + } else { + UnTrimBuddySeg(addr, blk_size, req_size); + } +} + +int BuddySpace::PercentFree() const { + uint64_t total_free_sz = 0; + uint64_t max_sz_in_unit = BitLeftShift(1, num_lvl_ - 1); + // Go through the count array without lock + for (int i = 0; i < num_lvl_; i++) { + int cnt = count_[i]; + if (cnt == 0) { + continue; + } + uint64_t blk_sz = BitLeftShift(1, i); + total_free_sz += (blk_sz * cnt); + } + return static_cast(static_cast(total_free_sz) / static_cast(max_sz_in_unit) * 100); +} + +BuddySpace::BuddySpace(int log_min, int num_lvl) + : hint_(nullptr), + count_(nullptr), + map_(nullptr), + log_min_(log_min), + num_lvl_(num_lvl), + min_(0), + max_(0), + ptr_(nullptr) {} + +BuddySpace::~BuddySpace() { + if (ptr_ != nullptr) { + free(ptr_); + } + hint_ = nullptr; + count_ = nullptr; + map_ = nullptr; +} + +Status BuddySpace::CreateBuddySpace(std::unique_ptr *out_bs, int log_min, int num_lvl) { + Status rc; + auto bs = new (std::nothrow) BuddySpace(log_min, num_lvl); + if (bs == nullptr) { + return Status(StatusCode::kOutOfMemory); + } + rc = bs->Init(); + if (rc.IsOk()) { + (*out_bs).reset(bs); + } else { + delete bs; + } + return rc; +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/util/buddy.h b/mindspore/ccsrc/dataset/util/buddy.h new file mode 100644 index 0000000000..08c05cbbdb --- /dev/null +++ b/mindspore/ccsrc/dataset/util/buddy.h @@ -0,0 +1,133 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_UTIL_BUDDY_H_ +#define DATASET_UTIL_BUDDY_H_ + +#include +#include +#include +#include +#include +#include +#include "dataset/util/status.h" + +using addr_t = int64_t; +using rel_addr_t = int32_t; +using log_t = int; +#define ALLOC_BIT 0x80 +#define ONE_BIT 0x40 +#define TWO_BIT 0x20 +#define MORE_BIT 0x10 +#define NOSPACE ((addr_t)(-1)) +namespace mindspore { +namespace dataset { +struct BSpaceDescriptor { + int32_t sig; + rel_addr_t addr; + size_t req_size; + size_t blk_size; +}; + +class BuddySpace { + public: + // C++11 feature. Change STATE into a type safe class with + // the keyword. Don't take out the keyword 'class' + enum class STATE { kFree, kAlloc, kEmpty }; + + BuddySpace(const BuddySpace &) = delete; + + BuddySpace &operator=(const BuddySpace &) = delete; + + virtual ~BuddySpace(); + + Status Alloc(uint64_t sz, BSpaceDescriptor *desc, addr_t *) noexcept; + + void Free(const BSpaceDescriptor *desc); + + uint64_t GetMinSize() const { return min_; } + + uint64_t GetMaxSize() const { return max_; } + + int PercentFree() const; + + friend std::ostream &operator<<(std::ostream &os, const BuddySpace &s); + + static uint64_t NextPowerOf2(uint64_t n) { + if (n <= 1) { + return 1; + } + n = n - 1; + while (n & (n - 1)) { + n = n & (n - 1); + } + return n << 1; + } + + static uint32_t Log2(uint64_t n) { + uint32_t cnt = 0; + while (n >>= 1) { + cnt++; + } + return cnt; + } + + static Status CreateBuddySpace(std::unique_ptr *out_bs, int log_min = 15, int num_lvl = 18); + + private: + rel_addr_t *hint_; + int *count_; + char *map_; + int log_min_; + int num_lvl_; + uint64_t min_; + uint64_t max_; + void *ptr_; + std::mutex mutex_; + + explicit BuddySpace(int log_min = 15, int num_lvl = 18); + + Status Init(); + + addr_t AllocNoLock(const uint64_t sz, BSpaceDescriptor *desc) noexcept; + + void FreeNoLock(const BSpaceDescriptor *desc); + + uint32_t SizeToBlock(const uint64_t sz) const { + uint32_t reqSize = (sz / min_); + if (sz % min_) { + reqSize++; + } + return reqSize; + } + + void GetBuddySegState(const rel_addr_t rel_addr, size_t *rel_sz, STATE *st) const; + + void SetBuddySegState(rel_addr_t rel_addr, size_t rel_sz, STATE st); + + void JoinBuddySeg(rel_addr_t addr, size_t blk_sz); + + void TrimBuddySeg(rel_addr_t addr, size_t blk_sz, size_t ask_sz); + + void UnTrimBuddySeg(rel_addr_t addr, size_t blk_sz, size_t ask_sz); + + rel_addr_t AllocBuddySeg(uint32_t req_size) noexcept; + + void FreeBuddySeg(rel_addr_t addr, size_t blk_size, size_t req_size); +}; +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_UTIL_BUDDY_H_ diff --git a/mindspore/ccsrc/dataset/util/cache_pool.cc b/mindspore/ccsrc/dataset/util/cache_pool.cc new file mode 100644 index 0000000000..92504cd063 --- /dev/null +++ b/mindspore/ccsrc/dataset/util/cache_pool.cc @@ -0,0 +1,202 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include "common/utils.h" +#include "dataset/util/cache_pool.h" +#include "dataset/util/services.h" + +namespace mindspore { +namespace dataset { +CachePool::CachePool(const value_allocator &alloc, const std::string &root) + : alloc_(alloc), root_(root), subfolder_(Services::GetUniqueID()), sm_(nullptr), tree_(nullptr) {} + +Status CachePool::DoServiceStart() { + tree_ = std::make_shared(); + // If we are given a disk path, set up the StorageManager + if (!root_.toString().empty()) { + Path spill = GetSpillPath(); + RETURN_IF_NOT_OK(spill.CreateDirectories()); + sm_ = std::make_shared(spill); + RETURN_IF_NOT_OK(sm_->ServiceStart()); + MS_LOG(INFO) << "CachePool will use disk folder: " << common::SafeCStr(spill.toString()); + } + return Status::OK(); +} +Status CachePool::DoServiceStop() { + Status rc; + Status rc2; + if (sm_ != nullptr) { + rc = sm_->ServiceStop(); + if (rc.IsError()) { + rc2 = rc; + } + } + sm_.reset(); + for (auto &bl : *tree_) { + if (bl.ptr != nullptr) { + alloc_.deallocate(bl.ptr, bl.sz); + } + } + tree_.reset(); + if (!root_.toString().empty()) { + Path spill = GetSpillPath(); + auto it = Path::DirIterator::OpenDirectory(&spill); + while (it->hasNext()) { + rc = it->next().Remove(); + if (rc.IsError() && rc2.IsOk()) { + rc2 = rc; + } + } + rc = spill.Remove(); + if (rc.IsError() && rc2.IsOk()) { + rc2 = rc; + } + } + return rc2; +} +CachePool::~CachePool() noexcept { (void)ServiceStop(); } +Status CachePool::Insert(const std::vector &buf, CachePool::key_type *key) { + DataLocator bl; + Status rc; + size_t sz = 0; + // We will consolidate all the slices into one piece. + for (auto &v : buf) { + sz += v.GetSize(); + } + bl.sz = sz; + try { + bl.ptr = alloc_.allocate(sz); + // We will do a piecewise copy. + WritableSlice dest(bl.ptr, bl.sz); + size_t pos = 0; + for (auto &v : buf) { + WritableSlice out(dest, pos); + rc = WritableSlice::Copy(&out, v); + if (rc.IsError()) { + break; + } + pos += v.GetSize(); + } + if (rc.IsError()) { + alloc_.deallocate(bl.ptr, sz); + bl.ptr = nullptr; + return rc; + } + } catch (std::bad_alloc &e) { + if (sm_ != nullptr) { + RETURN_IF_NOT_OK(sm_->Write(&bl.storage_key, buf)); + // We have an assumption 0 is not a valid key from the design of AutoIndexObj. + // Make sure it is not 0. + if (bl.storage_key == 0) { + RETURN_STATUS_UNEXPECTED("Key 0 is returned which is unexpected"); + } + } else { + return Status(StatusCode::kOutOfMemory, __LINE__, __FILE__); + } + } + rc = tree_->insert(bl, key); + if (rc.IsError() && bl.ptr != nullptr) { + alloc_.deallocate(bl.ptr, sz); + } + return rc; +} +Status CachePool::Read(CachePool::key_type key, WritableSlice *dest, size_t *bytesRead) const { + RETURN_UNEXPECTED_IF_NULL(dest); + auto r = tree_->Search(key); + if (r.second) { + auto &it = r.first; + if (it->ptr != nullptr) { + ReadableSlice src(it->ptr, it->sz); + RETURN_IF_NOT_OK(WritableSlice::Copy(dest, src)); + } else if (sm_ != nullptr) { + size_t expectedLength = 0; + RETURN_IF_NOT_OK(sm_->Read(it->storage_key, dest, &expectedLength)); + if (expectedLength != it->sz) { + MS_LOG(ERROR) << "Unexpected length. Read " << expectedLength << ". Expected " << it->sz << "." + << " Internal key: " << key << "\n"; + RETURN_STATUS_UNEXPECTED("Length mismatch. See log file for details."); + } + } + if (bytesRead != nullptr) { + *bytesRead = it->sz; + } + } else { + RETURN_STATUS_UNEXPECTED("Key not found"); + } + return Status::OK(); +} +const CachePool::value_allocator &CachePool::get_allocator() const { return alloc_; } +Path CachePool::GetSpillPath() const { + auto spill = Path(root_) / subfolder_; + return spill; +} +CachePool::CacheStat CachePool::GetStat() const { + CacheStat cs{0}; + for (auto &it : *tree_) { + if (it.ptr != nullptr) { + ++cs.num_mem_cached; + } else { + ++cs.num_disk_cached; + } + } + return cs; +} +Status CachePool::Spill(CachePool::DataLocator *dl) { + if (sm_ == nullptr) { + RETURN_STATUS_UNEXPECTED("No disk storage to spill"); + } + RETURN_UNEXPECTED_IF_NULL(dl); + RETURN_UNEXPECTED_IF_NULL(dl->ptr); + if (dl->storage_key == 0) { + ReadableSlice data(dl->ptr, dl->sz); + RETURN_IF_NOT_OK(sm_->Write(&dl->storage_key, {data})); + } + alloc_.deallocate(dl->ptr, dl->sz); + dl->ptr = nullptr; + return Status::OK(); +} +Status CachePool::Locate(CachePool::DataLocator *dl) { + RETURN_UNEXPECTED_IF_NULL(dl); + if (dl->ptr == nullptr) { + if (sm_ == nullptr) { + RETURN_STATUS_UNEXPECTED("No disk storage to locate the data"); + } + try { + dl->ptr = alloc_.allocate(dl->sz); + WritableSlice dest(dl->ptr, dl->sz); + Status rc = Read(dl->storage_key, &dest); + if (rc.IsError()) { + alloc_.deallocate(dl->ptr, dl->sz); + dl->ptr = nullptr; + return rc; + } + } catch (const std::bad_alloc &e) { + return Status(StatusCode::kOutOfMemory, __LINE__, __FILE__); + } + } + return Status::OK(); +} +size_t CachePool::GetSize(CachePool::key_type key) const { + auto r = tree_->Search(key); + if (r.second) { + auto &it = r.first; + return it->sz; + } else { + return 0; + } +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/util/cache_pool.h b/mindspore/ccsrc/dataset/util/cache_pool.h new file mode 100644 index 0000000000..d35617d0e4 --- /dev/null +++ b/mindspore/ccsrc/dataset/util/cache_pool.h @@ -0,0 +1,139 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_UTIL_CACHE_POOL_H_ +#define DATASET_UTIL_CACHE_POOL_H_ + +#include +#include +#include +#include +#include "dataset/util/allocator.h" +#include "dataset/util/service.h" +#include "dataset/util/slice.h" +#include "dataset/util/storage_manager.h" +#include "dataset/util/auto_index.h" + +namespace mindspore { +namespace dataset { +/// \brief A CachePool provides service for backup/restore a buffer. A buffer can be represented in a form of vector of +/// ReadableSlice where all memory blocks will be copied to one contiguous block which can be in memory or spilled to +/// disk (if a disk directory is provided). Every buffer insert will return a generated key which can be used to +/// restore the buffer. +/// \see ReadableSlice +class CachePool : public Service { + public: + using base_type = uint8_t; + using pointer = base_type *; + using const_pointer = const base_type *; + using reference = base_type &; + using const_reference = const base_type &; + using value_allocator = Allocator; + + // An internal class to locate the whereabouts of a backed up buffer which can be either in + class DataLocator { + public: + DataLocator() : ptr(nullptr), sz(0), storage_key(0) {} + ~DataLocator() = default; + DataLocator(const DataLocator &other) = default; + DataLocator &operator=(const DataLocator &other) = default; + DataLocator(DataLocator &&other) noexcept { + ptr = other.ptr; + sz = other.sz; + storage_key = other.storage_key; + other.ptr = nullptr; + other.sz = 0; + other.storage_key = 0; + } + DataLocator &operator=(DataLocator &&other) noexcept { + if (&other != this) { + ptr = other.ptr; + sz = other.sz; + storage_key = other.storage_key; + other.ptr = nullptr; + other.sz = 0; + other.storage_key = 0; + } + return *this; + } + pointer ptr; + size_t sz; + StorageManager::key_type storage_key; + }; + + using data_index = AutoIndexObj; + using key_type = data_index::key_type; + using bl_alloc_type = typename value_allocator::template rebind::other; + + /// \brief Simple statistics returned from CachePool like how many elements are cached in memory and + /// how many elements are spilled to disk. + struct CacheStat { + int64_t num_mem_cached; + int64_t num_disk_cached; + }; + + /// \brief Constructor + /// \param alloc Allocator to allocate memory from + /// \param root Optional disk folder to spill + explicit CachePool(const value_allocator &alloc, const std::string &root = ""); + + CachePool(const CachePool &) = delete; + CachePool(CachePool &&) = delete; + CachePool &operator=(const CachePool &) = delete; + CachePool &operator=(CachePool &&) = delete; + ~CachePool() noexcept; + + Status DoServiceStart() override; + Status DoServiceStop() override; + + Path GetSpillPath() const; + + /// \brief Insert a sequence of ReadableSlice objects into the pool. + /// All memory blocks will be consolidated into one contiguous block and be cached in either memory or on disk. + /// \param[in] buf A sequence of ReadableSlice objects. + /// \param[out] key Generated key + /// \return Error code + Status Insert(const std::vector &buf, key_type *key); + /// \brief Restore a cached buffer (from memory or disk) + /// \param[in] key A previous key returned from Insert + /// \param[out] dest The cached buffer will be copied to this destination represented by a WritableSlice + /// \param[out] bytesRead Optional. Number of bytes read. + /// \return Error code + Status Read(key_type key, WritableSlice *dest, size_t *bytesRead = nullptr) const; + + Status Spill(DataLocator *dl); + + Status Locate(DataLocator *dl); + + size_t GetSize(key_type key) const; + + /// \brief Get statistics. + /// \return CacheStat object + CacheStat GetStat() const; + + const value_allocator &get_allocator() const; + + std::string MyName() const { return subfolder_; } + + private: + value_allocator alloc_; + Path root_; + const std::string subfolder_; + std::shared_ptr sm_; + std::shared_ptr tree_; +}; +} // namespace dataset +} // namespace mindspore +#endif diff --git a/mindspore/ccsrc/dataset/util/circular_pool.cc b/mindspore/ccsrc/dataset/util/circular_pool.cc index 92b169c94a..0c68dab81b 100644 --- a/mindspore/ccsrc/dataset/util/circular_pool.cc +++ b/mindspore/ccsrc/dataset/util/circular_pool.cc @@ -19,8 +19,8 @@ #include #include #include "./securec.h" -#include "dataset/util/de_error.h" #include "dataset/util/system_pool.h" +#include "utils/log_adapter.h" namespace mindspore { namespace dataset { @@ -65,7 +65,7 @@ void CircularPool::CircularIterator::Reset() { auto list_end = dp_->mem_segments_.end(); auto it = std::find_if(dp_->mem_segments_.begin(), list_end, [this](const std::shared_ptr &b) { return b.get() == cur_tail_; }); - DS_ASSERT(it != list_end); + MS_ASSERT(it != list_end); start_ = std::distance(dp_->mem_segments_.begin(), it); cur_ = start_; has_next_ = true; @@ -153,7 +153,7 @@ Status CircularPool::Reallocate(void **pp, size_t old_sz, size_t new_sz) { return (q > base && q < base + b->get_max_size()); }); lock.Unlock(); - DS_ASSERT(it != mem_segments_.end()); + MS_ASSERT(it != mem_segments_.end()); Arena *ba = it->get(); Status rc = ba->Reallocate(pp, old_sz, new_sz); if (rc.IsOutofMemory()) { diff --git a/mindspore/ccsrc/dataset/util/list.h b/mindspore/ccsrc/dataset/util/list.h index 5a08f4514e..06f26ab57c 100644 --- a/mindspore/ccsrc/dataset/util/list.h +++ b/mindspore/ccsrc/dataset/util/list.h @@ -18,7 +18,8 @@ #include #include -#include "dataset/util/de_error.h" + +#include "utils/log_adapter.h" namespace mindspore { namespace dataset { @@ -90,7 +91,7 @@ struct List { // Insert elem2 after elem1 in the list. virtual void InsertAfter(pointer elem1, pointer elem2) { - DS_ASSERT(elem1 != elem2); + MS_ASSERT(elem1 != elem2); Node &elem1_node = elem1->*node; Node &elem2_node = elem2->*node; elem2_node.prev = elem1; @@ -106,6 +107,24 @@ struct List { ++count; } + // Insert elem2 before elem1 in the list. + virtual void InsertBefore(pointer elem1, pointer elem2) { + MS_ASSERT(elem1 != elem2); + Node &elem1_node = elem1->*node; + Node &elem2_node = elem2->*node; + elem2_node.next = elem1; + elem2_node.prev = elem1_node.prev; + if (elem1_node.prev != nullptr) { + Node &prev_node = elem1_node.prev->*node; + prev_node.next = elem2; + } + elem1_node.prev = elem2; + if (head == elem1) { + head = elem2; + } + ++count; + } + // Remove an element in the list virtual void Remove(pointer elem) noexcept { Node &elem_node = elem->*node; diff --git a/mindspore/ccsrc/dataset/util/lock.cc b/mindspore/ccsrc/dataset/util/lock.cc index 13a43e3e84..bde9d84005 100644 --- a/mindspore/ccsrc/dataset/util/lock.cc +++ b/mindspore/ccsrc/dataset/util/lock.cc @@ -14,7 +14,7 @@ * limitations under the License. */ #include "dataset/util/lock.h" -#include "dataset/util/de_error.h" +#include "utils/log_adapter.h" namespace mindspore { namespace dataset { @@ -63,7 +63,7 @@ void RWLock::Unlock() noexcept { void RWLock::Upgrade() { std::unique_lock lck(mtx_); - DS_ASSERT(status_); + MS_ASSERT(status_); if (status_ == -1) { // I am a writer already. return; @@ -81,7 +81,7 @@ void RWLock::Upgrade() { void RWLock::Downgrade() { std::unique_lock lck(mtx_); - DS_ASSERT(status_); + MS_ASSERT(status_); if (status_ == -1) { // If there are no other writers waiting, just change the status if (waiting_writers_ == 0) { @@ -111,24 +111,24 @@ SharedLock::~SharedLock() { } void SharedLock::Unlock() { - DS_ASSERT(ownlock_ == true); + MS_ASSERT(ownlock_ == true); rw_->Unlock(); ownlock_ = false; } void SharedLock::Lock() { - DS_ASSERT(ownlock_ == false); + MS_ASSERT(ownlock_ == false); rw_->LockShared(); ownlock_ = true; } void SharedLock::Upgrade() { - DS_ASSERT(ownlock_ == true); + MS_ASSERT(ownlock_ == true); rw_->Upgrade(); } void SharedLock::Downgrade() { - DS_ASSERT(ownlock_ == true); + MS_ASSERT(ownlock_ == true); rw_->Downgrade(); } @@ -146,13 +146,13 @@ UniqueLock::~UniqueLock() { } void UniqueLock::Unlock() { - DS_ASSERT(ownlock_ == true); + MS_ASSERT(ownlock_ == true); rw_->Unlock(); ownlock_ = false; } void UniqueLock::Lock() { - DS_ASSERT(ownlock_ == false); + MS_ASSERT(ownlock_ == false); rw_->LockExclusive(); ownlock_ = true; } @@ -171,13 +171,13 @@ LockGuard::~LockGuard() { } void LockGuard::Unlock() { - DS_ASSERT(own_lock_); + MS_ASSERT(own_lock_); lck_->Unlock(); own_lock_ = false; } void LockGuard::Lock() { - DS_ASSERT(own_lock_ == false); + MS_ASSERT(own_lock_ == false); lck_->Lock(); own_lock_ = true; } diff --git a/mindspore/ccsrc/dataset/util/memory_pool.h b/mindspore/ccsrc/dataset/util/memory_pool.h index 70876a8141..ee1da3bda1 100644 --- a/mindspore/ccsrc/dataset/util/memory_pool.h +++ b/mindspore/ccsrc/dataset/util/memory_pool.h @@ -44,20 +44,6 @@ class MemoryPool { virtual ~MemoryPool() {} }; -// Used by unique_ptr -template -class Deleter { - public: - explicit Deleter(std::shared_ptr &mp) : mp_(mp) {} - - ~Deleter() = default; - - void operator()(T *ptr) const { mp_->Deallocate(ptr); } - - private: - std::shared_ptr mp_; -}; - Status DeMalloc(std::size_t s, void **p, bool); } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/util/path.cc b/mindspore/ccsrc/dataset/util/path.cc index c37fdc17f1..cdd2343799 100644 --- a/mindspore/ccsrc/dataset/util/path.cc +++ b/mindspore/ccsrc/dataset/util/path.cc @@ -16,17 +16,18 @@ #include "dataset/util/path.h" #include +#include +#include #include #include #include #include "common/utils.h" -#include "dataset/util/de_error.h" #include "utils/log_adapter.h" namespace mindspore { namespace dataset { -#ifdef _WIN32 +#if defined(_WIN32) || defined(_WIN64) char Path::separator_ = '\\'; #else char Path::separator_ = '/'; @@ -132,7 +133,7 @@ Status Path::CreateDirectory() { #if defined(_WIN32) || defined(_WIN64) int rc = mkdir(common::SafeCStr(path_)); #else - int rc = mkdir(common::SafeCStr(path_), 0700); + int rc = mkdir(common::SafeCStr(path_), S_IRUSR | S_IWUSR | S_IXUSR); #endif if (rc) { std::ostringstream oss; @@ -182,6 +183,111 @@ Status Path::CreateDirectories() { return Status::OK(); } +Status Path::Remove() { + if (Exists()) { + if (IsDirectory()) { + errno_t err = rmdir(common::SafeCStr(path_)); + if (err == -1) { + std::ostringstream oss; + oss << "Unable to delete directory " << path_ << ". Errno = " << errno; + RETURN_STATUS_UNEXPECTED(oss.str()); + } + } else { + errno_t err = unlink(common::SafeCStr(path_)); + if (err == -1) { + std::ostringstream oss; + oss << "Unable to delete file " << path_ << ". Errno = " << errno; + RETURN_STATUS_UNEXPECTED(oss.str()); + } + } + } + return Status::OK(); +} + +Status Path::CreateFile(int *file_descriptor) { return OpenFile(file_descriptor, true); } + +Status Path::OpenFile(int *file_descriptor, bool create) { + int fd; + if (file_descriptor == nullptr) { + RETURN_STATUS_UNEXPECTED("null pointer"); + } + if (IsDirectory()) { + std::ostringstream oss; + oss << "Unable to create file " << path_ << " which is a directory."; + RETURN_STATUS_UNEXPECTED(oss.str()); + } + // Convert to canonical form. + if (strlen(common::SafeCStr(path_)) > PATH_MAX) { + RETURN_STATUS_UNEXPECTED(strerror(errno)); + } + char canonical_path[PATH_MAX + 1] = {0x00}; +#if defined(_WIN32) || defined(_WIN64) + if (_fullpath(canonical_path, common::SafeCStr(path_), PATH_MAX) == nullptr) { +#else + if (realpath(common::SafeCStr(path_), canonical_path) == nullptr) { +#endif + if (errno == ENOENT && create) { + // File doesn't exist and we are to create it. Let's break it down. + auto file_part = Basename(); + auto parent_part = ParentPath(); +#if defined(_WIN32) || defined(_WIN64) + if (_fullpath(canonical_path, common::SafeCStr(parent_part), PATH_MAX) == nullptr) { +#else + if (realpath(common::SafeCStr(parent_part), canonical_path) == nullptr) { +#endif + RETURN_STATUS_UNEXPECTED(strerror(errno)); + } + auto cur_inx = strlen(canonical_path); + if ((cur_inx + file_part.length() + 1) > PATH_MAX) { + RETURN_STATUS_UNEXPECTED(strerror(errno)); + } + canonical_path[cur_inx++] = separator_; + if (strncpy_s(canonical_path + cur_inx, PATH_MAX - cur_inx, common::SafeCStr(file_part), file_part.length()) != + EOK) { + RETURN_STATUS_UNEXPECTED(strerror(errno)); + } + } else { + RETURN_STATUS_UNEXPECTED(strerror(errno)); + } + } + if (create) { + fd = open(canonical_path, O_CREAT | O_TRUNC | O_RDWR, S_IRUSR | S_IWUSR | S_IRGRP); + } else { + fd = open(canonical_path, O_RDWR); + } + if (fd == -1) { + RETURN_STATUS_UNEXPECTED(strerror(errno)); + } + *file_descriptor = fd; + return Status::OK(); +} + +Status Path::CloseFile(int fd) const { + if (close(fd) < 0) { + RETURN_STATUS_UNEXPECTED(strerror(errno)); + } + return Status::OK(); +} + +Status Path::TruncateFile(int fd) const { + int rc; + rc = ftruncate(fd, 0); + if (rc == 0) { + return Status::OK(); + } else { + RETURN_STATUS_UNEXPECTED(strerror(errno)); + } +} + +std::string Path::Basename() { + std::size_t found = path_.find_last_of(separator_); + if (found != std::string::npos) { + return path_.substr(found + 1); + } else { + return path_; + } +} + std::shared_ptr Path::DirIterator::OpenDirectory(Path *f) { auto it = new (std::nothrow) DirIterator(f); @@ -208,7 +314,7 @@ Path::DirIterator::~DirIterator() { Path::DirIterator::DirIterator(Path *f) : dir_(f), dp_(nullptr), entry_(nullptr) { MS_LOG(DEBUG) << "Open directory " << f->toString() << "."; - dp_ = opendir(common::SafeCStr(f->toString())); + dp_ = opendir(f->toString().c_str()); } bool Path::DirIterator::hasNext() { @@ -225,5 +331,10 @@ bool Path::DirIterator::hasNext() { } Path Path::DirIterator::next() { return (*(this->dir_) / Path(entry_->d_name)); } + +std::ostream &operator<<(std::ostream &os, const Path &s) { + os << s.path_; + return os; +} } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/util/path.h b/mindspore/ccsrc/dataset/util/path.h index efe01a7d16..fbf65b8c23 100644 --- a/mindspore/ccsrc/dataset/util/path.h +++ b/mindspore/ccsrc/dataset/util/path.h @@ -90,6 +90,20 @@ class Path { std::string ParentPath(); + Status Remove(); + + Status CreateFile(int *fd); + + Status OpenFile(int *fd, bool create = false); + + Status CloseFile(int fd) const; + + Status TruncateFile(int fd) const; + + std::string Basename(); + + friend std::ostream &operator<<(std::ostream &os, const Path &s); + private: static char separator_; std::string path_; diff --git a/mindspore/ccsrc/dataset/util/queue.h b/mindspore/ccsrc/dataset/util/queue.h index 9a51565861..7fca93d944 100644 --- a/mindspore/ccsrc/dataset/util/queue.h +++ b/mindspore/ccsrc/dataset/util/queue.h @@ -25,7 +25,6 @@ #include #include "common/utils.h" -#include "dataset/util/de_error.h" #include "utils/log_adapter.h" #include "dataset/util/allocator.h" #include "dataset/util/services.h" @@ -72,7 +71,7 @@ class Queue { arr_(nullptr), head_(0), tail_(0), - my_name_(std::move(Services::GetUniqueID())), + my_name_(Services::GetUniqueID()), alloc_(Services::GetInstance().GetServiceMemPool()) { Init(); MS_LOG(DEBUG) << "Create Q with uuid " << my_name_ << " of size " << sz_ << "."; @@ -155,7 +154,16 @@ class Queue { uint32_t k = head_++ % sz_; *p = std::move(arr_[k]); if (std::is_destructible::value) { + // std::move above only changes arr_[k] from rvalue to lvalue. + // The real implementation of move constructor depends on T. + // It may be compiler generated or user defined. But either case + // the result of arr_[k] is still a valid object of type T, and + // we will not keep any extra copy in the queue. arr_[k].~T(); + // For gcc 9, an extra fix is needed here to clear the memory content + // of arr_[k] because this slot can be reused by another Add which can + // do another std::move. We have seen SEGV here in this case. + std::allocator_traits>::construct(alloc_, &(arr_[k])); } full_cv_.NotifyAll(); _lock.unlock(); diff --git a/mindspore/ccsrc/dataset/util/de_error.h b/mindspore/ccsrc/dataset/util/semaphore.cc similarity index 50% rename from mindspore/ccsrc/dataset/util/de_error.h rename to mindspore/ccsrc/dataset/util/semaphore.cc index d4988c58db..36ddf5511d 100644 --- a/mindspore/ccsrc/dataset/util/de_error.h +++ b/mindspore/ccsrc/dataset/util/semaphore.cc @@ -13,27 +13,29 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef DATASET_UTIL_DE_ERROR_H_ -#define DATASET_UTIL_DE_ERROR_H_ - -#ifdef DEBUG -#include -#define DS_ASSERT(f) assert(f) -#else -#define DS_ASSERT(f) ((void)0) -#endif - -#include -#include "utils/error_code.h" +#include "dataset/util/semaphore.h" +#include "dataset/util/task_manager.h" namespace mindspore { namespace dataset { -DE_ERRORNO_DATASET(CATCH_EXCEPTION, 0, "try catch exception error"); -DE_ERRORNO_DATASET(FILE_NOT_FOUND, 1, "file is not found"); -DE_ERRORNO_DATASET(PARSE_FAILED, 2, "parse failed"); -DE_ERRORNO_DATASET(COPY_ERROR, 3, "copy data error"); -DE_ERRORNO_DATASET(BOUND_ERROR, 4, "variable overflow or lost of precision"); -DE_ERRORNO_DATASET(ALLOC_FAILED, 5, "dynamic memory allocation failed"); +Status Semaphore::P() { + std::unique_lock lck(mutex_); + RETURN_IF_NOT_OK(wait_cond_.Wait(&lck, [this]() { return value_ > 0; })); + --value_; + return Status::OK(); +} +void Semaphore::V() { + std::unique_lock lck(mutex_); + ++value_; + wait_cond_.NotifyOne(); +} +int Semaphore::Peek() { + std::unique_lock lck(mutex_); + return value_; +} +Status Semaphore::Register(TaskGroup *vg) { return wait_cond_.Register(vg->GetIntrpService()); } +Status Semaphore::Deregister() { return (wait_cond_.Deregister()); } +void Semaphore::ResetIntrpState() { wait_cond_.ResetIntrpState(); } + } // namespace dataset } // namespace mindspore -#endif // DATASET_UTIL_DE_ERROR_H_ diff --git a/mindspore/ccsrc/dataset/util/semaphore.h b/mindspore/ccsrc/dataset/util/semaphore.h new file mode 100644 index 0000000000..07b9e83e7f --- /dev/null +++ b/mindspore/ccsrc/dataset/util/semaphore.h @@ -0,0 +1,54 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_UTIL_SEMAPHORE_H_ +#define DATASET_UTIL_SEMAPHORE_H_ + +#include "dataset/util/cond_var.h" + +namespace mindspore { +namespace dataset { +class TaskGroup; + +/// \brief A counting semaphore. There are two external functions P and V. P decrements the internal count and will be +/// blocked if the count is 0 (zero). V increments the internal count and wake up one of the waiters. +class Semaphore { + public: + /// \brief Constructor + /// \param init Initial value of the internal counter. + explicit Semaphore(int init) : value_(init) {} + + virtual ~Semaphore() {} + /// \brief Decrement the internal counter. Will be blocked if the value is 0. + /// \return Error code. Can get interrupt. + Status P(); + /// \brief Increment the internal counter. Wakeup on of the watiers if any. + void V(); + /// \brief Peek the internal value + /// \return The internal value + int Peek(); + Status Register(TaskGroup *vg); + Status Deregister(); + void ResetIntrpState(); + + private: + int value_; + + std::mutex mutex_; + CondVar wait_cond_; +}; +} // namespace dataset +} // namespace mindspore +#endif // DATASET_UTIL_SEMAPHORE_H_ diff --git a/mindspore/ccsrc/dataset/util/slice.cc b/mindspore/ccsrc/dataset/util/slice.cc new file mode 100644 index 0000000000..f1798b4f44 --- /dev/null +++ b/mindspore/ccsrc/dataset/util/slice.cc @@ -0,0 +1,38 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + + * http://www.apache.org/licenses/LICENSE-2.0 + + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ +#include "dataset/util/slice.h" + +namespace mindspore { +namespace dataset { +WritableSlice::WritableSlice(const WritableSlice &src, off64_t offset, size_t len) : ReadableSlice(src, offset, len) { + mutable_data_ = static_cast(src.mutable_data_) + offset; +} +WritableSlice::WritableSlice(const WritableSlice &src, off64_t offset) + : WritableSlice(src, offset, src.GetSize() - offset) {} +Status WritableSlice::Copy(WritableSlice *dest, const ReadableSlice &src) { + RETURN_UNEXPECTED_IF_NULL(dest); + RETURN_UNEXPECTED_IF_NULL(dest->GetMutablePointer()); + if (dest->GetSize() <= 0) { + RETURN_STATUS_UNEXPECTED("Destination length is non-positive"); + } + auto err = memcpy_s(dest->GetMutablePointer(), dest->GetSize(), src.GetPointer(), src.GetSize()); + if (err) { + RETURN_STATUS_UNEXPECTED(std::to_string(err)); + } + return Status::OK(); +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/util/slice.h b/mindspore/ccsrc/dataset/util/slice.h new file mode 100644 index 0000000000..127df23cfa --- /dev/null +++ b/mindspore/ccsrc/dataset/util/slice.h @@ -0,0 +1,122 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_UTIL_SLICE_H_ +#define DATASET_UTIL_SLICE_H_ + +#include +#include +#include +#include "./securec.h" +#include "dataset/util/allocator.h" +#include "dataset/util/status.h" +namespace mindspore { +namespace dataset { +/// \brief A ReadableSlice wraps a const pointer in memory and its size. +/// \see WritableSlice for a non-const version +/// +class ReadableSlice { + public: + ReadableSlice() : ptr_(nullptr), sz_(0) {} + ReadableSlice(const void *ptr, size_t sz) : ptr_(ptr), sz_(sz) {} + ReadableSlice(const ReadableSlice &src, off64_t offset, size_t len) { + ptr_ = static_cast(src.GetPointer()) + offset; + sz_ = len; + } + ReadableSlice(const ReadableSlice &src, off64_t offset) : ReadableSlice(src, offset, src.sz_ - offset) {} + ReadableSlice(const ReadableSlice &lhs) { + ptr_ = lhs.ptr_; + sz_ = lhs.sz_; + } + ReadableSlice &operator=(const ReadableSlice &lhs) { + if (this != &lhs) { + ptr_ = lhs.ptr_; + sz_ = lhs.sz_; + } + return *this; + } + ReadableSlice(ReadableSlice &&lhs) noexcept { + if (this != &lhs) { + ptr_ = lhs.ptr_; + sz_ = lhs.sz_; + lhs.ptr_ = nullptr; + lhs.sz_ = 0; + } + } + ReadableSlice &operator=(ReadableSlice &&lhs) noexcept { + if (this != &lhs) { + ptr_ = lhs.ptr_; + sz_ = lhs.sz_; + lhs.ptr_ = nullptr; + lhs.sz_ = 0; + } + return *this; + } + /// \brief Getter function + /// \return Const version of the pointer + const void *GetPointer() const { return ptr_; } + /// \brief Getter function + /// \return Size of the slice + size_t GetSize() const { return sz_; } + bool empty() const { return ptr_ == nullptr; } + + private: + const void *ptr_; + size_t sz_; +}; +/// \brief A WritableSlice inherits from ReadableSlice to allow +/// one to write to the address pointed to by the pointer. +/// +class WritableSlice : public ReadableSlice { + public: + friend class StorageContainer; + /// \brief Default constructor + WritableSlice() : ReadableSlice(), mutable_data_(nullptr) {} + /// \brief This form of a constructor takes a pointer and its size. + WritableSlice(void *ptr, size_t sz) : ReadableSlice(ptr, sz), mutable_data_(ptr) {} + WritableSlice(const WritableSlice &src, off64_t offset, size_t len); + WritableSlice(const WritableSlice &src, off64_t offset); + WritableSlice(const WritableSlice &lhs) : ReadableSlice(lhs) { mutable_data_ = lhs.mutable_data_; } + WritableSlice &operator=(const WritableSlice &lhs) { + if (this != &lhs) { + mutable_data_ = lhs.mutable_data_; + ReadableSlice::operator=(lhs); + } + return *this; + } + WritableSlice(WritableSlice &&lhs) noexcept : ReadableSlice(std::move(lhs)) { + if (this != &lhs) { + mutable_data_ = lhs.mutable_data_; + lhs.mutable_data_ = nullptr; + } + } + WritableSlice &operator=(WritableSlice &&lhs) noexcept { + if (this != &lhs) { + mutable_data_ = lhs.mutable_data_; + lhs.mutable_data_ = nullptr; + ReadableSlice::operator=(std::move(lhs)); + } + return *this; + } + /// \brief Copy the content from one slice onto another. + static Status Copy(WritableSlice *dest, const ReadableSlice &src); + + private: + void *mutable_data_; + void *GetMutablePointer() { return mutable_data_; } +}; +} // namespace dataset +} // namespace mindspore +#endif // DATASET_UTIL_SLICE_H_ diff --git a/mindspore/ccsrc/dataset/util/storage_container.cc b/mindspore/ccsrc/dataset/util/storage_container.cc new file mode 100644 index 0000000000..3a4c13e2d9 --- /dev/null +++ b/mindspore/ccsrc/dataset/util/storage_container.cc @@ -0,0 +1,163 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/util/storage_container.h" + +#include +#include +#include +#include +#include "common/utils.h" +#include "dataset/util/path.h" +#include "dataset/util/status.h" +#include "utils/log_adapter.h" + +namespace mindspore { +namespace dataset { +Status StorageContainer::Create() { + RETURN_IF_NOT_OK(BuddySpace::CreateBuddySpace(&bs_)); + RETURN_IF_NOT_OK(cont_.CreateFile(&fd_)); + is_open_ = true; + MS_LOG(INFO) << "Container " << cont_ << " created"; + return Status::OK(); +} + +Status StorageContainer::Open() noexcept { + std::lock_guard lck(mutex_); + // Check again + if (!is_open_) { + RETURN_IF_NOT_OK(cont_.OpenFile(&fd_)); + is_open_ = true; + } + return Status::OK(); +} + +Status StorageContainer::Close() noexcept { + if (is_open_) { + std::lock_guard lck(mutex_); + // Check again + if (is_open_) { + RETURN_IF_NOT_OK(cont_.CloseFile(fd_)); + is_open_ = false; + fd_ = -1; + } + } + return Status::OK(); +} + +Status StorageContainer::Read(WritableSlice *dest, off64_t offset) const noexcept { + MS_ASSERT(is_open_); + RETURN_UNEXPECTED_IF_NULL(dest); + auto sz = dest->GetSize(); +#if defined(_WIN32) || defined(_WIN64) + // Doesn't seem there is any pread64 on mingw. + // So we will do a seek and then a read under + // a protection of mutex. + std::lock_guard lck(mutex_); + auto seek_err = lseek(fd_, offset, SEEK_SET); + if (seek_err < 0) { + RETURN_STATUS_UNEXPECTED(strerror(errno)); + } + auto r_sz = read(fd_, dest->GetMutablePointer(), sz); +#else + auto r_sz = pread64(fd_, dest->GetMutablePointer(), sz, offset); +#endif + if (r_sz != sz) { + errno_t err = (r_sz == 0) ? EOF : errno; + RETURN_STATUS_UNEXPECTED(strerror(err)); + } + return Status::OK(); +} + +Status StorageContainer::Write(const ReadableSlice &dest, off64_t offset) const noexcept { + MS_ASSERT(is_open_); + auto sz = dest.GetSize(); +#if defined(_WIN32) || defined(_WIN64) + // Doesn't seem there is any pwrite64 on mingw. + // So we will do a seek and then a read under + // a protection of mutex. + std::lock_guard lck(mutex_); + auto seek_err = lseek(fd_, offset, SEEK_SET); + if (seek_err < 0) { + RETURN_STATUS_UNEXPECTED(strerror(errno)); + } + auto r_sz = write(fd_, dest.GetPointer(), sz); +#else + auto r_sz = pwrite64(fd_, dest.GetPointer(), sz, offset); +#endif + if (r_sz != sz) { + errno_t err = (r_sz == 0) ? EOF : errno; + RETURN_STATUS_UNEXPECTED(strerror(err)); + } + return Status::OK(); +} + +Status StorageContainer::Insert(const std::vector &buf, off64_t *offset) noexcept { + size_t sz = 0; + for (auto &v : buf) { + sz += v.GetSize(); + } + if (sz == 0) { + RETURN_STATUS_UNEXPECTED("Unexpected 0 length"); + } + if (sz > bs_->GetMaxSize()) { + RETURN_STATUS_UNEXPECTED("Request size too big"); + } + BSpaceDescriptor bspd{0}; + addr_t addr = 0; + RETURN_IF_NOT_OK(bs_->Alloc(sz, &bspd, &addr)); + *offset = static_cast(addr); + // We will do piecewise copy of the data to disk. + for (auto &v : buf) { + RETURN_IF_NOT_OK(Write(v, addr)); + addr += v.GetSize(); + } + return Status::OK(); +} + +Status StorageContainer::Truncate() const noexcept { + if (is_open_) { + RETURN_IF_NOT_OK(cont_.TruncateFile(fd_)); + MS_LOG(INFO) << "Container " << cont_ << " truncated"; + } + return Status::OK(); +} + +StorageContainer::~StorageContainer() noexcept { + (void)Truncate(); + (void)Close(); +} + +std::ostream &operator<<(std::ostream &os, const StorageContainer &s) { + os << "File path : " << s.cont_ << "\n" << *(s.bs_.get()); + return os; +} + +Status StorageContainer::CreateStorageContainer(std::shared_ptr *out_sc, const std::string &path) { + Status rc; + auto sc = new (std::nothrow) StorageContainer(path); + if (sc == nullptr) { + return Status(StatusCode::kOutOfMemory); + } + rc = sc->Create(); + if (rc.IsOk()) { + (*out_sc).reset(sc); + } else { + delete sc; + } + return rc; +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/util/storage_container.h b/mindspore/ccsrc/dataset/util/storage_container.h new file mode 100644 index 0000000000..07e41bd66a --- /dev/null +++ b/mindspore/ccsrc/dataset/util/storage_container.h @@ -0,0 +1,79 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_UTIL_STORAGE_CONTAINER_H_ +#define DATASET_UTIL_STORAGE_CONTAINER_H_ + +#include +#include +#include +#include +#include +#include +#include "dataset/util/system_pool.h" +#include "dataset/util/buddy.h" +#include "dataset/util/path.h" +#include "dataset/util/slice.h" +#include "dataset/util/status.h" + +namespace mindspore { +namespace dataset { +class StorageManager; + +class StorageContainer { + public: + friend class StorageManager; + + ~StorageContainer() noexcept; + + StorageContainer(const StorageContainer &) = delete; + + StorageContainer &operator=(const StorageContainer &) = delete; + + friend std::ostream &operator<<(std::ostream &os, const StorageContainer &s); + + Status Open() noexcept; + + Status Close() noexcept; + + Status Insert(const std::vector &buf, off64_t *offset) noexcept; + + Status Write(const ReadableSlice &dest, off64_t offset) const noexcept; + + Status Read(WritableSlice *dest, off64_t offset) const noexcept; + + Status Truncate() const noexcept; + + bool IsOpen() const { return is_open_; } + + static Status CreateStorageContainer(std::shared_ptr *out_sc, const std::string &path); + + private: + mutable std::mutex mutex_; + Path cont_; + int fd_; + bool is_open_; + std::unique_ptr bs_; + + // Use the default value of BuddySpace + // which can map upto 4G of space. + explicit StorageContainer(const std::string &path) : cont_(path), fd_(-1), is_open_(false), bs_(nullptr) {} + + Status Create(); +}; +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_UTIL_STORAGE_CONTAINER_H_ diff --git a/mindspore/ccsrc/dataset/util/storage_manager.cc b/mindspore/ccsrc/dataset/util/storage_manager.cc new file mode 100644 index 0000000000..1d958576ba --- /dev/null +++ b/mindspore/ccsrc/dataset/util/storage_manager.cc @@ -0,0 +1,166 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/util/storage_manager.h" + +#include +#include +#include +#include +#include "common/utils.h" +#include "dataset/util/path.h" +#include "dataset/util/services.h" +#include "utils/log_adapter.h" + +namespace mindspore { +namespace dataset { +std::string StorageManager::GetBaseName(const std::string &prefix, int32_t file_id) { + std::ostringstream oss; + oss << prefix << std::setfill('0') << std::setw(5) << file_id; + return oss.str(); +} + +std::string StorageManager::ConstructFileName(const std::string &prefix, int32_t file_id, const std::string &suffix) { + std::string base_name = GetBaseName(prefix, file_id); + return (base_name + "." + suffix); +} + +Status StorageManager::AddOneContainer() { + const std::string kPrefix = "IMG"; + const std::string kSuffix = "LB"; + Path container_name = root_ / ConstructFileName(kPrefix, file_id_, kSuffix); + std::shared_ptr sc; + RETURN_IF_NOT_OK(StorageContainer::CreateStorageContainer(&sc, container_name.toString())); + containers_.push_back(sc); + file_id_++; + return Status::OK(); +} + +Status StorageManager::DoServiceStart() { + containers_.reserve(1000); + if (root_.IsDirectory()) { + RETURN_IF_NOT_OK(AddOneContainer()); + } else { + RETURN_STATUS_UNEXPECTED("Not a directory"); + } + return Status::OK(); +} + +Status StorageManager::Write(key_type *key, const std::vector &buf) { + RETURN_UNEXPECTED_IF_NULL(key); + size_t sz = 0; + for (auto &v : buf) { + sz += v.GetSize(); + } + if (sz == 0) { + RETURN_STATUS_UNEXPECTED("Unexpected 0 length"); + } + std::shared_ptr cont; + key_type out_key; + value_type out_value; + bool create_new_container = false; + do { + SharedLock lock_s(&rw_lock_); + size_t num_containers = containers_.size(); + if (create_new_container) { + // Upgrade to exclusvie lock. + lock_s.Upgrade(); + create_new_container = false; + // Check again if someone has already added a + // new container after we got the x lock + if (containers_.size() == num_containers) { + RETURN_IF_NOT_OK(AddOneContainer()); + } + // Refresh how many containers there are. + num_containers = containers_.size(); + // Downgrade back to shared lock + lock_s.Downgrade(); + } + if (num_containers == 0) { + RETURN_STATUS_UNEXPECTED("num_containers is zero"); + } + // Go to the last container to insert. + cont = containers_.at(num_containers - 1); + off64_t offset; + Status rc = cont->Insert(buf, &offset); + if (rc.IsNoSpace()) { + create_new_container = true; + } else if (rc.IsOk()) { + out_value = std::make_pair(num_containers - 1, std::make_pair(offset, sz)); + RETURN_IF_NOT_OK(index_.insert(out_value, &out_key)); + *key = out_key; + break; + } else { + return rc; + } + } while (true); + return Status::OK(); +} + +Status StorageManager::Read(StorageManager::key_type key, WritableSlice *dest, size_t *bytesRead) const { + RETURN_UNEXPECTED_IF_NULL(dest); + auto r = index_.Search(key); + if (r.second) { + auto &it = r.first; + value_type v = *it; + int container_inx = v.first; + off_t offset = v.second.first; + size_t sz = v.second.second; + if (dest->GetSize() < sz) { + std::string errMsg = "Destination buffer too small. Expect at least " + std::to_string(sz) + + " but length = " + std::to_string(dest->GetSize()); + RETURN_STATUS_UNEXPECTED(errMsg); + } + if (bytesRead != nullptr) { + *bytesRead = sz; + } + auto cont = containers_.at(container_inx); + RETURN_IF_NOT_OK(cont->Read(dest, offset)); + } else { + RETURN_STATUS_UNEXPECTED("Key not found"); + } + return Status::OK(); +} + +Status StorageManager::DoServiceStop() noexcept { + Status rc; + Status rc1; + for (auto const &p : containers_) { + // The destructor of StorageContainer is not called automatically until the use + // count drops to 0. But it is not always the case. We will do it ourselves. + rc = p.get()->Truncate(); + if (rc.IsError()) { + rc1 = rc; + } + } + containers_.clear(); + file_id_ = 0; + return rc1; +} + +StorageManager::StorageManager(const Path &root) : root_(root), file_id_(0), index_() {} + +StorageManager::~StorageManager() { (void)StorageManager::DoServiceStop(); } + +std::ostream &operator<<(std::ostream &os, const StorageManager &s) { + os << "Dumping all containers ..." + << "\n"; + for (auto const &p : s.containers_) { + os << *(p.get()); + } + return os; +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/util/storage_manager.h b/mindspore/ccsrc/dataset/util/storage_manager.h new file mode 100644 index 0000000000..075ac713d2 --- /dev/null +++ b/mindspore/ccsrc/dataset/util/storage_manager.h @@ -0,0 +1,76 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_UTIL_STORAGE_MANAGER_H_ +#define DATASET_UTIL_STORAGE_MANAGER_H_ + +#include +#include +#include +#include +#include +#include "dataset/util/allocator.h" +#include "dataset/util/auto_index.h" +#include "dataset/util/lock.h" +#include "dataset/util/memory_pool.h" +#include "dataset/util/path.h" +#include "dataset/util/service.h" +#include "dataset/util/slice.h" +#include "dataset/util/storage_container.h" + +using ListOfContainers = std::vector>; +namespace mindspore { +namespace dataset { +class StorageManager : public Service { + public: + using storage_index = AutoIndexObj>>; + using key_type = storage_index::key_type; + using value_type = storage_index::value_type; + + explicit StorageManager(const Path &); + + ~StorageManager() override; + + StorageManager(const StorageManager &) = delete; + + StorageManager &operator=(const StorageManager &) = delete; + + Status Write(key_type *out_key, const std::vector &buf); + + Status Read(key_type key, WritableSlice *dest, size_t *bytesRead) const; + + Status DoServiceStart() override; + + Status DoServiceStop() noexcept override; + + friend std::ostream &operator<<(std::ostream &os, const StorageManager &s); + + private: + Path root_; + ListOfContainers containers_; + int file_id_; + RWLock rw_lock_; + storage_index index_; + + std::string GetBaseName(const std::string &prefix, int32_t file_id); + + std::string ConstructFileName(const std::string &prefix, int32_t file_id, const std::string &suffix); + + Status AddOneContainer(); +}; +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_UTIL_STORAGE_MANAGER_H_ diff --git a/mindspore/ccsrc/dataset/util/system_pool.h b/mindspore/ccsrc/dataset/util/system_pool.h index bd15ad11dd..286e30a615 100644 --- a/mindspore/ccsrc/dataset/util/system_pool.h +++ b/mindspore/ccsrc/dataset/util/system_pool.h @@ -19,8 +19,10 @@ #include #include #include +#include #include #include "./securec.h" +#include "dataset/util/allocator.h" #include "dataset/util/memory_pool.h" namespace mindspore { @@ -61,6 +63,11 @@ class SystemPool : public MemoryPool { uint64_t get_max_size() const override { return std::numeric_limits::max(); } int PercentFree() const override { return 100; } + + template + static Allocator GetAllocator() { + return Allocator(std::make_shared()); + } }; } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/util/task.cc b/mindspore/ccsrc/dataset/util/task.cc index f00f26f5ce..93db55d5f9 100644 --- a/mindspore/ccsrc/dataset/util/task.cc +++ b/mindspore/ccsrc/dataset/util/task.cc @@ -16,7 +16,6 @@ #include "dataset/util/task.h" #include "common/utils.h" #include "dataset/util/task_manager.h" -#include "dataset/util/de_error.h" #include "utils/log_adapter.h" namespace mindspore { diff --git a/mindspore/ccsrc/dataset/util/task.h b/mindspore/ccsrc/dataset/util/task.h index b120ed3d7c..49eb16b182 100644 --- a/mindspore/ccsrc/dataset/util/task.h +++ b/mindspore/ccsrc/dataset/util/task.h @@ -27,7 +27,6 @@ #include #include #include -#include "dataset/util/de_error.h" #include "dataset/util/intrp_resource.h" #include "dataset/util/list.h" #include "dataset/util/memory_pool.h" diff --git a/mindspore/ccsrc/debug/CMakeLists.txt b/mindspore/ccsrc/debug/CMakeLists.txt index 30b10a17fd..ba0c5e07ac 100644 --- a/mindspore/ccsrc/debug/CMakeLists.txt +++ b/mindspore/ccsrc/debug/CMakeLists.txt @@ -10,6 +10,15 @@ set(_DEBUG_SRC_LIST "${CMAKE_CURRENT_SOURCE_DIR}/trace.cc" ) +if (ENABLE_DEBUGGER) + list(APPEND _DEBUG_SRC_LIST + "${CMAKE_CURRENT_SOURCE_DIR}/debugger/debugger.cc" + "${CMAKE_CURRENT_SOURCE_DIR}/debugger/grpc_client.cc" + "${CMAKE_CURRENT_SOURCE_DIR}/debugger/proto_exporter.cc" + "${CMAKE_CURRENT_SOURCE_DIR}/debug_services.cc" + ) +endif (ENABLE_DEBUGGER) + if (ENABLE_DUMP_E2E) list(APPEND _DEBUG_SRC_LIST "${CMAKE_CURRENT_SOURCE_DIR}/e2e_dump.cc") endif (ENABLE_DUMP_E2E) diff --git a/mindspore/ccsrc/debug/anf_ir_utils.cc b/mindspore/ccsrc/debug/anf_ir_utils.cc index 2b8e61ab15..c797b8efea 100644 --- a/mindspore/ccsrc/debug/anf_ir_utils.cc +++ b/mindspore/ccsrc/debug/anf_ir_utils.cc @@ -27,6 +27,7 @@ #include "utils/symbolic.h" #include "ir/meta_func_graph.h" #include "ir/param_value_py.h" +#include "ir/tensor_py.h" #include "pipeline/parse/python_adapter.h" #include "pipeline/parse/resolve.h" #include "operator/composite/composite.h" @@ -39,6 +40,8 @@ #include "utils/context/ms_context.h" #include "operator/ops.h" +using mindspore::tensor::TensorPy; + namespace mindspore { // max number of elements in sequence const int NUM_MAX_SEQUENCE_ELEMS = 0x00FFFFFF; @@ -399,8 +402,8 @@ std::string AnfExporter::GetValueText(const FuncGraphPtr &func_graph, const Valu oss << value->DumpText(); } else if (value->isa()) { auto tensor_ptr = dyn_cast(value); - oss << value->DumpText() << "@" << DumpObject(tensor_ptr->data(), "T"); - } else if (value->isa() || value->isa() || value->isa()) { + oss << value->DumpText() << "@" << DumpObject(TensorPy::AsNumpy(*tensor_ptr), "T"); + } else if (value->isa() || value->isa() || value->isa()) { oss << value->DumpText(); } else if (value->isa()) { oss << GetSequenceText(func_graph, value); @@ -1813,7 +1816,7 @@ class IrParser { if (tensor_data == nullptr) { return TOK_ERROR; } - *val_ptr = std::make_shared(tensor_data, TypeIdToType(type)); + *val_ptr = TensorPy::MakeTensor(tensor_data, TypeIdToType(type)); return lexer_.GetNextToken(); } diff --git a/mindspore/ccsrc/debug/debug_services.cc b/mindspore/ccsrc/debug/debug_services.cc new file mode 100644 index 0000000000..cb883eef51 --- /dev/null +++ b/mindspore/ccsrc/debug/debug_services.cc @@ -0,0 +1,194 @@ +/** + * Copyright 2019-2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "debug/debug_services.h" +namespace mindspore { + +DebugServices::DebugServices() { + tensor_loader_ = new TensorLoader(); + uint32_t iter_num = -1; + tensor_loader_->set_iter_num(iter_num); +} + +DebugServices::DebugServices(const DebugServices &other) { + tensor_loader_ = other.tensor_loader_; + watchpoint_table = other.watchpoint_table; +} + +DebugServices &DebugServices::operator=(const DebugServices &other) { + if (this != &other) { + tensor_loader_ = other.tensor_loader_; + watchpoint_table = other.watchpoint_table; + } + return *this; +} + +DebugServices::~DebugServices() { delete tensor_loader_; } + +void DebugServices::add_watchpoint(unsigned int id, unsigned int watch_condition, + const std::vector> &check_node_list) { + std::lock_guard lg(lock_); + + watchpoint_t watchpoint_item; + + watchpoint_item.id = id; + + if (watch_condition == 0) { + watchpoint_item.conditions.nan.enabled = true; + } else if (watch_condition == 1) { + watchpoint_item.conditions.inf.enabled = true; + watchpoint_item.conditions.neg_inf.enabled = true; + } + + watchpoint_item.check_node_list = check_node_list; + + watchpoint_table[id] = watchpoint_item; +} + +void DebugServices::remove_watchpoint(unsigned int id) { + std::lock_guard lg(lock_); + watchpoint_table.erase(id); +} + +void DebugServices::check_watchpoints(std::vector *name, std::vector *slot, + std::vector *data_ptr, std::vector *data_size, + std::vector *condition, std::vector *wacthpoint_id) { + std::lock_guard lg(lock_); + + std::vector> tensor_list = tensor_loader_->GetTensor(); + + std::string current_tensor_name; + std::unordered_map watchpoints_to_check_table; + + for (std::size_t i = 0; i < tensor_list.size(); i++) { + current_tensor_name = tensor_list[i]->GetName(); + mindspore::tensor::TensorPtr tensor_ptr = tensor_list[i]->GetTensor(); + int tensor_data_type = tensor_ptr->data_type_c(); + + // check if we need to analyze this node and for which watchpoints we will check + // create a list of watchpoints to check + watchpoints_to_check_table.clear(); + for (auto w_table_item : watchpoint_table) { + // if the watchpoint is checking for a nan or inf and the current tensor is not of a float type, then + // don't check the watchpoint for this tensor + if (std::get<1>(w_table_item).conditions.inf.enabled || std::get<1>(w_table_item).conditions.neg_inf.enabled || + std::get<1>(w_table_item).conditions.nan.enabled) { + if (tensor_data_type != kNumberTypeFloat16 && tensor_data_type != kNumberTypeFloat && + tensor_data_type != kNumberTypeFloat32 && tensor_data_type != kNumberTypeFloat64) { + continue; + } + } + + auto check_node_list = std::get<1>(w_table_item).check_node_list; + + for (auto check_node : check_node_list) { + std::string w_name = std::get<0>(check_node); + bool w_type = std::get<1>(check_node); + + // check if the current node tensor name is included the watchpoint + std::string current_node_name = current_tensor_name.substr(0, current_tensor_name.find_first_of(":")); + if ((w_type == true && (current_tensor_name.find(w_name) != string::npos || w_name == "*")) || + (w_type == false && current_node_name == w_name)) { + watchpoints_to_check_table[w_table_item.second.id] = w_table_item.second; + break; + } + } + } + + // check if no watchpoints are valid for the current tensor + if (watchpoints_to_check_table.empty()) { + continue; + } + + // need to add support for float16 and float64, and other types when we support conditions beyond inf and nan + if (tensor_data_type != kNumberTypeFloat && tensor_data_type != kNumberTypeFloat32) { + continue; + } + + float *start_addr = reinterpret_cast(tensor_ptr->data_c()); + unsigned int num_elements = (tensor_ptr->data().nbytes()) / sizeof(float); + + std::unordered_map::iterator it_w_table_check; + std::vector hit_encountered; + + for (unsigned int index = 0; index < num_elements; index++) { + float x = start_addr[index]; + it_w_table_check = watchpoints_to_check_table.begin(); + + while (it_w_table_check != watchpoints_to_check_table.end()) { + if ((it_w_table_check->second.conditions.inf.enabled || it_w_table_check->second.conditions.neg_inf.enabled) && + isinf(x)) { + hit_encountered.push_back(it_w_table_check->second.id); + } else if (it_w_table_check->second.conditions.nan.enabled && isnan(x)) { + hit_encountered.push_back(it_w_table_check->second.id); + } + + ++it_w_table_check; + } + + if (hit_encountered.size()) { + for (auto it_hit_id = hit_encountered.begin(); it_hit_id != hit_encountered.end(); ++it_hit_id) { + std::string name_no_slot = current_tensor_name.substr(0, current_tensor_name.find_first_of(":")); + name->push_back(name_no_slot); + + slot->push_back(std::to_string(tensor_list[i]->GetSlot())); + data_ptr->push_back(reinterpret_cast(tensor_ptr->data_c())); + data_size->push_back(tensor_ptr->data().nbytes()); + + int condition_item = -1; + if (watchpoint_table[*it_hit_id].conditions.nan.enabled) { + condition_item = 0; + } else if (watchpoint_table[*it_hit_id].conditions.inf.enabled || + watchpoint_table[*it_hit_id].conditions.neg_inf.enabled) { + condition_item = 1; + } + condition->push_back(condition_item); + + wacthpoint_id->push_back(*it_hit_id); + + watchpoints_to_check_table.erase(*it_hit_id); + } + + hit_encountered.clear(); + } + + if (watchpoints_to_check_table.empty()) { + break; + } + } + } +} + +void DebugServices::read_nodes_tensors(std::vector name, std::vector *ret_name, + std::vector *data_ptr, std::vector *data_size, + std::vector *dtype, std::vector> *shape) { + std::vector>> result_list; + tensor_loader_->SearchTensors(name, &result_list); + + for (auto result : result_list) { + if (!std::get<1>(result)) { + continue; + } + ret_name->push_back(std::get<0>(result)); + data_ptr->push_back(reinterpret_cast(std::get<1>(result)->GetTensor()->data_c())); + data_size->push_back(std::get<1>(result)->GetTensor()->data().nbytes()); + dtype->push_back(std::get<1>(result)->GetTensor()->Dtype()); + shape->push_back(std::get<1>(result)->GetTensor()->shape()); + } +} + +TensorLoader *DebugServices::get_tensor_loader() const { return tensor_loader_; } + +} // namespace mindspore diff --git a/mindspore/ccsrc/debug/debug_services.h b/mindspore/ccsrc/debug/debug_services.h new file mode 100644 index 0000000000..b2fd41cd68 --- /dev/null +++ b/mindspore/ccsrc/debug/debug_services.h @@ -0,0 +1,95 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_DEBUG_DEBUG_SERVICES_H_ +#define MINDSPORE_CCSRC_DEBUG_DEBUG_SERVICES_H_ + +#include +#include +#include +#include +#include +#include +#include "debug/tensor_load.h" +#include "debug/tensor_data.h" +#include "ir/dtype.h" + +namespace mindspore { +class DebugServices { + public: + DebugServices(); + + DebugServices(const DebugServices &other); + + DebugServices &operator=(const DebugServices &other); + + ~DebugServices(); + + void add_watchpoint(unsigned int id, unsigned int watch_condition, + const std::vector> &check_node_list); + + void remove_watchpoint(unsigned int id); + + void check_watchpoints(std::vector *name, std::vector *slot, std::vector *data_ptr, + std::vector *data_size, std::vector *condition, + std::vector *wacthpoint_id); + + void read_nodes_tensors(std::vector name, std::vector *ret_name, + std::vector *data_ptr, std::vector *data_size, + std::vector *dtype, std::vector> *shape); + + TensorLoader *get_tensor_loader() const; + + private: + typedef struct condition_no_param { + bool enabled = false; + } condition_no_param_t; + + typedef struct condition_with_param { + bool enabled = false; + float parameter = 0; + } condition_with_param_t; + + typedef struct conditions { + condition_no_param_t inf; + condition_no_param_t neg_inf; + condition_no_param_t nan; + condition_with_param_t max_below; + condition_with_param_t max_above; + condition_with_param_t min_below; + condition_with_param_t min_above; + condition_with_param_t max_minus_min_below; + condition_with_param_t max_minus_min_above; + condition_with_param_t mean_below; + condition_with_param_t mean_above; + condition_with_param_t std_dev_below; + condition_with_param_t std_dev_above; + } conditions_t; + + typedef struct watchpoint { + unsigned int id; + conditions_t conditions; + std::vector> check_node_list; + } watchpoint_t; + + std::mutex lock_; + + std::unordered_map watchpoint_table; + + TensorLoader *tensor_loader_; +}; +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_DEBUG_DEBUG_SERVICES_H_ diff --git a/mindspore/ccsrc/debug/debugger/debug_graph.proto b/mindspore/ccsrc/debug/debugger/debug_graph.proto new file mode 100644 index 0000000000..042360fac3 --- /dev/null +++ b/mindspore/ccsrc/debug/debugger/debug_graph.proto @@ -0,0 +1,316 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto2"; + +package debugger; + +// Versioning +enum Version { + // unknown version + UNKNOWWN_VERSION = 0; + + // Initial version (IR VERSION 1), published on Sep 23, 2019 + IR_VERSION = 0x0000000000000001; +} + +// Data type definition +enum DataType { + DT_UNDEFINED = 0; + // Basic types. + DT_BOOL = 1; // bool + + DT_INT8 = 2; // int8_t + DT_INT16 = 3; // int16_t + DT_INT32 = 4; // int32_t + DT_INT64 = 5; // int64_t + + DT_UINT8 = 6; // uint8_t + DT_UINT16 = 7; // uint16_t + DT_UINT32 = 8; // uint32_t + DT_UINT64 = 9; // uint64_t + + DT_FLOAT16 = 10; // float 16 + DT_FLOAT32 = 11; // float 32 + DT_FLOAT64 = 12; // float 64 + + DT_STRING = 13; // string + DT_TENSOR = 14; // tensor + DT_GRAPH = 15; // graph + + // list type + DT_BOOLS = 16; // list of bool + + DT_INTS8 = 17; // list of int8_t + DT_INTS16 = 18; // list of int16_t + DT_INTS32 = 19; // list of int32_t + DT_INTS64 = 20; // list of int64_t + + DT_UINTS8 = 21; // list of uint8_t + DT_UINTS16 = 22; // list of uint16_t + DT_UINTS32 = 23; // list of uint32_t + DT_UINTS64 = 24; // list of uint64_t + + DT_FLOATS16 = 25; // list of float16 + DT_FLOATS32 = 26; // list of float32 + DT_FLOATS64 = 27; // list of float64 + + DT_STRINGS = 28; // list of string + DT_TENSORS = 29; // list of tensor + DT_GRAPHS = 30; // list of graph + + DT_TUPLE = 31; // tuple + DT_LIST = 32; // list + DT_DICT = 33; // dictionary + + // other types + DT_NONE = 34; // None + DT_SYM_INST = 35; // Symbolic Key Instance + + // type related type + DT_BASE_INT = 36; // type generic int + DT_BASE_UINT = 37; // type generate unsigned int + DT_BASE_FLOAT = 38; // type generate float + DT_TYPE = 39; // type type + DT_ANYTHING = 40; // type anything + DT_REFKEY = 41; // type refkey + DT_REF = 42; // type ref +} + +// Value definition for attribute value or parameter default value +message ValueProto { + // data type of value + optional DataType dtype = 1; // discriminator that indicates which field below is in use + + // Exactly ONE of the following fields must be present for this version of the IR + optional bool bool_val = 2; // bool + optional int64 int_val = 3; // int + optional uint64 uint_val = 4; // uint + optional float float_val = 5; // float + optional double double_val = 6; // double + optional string str_val = 7; // string + optional TensorProto tensor_val = 8; // tensor value + optional GraphProto graph = 9; // graph + + repeated bool bool_vals = 10; // list of bool + repeated int64 int_vals = 11; // list of int + repeated uint64 uint_vals = 12; // list of uint + repeated float float_vals = 13; // list of float + repeated double double_vals = 14; // list of double + repeated string str_vals = 15; // list of string + repeated TensorProto tensor_vals = 16; // list of tensor value + repeated GraphProto graphs = 17; // list of graph + + // tuple or list + repeated ValueProto values = 18; // tuple, list of value + + // dictionary + repeated NamedValueProto dict_val = 19; // dictionary info + + // filed for type type + optional TypeProto type_val = 20; // type type info +} + +message AttributeProto { + optional string name = 1; // attribute name + optional ValueProto value = 2; // attribute value +} + +message NamedValueProto { + optional string key = 1; // attribute name + optional ValueProto value = 2; // attribute value +} + +// Defines a tensor shape. +message TensorShapeProto { + // One dimension of the tensor. + message Dimension { + // Size of the tensor in that dimension. + // This value must be >= -1, but values of -1 are reserved for "unknown" + // shapes (values of -1 mean "unknown" dimension). + optional int64 size = 1; + + // Optional name of the tensor dimension. + optional string name = 2; + }; + + repeated Dimension dim = 1; +} + +// Types for graph input(parameter) and output +message TypeProto { + + message Tensor { + // This field MUST have a valid DataType value except DT_TENSOR + optional DataType elem_type = 1; + optional TensorShapeProto shape = 2; // for scalar, this field is not set + } + + // tuple type + message Sequence { + // The type and optional shape of elements of the tuple. + repeated TypeProto elem_types = 1; + }; + + // data type + optional DataType data_type = 1; + + oneof value { + // The type of a tensor. + Tensor tensor_type = 2; + + // The type of a tuple. + Sequence sequence_type = 3; + } +} + +// Defines information on graph parameters, including the name, the type, and +// the default value of parameter if exists. +message ParameterProto { + optional string name = 1; // parameter name + optional TypeProto type = 2; // parameter type + optional ValueProto default_val = 3; // default value of parameter if exists +} + +// Defines graph output information +message OutputProto { + optional string name = 1; // output node name + optional TypeProto type = 2; // output node type +} + +// Define node input information +message InputProto { + enum EdgeType { + DATA_EDGE = 0; // data edge + CONTROL_EDGE = 1; // control edge + } + + optional string name = 1; + optional EdgeType type = 2; +} + +// Nodes +// +// Computation graphs are made up of a DAG of nodes, which represent what is +// commonly called a "layer" or "pipeline stage" in machine learning frameworks. +// +// For example, it can be a node of type "Conv" that takes in an image, a filter +// tensor and a bias tensor, and produces the convolved output. +message NodeProto { + repeated InputProto input = 1; // namespace Value + optional string name = 2; // namespace Value + + // The symbolic identifier of the Operator to execute. + optional string op_type = 3; // namespace Operator + // The domain of the OperatorSet that specifies the operator named by op_type. + optional string scope = 4; // namespace Domain + + // Additional named attributes. + repeated AttributeProto attribute = 5; + + // Optional type info of this node + optional TypeProto output_type = 6; + + // other fields for debug + optional uint64 output_i = 7; + + // for debugger, full name with scope + optional string debug_name = 8; +} + +// Models +// +// ModelProto is a top-level file/container format for bundling a ML model and +// associating its computation graph with metadata. +// +// The semantics of the model are described by the associated GraphProto. +message ModelProto { + // ir version + optional int64 ir_version = 1; + + // Domain name of the model. + // We use reverse domain names as name space indicators. For example: + // `com.facebook.fair` or `com.microsoft.cognitiveservices` + // + // Together with `model_version` and GraphProto.name, this forms the unique identity of + // the graph. + optional string domain = 2; + + // The version of the graph encoded. See Version enum below. + optional int64 model_version = 3; + + // The parameterized graph that is evaluated to execute the model. + optional GraphProto graph = 4; + + // metadata info of opeartors + optional OperatorSetProto metadata_operators = 5; +}; + +message OperatorProto { + optional string name = 1; // used as key, must be distinct + optional bytes config = 2; // operator config info + optional bytes obj_info = 3; // operator related object info, e.g. content of operator binary or name +}; + +message OperatorSetProto { + repeated OperatorProto operators = 1; + optional string summary = 2; // summary info of operators, e.g. file position of operators file +} + +// Graphs +// +// A graph defines the computational logic of a model and is comprised of a parameterized +// list of nodes that form a directed acyclic graph based on their inputs and outputs. +// This is the equivalent of the "network" or "graph" in many deep learning +// frameworks. +message GraphProto { + // The nodes in the graph, sorted topologically. + repeated NodeProto node = 1; + + // The name of the graph. + optional string name = 2; // namespace Graph + + // The parameters(inputs) and outputs of the graph. + repeated ParameterProto parameters = 3; + repeated OutputProto outputs = 4; + + // Constants used in this graph + repeated NamedValueProto const_vals = 5; +} + +// Tensors +// +// A serialized tensor value. +message TensorProto { + // The node name of the tensor. + optional string node_name = 1; + + // The slot of the tensor in its node. + optional string slot = 2; + + // The serialized tensor content. + optional bytes tensor_content = 3; + + // The shape of the tensor. + repeated int64 dims = 4; + + // The data type of the tensor. + // This field MUST have a valid DataType value except DT_TENSOR + optional DataType data_type = 5; + + // If the tensor content transferring is finished. + optional bool finished = 6; +} \ No newline at end of file diff --git a/mindspore/ccsrc/debug/debugger/debug_grpc.proto b/mindspore/ccsrc/debug/debugger/debug_grpc.proto new file mode 100644 index 0000000000..f742987a4e --- /dev/null +++ b/mindspore/ccsrc/debug/debugger/debug_grpc.proto @@ -0,0 +1,81 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; + +package debugger; + +import "debug_graph.proto"; + +service EventListener { + rpc WaitCMD (Metadata) returns (EventReply) {}; + rpc SendMetadata (Metadata) returns (EventReply) {}; + rpc SendGraph (GraphProto) returns (EventReply) {}; + rpc SendTensors (stream TensorProto) returns (EventReply) {}; + rpc SendWatchpointHits (stream WatchpointHit) returns (EventReply) {}; +} + +message Metadata { + string device_name = 1; + int32 cur_step = 2; +} + +message EventReply { + enum Status { + OK = 0; + FAILED = 1; + PENDING = 2; + } + + Status status = 1; + + oneof cmd { + bool exit = 2; + int32 run_cmd = 3; + SetCMD set_cmd = 4; + ViewCMD view_cmd = 5; + } +} + +message SetCMD { + repeated WatchNode watch_nodes = 1; + WatchCondition watch_condition = 2; + bool delete = 3; + int32 id = 4; +} + +message ViewCMD { + repeated TensorProto tensors = 1; +} + +message WatchCondition { + enum Condition { + nan = 0; + inf = 1; + } + Condition condition = 1; +} + +message WatchNode { + string node_name = 1; + string node_type = 2; +} + +message WatchpointHit { + TensorProto tensor = 1; + WatchCondition watch_condition = 2; + int32 id = 3; +} diff --git a/mindspore/ccsrc/debug/debugger/debugger.cc b/mindspore/ccsrc/debug/debugger/debugger.cc new file mode 100644 index 0000000000..ea147a929f --- /dev/null +++ b/mindspore/ccsrc/debug/debugger/debugger.cc @@ -0,0 +1,488 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include "debug/debugger/debugger.h" +#include "pipeline/pipeline.h" +#include "session/anf_runtime_algorithm.h" + +using debugger::EventReply; +using debugger::GraphProto; +using debugger::ModelProto; +using debugger::TensorProto; +using debugger::WatchCondition; +using debugger::WatchCondition_Condition_inf; +using debugger::WatchCondition_Condition_nan; +using debugger::WatchNode; +using debugger::WatchpointHit; + +namespace mindspore { + +DebuggerPtr Debugger::debugger_ = nullptr; +std::mutex Debugger::instance_lock_; + +Debugger::Debugger() + : grpc_client_(nullptr), + debug_services_(nullptr), + device_id_(0), + num_step_(0), + debugger_enabled_(false), + is_dataset_graph_(false) {} + +void Debugger::Init(const uint32_t device_id) { + // access lock for public method + std::lock_guard a_lock(access_lock_); + // save device_id + MS_LOG(INFO) << "Debugger got device_id: " << device_id; + device_id_ = device_id; +} + +void Debugger::EnableDebugger() { + // reset some of the class members + num_step_ = 0; + debugger_enabled_ = false; + grpc_client_ = nullptr; + debug_services_ = nullptr; + + // get env variables to configure debugger + const char *env_enable_str = std::getenv("ENABLE_MS_DEBUGGER"); + if (env_enable_str != nullptr) { + MS_LOG(INFO) << "Getenv ENABLE_MS_DEBUGGER: " << env_enable_str; + if (std::strcmp(env_enable_str, "1") == 0) { + debugger_enabled_ = true; + } + } + if (!debugger_enabled_) { + MS_LOG(WARNING) << "Not enabling debugger. Set environment variable ENABLE_MS_DEBUGGER=1 to enable debugger."; + return; + } + // configure host + const char *env_host_str = std::getenv("MS_DEBUGGER_HOST"); + std::string host; + if (env_host_str != nullptr) { + MS_LOG(INFO) << "Getenv MS_DEBUGGER_HOST: " << env_host_str; + host = std::string(env_host_str); + } else { + MS_LOG(WARNING) << "Environment variable MS_DEBUGGER_HOST doesn't exist. Using default debugger host: localhost"; + host = "localhost"; + } + // configure port + const char *env_port_str = std::getenv("MS_DEBUGGER_PORT"); + std::string port; + if (env_port_str != nullptr) { + MS_LOG(INFO) << "Getenv MS_DEBUGGER_PORT: " << env_port_str; + port = std::string(env_port_str); + } else { + MS_LOG(WARNING) << "Environment variable MS_DEBUGGER_PORT doesn't exist. Using default debugger port: 50051"; + port = "50051"; + } + + // initialize grpc client + grpc_client_ = std::make_unique(host, port); + debug_services_ = std::make_unique(); +} + +void Debugger::Reset() { + // access lock for public method + std::lock_guard a_lock(access_lock_); + // reset components + device_id_ = 0; + num_step_ = 0; + debugger_enabled_ = false; + is_dataset_graph_ = false; + graph_ptr_ = nullptr; + grpc_client_ = nullptr; + debug_services_ = nullptr; +} + +void Debugger::PreExecute(const KernelGraphPtr &graph_ptr) { + // access lock for public method + std::lock_guard a_lock(access_lock_); + // check and save graph_ptr, suspend if graph is new + CheckGraphPtr(graph_ptr); +} + +void Debugger::PostExecute() { + // access lock for public method + std::lock_guard a_lock(access_lock_); + // analyze tensor data and send the watchpoints been hit + if (debugger_enabled_ && !is_dataset_graph_) { + num_step_++; + MS_LOG(INFO) << "Debugger suspend at end of step; number of steps executed: " << num_step_; + SendWatchpointsAndSuspend(CheckWatchpoints()); + } +} + +void Debugger::PostDebugOp() { + // access lock for public method + std::lock_guard a_lock(access_lock_); + // suspend if debugger is enabled + if (debugger_enabled_ && !is_dataset_graph_) { + MS_LOG(INFO) << "Debugger suspend at debug_op"; + CommandLoop(); + } +} + +void Debugger::CheckGraphPtr(const KernelGraphPtr &graph_ptr) { + if (graph_ptr_ != graph_ptr) { + MS_LOG(INFO) << "Debugger got new graph: " << graph_ptr->graph_id(); + // save new graph_ptr + graph_ptr_ = graph_ptr; + // check if it is dataset graph + CheckDatasetGraph(); + if (!is_dataset_graph_) { + // only try to enable debugger if it is not a dataset graph + EnableDebugger(); + if (debugger_enabled_) { + // get graph proto and send to mindinsight + SendGraphAndSuspend(GetGraphProto()); + } + } + } +} + +void Debugger::CheckDatasetGraph() { + // print parameter node names + const auto ¶ms = graph_ptr_->inputs(); + for (const auto ¶m : params) { + MS_LOG(INFO) << "param: " << param->fullname_with_scope(); + } + // check if there is GetNext or InitDataSetQueue node + const auto &nodes = graph_ptr_->execution_order(); + for (const auto &node : nodes) { + auto node_name = AnfAlgo::GetCNodeName(node); + MS_LOG(INFO) << "node: " << node->fullname_with_scope(); + if (node_name == "GetNext" || node_name == "InitDataSetQueue") { + MS_LOG(WARNING) << "Not enabling debugger for graph " << graph_ptr_->graph_id() << ": found dataset graph node " + << node_name; + is_dataset_graph_ = true; + return; + } + } + is_dataset_graph_ = false; +} + +GraphProto Debugger::GetGraphProto() { + // convert kernel graph to debugger modelproto + ModelProto model = GetDebuggerFuncGraphProto(graph_ptr_); + return model.graph(); +} + +void Debugger::SendGraphAndSuspend(const GraphProto &graph_proto) { + // prepare metadata + std::string device_name = std::to_string(device_id_) + ":" + std::to_string(graph_ptr_->graph_id()); + Metadata metadata; + metadata.set_device_name(device_name); + metadata.set_cur_step(num_step_); + EventReply reply_metadata = grpc_client_->SendMetadata(metadata); + if (reply_metadata.status() != reply_metadata.OK) { + MS_LOG(ERROR) << "Error: SendMetadata failed"; + } + // send graph to mindinght server + EventReply reply = grpc_client_->SendGraph(graph_proto); + if (reply.status() != reply.OK) { + MS_LOG(ERROR) << "Error: SendGraph failed"; + } + // enter command loop, wait and process commands + CommandLoop(); +} + +void Debugger::CommandLoop() { + // prepare metadata + std::string device_name = std::to_string(device_id_) + ":" + std::to_string(graph_ptr_->graph_id()); + Metadata metadata; + metadata.set_device_name(device_name); + metadata.set_cur_step(num_step_); + + // loop exit flag + bool run = false; + int num_wait_fail = 0; + const int max_num_wait_fail = 5; + + while (!run) { + // wait for command + EventReply reply = grpc_client_->WaitForCommand(metadata); + if (reply.status() != reply.OK) { + MS_LOG(ERROR) << "Error: WaitForCommand failed"; + num_wait_fail++; + if (num_wait_fail > max_num_wait_fail) { + MS_LOG(ERROR) << "Maximum number of WaitForCommand retry reached: exiting training session"; + Exit(); + } + MS_LOG(ERROR) << "Number of consecutive WaitForCommand fail:" << num_wait_fail << "; Retry after " + << num_wait_fail << "s"; + std::this_thread::sleep_for(std::chrono::milliseconds(1000 * num_wait_fail)); + continue; + } + + // get type of the command in reply + DebuggerCommand cmd = GetCommand(reply); + if (cmd == DebuggerCommand::kUnknownCMD) { + MS_LOG(ERROR) << "Error: debugger recieved unknown command"; + continue; + } + + MS_LOG(INFO) << "recieved command: "; + switch (cmd) { + case DebuggerCommand::kUnknownCMD: + MS_LOG(INFO) << "UnknownCMD"; + break; + case DebuggerCommand::kExitCMD: + MS_LOG(INFO) << "ExitCMD"; + Exit(); + break; + case DebuggerCommand::kRunCMD: + MS_LOG(INFO) << "RunCMD"; + // exit loop + run = true; + break; + case DebuggerCommand::kSetCMD: + MS_LOG(INFO) << "SetCMD"; + { + // print set cmd content + ProtoVector recieved_nodes = GetWatchnodes(reply); + for (auto node : recieved_nodes) { + MS_LOG(INFO) << "node name: " << node.node_name(); + MS_LOG(INFO) << "node type: " << node.node_type(); + } + WatchCondition recieved_condition = GetWatchcondition(reply); + MS_LOG(INFO) << "condition: " << recieved_condition.condition(); + int32_t id = GetWatchpointID(reply); + MS_LOG(INFO) << "id: " << id; + bool delete_ = GetWatchpointDelete(reply); + MS_LOG(INFO) << "delete: " << delete_; + } + MS_LOG(INFO) << "Setting watchpoint"; + if (GetWatchpointDelete(reply)) { + RemoveWatchpoint(GetWatchpointID(reply)); + } else { + SetWatchpoint(GetWatchnodes(reply), GetWatchcondition(reply), GetWatchpointID(reply)); + } + break; + case DebuggerCommand::kViewCMD: + MS_LOG(INFO) << "ViewCMD"; + { + // print view cmd content + ProtoVector received_tensors = GetTensors(reply); + for (auto tensor : received_tensors) { + MS_LOG(INFO) << "tensor node name: " << tensor.node_name(); + MS_LOG(INFO) << "tensor slot: " << tensor.slot(); + MS_LOG(INFO) << "tensor finished: " << std::boolalpha << tensor.finished() << std::noboolalpha; + } + } + MS_LOG(INFO) << "Sending tensors"; + std::list tensors = LoadTensors(GetTensors(reply)); + { + for (auto tensor : tensors) { + MS_LOG(INFO) << "tensor node name: " << tensor.node_name(); + MS_LOG(INFO) << "tensor slot: " << tensor.slot(); + MS_LOG(INFO) << "tensor finished: " << std::boolalpha << tensor.finished() << std::noboolalpha; + MS_LOG(INFO) << "tensor dims: "; + for (auto dim : tensor.dims()) { + MS_LOG(INFO) << dim << ","; + } + MS_LOG(INFO) << "tensor dtype: " << tensor.data_type(); + } + } + EventReply send_tensors_reply = grpc_client_->SendTensors(tensors); + if (send_tensors_reply.status() != send_tensors_reply.OK) { + MS_LOG(ERROR) << "Error: SendTensors failed"; + } + break; + } + } +} + +DebuggerCommand Debugger::GetCommand(const EventReply &reply) { + DebuggerCommand cmd = DebuggerCommand::kUnknownCMD; + switch (reply.cmd_case()) { + case debugger::EventReply::CmdCase::kExit: + cmd = DebuggerCommand::kExitCMD; + break; + case debugger::EventReply::CmdCase::kRunCmd: + cmd = DebuggerCommand::kRunCMD; + break; + case debugger::EventReply::CmdCase::kSetCmd: + cmd = DebuggerCommand::kSetCMD; + break; + case debugger::EventReply::CmdCase::kViewCmd: + cmd = DebuggerCommand::kViewCMD; + break; + default: + MS_LOG(ERROR) << "Error: UnknownCMD"; + break; + } + return cmd; +} + +ProtoVector Debugger::GetWatchnodes(const EventReply &reply) { + if (!reply.has_set_cmd()) { + MS_LOG(ERROR) << "Error: Not SetCMD, can not get WatchNodes. Returning default value: ProtoVector()."; + return ProtoVector(); + } + return reply.set_cmd().watch_nodes(); +} + +WatchCondition Debugger::GetWatchcondition(const EventReply &reply) { + if (!reply.has_set_cmd() || !reply.set_cmd().has_watch_condition()) { + MS_LOG(ERROR) << "Error: Can not get WatchCondition from command. Returning default value: WatchCondition()."; + return WatchCondition(); + } + return reply.set_cmd().watch_condition(); +} + +int32_t Debugger::GetWatchpointID(const EventReply &reply) { + if (!reply.has_set_cmd()) { + MS_LOG(ERROR) << "Error: Not SetCMD, can not get Watchpoint ID. Returning default value: 0."; + return 0; + } + return reply.set_cmd().id(); +} + +bool Debugger::GetWatchpointDelete(const EventReply &reply) { + if (!reply.has_set_cmd()) { + MS_LOG(ERROR) << "Error: Not SetCMD, can not get Watchpoint delete flag. Returning default value: false."; + return false; + } + return reply.set_cmd().delete_(); +} + +ProtoVector Debugger::GetTensors(const EventReply &reply) { + if (!reply.has_view_cmd()) { + MS_LOG(ERROR) << "Error: Not ViewCMD, can not get Tensors. Returning default value: ProtoVector()."; + return ProtoVector(); + } + return reply.view_cmd().tensors(); +} + +void Debugger::SetWatchpoint(const ProtoVector &nodes, const WatchCondition &condition, const int32_t id) { + std::vector> check_node_list; + std::transform(nodes.begin(), nodes.end(), std::back_inserter(check_node_list), + [](WatchNode node) -> std::tuple { + return make_tuple(node.node_name(), node.node_type() == "scope"); + }); + + debug_services_->add_watchpoint(id, condition.condition(), check_node_list); +} + +void Debugger::RemoveWatchpoint(const int32_t id) { debug_services_->remove_watchpoint(id); } + +std::list Debugger::LoadTensors(const ProtoVector &tensors) { + std::vector name; + std::vector ret_name; + std::vector data_ptr; + std::vector data_size; + std::vector dtype; + std::vector> shape; + + std::transform(tensors.begin(), tensors.end(), std::back_inserter(name), + [](TensorProto tensor) -> std::string { return tensor.node_name() + ":" + tensor.slot(); }); + + debug_services_->read_nodes_tensors(name, &ret_name, &data_ptr, &data_size, &dtype, &shape); + + std::list tensor_list; + unsigned int result_index = 0; + TensorProto tensor_item; + + for (auto tensor : tensors) { + tensor_item.set_node_name(tensor.node_name()); + tensor_item.set_slot(tensor.slot()); + tensor_item.set_finished(true); + + // return empty tensor if didn't find the requested tensor + if (result_index >= ret_name.size() || ret_name[result_index] != tensor.node_name() + ":" + tensor.slot()) { + tensor_list.push_back(tensor_item); + continue; + } + + tensor_item.set_tensor_content(data_ptr[result_index], data_size[result_index]); + tensor_item.set_data_type(GetDebuggerNumberDataType(dtype[result_index])); + tensor_item.clear_dims(); + for (auto &elem : shape[result_index]) { + tensor_item.add_dims(elem); + } + + tensor_list.push_back(tensor_item); + + result_index++; + } + + return tensor_list; +} + +void Debugger::Exit() { + // clear resource before exit + pipeline::ClearResAtexit(); + std::exit(EXIT_FAILURE); +} + +std::list Debugger::CheckWatchpoints() { + std::vector name; + std::vector slot; + std::vector data_ptr; + std::vector data_size; + std::vector condition; + std::vector watchpoint_id; + + debug_services_->check_watchpoints(&name, &slot, &data_ptr, &data_size, &condition, &watchpoint_id); + + std::list points; + + for (unsigned int i = 0; i < name.size(); i++) { + TensorProto *tensor_item; + tensor_item = new TensorProto(); + tensor_item->set_node_name(name[i]); + tensor_item->set_slot(slot[i]); + tensor_item->set_tensor_content(data_ptr[i], data_size[i]); + + // finished in TensorProto will always be true before we implement big tensor splitting + tensor_item->set_finished(true); + + WatchCondition *condition_item; + condition_item = new WatchCondition(); + condition_item->set_condition(debugger::WatchCondition_Condition(condition[i])); + + WatchpointHit point; + point.set_allocated_tensor(tensor_item); + point.set_allocated_watch_condition(condition_item); + point.set_id(watchpoint_id[i]); + + points.push_back(point); + } + + return points; +} + +void Debugger::SendWatchpointsAndSuspend(const std::list &points) { + // send info about watchpoint + if (!points.empty()) { + EventReply reply = grpc_client_->SendWatchpointHits(points); + if (reply.status() != reply.OK) { + MS_LOG(ERROR) << "Error: SendWatchpointHits failed"; + } + } + // enter command loop + CommandLoop(); +} + +DebugServices *Debugger::get_debug_services() { return debug_services_.get(); } + +bool Debugger::debugger_enabled() { return debugger_enabled_; } + +} // namespace mindspore diff --git a/mindspore/ccsrc/debug/debugger/debugger.h b/mindspore/ccsrc/debug/debugger/debugger.h new file mode 100644 index 0000000000..6ce7d03625 --- /dev/null +++ b/mindspore/ccsrc/debug/debugger/debugger.h @@ -0,0 +1,159 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_DEBUG_DEBUGGER_DEBUGGER_H_ +#define MINDSPORE_CCSRC_DEBUG_DEBUGGER_DEBUGGER_H_ + +#include +#include +#include +#include "session/kernel_graph.h" +#include "debug/debugger/grpc_client.h" +#include "debug/debug_services.h" + +using debugger::DataType; +using debugger::EventReply; +using debugger::GraphProto; +using debugger::ModelProto; +using debugger::TensorProto; +using debugger::WatchCondition; +using debugger::WatchNode; +using debugger::WatchpointHit; + +template +using ProtoVector = google::protobuf::RepeatedPtrField; + +namespace mindspore { +// different types of command recieved by debugger +// need to keep sync with client-side proto and server-side proto +enum class DebuggerCommand { kExitCMD = 2, kRunCMD = 3, kSetCMD = 4, kViewCMD = 5, kUnknownCMD = -1 }; + +class Debugger : public std::enable_shared_from_this { + public: + static std::shared_ptr GetInstance() { + std::lock_guard i_lock(instance_lock_); + if (debugger_ == nullptr) { + debugger_ = std::shared_ptr(new (std::nothrow) Debugger()); + } + return debugger_; + } + + // deconstructor + ~Debugger() = default; + + // init + // only save device_id + void Init(const uint32_t device_id); + + // reset debugger + void Reset(); + + // enable debugger + // send graph and wait for command + // do nothing if graph is set already + void PreExecute(const KernelGraphPtr &graph_ptr); + + // analyze tensors and wait for command + // don't need a graph_ptr because it is saved during pre_execute + void PostExecute(); + + // suspend the execution after a debug_op + void PostDebugOp(); + + DebugServices *get_debug_services(); + + bool debugger_enabled(); + + private: + // private constructor for singleton + Debugger(); + + // enable debugger + // instantiate class members + // read env variable for grpc client + void EnableDebugger(); + + // check and save graph pointer + void CheckGraphPtr(const KernelGraphPtr &graph_ptr); + + // check if the graph is a dataset graph + void CheckDatasetGraph(); + + // serialize graph and get proto + GraphProto GetGraphProto(); + + // send graph and enter command wait loop + void SendGraphAndSuspend(const GraphProto &graph_proto); + + // wait for command and process command + // send command request and process reply in a loop + // break if RunCMD + void CommandLoop(); + + // process reply and command type + DebuggerCommand GetCommand(const EventReply &reply); + + // parse other data out of EventReply + ProtoVector GetWatchnodes(const EventReply &reply); + WatchCondition GetWatchcondition(const EventReply &reply); + int32_t GetWatchpointID(const EventReply &reply); + bool GetWatchpointDelete(const EventReply &reply); + ProtoVector GetTensors(const EventReply &reply); + + // set what nodes and conditions to watch + void SetWatchpoint(const ProtoVector &nodes, const WatchCondition &condition, const int32_t id); + + // remove watchpoint with id + void RemoveWatchpoint(const int32_t id); + + // load tensor for view command + std::list LoadTensors(const ProtoVector &tensors); + + // terminate training process + void Exit(); + + // analyze tensors and check watchpoint conditions + // return names of tensors and what condition they hit + std::list CheckWatchpoints(); + + // send watchpoints that hit and enter command wait loop + void SendWatchpointsAndSuspend(const std::list &points); + + // class members + std::unique_ptr grpc_client_; + std::unique_ptr debug_services_; + KernelGraphPtr graph_ptr_; + uint32_t device_id_; + int32_t num_step_; + bool debugger_enabled_; + bool is_dataset_graph_; + std::mutex access_lock_; + + // singleton + static std::mutex instance_lock_; + static std::shared_ptr debugger_; +}; + +using DebuggerPtr = std::shared_ptr; + +// get debugger ModelProto +std::string GetDebuggerFuncGraphProtoString(const FuncGraphPtr &func_graph); +ModelProto GetDebuggerFuncGraphProto(const FuncGraphPtr &func_graph); + +// for getting proto DataType from Type of Tensor +DataType GetDebuggerNumberDataType(const TypePtr &type); + +} // namespace mindspore +#endif // MINDSPORE_CCSRC_DEBUG_DEBUGGER_DEBUGGER_H_ diff --git a/mindspore/ccsrc/debug/debugger/grpc_client.cc b/mindspore/ccsrc/debug/debugger/grpc_client.cc new file mode 100644 index 0000000000..7709f4c0d1 --- /dev/null +++ b/mindspore/ccsrc/debug/debugger/grpc_client.cc @@ -0,0 +1,124 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include "debug/debugger/grpc_client.h" +#include "utils/log_adapter.h" + +using debugger::EventListener; +using debugger::EventReply; +using debugger::EventReply_Status_FAILED; +using debugger::GraphProto; +using debugger::Metadata; +using debugger::TensorProto; +using debugger::WatchpointHit; + +namespace mindspore { +GrpcClient::GrpcClient(const std::string &host, const std::string &port) : stub_(nullptr) { Init(host, port); } + +void GrpcClient::Init(const std::string &host, const std::string &port) { + std::string target_str = host + ":" + port; + MS_LOG(INFO) << "GrpcClient connecting to: " << target_str; + + std::shared_ptr channel = grpc::CreateChannel(target_str, grpc::InsecureChannelCredentials()); + stub_ = EventListener::NewStub(channel); +} + +void GrpcClient::Reset() { stub_ = nullptr; } + +EventReply GrpcClient::WaitForCommand(const Metadata &metadata) { + EventReply reply; + grpc::ClientContext context; + grpc::Status status = stub_->WaitCMD(&context, metadata, &reply); + + if (!status.ok()) { + MS_LOG(ERROR) << "RPC failed: WaitForCommand"; + MS_LOG(ERROR) << status.error_code() << ": " << status.error_message(); + reply.set_status(EventReply_Status_FAILED); + } + return reply; +} + +EventReply GrpcClient::SendMetadata(const Metadata &metadata) { + EventReply reply; + grpc::ClientContext context; + grpc::Status status = stub_->SendMetadata(&context, metadata, &reply); + + if (!status.ok()) { + MS_LOG(ERROR) << "RPC failed: SendMetadata"; + MS_LOG(ERROR) << status.error_code() << ": " << status.error_message(); + reply.set_status(EventReply_Status_FAILED); + } + return reply; +} + +EventReply GrpcClient::SendGraph(const GraphProto &graph) { + EventReply reply; + grpc::ClientContext context; + grpc::Status status = stub_->SendGraph(&context, graph, &reply); + + if (!status.ok()) { + MS_LOG(ERROR) << "RPC failed: SendGraph"; + MS_LOG(ERROR) << status.error_code() << ": " << status.error_message(); + reply.set_status(EventReply_Status_FAILED); + } + return reply; +} + +EventReply GrpcClient::SendTensors(const std::list &tensors) { + EventReply reply; + grpc::ClientContext context; + + std::unique_ptr > writer(stub_->SendTensors(&context, &reply)); + for (const auto &tensor : tensors) { + if (!writer->Write(tensor)) { + break; + } + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + writer->WritesDone(); + grpc::Status status = writer->Finish(); + + if (!status.ok()) { + MS_LOG(ERROR) << "RPC failed: SendTensors"; + MS_LOG(ERROR) << status.error_code() << ": " << status.error_message(); + reply.set_status(EventReply_Status_FAILED); + } + return reply; +} + +EventReply GrpcClient::SendWatchpointHits(const std::list &watchpoints) { + EventReply reply; + grpc::ClientContext context; + + std::unique_ptr > writer(stub_->SendWatchpointHits(&context, &reply)); + for (const auto &watchpoint : watchpoints) { + if (!writer->Write(watchpoint)) { + break; + } + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + writer->WritesDone(); + grpc::Status status = writer->Finish(); + + if (!status.ok()) { + MS_LOG(ERROR) << "RPC failed: SendWatchpointHits"; + MS_LOG(ERROR) << status.error_code() << ": " << status.error_message(); + reply.set_status(EventReply_Status_FAILED); + } + return reply; +} +} // namespace mindspore diff --git a/mindspore/ccsrc/debug/debugger/grpc_client.h b/mindspore/ccsrc/debug/debugger/grpc_client.h new file mode 100644 index 0000000000..0b5359e444 --- /dev/null +++ b/mindspore/ccsrc/debug/debugger/grpc_client.h @@ -0,0 +1,61 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_DEBUG_DEBUGGER_GRPC_CLIENT_H_ +#define MINDSPORE_CCSRC_DEBUG_DEBUGGER_GRPC_CLIENT_H_ + +#include +#include +#include +#include +#include "proto/debug_grpc.grpc.pb.h" + +using debugger::EventListener; +using debugger::EventReply; +using debugger::GraphProto; +using debugger::Metadata; +using debugger::TensorProto; +using debugger::WatchpointHit; + +namespace mindspore { +class GrpcClient { + public: + // constructor + GrpcClient(const std::string &host, const std::string &port); + + // deconstructor + ~GrpcClient() = default; + + // init + void Init(const std::string &host, const std::string &port); + + // reset + void Reset(); + + EventReply WaitForCommand(const Metadata &metadata); + + EventReply SendMetadata(const Metadata &metadata); + + EventReply SendGraph(const GraphProto &graph); + + EventReply SendTensors(const std::list &tensors); + + EventReply SendWatchpointHits(const std::list &watchpoints); + + private: + std::unique_ptr stub_; +}; +} // namespace mindspore +#endif // MINDSPORE_CCSRC_DEBUG_DEBUGGER_GRPC_CLIENT_H_ diff --git a/mindspore/ccsrc/debug/debugger/proto_exporter.cc b/mindspore/ccsrc/debug/debugger/proto_exporter.cc new file mode 100644 index 0000000000..b4b4de9d99 --- /dev/null +++ b/mindspore/ccsrc/debug/debugger/proto_exporter.cc @@ -0,0 +1,542 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "debug/debugger/debugger.h" +#include "proto/debug_graph.pb.h" +#include "utils/graph_utils.h" +#include "utils/symbolic.h" + +namespace mindspore { +class DebuggerProtoExporter { + public: + DebuggerProtoExporter() {} + ~DebuggerProtoExporter() {} + + std::string GetFuncGraphProtoString(const FuncGraphPtr &func_graph); + debugger::ModelProto GetFuncGraphProto(const FuncGraphPtr &func_graph); + + private: + void InitModelInfo(); + void GetOpNodeTypeAndAttrs(const FuncGraphPtr &func_graph, const AnfNodePtr &node, debugger::NodeProto *node_proto); + std::string GetOpNodeInputId(const FuncGraphPtr &func_graph, const AnfNodePtr &node, + const std::map &apply_map, + std::map *const_map_ptr); + void SetValueToProto(const ValuePtr &attr_value, debugger::ValueProto *value_proto); + void SetScalarToProto(const ScalarPtr &val, debugger::ValueProto *value_proto); + void SetSequenceToProto(const ValueSequeuePtr &val, debugger::ValueProto *value_proto); + void SetDictionaryToProto(const ValueDictionaryPtr &val, debugger::ValueProto *value_proto); + void SetNodeOutputType(const AnfNodePtr &node, debugger::TypeProto *type_proto); + void SetNodeOutputType(const TypePtr &node, const BaseShapePtr &shape, debugger::TypeProto *type_proto); + + void ExportFuncGraph(const FuncGraphPtr &func_graph, debugger::GraphProto *graph_proto); + void ExportParameters(const FuncGraphPtr &func_graph, debugger::GraphProto *graph_proto); + void ExportCNodes(const FuncGraphPtr &func_graph, debugger::GraphProto *graph_proto, + std::map *const_map_ptr); + void ExportCNode(const FuncGraphPtr &func_graph, const CNodePtr &node, std::map *apply_map_ptr, + std::map *const_map_ptr, debugger::GraphProto *graph_proto); + void ExportFuncGraphOutput(const FuncGraphPtr &func_graph, const CNodePtr &ret_node, + const std::map &apply_map, std::map *const_map_ptr, + debugger::GraphProto *graph_proto); + void ExportValueNodes(const std::map &const_map, debugger::GraphProto *graph_proto); + + static std::string GetConstNodeId(size_t idx) { return std::string("cst") + std::to_string(idx); } + + debugger::ModelProto model_; +}; + +void DebuggerProtoExporter::SetNodeOutputType(const TypePtr &type, const BaseShapePtr &shape, + debugger::TypeProto *type_proto) { + if (type_proto == nullptr) { + return; + } + + if (type == nullptr) { + type_proto->set_data_type(debugger::DT_UNDEFINED); + } else if (type->isa()) { + type_proto->set_data_type(GetDebuggerNumberDataType(type)); + } else if (type->isa()) { + TypePtr elem_type = dyn_cast(type)->element(); + type_proto->mutable_tensor_type()->set_elem_type(GetDebuggerNumberDataType(elem_type)); + type_proto->set_data_type(debugger::DT_TENSOR); + if (shape != nullptr && shape->isa()) { + abstract::ShapePtr shape_info = dyn_cast(shape); + for (const auto &elem : shape_info->shape()) { + type_proto->mutable_tensor_type()->mutable_shape()->add_dim()->set_size(elem); + } + } + } else if (type->isa()) { + TuplePtr tuple_type = dyn_cast(type); + type_proto->set_data_type(debugger::DT_TUPLE); + for (const auto &elem_type : tuple_type->elements()) { + SetNodeOutputType(elem_type, nullptr, type_proto->mutable_sequence_type()->add_elem_types()); + } + } else if (type->isa()) { + type_proto->set_data_type(debugger::DT_TYPE); + } else if (type->isa()) { + ListPtr list_type = dyn_cast(type); + type_proto->set_data_type(debugger::DT_LIST); + for (const auto &elem_type : list_type->elements()) { + SetNodeOutputType(elem_type, nullptr, type_proto->mutable_sequence_type()->add_elem_types()); + } + } else if (type->isa()) { + type_proto->set_data_type(debugger::DT_ANYTHING); + } else if (type->isa()) { + type_proto->set_data_type(debugger::DT_REFKEY); + } else if (type->isa()) { + type_proto->set_data_type(debugger::DT_REF); + } else if (type->isa()) { + type_proto->set_data_type(debugger::DT_GRAPH); + } else if (type->isa()) { + type_proto->set_data_type(debugger::DT_NONE); + } else if (type->isa()) { + type_proto->set_data_type(debugger::DT_STRING); + } else if (type->isa()) { + // Do Nothing. + } else { + MS_LOG(EXCEPTION) << "Unknown type: " << type->type_name(); + } +} + +void DebuggerProtoExporter::SetNodeOutputType(const AnfNodePtr &node, debugger::TypeProto *type_proto) { + if (node == nullptr || type_proto == nullptr) { + return; + } + SetNodeOutputType(node->Type(), node->Shape(), type_proto); +} + +void DebuggerProtoExporter::SetValueToProto(const ValuePtr &val, debugger::ValueProto *value_proto) { + if (val == nullptr || value_proto == nullptr) { + return; + } + + if (val->isa()) { + const StringImmPtr &value = dyn_cast(val); + value_proto->set_dtype(debugger::DT_STRING); + value_proto->set_str_val(value->value()); + } else if (val->isa()) { + SetScalarToProto(dyn_cast(val), value_proto); + } else if (val->isa()) { + value_proto->set_dtype(debugger::DT_TYPE); + value_proto->mutable_type_val()->set_data_type(debugger::DT_BOOL); + } else if (val->isa()) { + value_proto->set_dtype(debugger::DT_TYPE); + value_proto->mutable_type_val()->set_data_type(debugger::DT_BASE_INT); + } else if (val->isa()) { + value_proto->set_dtype(debugger::DT_TYPE); + value_proto->mutable_type_val()->set_data_type(debugger::DT_BASE_FLOAT); + } else if (val->isa()) { + SetSequenceToProto(dyn_cast(val), value_proto); + } else if (val->isa()) { + value_proto->set_dtype(debugger::DT_NONE); + value_proto->set_str_val("None"); + } else if (val->isa()) { + SymbolicKeyInstancePtr sym_inst = dyn_cast(val); + ParameterPtr sym_node = dyn_cast(sym_inst->node()); + value_proto->set_dtype(debugger::DT_SYM_INST); + value_proto->set_str_val(sym_node == nullptr ? std::string("nullptr") : sym_node->ToString()); + } else if (val->isa()) { + SetDictionaryToProto(dyn_cast(val), value_proto); + } else if (val->isa()) { + tensor::TensorPtr tensor_ptr = dyn_cast(val); + value_proto->set_dtype(debugger::DT_TENSOR); + debugger::TensorProto *tensor_proto = value_proto->mutable_tensor_val(); + tensor_proto->set_data_type(GetDebuggerNumberDataType(tensor_ptr->Dtype())); + for (auto &elem : tensor_ptr->shape()) { + tensor_proto->add_dims(elem); + } + tensor_proto->set_tensor_content(tensor_ptr->data_c(), tensor_ptr->data().nbytes()); + } else if (val->isa()) { + value_proto->set_dtype(debugger::DT_TYPE); + + debugger::TypeProto *type_proto = value_proto->mutable_type_val(); + type_proto->set_data_type(debugger::DT_TENSOR); + TypePtr elem_type = dyn_cast(val)->element(); + type_proto->mutable_tensor_type()->set_elem_type(GetDebuggerNumberDataType(elem_type)); + } else { + MS_LOG(WARNING) << "Unsupported type " << val->type_name(); + } +} + +void DebuggerProtoExporter::SetScalarToProto(const ScalarPtr &val, debugger::ValueProto *value_proto) { + if (val == nullptr || value_proto == nullptr) { + return; + } + + if (val->isa()) { + const BoolImmPtr &value = dyn_cast(val); + value_proto->set_dtype(debugger::DT_BOOL); + value_proto->set_bool_val(value->value()); + } else if (val->isa()) { + const Int8ImmPtr &value = dyn_cast(val); + value_proto->set_dtype(debugger::DT_INT8); + value_proto->set_int_val(value->value()); + } else if (val->isa()) { + const Int16ImmPtr &value = dyn_cast(val); + value_proto->set_dtype(debugger::DT_INT16); + value_proto->set_int_val(value->value()); + } else if (val->isa()) { + const Int32ImmPtr &value = dyn_cast(val); + value_proto->set_dtype(debugger::DT_INT32); + value_proto->set_int_val(value->value()); + } else if (val->isa()) { + const Int64ImmPtr &value = dyn_cast(val); + value_proto->set_dtype(debugger::DT_INT64); + value_proto->set_int_val(value->value()); + } else if (val->isa()) { + const UInt8ImmPtr &value = dyn_cast(val); + value_proto->set_dtype(debugger::DT_UINT8); + value_proto->set_uint_val(value->value()); + } else if (val->isa()) { + const UInt16ImmPtr &value = dyn_cast(val); + value_proto->set_dtype(debugger::DT_UINT16); + value_proto->set_uint_val(value->value()); + } else if (val->isa()) { + const UInt32ImmPtr &value = dyn_cast(val); + value_proto->set_dtype(debugger::DT_UINT32); + value_proto->set_uint_val(value->value()); + } else if (val->isa()) { + const UInt64ImmPtr &value = dyn_cast(val); + value_proto->set_dtype(debugger::DT_UINT64); + value_proto->set_uint_val(value->value()); + } else if (val->isa()) { + const FP32ImmPtr &value = dyn_cast(val); + value_proto->set_dtype(debugger::DT_FLOAT32); + value_proto->set_float_val(value->value()); + } else if (val->isa()) { + const FP64ImmPtr &value = dyn_cast(val); + value_proto->set_dtype(debugger::DT_FLOAT64); + value_proto->set_double_val(value->value()); + } else { + MS_LOG(EXCEPTION) << "Unknown scalar type " << val->ToString(); + } +} + +void DebuggerProtoExporter::SetSequenceToProto(const ValueSequeuePtr &val, debugger::ValueProto *value_proto) { + if (val == nullptr || value_proto == nullptr) { + return; + } + + if (val->isa()) { + const ValueTuplePtr &value = dyn_cast(val); + value_proto->set_dtype(debugger::DT_TUPLE); + for (const auto &item : value->value()) { + SetValueToProto(item, value_proto->add_values()); + } + } else if (val->isa()) { + const ValueListPtr &value = dyn_cast(val); + value_proto->set_dtype(debugger::DT_LIST); + for (const auto &item : value->value()) { + SetValueToProto(item, value_proto->add_values()); + } + } +} + +void DebuggerProtoExporter::SetDictionaryToProto(const ValueDictionaryPtr &val, debugger::ValueProto *value_proto) { + if (val == nullptr || value_proto == nullptr) { + return; + } + + value_proto->set_dtype(debugger::DT_DICT); + for (const auto &item : val->value()) { + debugger::NamedValueProto *named_val = value_proto->add_dict_val(); + named_val->set_key(item.first); + SetValueToProto(item.second, named_val->mutable_value()); + } +} + +void DebuggerProtoExporter::GetOpNodeTypeAndAttrs(const FuncGraphPtr &, const AnfNodePtr &node, + debugger::NodeProto *node_proto) { + if (node == nullptr || node_proto == nullptr) { + return; + } + + if (node->isa() || node->isa() || IsValueNode(node)) { + MS_LOG(EXCEPTION) << "Op node can not be CNode, Parameter or ValueNode Graph. But got " << node->ToString(); + } + + if (!IsValueNode(node)) { + MS_LOG(EXCEPTION) << "Op node is not primitive: " << node->ToString(); + } + + const PrimitivePtr &prim = GetValueNode(node); + node_proto->set_op_type(prim->name()); + for (const auto &attr : prim->attrs()) { + debugger::AttributeProto *attr_proto = node_proto->add_attribute(); + attr_proto->set_name(attr.first); + SetValueToProto(attr.second, attr_proto->mutable_value()); + } + node_proto->set_scope(node->scope()->name()); +} + +std::string DebuggerProtoExporter::GetOpNodeInputId(const FuncGraphPtr &, const AnfNodePtr &node, + const std::map &apply_map, + std::map *const_map_ptr) { + if (node == nullptr || const_map_ptr == nullptr) { + return ""; + } + + if (node->isa()) { + auto iter = apply_map.find(node); + if (iter == apply_map.end()) { + MS_LOG(EXCEPTION) << "Can not find node '" << node->ToString() << "' in apply_map"; + } + return std::to_string(iter->second); + } + + if (node->isa()) { + return node->ToString(); + } + + if (node->isa()) { + auto iter = const_map_ptr->find(node); + if (iter == const_map_ptr->end()) { + // Start index number from 1 + auto const_idx = const_map_ptr->size() + 1; + (*const_map_ptr)[node] = const_idx; + } + return GetConstNodeId((*const_map_ptr)[node]); + } + + MS_LOG(EXCEPTION) << "Unknown node type. node is '" << node->ToString() << "'"; +} + +std::string DebuggerProtoExporter::GetFuncGraphProtoString(const FuncGraphPtr &func_graph) { + if (func_graph == nullptr) { + return ""; + } + + InitModelInfo(); + debugger::GraphProto *graph_proto = model_.mutable_graph(); + ExportFuncGraph(func_graph, graph_proto); + return model_.SerializeAsString(); +} + +debugger::ModelProto DebuggerProtoExporter::GetFuncGraphProto(const FuncGraphPtr &func_graph) { + if (func_graph == nullptr) { + return ModelProto(); + } + + InitModelInfo(); + debugger::GraphProto *graph_proto = model_.mutable_graph(); + ExportFuncGraph(func_graph, graph_proto); + return model_; +} + +void DebuggerProtoExporter::ExportFuncGraph(const FuncGraphPtr &func_graph, debugger::GraphProto *graph_proto) { + if (func_graph == nullptr || graph_proto == nullptr) { + return; + } + + // map for store ValueNodes of this graph + std::map const_map; + + // set graph name + graph_proto->set_name(func_graph->ToString()); + + ExportParameters(func_graph, graph_proto); + + ExportCNodes(func_graph, graph_proto, &const_map); + + ExportValueNodes(const_map, graph_proto); +} + +void DebuggerProtoExporter::ExportParameters(const FuncGraphPtr &func_graph, debugger::GraphProto *graph_proto) { + if (func_graph == nullptr || graph_proto == nullptr) { + return; + } + + // cast FuncGraph to KernelGraph to access inputs() + std::vector parameters = static_cast(func_graph.get())->inputs(); + + for (auto ¶m : parameters) { + debugger::ParameterProto *param_proto = graph_proto->add_parameters(); + param_proto->set_name(param->ToString()); + + SetNodeOutputType(param, param_proto->mutable_type()); + + const ParameterPtr param_ptr = dyn_cast(param); + if (param_ptr == nullptr) { + MS_LOG(EXCEPTION) << "Parameter '" << param->ToString() << "' could not cast to parameter."; + } + } +} + +void DebuggerProtoExporter::ExportCNodes(const FuncGraphPtr &func_graph, debugger::GraphProto *graph_proto, + std::map *const_map_ptr) { + if (func_graph == nullptr || graph_proto == nullptr || const_map_ptr == nullptr) { + return; + } + // topo sort nodes + std::vector nodes = TopoSort(func_graph->get_return(), SuccIncoming, AlwaysInclude); + std::map apply_map; + for (const AnfNodePtr &node : nodes) { + MS_EXCEPTION_IF_NULL(node); + if (!node->isa()) { + continue; + } + auto cnode = node->cast(); + if (cnode != func_graph->get_return()) { + ExportCNode(func_graph, cnode, &apply_map, const_map_ptr, graph_proto); + } else { + ExportFuncGraphOutput(func_graph, cnode, apply_map, const_map_ptr, graph_proto); + } + } +} + +void DebuggerProtoExporter::ExportCNode(const FuncGraphPtr &func_graph, const CNodePtr &node, + std::map *apply_map_ptr, + std::map *const_map_ptr, + debugger::GraphProto *graph_proto) { + if (func_graph == nullptr || node == nullptr || apply_map_ptr == nullptr || const_map_ptr == nullptr || + graph_proto == nullptr) { + return; + } + + auto apply_idx = apply_map_ptr->size() + 1; + (*apply_map_ptr)[node] = apply_idx; + + auto &inputs = node->inputs(); + if (inputs.size() < 1) { + MS_LOG(EXCEPTION) << "Inputs of apply node is empty"; + } + AnfNodePtr op = inputs[0]; + debugger::NodeProto *node_proto = graph_proto->add_node(); + + // CNode/ConstGraph/Const/Parameter + if (op->isa() || IsValueNode(op) || op->isa()) { + MS_LOG(WARNING) << "Operator must be a primitive"; + } else { + GetOpNodeTypeAndAttrs(func_graph, op, node_proto); + node_proto->set_name(std::to_string(apply_idx)); + node_proto->set_scope(node->scope()->name()); + + // add debug_name for debugger + node_proto->set_debug_name(node->fullname_with_scope()); + + // process OP inputs + for (size_t i = 1; i < inputs.size(); ++i) { + debugger::InputProto *input_proto = node_proto->add_input(); + input_proto->set_type(debugger::InputProto_EdgeType_DATA_EDGE); + std::string id = GetOpNodeInputId(func_graph, inputs[i], *apply_map_ptr, const_map_ptr); + input_proto->set_name(id); + } + + // set node output type + SetNodeOutputType(node, node_proto->mutable_output_type()); + } +} + +void DebuggerProtoExporter::ExportFuncGraphOutput(const FuncGraphPtr &func_graph, const CNodePtr &ret_node, + const std::map &apply_map, + std::map *const_map_ptr, + debugger::GraphProto *graph_proto) { + if (ret_node == nullptr || !ret_node->isa()) { + MS_LOG(EXCEPTION) << "Graph return node is illegal"; + } + AnfNodePtr arg = ret_node->input(1); + if (graph_proto == nullptr) { + MS_LOG(EXCEPTION) << "graph_proto is nullptr"; + } + debugger::OutputProto *output_proto = graph_proto->add_outputs(); + if (output_proto == nullptr) { + MS_LOG(EXCEPTION) << "output_proto is nullptr"; + } + std::string id = GetOpNodeInputId(func_graph, arg, apply_map, const_map_ptr); + output_proto->set_name(id); + SetNodeOutputType(arg, output_proto->mutable_type()); +} + +static bool CompareValue(const std::pair &x, const std::pair &y) { + return x.second < y.second; +} + +void DebuggerProtoExporter::ExportValueNodes(const std::map &const_map, + debugger::GraphProto *graph_proto) { + std::vector> nodes; + (void)std::transform(const_map.cbegin(), const_map.cend(), std::back_inserter(nodes), + [](const std::pair &item) { return item; }); + + sort(nodes.begin(), nodes.end(), CompareValue); + + for (auto &item : nodes) { + if (graph_proto == nullptr) { + MS_LOG(EXCEPTION) << "graph_proto is nullptr"; + } + debugger::NamedValueProto *named_value = graph_proto->add_const_vals(); + MS_EXCEPTION_IF_NULL(named_value); + named_value->set_key(GetConstNodeId(item.second)); + SetValueToProto(GetValueNode(item.first), named_value->mutable_value()); + } +} + +void DebuggerProtoExporter::InitModelInfo() { model_.set_ir_version(debugger::IR_VERSION); } + +std::string GetDebuggerFuncGraphProtoString(const FuncGraphPtr &func_graph) { + DebuggerProtoExporter exporter; + return exporter.GetFuncGraphProtoString(func_graph); +} + +debugger::ModelProto GetDebuggerFuncGraphProto(const FuncGraphPtr &func_graph) { + DebuggerProtoExporter exporter; + return exporter.GetFuncGraphProto(func_graph); +} + +debugger::DataType GetDebuggerNumberDataType(const TypePtr &type) { + switch (type->type_id()) { + case kNumberTypeBool: + return debugger::DT_BOOL; + case kNumberTypeInt8: + return debugger::DT_INT8; + case kNumberTypeInt16: + return debugger::DT_INT16; + case kNumberTypeInt32: + return debugger::DT_INT32; + case kNumberTypeInt64: + return debugger::DT_INT64; + case kNumberTypeUInt8: + return debugger::DT_UINT8; + case kNumberTypeUInt16: + return debugger::DT_UINT16; + case kNumberTypeUInt32: + return debugger::DT_UINT32; + case kNumberTypeUInt64: + return debugger::DT_UINT64; + case kNumberTypeFloat16: + return debugger::DT_FLOAT16; + case kNumberTypeFloat32: + return debugger::DT_FLOAT32; + case kNumberTypeFloat64: + return debugger::DT_FLOAT64; + case kNumberTypeInt: + return debugger::DT_BASE_INT; + case kNumberTypeUInt: + return debugger::DT_BASE_UINT; + case kNumberTypeFloat: + return debugger::DT_BASE_FLOAT; + default: + MS_LOG(EXCEPTION) << "Unexpected type " << type->type_name(); + } +} + +} // namespace mindspore diff --git a/mindspore/ccsrc/debug/dump_proto.cc b/mindspore/ccsrc/debug/dump_proto.cc index ab2ce1322a..99440537c7 100644 --- a/mindspore/ccsrc/debug/dump_proto.cc +++ b/mindspore/ccsrc/debug/dump_proto.cc @@ -120,6 +120,10 @@ void ProtoExporter::SetNodeOutputType(const TypePtr &type, const BaseShapePtr &s type_proto->mutable_tensor_type()->mutable_shape()->add_dim()->set_size(elem); } } + } else if (type->isa()) { + // Do Nothing + } else if (type->isa()) { + // Do Nothing } else if (type->isa()) { TuplePtr tuple_type = dyn_cast(type); type_proto->set_data_type(irpb::DT_TUPLE); diff --git a/mindspore/ccsrc/debug/tensor_data.h b/mindspore/ccsrc/debug/tensor_data.h new file mode 100644 index 0000000000..9704d69089 --- /dev/null +++ b/mindspore/ccsrc/debug/tensor_data.h @@ -0,0 +1,75 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_DEBUG_TENSOR_DATA_H_ +#define MINDSPORE_CCSRC_DEBUG_TENSOR_DATA_H_ + +#include +#include +#include +#include +#include "ir/tensor.h" + +namespace mindspore { +class TensorData { + private: + mindspore::tensor::TensorPtr tensor_ptr; + std::string name; + size_t slot; + int execution_order; + + public: + TensorData() : slot(0), execution_order(-1) {} + + TensorData(const TensorData &obj) { + std::cout << "Copy Constructor" << std::endl; + this->name = obj.name; + this->execution_order = obj.execution_order; + this->slot = obj.slot; + this->tensor_ptr = obj.tensor_ptr; + } + + ~TensorData() {} + + std::string GetName() { return this->name; } + + mindspore::tensor::TensorPtr GetTensor() { return this->tensor_ptr; } + + size_t GetSlot() { return this->slot; } + + int GetExecutionOrder() { return this->execution_order; } + + int SetExecutionOrder(int execution_order) { + this->execution_order = execution_order; + return true; + } + + int SetName(const std::string &name) { + this->name = name; + return true; + } + + bool SetTensor(mindspore::tensor::TensorPtr out_tensor) { + this->tensor_ptr = out_tensor; + return true; + } + + bool SetSlot(size_t slot) { + this->slot = slot; + return true; + } +}; +} // namespace mindspore +#endif // MINDSPORE_CCSRC_DEBUG_TENSOR_DATA_H_ diff --git a/mindspore/ccsrc/debug/tensor_load.h b/mindspore/ccsrc/debug/tensor_load.h new file mode 100644 index 0000000000..6c3ea67a78 --- /dev/null +++ b/mindspore/ccsrc/debug/tensor_load.h @@ -0,0 +1,69 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_DEBUG_TENSOR_LOAD_H_ +#define MINDSPORE_CCSRC_DEBUG_TENSOR_LOAD_H_ + +#include +#include +#include +#include +#include +#include "debug/tensor_data.h" +namespace mindspore { +class TensorLoader { + public: + TensorLoader() : iter_num(-1) {} + + ~TensorLoader() {} + + bool LoadNewTensor(std::shared_ptr tensor) { + tensor_list.push_back(tensor); + tensor_list_map.insert({tensor->GetName(), tensor}); + return true; + } + std::vector> GetTensor() { return tensor_list; } + + uint32_t GetIterNum() { return iter_num; } + + std::map> GetTensorMap() { return tensor_list_map; } + void SearchTensors(const std::vector &search_list, + std::vector>> *result_list) { + for (auto i : search_list) { + std::map>::iterator iter; + iter = tensor_list_map.find(i); + if (iter != tensor_list_map.end()) { + result_list->push_back(std::make_tuple(i, iter->second)); + } else { + result_list->push_back(std::make_tuple(i, nullptr)); + } + } + } + + bool EmptyTensor() { + tensor_list_map.clear(); + tensor_list.clear(); + return true; + } + + void set_iter_num(uint32_t iter_num) { this->iter_num = iter_num; } + + private: + std::vector> tensor_list; + std::map> tensor_list_map; + uint32_t iter_num; +}; +} // namespace mindspore +#endif // MINDSPORE_CCSRC_DEBUG_TENSOR_LOAD_H_ diff --git a/mindspore/ccsrc/device/CMakeLists.txt b/mindspore/ccsrc/device/CMakeLists.txt index 7178a01ce6..652c04d4cd 100644 --- a/mindspore/ccsrc/device/CMakeLists.txt +++ b/mindspore/ccsrc/device/CMakeLists.txt @@ -14,13 +14,17 @@ endif () if (ENABLE_CPU) file(GLOB_RECURSE CPU_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "cpu/*.cc") - if (NOT ENABLE_MPI) - list(REMOVE_ITEM CPU_SRC_LIST "cpu/mpi/mpi_adapter.cc") - endif () + list(REMOVE_ITEM CPU_SRC_LIST "cpu/mpi/mpi_adapter.cc") endif () if (ENABLE_MPI) # _ms_mpi + file(GLOB_RECURSE MPI_SRC_LIST "cpu/mpi/mpi_adapter.cc") + set_property(SOURCE ${MPI_SRC_LIST} + PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_DEVICE) + add_library(mpi_adapter SHARED ${MPI_SRC_LIST}) + target_link_libraries(mpi_adapter PRIVATE mindspore::ompi) + set_property(SOURCE "gpu/mpi/mpi_initializer.cc" PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_DEVICE) pybind11_add_module(_ms_mpi "gpu/mpi/mpi_initializer.cc") diff --git a/mindspore/ccsrc/device/ascend/ascend_device_address.cc b/mindspore/ccsrc/device/ascend/ascend_device_address.cc index a47c482c0e..c4b8717fa5 100644 --- a/mindspore/ccsrc/device/ascend/ascend_device_address.cc +++ b/mindspore/ccsrc/device/ascend/ascend_device_address.cc @@ -30,6 +30,10 @@ #ifdef ENABLE_DUMP_E2E #include "debug/e2e_dump.h" #endif +#ifdef ENABLE_DEBUGGER +#include "debug/tensor_load.h" +#endif + namespace mindspore { namespace device { namespace ascend { @@ -170,7 +174,13 @@ bool AscendDeviceAddress::SyncDeviceToHostAndConvertFormat(const std::vector(host_type, host_shape); size_t host_size = out_tensor->data().nbytes(); - ret = SyncDeviceToHost(host_shape, host_size, host_type, out_tensor->data_c(true)); + ret = SyncDeviceToHost(host_shape, host_size, host_type, out_tensor->data_c()); if (!ret) { MS_LOG(ERROR) << "Copy device mem to host failed"; return ret; } - ret = mindspore::Dump::DumpToFile(path, out_tensor->data_c(false), host_size); + ret = mindspore::Dump::DumpToFile(path, out_tensor->data_c(), host_size); } else { auto host_tmp = std::vector(size_); auto ret_rt_memcpy = rtMemcpy(host_tmp.data(), size_, ptr_, size_, RT_MEMCPY_DEVICE_TO_HOST); @@ -346,6 +356,50 @@ bool AscendDeviceAddress::DumpMemToFile(bool trans_flag, const std::string &file return ret; } #endif + +#ifdef ENABLE_DEBUGGER +bool AscendDeviceAddress::LoadMemToHost(bool trans_flag, const std::string &tensor_name, int execution_order, + const std::string &host_fmt, const std::vector &host_shape, + TypeId host_type, size_t slot, Debugger *debugger) const { + bool ret = false; + + DebugServices *debug_services = debugger->get_debug_services(); + TensorLoader *tensor_loader = debug_services->get_tensor_loader(); + + if (trans_flag) { + MS_LOG(INFO) << "E2E tensor name is " << tensor_name; + mindspore::tensor::TensorPtr out_tensor = std::make_shared(host_type, host_shape); + size_t host_size = out_tensor->data().nbytes(); + ret = SyncDeviceToHost(host_shape, host_size, host_type, out_tensor->data_c()); + if (!ret) { + MS_LOG(ERROR) << "Copy device mem to host failed"; + return ret; + } + auto tensor_data = std::make_shared(); + tensor_data->SetName(tensor_name); + tensor_data->SetExecutionOrder(execution_order); + tensor_data->SetTensor(out_tensor); + tensor_data->SetSlot(slot); + ret = tensor_loader->LoadNewTensor(tensor_data); + } else { + mindspore::tensor::TensorPtr out_tensor = std::make_shared(type_id_, host_shape); + size_t host_size = out_tensor->data().nbytes(); + auto ret_rt_memcpy = rtMemcpy(out_tensor->data_c(), host_size, ptr_, host_size, RT_MEMCPY_DEVICE_TO_HOST); + + auto tensor_data = std::make_shared(); + tensor_data->SetName(tensor_name); + tensor_data->SetExecutionOrder(execution_order); + tensor_data->SetTensor(out_tensor); + tensor_data->SetSlot(slot); + ret = tensor_loader->LoadNewTensor(tensor_data); + if (ret_rt_memcpy != RT_ERROR_NONE) { + MS_LOG(ERROR) << "SyncDeviceToHost: rtMemcpy mem size[" << size_ << "] fail, ret[" << ret_rt_memcpy << "]"; + } + MS_LOG(INFO) << "E2E tensor name is " << tensor_name; + } + return ret; +} +#endif } // namespace ascend } // namespace device } // namespace mindspore diff --git a/mindspore/ccsrc/device/ascend/ascend_device_address.h b/mindspore/ccsrc/device/ascend/ascend_device_address.h index 364f9e95fd..16b9f7817a 100644 --- a/mindspore/ccsrc/device/ascend/ascend_device_address.h +++ b/mindspore/ccsrc/device/ascend/ascend_device_address.h @@ -25,6 +25,9 @@ #include "ir/dtype.h" namespace mindspore { +#ifdef ENABLE_DEBUGGER +class Debugger; +#endif namespace device { namespace ascend { class AscendDeviceAddress : public DeviceAddress { @@ -40,6 +43,11 @@ class AscendDeviceAddress : public DeviceAddress { bool DumpMemToFile(bool dump_mode, const std::string &filepath, const std::string &host_fmt, const std::vector &host_shape, TypeId host_type) const; #endif +#ifdef ENABLE_DEBUGGER + bool LoadMemToHost(bool dump_mode, const std::string &tensor_name, int execution_order, const std::string &host_fmt, + const std::vector &host_shape, TypeId host_type, size_t slot, Debugger *debugger) const; +#endif + private: bool SyncDeviceToHostAndConvertFormat(const std::vector &shape, size_t size, TypeId type, void *host_ptr) const; bool ConvertFormatAndSyncHostToDevice(const std::vector &shape, size_t size, TypeId type, diff --git a/mindspore/ccsrc/device/ascend/ascend_kernel_runtime.cc b/mindspore/ccsrc/device/ascend/ascend_kernel_runtime.cc index fb2a3f350b..efdcb98755 100644 --- a/mindspore/ccsrc/device/ascend/ascend_kernel_runtime.cc +++ b/mindspore/ccsrc/device/ascend/ascend_kernel_runtime.cc @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - +#define PATH_MAX 0x3ffff #include "device/ascend/ascend_kernel_runtime.h" #include #include @@ -21,7 +21,6 @@ #include #include #include - #include "device/ascend/ascend_device_address.h" #include "device/cpu/mpi/mpi_adapter.h" #include "utils/context/ms_context.h" @@ -41,6 +40,7 @@ #include "kernel/tbe/tbe_python_funcs.h" #include "pre_activate/mem_reuse/mem_reuse_checker.h" #include "device/ascend/ascend_memory_manager.h" +#include "debug/tensor_load.h" using mindspore::device::ascend::ProfilingManager; using mindspore::device::ascend::ProfilingUtils; @@ -59,16 +59,18 @@ std::string GetRankId() { auto mpi_config_ptr = MpiConfig::GetInstance(); MS_EXCEPTION_IF_NULL(mpi_config_ptr); if (mpi_config_ptr->enable_mpi()) { - int rank_id = device::cpu::MPIAdapter::Instance().GetRankId(); + auto mpi_instance = device::cpu::MPIAdapter::Instance(); + MS_EXCEPTION_IF_NULL(mpi_instance); + int rank_id = mpi_instance->GetRankId(); const char *offset = std::getenv("RANK_OFFSET"); if (offset != nullptr) { try { int rank_offset = std::stoi(offset); rank_id += rank_offset; } catch (std::invalid_argument) { - MS_LOG(EXCEPTION) << "stoi invalid argument:" << offset; + MS_LOG(EXCEPTION) << "Call stoi invalid argument:" << offset; } catch (std::out_of_range) { - MS_LOG(EXCEPTION) << "stoi out_of_range:" << offset; + MS_LOG(EXCEPTION) << "Call stoi out_of_range:" << offset; } } rank_id_str = std::to_string(rank_id); @@ -79,7 +81,7 @@ std::string GetRankId() { rank_id_str = std::getenv("RANK_ID"); #endif if (rank_id_str.empty()) { - MS_LOG(ERROR) << "get hccl rankid failed, please set env RANK_ID"; + MS_LOG(ERROR) << "Get hccl rankid failed, please set env RANK_ID"; } return rank_id_str; } @@ -98,7 +100,7 @@ void AscendKernelRuntime::ClearGraphModelMap() { } void AscendKernelRuntime::ClearGraphRuntimeResource(uint32_t graph_id) { - MS_LOG(DEBUG) << "clear graph:" << graph_id << " runtime resource"; + MS_LOG(DEBUG) << "Clear graph:" << graph_id << " runtime resource"; auto iter = graph_model_map_.find(graph_id); if (iter == graph_model_map_.end()) { MS_LOG(DEBUG) << "GraphId:" << graph_id << " not found"; @@ -116,7 +118,7 @@ bool AscendKernelRuntime::NeedDestroyHccl() { auto context_ptr = MsContext::GetInstance(); MS_EXCEPTION_IF_NULL(context_ptr); if (!context_ptr->enable_hccl()) { - MS_LOG(INFO) << "hccl is not enabled"; + MS_LOG(INFO) << "Hccl is not enabled"; return false; } // Note: make sure hcom_connectivity_detection api never be used. @@ -124,7 +126,7 @@ bool AscendKernelRuntime::NeedDestroyHccl() { } void AscendKernelRuntime::ReleaseDeviceRes() { - MS_LOG(INFO) << "ascend finalize start"; + MS_LOG(INFO) << "Ascend finalize start"; // release ge runtime ClearGraphModelMap(); @@ -132,7 +134,7 @@ void AscendKernelRuntime::ReleaseDeviceRes() { MS_EXCEPTION_IF_NULL(context_ptr); auto ret = rtSetDevice(context_ptr->device_id()); if (ret != RT_ERROR_NONE) { - MS_EXCEPTION(DeviceProcessError) << "rtSetDevice, ret[" << static_cast(ret) << "]"; + MS_EXCEPTION(DeviceProcessError) << "Call rtSetDevice, ret[" << static_cast(ret) << "]"; } if (mem_manager_ != nullptr) { @@ -142,7 +144,7 @@ void AscendKernelRuntime::ReleaseDeviceRes() { (void)DestroyHccl(); (void)ResetDevice(); (void)ProfilingManager::GetInstance().StopProfiling(); - MS_LOG(INFO) << "ascend finalize end"; + MS_LOG(INFO) << "Ascend finalize end"; } bool AscendKernelRuntime::Init() { @@ -153,7 +155,7 @@ bool AscendKernelRuntime::Init() { #ifdef ENABLE_DUMP_E2E ret = SetDumpConf(); if (!ret) { - MS_LOG(INFO) << "no dump conf to set!"; + MS_LOG(INFO) << "No dump conf to set!"; } #endif @@ -261,13 +263,13 @@ void DumpParameters(mindspore::session::KernelGraph *graph, const string &dump_p bool AscendKernelRuntime::DumpData(mindspore::session::KernelGraph *graph) { MS_EXCEPTION_IF_NULL(graph); #ifdef ENABLE_DUMP_E2E - MS_LOG(INFO) << "start dump step"; + MS_LOG(INFO) << "Start dump step"; DumpConfPtr dump_conf = GetDumpConf(); MS_EXCEPTION_IF_NULL(dump_conf); dump_conf->UpdataCurIter(); bool dump_flag = dump_conf->dump_enable(); if (!dump_flag) { - MS_LOG(INFO) << "dump flag is disable, pass dump step"; + MS_LOG(INFO) << "Dump flag is disable, pass dump step"; return true; } uint32_t cur_iter = dump_conf->cur_iter(); @@ -276,7 +278,7 @@ bool AscendKernelRuntime::DumpData(mindspore::session::KernelGraph *graph) { return true; } } - MS_LOG(INFO) << "cur iter is " << cur_iter; + MS_LOG(INFO) << "Cur iter is " << cur_iter; std::string net_name = dump_conf->dump_net_name(); std::string iterator = to_string(cur_iter); std::string dump_path = dump_conf->dump_path(); @@ -293,6 +295,91 @@ bool AscendKernelRuntime::DumpData(mindspore::session::KernelGraph *graph) { return true; } +#ifdef ENABLE_DEBUGGER +namespace { +void LoadOutput(mindspore::session::KernelGraph *graph, Debugger *debugger) { + MS_EXCEPTION_IF_NULL(graph); + bool trans_flag = false; + const auto &apply_kernels = graph->execution_order(); + // for kernels, execution order starts from 1 + int exec_order = 1; + for (const auto &node : apply_kernels) { + MS_EXCEPTION_IF_NULL(node); + auto node_name = AnfAlgo::GetCNodeName(node); + std::string kernel_name = node->fullname_with_scope(); + auto output_size = AnfAlgo::GetOutputTensorNum(node); + for (size_t j = 0; j < output_size; ++j) { + auto addr = AnfAlgo::GetOutputAddr(node, j); + auto type = AnfAlgo::GetOutputInferDataType(node, j); + auto format = kOpFormat_DEFAULT; + string tensor_name = kernel_name + ':' + std::to_string(j); + auto ascend_addr = dynamic_cast(addr); + std::vector int_shapes; + if (trans_flag) { + int_shapes = trans::GetRuntimePaddingShape(node, j); + } else { + auto shape = AnfAlgo::GetOutputDeviceShape(node, j); + (void)std::transform(shape.begin(), shape.end(), std::back_inserter(int_shapes), + [](size_t inner_item) { return SizeToInt(inner_item); }); + } + auto ret = ascend_addr->LoadMemToHost(trans_flag, tensor_name, exec_order, format, int_shapes, type, j, debugger); + if (!ret) { + MS_LOG(ERROR) << "LoadMemToHost: flag:" << trans_flag << ", tensor_name:" << tensor_name + << ", host_format:" << format << ".!"; + } + } + exec_order = exec_order + 1; + } +} + +void LoadParameters(mindspore::session::KernelGraph *graph, Debugger *debugger) { + MS_EXCEPTION_IF_NULL(graph); + bool trans_flag = false; + const auto ¶meters = graph->inputs(); + // for parameters, set its execution order to be 0; + int exec_order = 0; + for (auto &item : parameters) { + if (!item->isa()) { + continue; + } + std::string parameter_name = item->fullname_with_scope(); + auto addr = AnfAlgo::GetOutputAddr(item, PRAMATER_OUTPUT_INDEX); + auto type = AnfAlgo::GetOutputInferDataType(item, PRAMATER_OUTPUT_INDEX); + auto format = kOpFormat_DEFAULT; + string tensor_name = parameter_name + ':' + "0"; + auto ascend_addr = dynamic_cast(addr); + std::vector int_shapes; + if (trans_flag) { + int_shapes = trans::GetRuntimePaddingShape(item, PRAMATER_OUTPUT_INDEX); + } else { + auto shape = AnfAlgo::GetOutputDeviceShape(item, PRAMATER_OUTPUT_INDEX); + (void)std::transform(shape.begin(), shape.end(), std::back_inserter(int_shapes), + [](size_t inner_item) { return SizeToInt(inner_item); }); + } + auto ret = ascend_addr->LoadMemToHost(trans_flag, tensor_name, exec_order, format, int_shapes, type, 0, debugger); + if (!ret) { + MS_LOG(ERROR) << "LoadMemToHost Failed: flag:" << trans_flag << ", path:" << tensor_name + << ", host_format:" << format << ".!"; + } + } +} +} // namespace +#endif + +bool AscendKernelRuntime::LoadData(mindspore::session::KernelGraph *graph, Debugger *debugger) { + MS_EXCEPTION_IF_NULL(graph); +#ifdef ENABLE_DEBUGGER + MS_LOG(INFO) << "Start load step"; + uint32_t cur_iter = 0; + MS_LOG(INFO) << "Cur iter is " << cur_iter; + // load output + LoadOutput(graph, debugger); + // load parameters + LoadParameters(graph, debugger); +#endif + return true; +} + bool AscendKernelRuntime::NodeOutputDeviceAddressExist(const AnfNodePtr &kernel, size_t index) { if (AnfAlgo::OutputAddrExist(kernel, index)) { auto address = AnfAlgo::GetOutputAddr(kernel, index); @@ -334,27 +421,27 @@ bool AscendKernelRuntime::GenTask(const session::KernelGraph *graph) { } // Graph may have no compute node, such TensorAddGrad. if (task_info_list.empty()) { - MS_LOG(WARNING) << "graph " << graph->graph_id() << " have no compute node"; + MS_LOG(WARNING) << "Graph " << graph->graph_id() << " have no compute node"; return true; } AscendStreamAssign &assign_instance = AscendStreamAssign::GetInstance(); - AscendStreamMng &stream_manager = AscendStreamMng::GetInstance(); + AscendResourceMng &resource_manager = AscendResourceMng::GetInstance(); AscendLabelAssign &label_assign_instance = AscendLabelAssign::GetInstance(); // the streams' flag not HEAD_STREAM std::vector wait_active_stream_list; assign_instance.GetWaitStreams(&wait_active_stream_list); std::vector force_copy_stream_list; assign_instance.GetHcomStreams(&force_copy_stream_list); - MS_LOG(INFO) << "call DavinciModel total stream num:" << stream_manager.GetCurAllocStreamNum() - << ", total event num:" << assign_instance.total_event_num() + MS_LOG(INFO) << "Call DavinciModel total stream num:" << resource_manager.get_cur_stream_num() + << ", total event num:" << resource_manager.get_cur_event_num() << ", total label num:" << label_assign_instance.GetLabelNum(NOT_NULL(graph)) << ", wait_active_stream_list size:" << wait_active_stream_list.size() << ", force_copy_stream_list size:" << force_copy_stream_list.size(); std::vector> empty_list; std::shared_ptr model = std::make_shared( task_info_list, empty_list, empty_list, empty_list, empty_list, wait_active_stream_list, force_copy_stream_list, 0, - 0, 0, 0, 0, 0, stream_manager.GetCurAllocStreamNum(), label_assign_instance.GetLabelNum(NOT_NULL(graph)), - assign_instance.total_event_num(), 0); + 0, 0, 0, 0, 0, resource_manager.get_cur_stream_num(), label_assign_instance.GetLabelNum(NOT_NULL(graph)), + resource_manager.get_cur_event_num(), 0); auto ret = graph_model_map_.insert(std::make_pair(graph->graph_id(), model)); if (!ret.second) { MS_LOG(EXCEPTION) << "Duplicate GraphId! Please check in ascend_session."; @@ -437,7 +524,7 @@ bool AscendKernelRuntime::RunTask(const session::KernelGraph *graph) { bool status = ge::model_runner::ModelRunner::Instance().RunModel(graph->graph_id(), input_tensors, output_tensors); if (!status) { - MS_LOG(ERROR) << "run task failed"; + MS_LOG(ERROR) << "Run task failed"; DebugTaskIdName(graph->graph_id()); return false; } @@ -456,18 +543,18 @@ bool AscendKernelRuntime::InitDevice() { int device_count = 0; auto ret = rtGetDeviceCount(&device_count); if (ret != RT_ERROR_NONE) { - MS_EXCEPTION(DeviceProcessError) << "rtGetDeviceCount, ret[" << static_cast(ret) << "]"; + MS_EXCEPTION(DeviceProcessError) << "Call rtGetDeviceCount, ret[" << static_cast(ret) << "]"; } ret = rtSetDevice(device_id_); if (ret != RT_ERROR_NONE) { - MS_EXCEPTION(DeviceProcessError) << "rtSetDevice, ret[" << static_cast(ret) << "]"; + MS_EXCEPTION(DeviceProcessError) << "Call rtSetDevice, ret[" << static_cast(ret) << "]"; } auto context_ptr = MsContext::GetInstance(); MS_EXCEPTION_IF_NULL(context_ptr); if (context_ptr == nullptr) { - MS_LOG(ERROR) << "get MsContext instance failed"; + MS_LOG(ERROR) << "Get MsContext instance failed"; return false; } if (context_ptr->enable_hccl()) { @@ -479,17 +566,17 @@ bool AscendKernelRuntime::InitDevice() { ret = rtCtxCreate(&rt_context_, 0, device_id_); if (ret != RT_ERROR_NONE) { - MS_EXCEPTION(DeviceProcessError) << "rtCtxCreate, ret[" << static_cast(ret) << "]"; + MS_EXCEPTION(DeviceProcessError) << "Call rtCtxCreate, ret[" << static_cast(ret) << "]"; } ret = rtCtxSetCurrent(rt_context_); if (ret != RT_ERROR_NONE) { - MS_EXCEPTION(DeviceProcessError) << "rtCtxSetCurrent, ret[" << ret << "]"; + MS_EXCEPTION(DeviceProcessError) << "Call rtCtxSetCurrent, ret[" << ret << "]"; } ret = rtStreamCreate(&stream_, 0); if (ret != RT_ERROR_NONE) { - MS_LOG(EXCEPTION) << "rtStreamCreate, ret[" << ret << "]"; + MS_LOG(EXCEPTION) << "Call rtStreamCreate, ret[" << ret << "]"; } return true; @@ -498,14 +585,14 @@ bool AscendKernelRuntime::InitDevice() { bool AscendKernelRuntime::ResetDevice() { auto ret = rtCtxSetCurrent(rt_context_); if (ret != RT_ERROR_NONE) { - MS_LOG(ERROR) << "call rtCtxSetCurrent failed"; + MS_LOG(ERROR) << "Call rtCtxSetCurrent failed"; return false; } if (stream_ != nullptr) { ret = rtStreamDestroy(stream_); if (ret != RT_ERROR_NONE) { - MS_LOG(EXCEPTION) << "rtStreamDestroy, ret[" << ret << "]"; + MS_LOG(EXCEPTION) << "Call rtStreamDestroy, ret[" << ret << "]"; } stream_ = nullptr; } @@ -513,7 +600,7 @@ bool AscendKernelRuntime::ResetDevice() { if (rt_context_ != nullptr) { ret = rtCtxDestroy(rt_context_); if (ret != RT_ERROR_NONE) { - MS_EXCEPTION(DeviceProcessError) << "rtCtxDestroy, ret[" << ret << "]"; + MS_EXCEPTION(DeviceProcessError) << "Call rtCtxDestroy, ret[" << ret << "]"; } rt_context_ = nullptr; } @@ -526,26 +613,30 @@ bool AscendKernelRuntime::HcclInit() { if (!context_ptr->IsTsdOpened()) { MS_LOG(EXCEPTION) << "Hccl dependent tsd is not open"; } - MS_LOG(INFO) << "do hcom init"; + MS_LOG(INFO) << "Do hcom init"; auto config_path_str = std::getenv("MINDSPORE_HCCL_CONFIG_PATH"); if (config_path_str == nullptr) { config_path_str = std::getenv("RANK_TABLE_FILE"); if (config_path_str == nullptr) { - MS_LOG(ERROR) << "get hccl json config failed, please set env MINDSPORE_HCCL_CONFIG_PATH or RANK_TABLE_FILE"; + MS_LOG(ERROR) << "Get hccl json config failed, please set env MINDSPORE_HCCL_CONFIG_PATH or RANK_TABLE_FILE"; return false; } } + if (strlen(config_path_str) > PATH_MAX) { + MS_LOG(ERROR) << "File path oversize"; + return false; + } std::string rank_id_str = GetRankId(); auto full_path = realpath(config_path_str, nullptr); if (full_path == nullptr) { - MS_LOG(ERROR) << "file path " << config_path_str << " does not exist"; + MS_LOG(ERROR) << "File path " << config_path_str << " does not exist"; return false; } MS_LOG(INFO) << "MINDSPORE_HCCL_CONFIG_PATH : " << full_path << ", RANK_ID: " << rank_id_str; hcclResult_t res = hcom_init(full_path, rank_id_str.c_str()); free(full_path); if (res != HCCL_SUCCESS) { - MS_LOG(ERROR) << "hcom init failed, res is " << static_cast(res); + MS_LOG(ERROR) << "Hcom init failed, res is " << static_cast(res); return false; } return true; @@ -555,15 +646,15 @@ bool AscendKernelRuntime::DestroyHccl() { auto context_ptr = MsContext::GetInstance(); MS_EXCEPTION_IF_NULL(context_ptr); if (!NeedDestroyHccl()) { - MS_LOG(INFO) << "hccl is not enable, no need to close."; + MS_LOG(INFO) << "Hccl is not enable, no need to close."; return true; } hcclResult_t res = hcom_destroy(); if (res != HCCL_SUCCESS) { - MS_LOG(ERROR) << "hccl destroy failed"; + MS_LOG(ERROR) << "Hccl destroy failed"; return false; } - MS_LOG(INFO) << "hccl destroy successful, status = " << res << "."; + MS_LOG(INFO) << "Hccl destroy successful, status = " << res << "."; context_ptr->set_enable_hccl(false); return true; } diff --git a/mindspore/ccsrc/device/ascend/ascend_kernel_runtime.h b/mindspore/ccsrc/device/ascend/ascend_kernel_runtime.h index 28076f95b7..69ba8b295a 100644 --- a/mindspore/ccsrc/device/ascend/ascend_kernel_runtime.h +++ b/mindspore/ccsrc/device/ascend/ascend_kernel_runtime.h @@ -37,6 +37,7 @@ class AscendKernelRuntime : public KernelRuntime { ~AscendKernelRuntime() override; bool Init() override; bool DumpData(session::KernelGraph *graph) override; + bool LoadData(session::KernelGraph *graph, Debugger *debugger) override; bool GenTask(const session::KernelGraph *graph) override; bool RunTask(const session::KernelGraph *graph) override; bool LoadTask(const session::KernelGraph *graph) override; diff --git a/mindspore/ccsrc/device/ascend/ascend_label_assign.cc b/mindspore/ccsrc/device/ascend/ascend_label_assign.cc index 7af615f448..2db81a1725 100644 --- a/mindspore/ccsrc/device/ascend/ascend_label_assign.cc +++ b/mindspore/ccsrc/device/ascend/ascend_label_assign.cc @@ -102,7 +102,7 @@ static void AssignLabelForGotoSwitch(NotNullinsert(graph.get()); MS_LOG(INFO) << "Process label goto/switch for " << graph->ToString(); - graph->SetExecOrderByDefault(); + auto nodes = graph->execution_order(); auto end_goto = graph->get_end_goto(); if (end_goto != nullptr) { @@ -128,6 +128,7 @@ static void AssignLabelForGotoSwitch(NotNullchild_graph_order()) { AssignLabelForGotoSwitch(NOT_NULL(cg), memo); } + graph->SetExecOrderByDefault(); } void AscendLabelAssign::AssignLabel(NotNull> graph) { diff --git a/mindspore/ccsrc/device/ascend/ascend_stream_assign.cc b/mindspore/ccsrc/device/ascend/ascend_stream_assign.cc index f0bad6b492..736d6203e9 100644 --- a/mindspore/ccsrc/device/ascend/ascend_stream_assign.cc +++ b/mindspore/ccsrc/device/ascend/ascend_stream_assign.cc @@ -34,110 +34,136 @@ namespace ascend { const uint32_t kHcomMaxTask = 5; const uint32_t kCommonMaxTask = 350; -void AscendStreamAssign::AssignStream(const shared_ptr &graph_ptr) { +void AscendStreamAssign::AssignStream(const NotNull &graph_ptr) { if (IsTaskSink()) { Reset(); ReorderIndependentOrders(graph_ptr); AssignAllNodesStream(graph_ptr); UpdateAtomicAddrCleanStreamId(graph_ptr); - FindHcomParallelStreams(graph_ptr); InsertStreamActive(graph_ptr); - InsertSendRecvForHcomParallel(graph_ptr); - InsertSendRecvForIndependent(graph_ptr); - UpdateEventId(graph_ptr); + InsertEventForHcomParallel(graph_ptr); + InsertEventForIndependentParallel(graph_ptr); GetNeedActiveStreams(graph_ptr); graph_ptr->PrintGraphExecuteOrder(); - CheckStreamAssign(graph_ptr); - MS_LOG(INFO) << "after finish stream assign"; + CheckResourceAssign(graph_ptr); + MS_LOG(INFO) << "After finish stream assign"; // Get info for D Model - AscendStreamMng &stream_manager = AscendStreamMng::GetInstance(); - generator::IRModelUtil::GetInstance().set_event_num(total_event_num()); - generator::IRModelUtil::GetInstance().set_stream_num(stream_manager.GetCurAllocStreamNum()); + AscendResourceMng &resource_manager = AscendResourceMng::GetInstance(); + generator::IRModelUtil::GetInstance().set_event_num(resource_manager.get_cur_event_num()); + generator::IRModelUtil::GetInstance().set_stream_num(resource_manager.get_cur_stream_num()); // Init to 1,temporarily generator::IRModelUtil::GetInstance().set_batch_num(1); } } -// section 0 -void AscendStreamAssign::CheckStreamAssign(const shared_ptr &graph_ptr) { - MS_EXCEPTION_IF_NULL(graph_ptr); - std::set streams; - uint32_t max_stream = 0; - uint32_t min_stream = kInvalidStreamId; - const std::vector &cnode_ptr_list = graph_ptr->execution_order(); +// section 1 +void AscendStreamAssign::ReorderIndependentOrders(const NotNull &graph_ptr) { + std::vector exe_orders; + std::vector independents; + std::vector others; + + auto cnode_ptr_list = graph_ptr->execution_order(); + MS_LOG(INFO) << "Before reorder, graph orders size:" << cnode_ptr_list.size(); for (size_t i = 0; i < cnode_ptr_list.size(); ++i) { - CNodePtr cur_cnode_ptr = cnode_ptr_list[i]; + auto cur_cnode_ptr = cnode_ptr_list[i]; MS_EXCEPTION_IF_NULL(cur_cnode_ptr); - uint32_t stream_id = AnfAlgo::GetStreamId(cur_cnode_ptr); - if (stream_id == kInvalidStreamId) { - MS_LOG(EXCEPTION) << "node [" << AnfAlgo::GetCNodeName(cur_cnode_ptr) << "] had not been assigned streams"; + if (IsIndependentNode(cur_cnode_ptr)) { + independents.emplace_back(cur_cnode_ptr); + } else { + others.emplace_back(cur_cnode_ptr); } + } - streams.emplace(stream_id); - if (stream_id > max_stream) { - max_stream = stream_id; - } - if (stream_id < min_stream) { - min_stream = stream_id; - } + if (others.empty() || independents.empty()) { + MS_LOG(INFO) << "Independent or others is empty, no need reorder"; + return; } - if (!streams.empty()) { - if (min_stream != 0) { - MS_LOG(EXCEPTION) << "before stream assign, assigned stream should start from 0, now is from " << min_stream; + std::set processed; + for (size_t i = 0; i < others.size(); i++) { + auto begin = others.begin() + i; + auto end = begin + 1; + bool flag = false; + for (size_t j = 0; j < independents.size(); j++) { + auto cur_independent = independents[j]; + auto it = std::find(processed.begin(), processed.end(), cur_independent.get()); + if (it != processed.end()) { + continue; + } + + auto res = FindTargetOp(begin, end, cur_independent); + if (res != end) { + flag = true; + exe_orders.emplace_back(cur_independent); + exe_orders.emplace_back(*begin); + processed.emplace(cur_independent.get()); + break; + } } - if (max_stream != (streams.size() - 1)) { - MS_LOG(EXCEPTION) << "before stream assign, assigned stream should be consecutive"; + + if (!flag) { + exe_orders.emplace_back(*begin); } } + + MS_LOG(INFO) << "After reorder, graph orders size:" << exe_orders.size(); + if (processed.size() != independents.size()) { + MS_LOG(WARNING) << "Processed independent nodes size is not equal to exiting independent nodes size"; + return; + } + + graph_ptr->set_execution_order(exe_orders); } -// section 1 -void AscendStreamAssign::AssignAllNodesStream(const shared_ptr &graph_ptr) { - MS_EXCEPTION_IF_NULL(graph_ptr); +// section 2 +void AscendStreamAssign::AssignAllNodesStream(const NotNull &graph_ptr) { auto cnode_ptr_list = graph_ptr->execution_order(); - CNodePtr pre_cnode_ptr = nullptr; - uint32_t cur_index = 0; - uint32_t cur_stream_id = 0; - bool exit_independent = false; - AscendStreamMng &stream_manager = AscendStreamMng::GetInstance(); + bool exit_hcom = false; + AscendResourceMng &resource_manager = AscendResourceMng::GetInstance(); for (size_t i = 0; i < cnode_ptr_list.size(); ++i) { CNodePtr cur_cnode_ptr = cnode_ptr_list[i]; MS_EXCEPTION_IF_NULL(cur_cnode_ptr); + // node has been assigned stream before if (AnfAlgo::GetStreamId(cur_cnode_ptr) != kInvalidStreamId) { continue; } + + if (IsHcom(cur_cnode_ptr)) { + exit_hcom = true; + continue; + } + if (IsIndependentNode(cur_cnode_ptr)) { exit_independent = true; continue; } - // first common node, only exe one time - if (pre_cnode_ptr == nullptr) { - uint32_t cur_stream_num = stream_manager.GetCurAllocStreamNum(); - if (cur_stream_num == 0) { - cur_stream_id = stream_manager.ApplyNewStream(); - } else { - cur_stream_id = stream_manager.GetCurAllocStream(); + + AssignCommonStreamId(cur_cnode_ptr); + } + MS_LOG(INFO) << "Common start from 0, common stream nums:" << resource_manager.get_cur_stream_num(); + + if (exit_hcom) { + uint32_t first_hcom_stream_id = resource_manager.ApplyNewStream(); + for (size_t i = 0; i < cnode_ptr_list.size(); ++i) { + CNodePtr cur_cnode_ptr = cnode_ptr_list[i]; + // node has been assigned stream before + if (AnfAlgo::GetStreamId(cur_cnode_ptr) != kInvalidStreamId) { + continue; } - ++cur_index; - pre_cnode_ptr = cur_cnode_ptr; - AnfAlgo::SetStreamId(cur_stream_id, cur_cnode_ptr.get()); + if (IsHcom(cur_cnode_ptr)) { - hcom_stream_list_.emplace(cur_stream_id); + AssignHcomStreamId(cur_cnode_ptr); } - continue; } - - AssignCommonStreamId(cur_cnode_ptr, &pre_cnode_ptr, &cur_index, &cur_stream_id); + MS_LOG(INFO) << "Hcom start from :" << first_hcom_stream_id << ", hcom stream nums:" << hcom_stream_map_.size(); } if (exit_independent) { - uint32_t first_independent_stream_id = stream_manager.ApplyNewStream(); + uint32_t first_independ = resource_manager.ApplyNewStream(); for (size_t i = 0; i < cnode_ptr_list.size(); ++i) { CNodePtr cur_cnode_ptr = cnode_ptr_list[i]; - MS_EXCEPTION_IF_NULL(cur_cnode_ptr); if (AnfAlgo::GetStreamId(cur_cnode_ptr) != kInvalidStreamId) { continue; } @@ -145,28 +171,75 @@ void AscendStreamAssign::AssignAllNodesStream(const shared_ptrsecond < kCommonMaxTask) { + AnfAlgo::SetStreamId(it->first, cur_cnode_ptr.get()); + it->second++; + } else { + cur_common_stream_id = resource_manager.ApplyNewStream(); + AnfAlgo::SetStreamId(cur_common_stream_id, cur_cnode_ptr.get()); + common_stream_map_.insert(std::make_pair(cur_common_stream_id, 1)); + } + } +} + +void AscendStreamAssign::AssignHcomStreamId(const CNodePtr &cur_cnode_ptr) { + MS_EXCEPTION_IF_NULL(cur_cnode_ptr); + AscendResourceMng &resource_manager = AscendResourceMng::GetInstance(); + uint32_t cur_hcom_stream_id = resource_manager.GetCurAllocStreamId(); + auto it = hcom_stream_map_.find(cur_hcom_stream_id); + if (it == hcom_stream_map_.end()) { + AnfAlgo::SetStreamId(cur_hcom_stream_id, cur_cnode_ptr.get()); + hcom_stream_map_.insert(std::make_pair(cur_hcom_stream_id, 1)); + } else { + if (it->second < kHcomMaxTask) { + AnfAlgo::SetStreamId(it->first, cur_cnode_ptr.get()); + it->second++; + } else { + cur_hcom_stream_id = resource_manager.ApplyNewStream(); + AnfAlgo::SetStreamId(cur_hcom_stream_id, cur_cnode_ptr.get()); + hcom_stream_map_.insert(std::make_pair(cur_hcom_stream_id, 1)); + } + } } void AscendStreamAssign::AssignIndependentStreamId(const CNodePtr &cur_cnode_ptr) { MS_EXCEPTION_IF_NULL(cur_cnode_ptr); - AscendStreamMng &stream_manager = AscendStreamMng::GetInstance(); - uint32_t cur_independent_id = stream_manager.GetCurAllocStream(); + AscendResourceMng &resource_manager = AscendResourceMng::GetInstance(); + uint32_t cur_independent_id = resource_manager.GetCurAllocStreamId(); auto it = independent_stream_map_.find(cur_independent_id); if (it == independent_stream_map_.end()) { AnfAlgo::SetStreamId(cur_independent_id, cur_cnode_ptr.get()); - independent_stream_map_.emplace(cur_independent_id, 1); + independent_stream_map_.insert(std::make_pair(cur_independent_id, 1)); } else { if (it->second < kCommonMaxTask) { AnfAlgo::SetStreamId(it->first, cur_cnode_ptr.get()); it->second++; } else { - cur_independent_id = stream_manager.ApplyNewStream(); + cur_independent_id = resource_manager.ApplyNewStream(); AnfAlgo::SetStreamId(cur_independent_id, cur_cnode_ptr.get()); - independent_stream_map_.emplace(cur_independent_id, 1); + independent_stream_map_.insert(std::make_pair(cur_independent_id, 1)); } } } @@ -184,310 +257,390 @@ bool AscendStreamAssign::IsIndependentNode(const CNodePtr &node_ptr) { uint32_t input_nums = AnfAlgo::GetInputTensorNum(node_ptr); if (input_nums == 0) { - MS_LOG(INFO) << "node " << node_ptr->fullname_with_scope() << " is independent, as inputs nums is zero"; + MS_LOG(INFO) << "Node " << node_ptr->fullname_with_scope() << " is independent, as inputs nums is zero"; return true; } - const std::vector &inputs = node_ptr->inputs(); + auto inputs = node_ptr->inputs(); for (size_t i = 1; i < inputs.size(); i++) { if (!inputs[i]->isa()) { return false; } } - MS_LOG(INFO) << "node " << node_ptr->fullname_with_scope() << " is independent, as inputs is all value node"; + MS_LOG(INFO) << "Node " << node_ptr->fullname_with_scope() << " is independent, as inputs is all value node"; return true; } -void AscendStreamAssign::AssignCommonStreamId(const CNodePtr &cur_cnode_ptr, CNodePtr *pre_cnode_ptr, - uint32_t *cur_index, uint32_t *cur_stream_id) { - MS_EXCEPTION_IF_NULL(cur_cnode_ptr); - MS_EXCEPTION_IF_NULL(pre_cnode_ptr); - MS_EXCEPTION_IF_NULL(*pre_cnode_ptr); - AscendStreamMng &stream_manager = AscendStreamMng::GetInstance(); - bool over_max_hcom_task = (IsHcom(cur_cnode_ptr) && (*cur_index) % kHcomMaxTask == 0); - bool over_max_common_task = (!IsHcom(cur_cnode_ptr) && (*cur_index) % kCommonMaxTask == 0); - bool pre_common_cur_hcom = (IsHcom(cur_cnode_ptr) && !IsHcom(*pre_cnode_ptr)); - bool pre_hcom_cur_common = (!IsHcom(cur_cnode_ptr) && IsHcom(*pre_cnode_ptr)); - if (over_max_hcom_task || over_max_common_task || pre_common_cur_hcom || pre_hcom_cur_common) { - *cur_index = 0; - *cur_stream_id = stream_manager.ApplyNewStream(); - } - - ++(*cur_index); - AnfAlgo::SetStreamId(*cur_stream_id, cur_cnode_ptr.get()); - *pre_cnode_ptr = cur_cnode_ptr; - - // record ll hcom streams as hcom stream has different stream flag - if (IsHcom(cur_cnode_ptr)) { - auto it = std::find(hcom_stream_list_.begin(), hcom_stream_list_.end(), *cur_stream_id); - if (it == hcom_stream_list_.end()) { - MS_LOG(INFO) << "hcom stream id:" << *cur_stream_id; - hcom_stream_list_.emplace(*cur_stream_id); - } - } -} - -// section 2: -void AscendStreamAssign::UpdateAtomicAddrCleanStreamId(const shared_ptr &graph_ptr) { - MS_LOG(INFO) << "start"; - MS_EXCEPTION_IF_NULL(graph_ptr); - const std::vector &cnode_ptr_list = graph_ptr->execution_order(); +// section 3: +void AscendStreamAssign::UpdateAtomicAddrCleanStreamId(const NotNull &graph_ptr) { + MS_LOG(INFO) << "Start"; + auto cnode_ptr_list = graph_ptr->execution_order(); for (size_t i = 0; i < cnode_ptr_list.size(); ++i) { CNodePtr cur_cnode_ptr = cnode_ptr_list[i]; MS_EXCEPTION_IF_NULL(cur_cnode_ptr); // update AtomicAddrClean stream same witch the next node if (i > 0 && AnfAlgo::GetCNodeName(cnode_ptr_list[i - 1]) == kAtomicAddrCleanOpName) { - MS_LOG(INFO) << "update AtomicAddrClean stream id from[" << AnfAlgo::GetStreamId(cnode_ptr_list[i - 1]) - << "] to [" << AnfAlgo::GetStreamId(cur_cnode_ptr) << "]"; AnfAlgo::SetStreamId(AnfAlgo::GetStreamId(cur_cnode_ptr), cnode_ptr_list[i - 1].get()); } } - MS_LOG(INFO) << "end"; + MS_LOG(INFO) << "End"; } -// section 3 -void AscendStreamAssign::FindHcomParallelStreams(const shared_ptr &graph_ptr) { - MS_EXCEPTION_IF_NULL(graph_ptr); +// section 4 +void AscendStreamAssign::InsertStreamActive(const NotNull &graph_ptr) { + MS_LOG(INFO) << "Start"; + GetProcessedStream(graph_ptr); + std::vector update_cnode_list; CNodePtr cur_cnode_ptr = nullptr; CNodePtr pre_cnode_ptr = nullptr; uint32_t pre_stream_id = UINT32_MAX; + + bool independent_flag = !(independent_stream_map_.empty()); + bool hcom_flag = !(hcom_stream_map_.empty()); auto cnode_ptr_list = graph_ptr->execution_order(); - for (uint32_t i = 0; i < cnode_ptr_list.size(); ++i) { + for (size_t i = 0; i < cnode_ptr_list.size(); ++i) { cur_cnode_ptr = cnode_ptr_list[i]; MS_EXCEPTION_IF_NULL(cur_cnode_ptr); - uint32_t cur_stream_id = AnfAlgo::GetStreamId(cur_cnode_ptr); - if (i == 0) { - pre_cnode_ptr = cur_cnode_ptr; - pre_stream_id = cur_stream_id; + if (IsIndependentNode(cur_cnode_ptr)) { + update_cnode_list.emplace_back(cur_cnode_ptr); continue; } - bool pre_fusion_hcom = IsFusionHcom(pre_cnode_ptr); - bool diff_stream = (pre_stream_id != cur_stream_id); - if (diff_stream && pre_fusion_hcom) { - inner_parallel_streams_.emplace_back(std::vector{pre_stream_id, cur_stream_id}); + if (IsHcom(cur_cnode_ptr)) { + update_cnode_list.emplace_back(cur_cnode_ptr); + continue; + } + uint32_t cur_stream_id = AnfAlgo::GetStreamId(cur_cnode_ptr); + bool processed = IsProcessedStream(cur_stream_id); + // 1)inner stream assign, need insert active op + if (!processed) { + MS_LOG(INFO) << "Common stream active info:" << pre_stream_id << "->active" << cur_stream_id; + CNodePtr active_ptr = KernelAdjust::GetInstance().CreateStreamActiveOp(graph_ptr); + // 1.set stream id + AnfAlgo::SetStreamId(pre_stream_id, active_ptr.get()); + // 2.set active stream ids + std::vector active_index_list{cur_stream_id}; + AnfAlgo::SetNodeAttr(kAttrActiveStreamList, MakeValue>(active_index_list), active_ptr); + update_cnode_list.emplace_back(active_ptr); } - pre_cnode_ptr = cur_cnode_ptr; + if ((independent_flag || hcom_flag) && (AnfAlgo::GetCNodeName(cur_cnode_ptr) == kStreamSwitchOpName)) { + MS_LOG(INFO) << "Insert StreamActive op after FP StreamSwitch for stream parallel"; + UpdateStreamSwitch(graph_ptr, cur_cnode_ptr, &update_cnode_list); + } else { + update_cnode_list.emplace_back(cur_cnode_ptr); + } + + processed_streams_.emplace(cur_stream_id); pre_stream_id = cur_stream_id; + pre_cnode_ptr = cur_cnode_ptr; } + graph_ptr->set_execution_order(update_cnode_list); + MS_LOG(INFO) << "End"; } -// section 4 -void AscendStreamAssign::UpdateStreamSwitch(const std::shared_ptr &graph_ptr, - const CNodePtr &switch_ptr, const vector &independent_stream, +void AscendStreamAssign::GetProcessedStream(const NotNull &graph_ptr) { + // 0 stream is activated at first + processed_streams_.emplace(0); + auto cnode_ptr_list = graph_ptr->execution_order(); + for (size_t i = 0; i < cnode_ptr_list.size(); ++i) { + auto cur_cnode_ptr = cnode_ptr_list[i]; + uint32_t cur_stream_id = AnfAlgo::GetStreamId(cur_cnode_ptr); + + if (AnfAlgo::GetCNodeName(cur_cnode_ptr) == kStreamSwitchOpName) { + auto true_stream_id = AnfAlgo::GetNodeAttr(cur_cnode_ptr, kAttrTrueBranchStream); + processed_streams_.emplace(true_stream_id); + + if (!AnfAlgo::HasNodeAttr(kStreamNeedActivedFirst, cur_cnode_ptr)) { + continue; + } + auto need_active = AnfAlgo::GetNodeAttr(cur_cnode_ptr, kStreamNeedActivedFirst); + if (need_active) { + processed_streams_.emplace(cur_stream_id); + } + } + } + for (const auto &item : processed_streams_) { + MS_LOG(INFO) << "Before active:" << item << " is been processed"; + } +} + +void AscendStreamAssign::UpdateStreamSwitch(const NotNull &graph_ptr, const CNodePtr &switch_ptr, vector *orders) { - MS_EXCEPTION_IF_NULL(orders); orders->emplace_back(switch_ptr); - auto primitive = AnfAlgo::GetCNodePrimitive(switch_ptr); - MS_EXCEPTION_IF_NULL(primitive); - auto value_ptr = primitive->GetAttr(kStreamNeedActivedFirst); - if (value_ptr == nullptr) { + if (!AnfAlgo::HasNodeAttr(kStreamNeedActivedFirst, switch_ptr)) { return; } - auto need_active = GetValue(value_ptr); + auto need_active = AnfAlgo::GetNodeAttr(switch_ptr, kStreamNeedActivedFirst); if (!need_active) { return; } - MS_LOG(INFO) << "start update switch op[" << switch_ptr->DebugString() << "]"; MS_EXCEPTION_IF_NULL(switch_ptr); - auto true_stream_id = GetValue(primitive->GetAttr(kAttrTrueBranchStream)); - MS_LOG(INFO) << "streamswtich stream id[" << AnfAlgo::GetStreamId(switch_ptr) << "], true_logic_id[" << true_stream_id - << "]"; + auto true_stream_id = AnfAlgo::GetNodeAttr(switch_ptr, kAttrTrueBranchStream); + MS_LOG(INFO) << "Streamswtich stream id:" << AnfAlgo::GetStreamId(switch_ptr) + << "; active stream id:" << true_stream_id; CNodePtr active_ptr = KernelAdjust::GetInstance().CreateStreamActiveOp(graph_ptr); - MS_LOG(INFO) << "start update StreamActive op[" << active_ptr->DebugString() << "]"; AnfAlgo::SetStreamId(true_stream_id, active_ptr.get()); - AnfAlgo::SetNodeAttr(kAttrActiveStreamList, MakeValue>(independent_stream), active_ptr); - independent_stream_activated_ = true; + vector active_ids; + // active indepdent stream + for (const auto &item : independent_stream_map_) { + active_ids.emplace_back(item.first); + } + // active hcom stream + for (const auto &item : hcom_stream_map_) { + active_ids.emplace_back(item.first); + } + AnfAlgo::SetNodeAttr(kAttrActiveStreamList, MakeValue>(active_ids), active_ptr); // update processed stream - for (auto &item : independent_stream) { - processed_streams_.emplace(item); + independent_stream_activated_ = true; + for (const auto &item : independent_stream_map_) { + processed_streams_.emplace(item.first); + } + + hcom_stream_activated_ = true; + for (const auto &item : hcom_stream_map_) { + processed_streams_.emplace(item.first); } orders->emplace_back(active_ptr); -} // namespace ascend +} -void AscendStreamAssign::InsertStreamActive(const std::shared_ptr &graph_ptr) { - MS_LOG(INFO) << "start"; - MS_EXCEPTION_IF_NULL(graph_ptr); - std::vector update_cnode_list; - CNodePtr cur_cnode_ptr = nullptr; - CNodePtr pre_cnode_ptr = nullptr; - uint32_t pre_stream_id = UINT32_MAX; - std::vector independent_stream; - MS_LOG(INFO) << "independent stream size:" << independent_stream_map_.size(); - for (auto item : independent_stream_map_) { - independent_stream.emplace_back(item.first); +bool AscendStreamAssign::IsProcessedStream(uint32_t stream_id) { + auto it = std::find(processed_streams_.begin(), processed_streams_.end(), stream_id); + if (it != processed_streams_.end()) { + return true; } + return false; +} - bool independent_flag = !(independent_stream.empty()); +// section5 +void AscendStreamAssign::InsertEventForHcomParallel(const NotNull &graph_ptr) { + MS_LOG(INFO) << "Start"; + InsertEventCommonDependHcom(graph_ptr); + InsertEventHcomDependCommon(graph_ptr); + InsertEventHcomDependHcom(graph_ptr); + MS_LOG(INFO) << "End"; +} + +void AscendStreamAssign::InsertEventCommonDependHcom(const NotNull &graph_ptr) { + AscendResourceMng &resource_manager = AscendResourceMng::GetInstance(); + auto cnode_ptr_list = graph_ptr->execution_order(); + vector cnodes = cnode_ptr_list; + uint32_t cur_event_id = resource_manager.ApplyNewEvent(); + auto it = cnodes.begin(); + while (it != cnodes.end() && (it + 1) != cnodes.end()) { + MS_EXCEPTION_IF_NULL(*it); + MS_EXCEPTION_IF_NULL(*(it + 1)); + if (IsHcom(*it) && !IsHcom(*(it + 1))) { + CNodePtr send_cnode_ptr = CreateSendApplyKernel(graph_ptr, cur_event_id, AnfAlgo::GetStreamId(*it)); + it = cnodes.insert(it + 1, send_cnode_ptr); + + auto target = FindTargetOp(it, cnodes.end(), *(it - 1)); + if (target == cnodes.end()) { + MS_LOG(WARNING) << "Hcom node:" << (*(it - 1))->fullname_with_scope() + << ", can't find target for insert recv op, no insert send/recv"; + it = cnodes.erase(it); + continue; + } + + if (IsHcom(*target)) { + it = cnodes.erase(it); + continue; + } + + // deal recv op + uint32_t stream_id = AnfAlgo::GetStreamId(*target); + CNodePtr recv_cnode_ptr = CreateRecvApplyKernel(graph_ptr, cur_event_id, stream_id); + (void)cnodes.insert(target, recv_cnode_ptr); + cur_event_id = resource_manager.ApplyNewEvent(); + } + ++it; + } + // one event allocated additional, should delete + resource_manager.DeleteEvent(); + graph_ptr->set_execution_order(cnodes); + MS_LOG(INFO) << "After common depend hcom, total event nums:" << resource_manager.get_cur_event_num(); +} - const std::vector &cnode_ptr_list = graph_ptr->execution_order(); +void AscendStreamAssign::InsertEventHcomDependCommon(const NotNull &graph_ptr) { + AscendResourceMng &resource_manager = AscendResourceMng::GetInstance(); + auto cnode_ptr_list = graph_ptr->execution_order(); + vector cnodes; + CNodePtr cur_cnode_ptr = nullptr; + uint32_t pre_stream_id = UINT32_MAX; for (size_t i = 0; i < cnode_ptr_list.size(); ++i) { cur_cnode_ptr = cnode_ptr_list[i]; - MS_EXCEPTION_IF_NULL(cur_cnode_ptr); uint32_t cur_stream_id = AnfAlgo::GetStreamId(cur_cnode_ptr); - if (IsIndependentNode(cur_cnode_ptr)) { - update_cnode_list.emplace_back(cur_cnode_ptr); + MS_EXCEPTION_IF_NULL(cur_cnode_ptr); + if (i == 0) { + cnodes.emplace_back(cur_cnode_ptr); + pre_stream_id = cur_stream_id; continue; } - bool inner_active = false; - if (pre_cnode_ptr != nullptr) { - inner_active = pre_stream_id != cur_stream_id && AnfAlgo::GetCNodeName(pre_cnode_ptr) != kStreamSwitchOpName && - AnfAlgo::GetCNodeName(pre_cnode_ptr) != kSendOpName; + if (!IsHcom(cur_cnode_ptr)) { + cnodes.emplace_back(cur_cnode_ptr); + pre_stream_id = cur_stream_id; + continue; } - bool processed = IsProcessedStream(cur_stream_id); - // 1)inner stream assign, need insert active op - if (inner_active && !processed) { - MS_LOG(INFO) << "Inner insert active op, self stream id[" << pre_stream_id << "]"; - CNodePtr active_ptr = KernelAdjust::GetInstance().CreateStreamActiveOp(graph_ptr); - // 1.set stream id - AnfAlgo::SetStreamId(pre_stream_id, active_ptr.get()); - // 2.set active stream ids - std::vector active_index_list; - GetParallelStream(cur_stream_id, pre_stream_id, &active_index_list); - AnfAlgo::SetNodeAttr(kAttrActiveStreamList, MakeValue>(active_index_list), active_ptr); - update_cnode_list.emplace_back(active_ptr); + if (cur_stream_id == pre_stream_id) { + cnodes.emplace_back(cur_cnode_ptr); + pre_stream_id = cur_stream_id; + continue; } - if (independent_flag && (AnfAlgo::GetCNodeName(cur_cnode_ptr) == kStreamSwitchOpName)) { - MS_LOG(INFO) << "Insert StreamActive op after FP StreamSwitch for stream parallel"; - UpdateStreamSwitch(graph_ptr, cur_cnode_ptr, independent_stream, &update_cnode_list); - } else { - update_cnode_list.emplace_back(cur_cnode_ptr); + if (!IsHcom(cnode_ptr_list[i - 1])) { + uint32_t cur_event_id = resource_manager.ApplyNewEvent(); + auto send = CreateSendApplyKernel(graph_ptr, cur_event_id, pre_stream_id); + cnodes.emplace_back(send); + auto recv = CreateRecvApplyKernel(graph_ptr, cur_event_id, cur_stream_id); + cnodes.emplace_back(recv); + cnodes.emplace_back(cur_cnode_ptr); } - - processed_streams_.emplace(cur_stream_id); pre_stream_id = cur_stream_id; - pre_cnode_ptr = cur_cnode_ptr; - } - graph_ptr->set_execution_order(update_cnode_list); - MS_LOG(INFO) << "end"; -} - -bool AscendStreamAssign::IsProcessedStream(uint32_t stream_id) { - auto it = std::find(processed_streams_.begin(), processed_streams_.end(), stream_id); - if (it != processed_streams_.end()) { - return true; - } - return false; -} - -void AscendStreamAssign::GetParallelStream(uint32_t cur_stream_id, uint32_t stream_acitve_id, - vector *parallel_streams) { - MS_EXCEPTION_IF_NULL(parallel_streams); - for (size_t i = 0; i < inner_parallel_streams_.size(); i++) { - const auto &cur_parallel_streams = inner_parallel_streams_[i]; - auto it = std::find(cur_parallel_streams.begin(), cur_parallel_streams.end(), cur_stream_id); - if (it != cur_parallel_streams.end()) { - MS_LOG(INFO) << "stream id:" << cur_stream_id << " is parallel stream"; - for (size_t j = 0; j < cur_parallel_streams.size(); j++) { - if (cur_parallel_streams[j] == stream_acitve_id) { - MS_LOG(INFO) << "one of parallel stream id" << cur_parallel_streams[j] - << "is same with streamacvite stream id" << stream_acitve_id; - continue; - } - (*parallel_streams).emplace_back(cur_parallel_streams[j]); - processed_streams_.emplace(cur_parallel_streams[j]); - } - return; - } } - processed_streams_.emplace(cur_stream_id); - (*parallel_streams).push_back(cur_stream_id); + graph_ptr->set_execution_order(cnodes); + MS_LOG(INFO) << "After hcom depend common, total event nums:" << resource_manager.get_cur_event_num(); } -// section5 -void AscendStreamAssign::InsertSendRecvForDiffHcom(const shared_ptr &graph_ptr) { - MS_LOG(INFO) << "start"; - MS_EXCEPTION_IF_NULL(graph_ptr); +void AscendStreamAssign::InsertEventHcomDependHcom(const NotNull &graph_ptr) { + AscendResourceMng &resource_manager = AscendResourceMng::GetInstance(); auto cnode_ptr_list = graph_ptr->execution_order(); - vector fusion_hcom_index; - vector orders; + uint32_t first_hcom_stream = kInvalidStreamId; + uint32_t last_hcom_stream = kInvalidStreamId; + // key: stream id, value:hcom index + std::map> hcom_index; for (size_t i = 0; i < cnode_ptr_list.size(); i++) { auto cur_cnode = cnode_ptr_list[i]; - if (IsFusionHcom(cur_cnode)) { - fusion_hcom_index.emplace_back(i); + if (!IsHcom(cur_cnode)) { + continue; + } + uint32_t cur_stream_id = AnfAlgo::GetStreamId(cur_cnode); + auto it = hcom_index.find(cur_stream_id); + if (it != hcom_index.end()) { + hcom_index[cur_stream_id].emplace_back(i); + } else { + hcom_index[cur_stream_id] = {i}; + } + + // record first hcom stream id + if (first_hcom_stream == kInvalidStreamId) { + first_hcom_stream = cur_stream_id; + } + + // record last hcom stream id + if (cur_stream_id != last_hcom_stream) { + last_hcom_stream = cur_stream_id; } } - if (fusion_hcom_index.size() < 2) { - MS_LOG(INFO) << "fusion hcom size is less than 2, no need insert event between them"; + + if (hcom_index.size() < 2) { + MS_LOG(INFO) << "Different stream hcom size is less than 2, no need insert event between them"; return; } - uint32_t first_index = fusion_hcom_index[0]; - uint32_t last_index = fusion_hcom_index[fusion_hcom_index.size() - 1]; - uint32_t cur_event_id = total_event_num_; - uint32_t pre_hcom_stream_id = kInvalidStreamId; - std::copy(cnode_ptr_list.begin(), cnode_ptr_list.begin() + first_index, std::back_inserter(orders)); - for (size_t i = first_index; i <= last_index; i++) { + InsertEventBetweenHcom(graph_ptr, hcom_index, first_hcom_stream, last_hcom_stream); + MS_LOG(INFO) << "After hcom depend hcom, total event nums:" << resource_manager.get_cur_event_num(); +} + +void AscendStreamAssign::InsertEventBetweenHcom(const NotNull &graph_ptr, + const map> &hcom_index, + uint32_t first_hcom_stream, uint32_t last_hcom_stream) { + vector orders; + AscendResourceMng &resource_manager = AscendResourceMng::GetInstance(); + auto cnode_ptr_list = graph_ptr->execution_order(); + uint32_t cur_event_id = resource_manager.ApplyNewEvent(); + size_t first_stream_last_index = hcom_index.at(first_hcom_stream).back(); + size_t last_stream_first_index = hcom_index.at(last_hcom_stream).front(); + std::copy(cnode_ptr_list.begin(), cnode_ptr_list.begin() + first_stream_last_index, std::back_inserter(orders)); + for (size_t i = first_stream_last_index; i <= last_stream_first_index; i++) { auto cur_cnode = cnode_ptr_list[i]; - auto it = std::find(fusion_hcom_index.begin(), fusion_hcom_index.end(), i); - if (it == fusion_hcom_index.end()) { + if (!IsSatisfiedHcom(hcom_index, cur_cnode, i)) { orders.emplace_back(cur_cnode); continue; } auto cur_hcom_stream_id = AnfAlgo::GetStreamId(cur_cnode); - if (cur_hcom_stream_id == pre_hcom_stream_id) { - orders.emplace_back(cur_cnode); - continue; - } - if (i == first_index) { + if (i == first_stream_last_index) { // first fusion hcom orders.emplace_back(cur_cnode); auto send = CreateSendApplyKernel(graph_ptr, cur_event_id, cur_hcom_stream_id); orders.emplace_back(send); - } else if (i == last_index) { + } else if (i == last_stream_first_index) { // last fusion hcom auto recv = CreateRecvApplyKernel(graph_ptr, cur_event_id, cur_hcom_stream_id); orders.emplace_back(recv); orders.emplace_back(cur_cnode); - cur_event_id++; } else { - auto recv = CreateRecvApplyKernel(graph_ptr, cur_event_id, cur_hcom_stream_id); - orders.emplace_back(recv); - cur_event_id++; - orders.emplace_back(cur_cnode); - auto send = CreateSendApplyKernel(graph_ptr, cur_event_id, cur_hcom_stream_id); - orders.emplace_back(send); + auto cur_stream_hcom_size = hcom_index.at(cur_hcom_stream_id).size(); + if (cur_stream_hcom_size == 1) { + auto recv = CreateRecvApplyKernel(graph_ptr, cur_event_id, cur_hcom_stream_id); + orders.emplace_back(recv); + cur_event_id = resource_manager.ApplyNewEvent(); + orders.emplace_back(cur_cnode); + auto send = CreateSendApplyKernel(graph_ptr, cur_event_id, cur_hcom_stream_id); + orders.emplace_back(send); + } else { + // current stream, first hcom:add recv op + if (i == hcom_index.at(cur_hcom_stream_id).front()) { + auto recv = CreateRecvApplyKernel(graph_ptr, cur_event_id, cur_hcom_stream_id); + orders.emplace_back(recv); + cur_event_id = resource_manager.ApplyNewEvent(); + orders.emplace_back(cur_cnode); + } else if (i == hcom_index.at(cur_hcom_stream_id).back()) { + // current stream, last hcom:add send op + orders.emplace_back(cur_cnode); + auto send = CreateSendApplyKernel(graph_ptr, cur_event_id, cur_hcom_stream_id); + orders.emplace_back(send); + } else { + // current stream, not first and last op + orders.emplace_back(cur_cnode); + } + } } - pre_hcom_stream_id = cur_hcom_stream_id; } - std::copy(cnode_ptr_list.begin() + last_index + 1, cnode_ptr_list.end(), std::back_inserter(orders)); + std::copy(cnode_ptr_list.begin() + last_stream_first_index + 1, cnode_ptr_list.end(), std::back_inserter(orders)); graph_ptr->set_execution_order(orders); - total_event_num_ = cur_event_id; - MS_LOG(INFO) << "after indsert between allreduce, total event nums[" << total_event_num_ << "]\n end"; } -void AscendStreamAssign::InsertSendRecvForHcomParallel(const shared_ptr &graph_ptr) { - MS_LOG(INFO) << "start"; - MS_EXCEPTION_IF_NULL(graph_ptr); +bool AscendStreamAssign::IsSatisfiedHcom(const std::map> &hcom_index, const CNodePtr &node_ptr, + size_t index) { + MS_EXCEPTION_IF_NULL(node_ptr); + auto cur_hcom_stream_id = AnfAlgo::GetStreamId(node_ptr); + auto it = hcom_index.find(cur_hcom_stream_id); + if (it == hcom_index.end()) { + return false; + } + auto iter = std::find(hcom_index.at(cur_hcom_stream_id).begin(), hcom_index.at(cur_hcom_stream_id).end(), index); + if (iter == hcom_index.at(cur_hcom_stream_id).end()) { + return false; + } + return true; +} + +// section6 +void AscendStreamAssign::InsertEventForIndependentParallel(const NotNull &graph_ptr) { + MS_LOG(INFO) << "Start"; + AscendResourceMng &resource_manager = AscendResourceMng::GetInstance(); auto cnode_ptr_list = graph_ptr->execution_order(); vector cnodes = cnode_ptr_list; - uint32_t cur_event_id = 0; + uint32_t cur_event_id = resource_manager.ApplyNewEvent(); auto it = cnodes.begin(); - while (it != cnodes.end() && (it + 1) != cnodes.end()) { + while (it != cnodes.end()) { MS_EXCEPTION_IF_NULL(*it); - MS_EXCEPTION_IF_NULL(*(it + 1)); - if (IsHcom(*it) && !IsHcom(*(it + 1))) { - bool is_fusion = IsFusionHcom(*it); - if (!is_fusion) { - ++it; - continue; - } + if (IsIndependentNode(*it)) { + MS_LOG(INFO) << "Deal independent op[" << (*it)->DebugString() << "]"; CNodePtr send_cnode_ptr = CreateSendApplyKernel(graph_ptr, cur_event_id, AnfAlgo::GetStreamId(*it)); it = cnodes.insert(it + 1, send_cnode_ptr); auto target = FindTargetOp(it, cnodes.end(), *(it - 1)); if (target == cnodes.end()) { - MS_LOG(WARNING) << "hcom node[" << (*(it - 1))->fullname_with_scope() - << "] can't find target for insert recv op, no insert send/recv"; + MS_LOG(DEBUG) << "Independ node[" << (*(it - 1))->fullname_with_scope() + << "] can't find target for insert recv op, no insert send/recv"; it = cnodes.erase(it); continue; } @@ -496,93 +649,149 @@ void AscendStreamAssign::InsertSendRecvForHcomParallel(const shared_ptrset_execution_order(cnodes); - total_event_num_ = cur_event_id; - MS_LOG(INFO) << "after insert send/recv for hcom parallel, total event nums[" << total_event_num_ << "]"; - - // Insert Send/Recv between Hcom(such as:AllReduce1 Send1 Common Recv1 AllReduce2) - InsertSendRecvForDiffHcom(graph_ptr); - MS_LOG(INFO) << "end"; + MS_LOG(INFO) << "After independent parallel, total event nums:" << resource_manager.get_cur_event_num(); + MS_LOG(INFO) << "End"; } -void AscendStreamAssign::UpdateEventId(const shared_ptr &graph_ptr) { - MS_LOG(INFO) << "start"; - MS_EXCEPTION_IF_NULL(graph_ptr); +// section7 +void AscendStreamAssign::GetNeedActiveStreams(const NotNull &graph_ptr) { CNodePtr cur_cnode_ptr = nullptr; - // key:virutal event id, value:real event id - std::unordered_map event_id_map; - uint32_t event_id; auto cnode_ptr_list = graph_ptr->execution_order(); - for (size_t i = 0; i < cnode_ptr_list.size(); ++i) { - cur_cnode_ptr = cnode_ptr_list[i]; - MS_EXCEPTION_IF_NULL(cur_cnode_ptr); - if (AnfAlgo::GetCNodeName(cur_cnode_ptr) == kSendOpName || AnfAlgo::GetCNodeName(cur_cnode_ptr) == kRecvOpName) { - auto primitive = AnfAlgo::GetCNodePrimitive(cur_cnode_ptr); - MS_EXCEPTION_IF_NULL(primitive); - event_id = GetValue(primitive->GetAttr(kAttrEventId)); - // before stream assign, send/recv event_id assign from kFirstEventId - if (event_id < kFirstEventId) { - continue; - } - auto it = event_id_map.find(event_id); - if (it == event_id_map.end()) { - event_id_map.insert(std::make_pair(event_id, total_event_num_)); - AnfAlgo::SetNodeAttr(kAttrEventId, MakeValue(total_event_num_), cur_cnode_ptr); - total_event_num_++; - } else { - AnfAlgo::SetNodeAttr(kAttrEventId, MakeValue(it->second), cur_cnode_ptr); - } - } - } -} + // 1)first stream 0 should be actived first; + need_first_active_streams_.emplace_back(0); -void AscendStreamAssign::GetNeedActiveStreams(const shared_ptr &graph_ptr) { - MS_EXCEPTION_IF_NULL(graph_ptr); - CNodePtr cur_cnode_ptr = nullptr; - auto cnode_ptr_list = graph_ptr->execution_order(); - // 1)stream witch kStreamNeedActivedFirst attr should be actived; + // 2)stream witch kStreamNeedActivedFirst attr should be actived; for (size_t i = 0; i < cnode_ptr_list.size(); ++i) { cur_cnode_ptr = cnode_ptr_list[i]; MS_EXCEPTION_IF_NULL(cur_cnode_ptr); - ValuePtr value_ptr = nullptr; - auto primitive = AnfAlgo::GetCNodePrimitive(cur_cnode_ptr); - if (primitive != nullptr) { - value_ptr = primitive->GetAttr(kStreamNeedActivedFirst); - } else { - auto func_graph = AnfAlgo::GetCNodeFuncGraphPtr(cur_cnode_ptr); - MS_EXCEPTION_IF_NULL(func_graph); - value_ptr = func_graph->get_attr(kStreamNeedActivedFirst); - } - if (value_ptr == nullptr) { + if (!AnfAlgo::HasNodeAttr(kStreamNeedActivedFirst, cur_cnode_ptr)) { continue; } - auto need_active = GetValue(value_ptr); + auto need_active = AnfAlgo::GetNodeAttr(cur_cnode_ptr, kStreamNeedActivedFirst); if (need_active) { auto stream_id = AnfAlgo::GetStreamId(cur_cnode_ptr); - MS_LOG(INFO) << "stream id:" << stream_id << " is need actived at first"; + MS_LOG(INFO) << "Stream id:" << stream_id << " is need actived at first"; need_first_active_streams_.push_back(stream_id); } } - // 2)first stream 0 should be actived first; - need_first_active_streams_.emplace_back(0); - // 3)independent stream:if has not been activate, push to need active vector if (!independent_stream_activated_) { for (auto &item : independent_stream_map_) { need_first_active_streams_.emplace_back(item.first); } } + + // 4)hcom stream:if has not been activate, push to need active vector + if (!hcom_stream_activated_) { + for (auto &item : hcom_stream_map_) { + need_first_active_streams_.emplace_back(item.first); + } + } +} + +// section8 +void AscendStreamAssign::CheckResourceAssign(const NotNull &graph_ptr) { + CheckStreamAssign(graph_ptr); + CheckEventAssign(graph_ptr); +} + +void AscendStreamAssign::CheckStreamAssign(const NotNull &graph_ptr) { + AscendResourceMng &resource_manager = AscendResourceMng::GetInstance(); + std::set streams; + uint32_t max_stream = 0; + uint32_t min_stream = kInvalidStreamId; + auto cnode_ptr_list = graph_ptr->execution_order(); + for (size_t i = 0; i < cnode_ptr_list.size(); ++i) { + CNodePtr cur_cnode_ptr = cnode_ptr_list[i]; + MS_EXCEPTION_IF_NULL(cur_cnode_ptr); + uint32_t stream_id = AnfAlgo::GetStreamId(cur_cnode_ptr); + if (stream_id == kInvalidStreamId) { + MS_LOG(EXCEPTION) << "Node:" << AnfAlgo::GetCNodeName(cur_cnode_ptr) << "had not been assigned stream"; + } + + (void)streams.emplace(stream_id); + if (stream_id > max_stream) { + max_stream = stream_id; + } + if (stream_id < min_stream) { + min_stream = stream_id; + } + } + + // check stream assign + if (!streams.empty()) { + if (min_stream != 0) { + MS_LOG(EXCEPTION) << "Stream should start from 0, now is from " << min_stream; + } + uint32_t assigned_stream_num = resource_manager.get_cur_stream_num(); + if ((max_stream != assigned_stream_num - 1) || (streams.size() != assigned_stream_num)) { + MS_LOG(EXCEPTION) << "Stream should be consecutive, max stream id:" << max_stream + << "; alloc stream nums:" << assigned_stream_num << "; streams size:" << streams.size(); + } + } +} + +void AscendStreamAssign::CheckEventAssign(const NotNull &graph_ptr) { + AscendResourceMng &resource_manager = AscendResourceMng::GetInstance(); + std::map> event_map; + uint32_t max_event_id = 0; + uint32_t min_event_id = kInvalidEventId; + auto cnode_ptr_list = graph_ptr->execution_order(); + for (size_t i = 0; i < cnode_ptr_list.size(); ++i) { + CNodePtr cur_cnode_ptr = cnode_ptr_list[i]; + MS_EXCEPTION_IF_NULL(cur_cnode_ptr); + auto name = AnfAlgo::GetCNodeName(cur_cnode_ptr); + if (name == kSendOpName || name == kRecvOpName) { + uint32_t event_id = AnfAlgo::GetNodeAttr(cur_cnode_ptr, kAttrEventId); + if (event_id > max_event_id) { + max_event_id = event_id; + } + + if (event_id < min_event_id) { + min_event_id = event_id; + } + auto it = event_map.find(event_id); + if (it == event_map.end()) { + event_map[event_id] = {cur_cnode_ptr}; + } else { + event_map[event_id].emplace_back(cur_cnode_ptr); + } + } + } + // check event assign + if (!event_map.empty()) { + if (min_event_id != 0) { + MS_LOG(EXCEPTION) << "Event should start from 0, now is from " << min_event_id; + } + uint32_t assigned_event_num = resource_manager.get_cur_event_num(); + if ((max_event_id != assigned_event_num - 1) || (event_map.size() != assigned_event_num)) { + MS_LOG(EXCEPTION) << "Event should be consecutive"; + } + for (const auto &item : event_map) { + if (item.second.size() != 2) { + MS_LOG(EXCEPTION) << "Send/recv should be in pair and share one event id"; + } + auto first_name = AnfAlgo::GetCNodeName(item.second[0]); + auto second_name = AnfAlgo::GetCNodeName(item.second[1]); + if (!(first_name == kSendOpName && second_name == kRecvOpName)) { + MS_LOG(EXCEPTION) << "Send should be before recv"; + } + } + } } -CNodePtr AscendStreamAssign::CreateSendApplyKernel(const std::shared_ptr &graph_ptr, - uint32_t event_id, uint32_t stream_id) { - MS_EXCEPTION_IF_NULL(graph_ptr); +// section9 +CNodePtr AscendStreamAssign::CreateSendApplyKernel(const NotNull &graph_ptr, uint32_t event_id, + uint32_t stream_id) { auto send_op = std::make_shared(kSendOpName); MS_EXCEPTION_IF_NULL(send_op); auto send_apply = std::make_shared(send_op); @@ -601,9 +810,8 @@ CNodePtr AscendStreamAssign::CreateSendApplyKernel(const std::shared_ptr &graph_ptr, - uint32_t event_id, uint32_t stream_id) { - MS_EXCEPTION_IF_NULL(graph_ptr); +CNodePtr AscendStreamAssign::CreateRecvApplyKernel(const NotNull &graph_ptr, uint32_t event_id, + uint32_t stream_id) { auto recv_op = std::make_shared(kRecvOpName); MS_EXCEPTION_IF_NULL(recv_op); auto recv_apply = std::make_shared(recv_op); @@ -641,7 +849,7 @@ vector::iterator AscendStreamAssign::FindTargetOp(vector::it } else { auto real_input = AnfAlgo::VisitKernel(input, 0); if (node == real_input.first) { - MS_LOG(INFO) << "find target op[" << (*begin)->DebugString() << "]"; + MS_LOG(INFO) << "Find target op[" << (*begin)->DebugString() << "]"; return begin; } } @@ -649,62 +857,26 @@ vector::iterator AscendStreamAssign::FindTargetOp(vector::it ++begin; } return end; -} // namespace ascend - -void AscendStreamAssign::InsertSendRecvForIndependent(const shared_ptr &graph_ptr) { - MS_LOG(INFO) << "start"; - MS_EXCEPTION_IF_NULL(graph_ptr); - auto cnode_ptr_list = graph_ptr->execution_order(); - vector cnodes = cnode_ptr_list; - uint32_t cur_event_id = total_event_num_; - auto it = cnodes.begin(); - while (it != cnodes.end()) { - MS_EXCEPTION_IF_NULL(*it); - if (IsIndependentNode(*it)) { - MS_LOG(INFO) << "deal independent op[" << (*it)->DebugString() << "]"; - CNodePtr send_cnode_ptr = CreateSendApplyKernel(graph_ptr, cur_event_id, AnfAlgo::GetStreamId(*it)); - it = cnodes.insert(it + 1, send_cnode_ptr); - - auto target = FindTargetOp(it, cnodes.end(), *(it - 1)); - if (target == cnodes.end()) { - MS_LOG(DEBUG) << "independ node[" << (*(it - 1))->fullname_with_scope() - << "] can't find target for insert recv op, no insert send/recv"; - it = cnodes.erase(it); - continue; - } - - // deal recv op - uint32_t stream_id = AnfAlgo::GetStreamId(*target); - CNodePtr recv_cnode_ptr = CreateRecvApplyKernel(graph_ptr, cur_event_id, stream_id); - (void)cnodes.insert(target, recv_cnode_ptr); - ++cur_event_id; - } - ++it; - } - graph_ptr->set_execution_order(cnodes); - total_event_num_ = cur_event_id; - MS_LOG(INFO) << "total event nums[" << total_event_num_ << "]"; - MS_LOG(INFO) << "end"; } bool AscendStreamAssign::IsTaskSink() { auto ms_context = MsContext::GetInstance(); MS_EXCEPTION_IF_NULL(ms_context); if (!ms_context->enable_task_sink()) { - MS_LOG(INFO) << "task sink mode is not enable"; + MS_LOG(INFO) << "Task sink mode is not enable"; return false; } else { - MS_LOG(INFO) << "task sink mode is enable"; + MS_LOG(INFO) << "Task sink mode is enable"; return true; } } void AscendStreamAssign::GetWaitStreams(vector *wait_active_stream_list) { MS_EXCEPTION_IF_NULL(wait_active_stream_list); - AscendStreamMng &stream_manager = AscendStreamMng::GetInstance(); - uint32_t total_stream_num = stream_manager.GetCurAllocStreamNum(); + AscendResourceMng &resource_manager = AscendResourceMng::GetInstance(); + uint32_t total_stream_num = resource_manager.get_cur_stream_num(); if (total_stream_num == 0) { - MS_LOG(INFO) << "total_common_stream_num is zero"; + MS_LOG(INFO) << "The total_common_stream_num is zero"; return; } @@ -712,8 +884,8 @@ void AscendStreamAssign::GetWaitStreams(vector *wait_active_stream_lis for (uint32_t i = 0; i < total_stream_num; i++) { auto it = std::find(need_first_active_streams_.begin(), need_first_active_streams_.end(), i); if (it == need_first_active_streams_.end()) { - MS_LOG(INFO) << "wait common stream id = " << i; - (*wait_active_stream_list).push_back(i); + MS_LOG(INFO) << "Wait common stream id = " << i; + wait_active_stream_list->push_back(i); } } } @@ -723,94 +895,21 @@ bool AscendStreamAssign::IsHcom(const CNodePtr &apply_kernel) { return AnfAlgo::GetKernelType(apply_kernel) == HCCL_KERNEL; } -bool AscendStreamAssign::IsFusionHcom(const CNodePtr &cur_cnode_ptr) { - MS_EXCEPTION_IF_NULL(cur_cnode_ptr); - bool is_hcom = IsHcom(cur_cnode_ptr); - if (!is_hcom) { - return false; - } - - if (!AnfAlgo::HasNodeAttr(kAttrFusion, cur_cnode_ptr)) { - return false; - } - - if (AnfAlgo::GetNodeAttr(cur_cnode_ptr, kAttrFusion) == 0) { - return false; - } - - return true; -} - void AscendStreamAssign::GetHcomStreams(std::vector *streams) { MS_EXCEPTION_IF_NULL(streams); - for (const auto &stream : hcom_stream_list_) { - (*streams).emplace_back(stream); - } -} - -void AscendStreamAssign::ReorderIndependentOrders(const shared_ptr &graph_ptr) { - MS_EXCEPTION_IF_NULL(graph_ptr); - CNodePtr cur_cnode_ptr = nullptr; - std::vector exe_orders; - std::vector independents; - std::vector others; - auto cnode_ptr_list = graph_ptr->execution_order(); - MS_LOG(INFO) << "before reorder, graph orders size:" << cnode_ptr_list.size(); - for (size_t i = 0; i < cnode_ptr_list.size(); ++i) { - cur_cnode_ptr = cnode_ptr_list[i]; - MS_EXCEPTION_IF_NULL(cur_cnode_ptr); - if (IsIndependentNode(cur_cnode_ptr)) { - independents.emplace_back(cur_cnode_ptr); - } else { - others.emplace_back(cur_cnode_ptr); - } - } - if (others.empty() || independents.empty()) { - MS_LOG(INFO) << "independent or others is empty, no need reorder"; - return; - } - - std::set processed; - for (size_t i = 0; i < others.size(); i++) { - auto begin = others.begin() + i; - auto end = begin + 1; - bool flag = false; - for (size_t j = 0; j < independents.size(); j++) { - auto cur_independent = independents[j]; - auto it = std::find(processed.begin(), processed.end(), cur_independent.get()); - if (it != processed.end()) { - continue; - } - auto res = FindTargetOp(begin, end, cur_independent); - if (res != end) { - flag = true; - exe_orders.emplace_back(cur_independent); - exe_orders.emplace_back(*begin); - processed.emplace(cur_independent.get()); - break; - } - } - if (!flag) { - exe_orders.emplace_back(*begin); - } - } - MS_LOG(INFO) << "after reorder, graph orders size:" << exe_orders.size(); - if (processed.size() != independents.size()) { - MS_LOG(WARNING) << "processed independent nodes size is not equal to exiting independent nodes size"; - return; + for (const auto &item : hcom_stream_map_) { + streams->emplace_back(item.first); } - - graph_ptr->set_execution_order(exe_orders); } void AscendStreamAssign::Reset() { - total_event_num_ = 0; independent_stream_activated_ = false; + hcom_stream_activated_ = false; independent_stream_map_.clear(); + hcom_stream_map_.clear(); + common_stream_map_.clear(); processed_streams_.clear(); - hcom_stream_list_.clear(); need_first_active_streams_.clear(); - inner_parallel_streams_.clear(); } } // namespace ascend } // namespace device diff --git a/mindspore/ccsrc/device/ascend/ascend_stream_assign.h b/mindspore/ccsrc/device/ascend/ascend_stream_assign.h index bb918cfc79..625ab6ad6e 100644 --- a/mindspore/ccsrc/device/ascend/ascend_stream_assign.h +++ b/mindspore/ccsrc/device/ascend/ascend_stream_assign.h @@ -29,6 +29,7 @@ #include "runtime/rt_model.h" #include "runtime/stream.h" #include "session/kernel_graph.h" +#include "utils/contract.h" namespace mindspore { namespace device { @@ -38,35 +39,59 @@ using std::shared_ptr; using std::unordered_map; using std::unordered_set; using std::vector; -using CnodeKey = void *; const uint32_t kInvalidStreamId = UINT32_MAX; -class AscendStreamMng { +const uint32_t kInvalidEventId = UINT32_MAX; +class AscendResourceMng { public: - static AscendStreamMng &GetInstance() { - static AscendStreamMng instance; + static AscendResourceMng &GetInstance() { + static AscendResourceMng instance; return instance; } - void Reset() { - cur_stream_id = 0; - cur_stream_num = 0; + void ResetResource() { + cur_stream_num_ = 0; + cur_event_num_ = 0; } uint32_t ApplyNewStream() { - if (!cur_stream_num) { - cur_stream_num++; + if (!cur_stream_num_) { + uint32_t cur_stream_id = cur_stream_num_; + cur_stream_num_++; return cur_stream_id; } - cur_stream_num++; - cur_stream_id++; + uint32_t cur_stream_id = cur_stream_num_; + cur_stream_num_++; return cur_stream_id; } + uint32_t ApplyNewEvent() { + if (!cur_event_num_) { + uint32_t cur_event_id = cur_event_num_; + cur_event_num_++; + return cur_event_id; + } + uint32_t cur_event_id = cur_event_num_; + cur_event_num_++; + return cur_event_id; + } - uint32_t GetCurAllocStream() { return cur_stream_id; } - uint32_t GetCurAllocStreamNum() { return cur_stream_num; } + void DeleteEvent() { + if (!cur_event_num_) { + MS_LOG(WARNING) << "total event num is 0, no event to delete"; + } else { + --cur_event_num_; + } + } + uint32_t get_cur_stream_num() { return cur_stream_num_; } + uint32_t GetCurAllocStreamId() { + if (!cur_stream_num_) { + MS_LOG(EXCEPTION) << "stream nums is 0, no stream id should be get"; + } + return cur_stream_num_ - 1; + } + uint32_t get_cur_event_num() { return cur_event_num_; } private: - uint32_t cur_stream_num{0}; - uint32_t cur_stream_id{0}; + uint32_t cur_stream_num_{0}; + uint32_t cur_event_num_{0}; }; class AscendStreamAssign { @@ -79,39 +104,42 @@ class AscendStreamAssign { AscendStreamAssign(const AscendStreamAssign &) = delete; AscendStreamAssign &operator=(const AscendStreamAssign &) = delete; - uint32_t total_event_num() const { return total_event_num_; } + void AssignStream(const NotNull &graph_ptr); void GetHcomStreams(std::vector *streams); - - void AssignStream(const std::shared_ptr &graph_ptr); void GetWaitStreams(vector *wait_active_stream_list); - CNodePtr CreateSendApplyKernel(const std::shared_ptr &graph_ptr, uint32_t event_id, - uint32_t stream_id); - CNodePtr CreateRecvApplyKernel(const std::shared_ptr &graph_ptr, uint32_t event_id, - uint32_t stream_id); + CNodePtr CreateSendApplyKernel(const NotNull &graph_ptr, uint32_t event_id, uint32_t stream_id); + CNodePtr CreateRecvApplyKernel(const NotNull &graph_ptr, uint32_t event_id, uint32_t stream_id); private: AscendStreamAssign() = default; ~AscendStreamAssign() = default; void Reset(); - void CheckStreamAssign(const std::shared_ptr &graph_ptr); - void AssignAllNodesStream(const std::shared_ptr &graph_ptr); - void AssignCommonStreamId(const CNodePtr &cur_cnode_ptr, CNodePtr *pre_cnode_ptr, uint32_t *cur_index, - uint32_t *cur_stream_id); + void CheckResourceAssign(const NotNull &graph_ptr); + void CheckStreamAssign(const NotNull &graph_ptr); + void CheckEventAssign(const NotNull &graph_ptr); + void AssignAllNodesStream(const NotNull &graph_ptr); + void AssignCommonStreamId(const CNodePtr &cur_cnode_ptr); + void AssignHcomStreamId(const CNodePtr &cur_cnode_ptr); void AssignIndependentStreamId(const CNodePtr &cur_cnode_ptr); - void UpdateAtomicAddrCleanStreamId(const std::shared_ptr &graph_ptr); - void FindHcomParallelStreams(const std::shared_ptr &graph_ptr); - void InsertStreamActive(const std::shared_ptr &graph_ptr); - void UpdateStreamSwitch(const std::shared_ptr &graph_ptr, const CNodePtr &switch_ptr, - const vector &independent_stream, vector *orders); - void InsertSendRecvForIndependent(const std::shared_ptr &graph_ptr); - void InsertSendRecvForHcomParallel(const std::shared_ptr &graph_ptr); - void InsertSendRecvForDiffHcom(const shared_ptr &graph_ptr); - void UpdateEventId(const std::shared_ptr &graph_ptr); - void GetNeedActiveStreams(const std::shared_ptr &graph_ptr); - void ReorderIndependentOrders(const std::shared_ptr &graph_ptr); + void UpdateAtomicAddrCleanStreamId(const NotNull &graph_ptr); + void FindHcomParallelStreams(const NotNull &graph_ptr); + void InsertStreamActive(const NotNull &graph_ptr); + void UpdateStreamSwitch(const NotNull &graph_ptr, const CNodePtr &switch_ptr, + vector *orders); + void InsertEventForIndependentParallel(const NotNull &graph_ptr); + void InsertEventForHcomParallel(const NotNull &graph_ptr); + void InsertEventCommonDependHcom(const NotNull &graph_ptr); + void InsertEventHcomDependCommon(const NotNull &graph_ptr); + void InsertEventHcomDependHcom(const NotNull &graph_ptr); + void InsertEventBetweenHcom(const NotNull &graph_ptr, const map> &hcom_index, + uint32_t first_hcom_stream, uint32_t last_hcom_stream); + bool IsSatisfiedHcom(const std::map> &hcom_index, const CNodePtr &node_ptr, size_t index); + + void GetProcessedStream(const NotNull &graph_ptr); + void GetNeedActiveStreams(const NotNull &graph_ptr); + void ReorderIndependentOrders(const NotNull &graph_ptr); bool IsTaskSink(); - bool IsFusionHcom(const CNodePtr &cur_cnode_ptr); bool IsHcom(const CNodePtr &cur_cnode_ptr); bool IsIndependentNode(const CNodePtr &node_ptr); bool IsProcessedStream(uint32_t stream_id); @@ -119,14 +147,13 @@ class AscendStreamAssign { const CNodePtr &node); void GetParallelStream(uint32_t cur_stream_id, uint32_t stream_acitve_id, std::vector *parallel_streams); - uint32_t total_event_num_{0}; bool independent_stream_activated_{false}; + bool hcom_stream_activated_{false}; std::map independent_stream_map_{}; + std::map hcom_stream_map_{}; + std::map common_stream_map_{}; std::set processed_streams_{}; - std::set hcom_stream_list_{}; std::vector need_first_active_streams_{}; - std::vector> inner_parallel_streams_{}; - // new policy end }; } // namespace ascend diff --git a/mindspore/ccsrc/device/ascend/kernel_build_ascend.cc b/mindspore/ccsrc/device/ascend/kernel_build_ascend.cc index 81d5be6731..bd0b436344 100644 --- a/mindspore/ccsrc/device/ascend/kernel_build_ascend.cc +++ b/mindspore/ccsrc/device/ascend/kernel_build_ascend.cc @@ -207,7 +207,7 @@ static bool IsAtomicNode(const CNodePtr &kernel_node) { } } // process output - std::vector output_indexs; + std::vector output_indexs = {}; for (size_t i = 0; i < output_num; ++i) { auto param_output = parameters_indexs.at(input_num + workspace_num + i); if (param_output == 1) { @@ -215,9 +215,11 @@ static bool IsAtomicNode(const CNodePtr &kernel_node) { MS_LOG(INFO) << "Atomic clear output index: " << i; } } - AnfAlgo::SetNodeAttr(kAttrAtomicOutputIndexs, MakeValue(output_indexs), kernel_node); + if (!output_indexs.empty()) { + AnfAlgo::SetNodeAttr(kAttrAtomicOutputIndexs, MakeValue(output_indexs), kernel_node); + } // process workspace - std::vector workspace_indexs; + std::vector workspace_indexs = {}; for (size_t k = 0; k < workspace_num; ++k) { auto param_workspace = parameters_indexs.at(input_num + k); if (param_workspace == 1) { @@ -225,8 +227,9 @@ static bool IsAtomicNode(const CNodePtr &kernel_node) { MS_LOG(INFO) << "Atomic clear workspace index: " << k; } } - AnfAlgo::SetNodeAttr(kAttrAtomicWorkspaceIndexs, MakeValue(workspace_indexs), kernel_node); - + if (!workspace_indexs.empty()) { + AnfAlgo::SetNodeAttr(kAttrAtomicWorkspaceIndexs, MakeValue(workspace_indexs), kernel_node); + } return !(workspace_indexs.empty() && output_indexs.empty()); } diff --git a/mindspore/ccsrc/device/ascend/kernel_select_ascend.cc b/mindspore/ccsrc/device/ascend/kernel_select_ascend.cc index 4e56721fe0..cde79a18f7 100644 --- a/mindspore/ccsrc/device/ascend/kernel_select_ascend.cc +++ b/mindspore/ccsrc/device/ascend/kernel_select_ascend.cc @@ -24,7 +24,6 @@ #include #include #include - #include "common/utils.h" #include "debug/anf_ir_dump.h" #include "operator/ops.h" @@ -566,10 +565,17 @@ KernelSelectStatus SelectKernelInfo(const CNodePtr &kernel_node, KernelType kern MS_LOG(WARNING) << "Kernel [" << (kernel_info_list.size() + index) << "] :" << aicpu_kernel_info_list[index]->ToString(); } - MS_LOG(WARNING) << " <<<"; - MS_EXCEPTION(TypeError) << "The node [" << kernel_node->DebugString() - << "] cannot find valid kernel info, not supported the type:" << buffer.str() - << ", please refer to the supported dtypes in candidates kernel info list"; + if (IsPrimitiveCNode(kernel_node, prim::kPrimLabelSwitch)) { + auto selected_kernel_info = ChooseMatchedKernelInfo(kernel_node, kernel_info_list); + AnfAlgo::SetSelectKernelBuildInfo(selected_kernel_info, kernel_node.get()); + // Set format and data type for input tensor. + SetTensorDeviceInfo(*selected_kernel_info, kernel_node); + } else { + MS_LOG(WARNING) << " <<<"; + MS_EXCEPTION(TypeError) << "The node [" << kernel_node->DebugString() + << "] cannot find valid kernel info, not supported the type:" << buffer.str() + << ", please refer to the supported dtypes in candidates kernel info list"; + } } return select_status; } diff --git a/mindspore/ccsrc/device/ascend/kernel_select_graph_kernel.cc b/mindspore/ccsrc/device/ascend/kernel_select_graph_kernel.cc index b57ed1cd1b..db31460d31 100644 --- a/mindspore/ccsrc/device/ascend/kernel_select_graph_kernel.cc +++ b/mindspore/ccsrc/device/ascend/kernel_select_graph_kernel.cc @@ -25,6 +25,18 @@ namespace mindspore { namespace device { namespace ascend { +namespace { +// sort format according the number of occurrences. +bool cmp_format_num(const std::pair &a, const std::pair &b) { + if (a.second != b.second) { + return a.second > b.second; + } else if (a.first == kOpFormat_DEFAULT) { + return a.second + 1 > b.second; + } else if (b.first == kOpFormat_DEFAULT) { + return a.second > b.second + 1; + } + return a.second > b.second; +} TypeId GetPrimitivePrecision(const CNodePtr &cnode) { auto primitive = AnfAlgo::GetCNodePrimitive(cnode); @@ -44,6 +56,7 @@ TypeId GetPrimitivePrecision(const CNodePtr &cnode) { return except_type; } +} // namespace void ResetKernelBuildInfo(const CNodePtr &kernel_node) { size_t input_num = AnfAlgo::GetInputTensorNum(kernel_node); @@ -185,15 +198,12 @@ void GetDefaultFormat(const CNodePtr &kernel_node, std::string *default_format, auto input_kernel_node = AnfAlgo::VisitKernel(kernel_node->input(i + 1), 0).first; MS_EXCEPTION_IF_NULL(input_kernel_node); if (!input_kernel_node->isa()) { - auto pre_format = AnfAlgo::GetPrevNodeOutputFormat(kernel_node, i); - ++all_input_formats[pre_format]; + ++all_input_formats[AnfAlgo::GetPrevNodeOutputFormat(kernel_node, i)]; continue; } auto para = input_kernel_node->cast(); - MS_EXCEPTION_IF_NULL(para); if (AnfAlgo::GetOutputDeviceDataType(para, 0) != kTypeUnknown) { - auto pre_format = AnfAlgo::GetOutputFormat(para, 0); - ++all_input_formats[pre_format]; + ++all_input_formats[AnfAlgo::GetOutputFormat(para, 0)]; continue; } *use_same_format = false; @@ -207,17 +217,8 @@ void GetDefaultFormat(const CNodePtr &kernel_node, std::string *default_format, for (auto iter = all_input_formats.begin(); iter != all_input_formats.end(); ++iter) { pairs.push_back(std::make_pair(iter->first, iter->second)); } - auto cmp_func = [](const std::pair &a, const std::pair &b) { - if (a.second != b.second) { - return a.second > b.second; - } else if (a.first == kOpFormat_DEFAULT) { - return a.second + 1 > b.second; - } else if (b.first == kOpFormat_DEFAULT) { - return a.second > b.second + 1; - } - return a.second > b.second; - }; - std::sort(pairs.begin(), pairs.end(), cmp_func); + + std::sort(pairs.begin(), pairs.end(), cmp_format_num); *default_format = pairs.begin()->first; } @@ -237,10 +238,9 @@ void GetDefaultFormat(const CNodePtr &kernel_node, std::string *default_format, } } -void UpdateGraphKernelInputsKernelInfo(const CNodePtr &kernel_node, const std::vector &input_list, - const std::string &default_format, bool use_same_format, - std::vector *graph_input_format, - std::vector *graph_input_type) { +void UpdateInputsKernelInfo(const CNodePtr &kernel_node, const std::vector &input_list, + const std::string &default_format, bool use_same_format, + std::vector *graph_input_format, std::vector *graph_input_type) { MS_EXCEPTION_IF_NULL(graph_input_format); MS_EXCEPTION_IF_NULL(graph_input_type); // We set same format to all inputs of graph kernel subgraph, and process this latter. @@ -338,21 +338,22 @@ void UpdateEquivFormat(const std::vector> &output_ } } -void UpdateFormatsAndDtypes(const CNodePtr &kernel_node, const std::vector &node_list, - const std::vector &input_list, const FuncGraphManagerPtr &mng, - const std::string &default_format, std::vector *graph_input_format, - std::vector *graph_input_type) { +void CheckFormatsAndDtypes(const CNodePtr &kernel_node, const std::vector &input_list, + const FuncGraphManagerPtr &mng, const std::string &default_format, + std::vector *graph_input_format, std::vector *graph_input_type, + std::vector *need_update) { MS_EXCEPTION_IF_NULL(kernel_node); MS_EXCEPTION_IF_NULL(mng); MS_EXCEPTION_IF_NULL(graph_input_format); MS_EXCEPTION_IF_NULL(graph_input_type); - // update graph input format and dtype use inner ops. + MS_EXCEPTION_IF_NULL(need_update); + // check graph input format and dtype use inner ops. size_t input_num = AnfAlgo::GetInputTensorNum(kernel_node); - if (graph_input_format->size() != input_num) { + if (graph_input_format->size() != input_num || graph_input_type->size() != input_num || + need_update->size() != input_num) { MS_LOG(EXCEPTION) << "Graph input format size is not equal to input num of cnode[" << kernel_node->DebugString() - << "], [%" << graph_input_format->size() << "] != [%" << input_num << "]"; + << "], [" << graph_input_format->size() << "] != [" << input_num << "]"; } - std::vector need_update(input_num, false); auto &node_users = mng->node_users(); for (size_t i = 0; i < input_num; ++i) { auto &input = input_list[i]; @@ -372,36 +373,48 @@ void UpdateFormatsAndDtypes(const CNodePtr &kernel_node, const std::vectorDebugString() << "] selected different format. we use defult: " << default_format; (*graph_input_format)[i] = default_format; - need_update[i] = true; + (*need_update)[i] = true; } - if (kernel_node->input(i + 1)->isa()) { - auto user_dtype = AnfAlgo::GetInputDeviceDataType(node_user.first, IntToSize(node_user.second - 1)); - if (user_dtype != (*graph_input_type)[i]) { - TypeId default_dtype = AnfAlgo::GetOutputInferDataType(input, 0); - MS_LOG(WARNING) << "Users of input: [" << i << "][" << input->DebugString(2) << " of [" - << kernel_node->DebugString() - << "] selected different dtype. we use default: " << TypeIdLabel(default_dtype); - (*graph_input_type)[i] = default_dtype; - need_update[i] = true; - } + if (kernel_node->input(i + 1)->isa() || + AnfAlgo::GetInputDeviceDataType(node_user.first, IntToSize(node_user.second - 1)) == (*graph_input_type)[i]) { + continue; } + + TypeId default_dtype = AnfAlgo::GetOutputInferDataType(input, 0); + MS_LOG(WARNING) << "Users of input: [" << i << "][" << input->DebugString(2) << " of [" + << kernel_node->DebugString() + << "] selected different dtype. we use default: " << TypeIdLabel(default_dtype); + (*graph_input_type)[i] = default_dtype; + (*need_update)[i] = true; } } +} +void UpdateFormatsAndDtypes(const CNodePtr &kernel_node, const std::vector &node_list, + const std::vector &input_list, const std::vector &need_update, + const std::vector &graph_input_format, + const std::vector &graph_input_type) { + MS_EXCEPTION_IF_NULL(kernel_node); + // update graph input format and dtype use inner ops. + size_t input_num = AnfAlgo::GetInputTensorNum(kernel_node); + if (graph_input_format.size() != input_num || graph_input_type.size() != input_num || + need_update.size() != input_num) { + MS_LOG(EXCEPTION) << "Graph input format size is not equal to input num of cnode[" << kernel_node->DebugString() + << "], [" << graph_input_format.size() << "] != [" << input_num << "]"; + } for (size_t i = 0; i < input_num; ++i) { if (!need_update[i]) { continue; } - need_update[i] = false; MS_LOG(DEBUG) << "Update input format: " << i << " of: [" << kernel_node->DebugString() - << "] to: " << (*graph_input_format)[i]; + << "] to: " << graph_input_format[i]; MS_LOG(DEBUG) << "Update input dtype: " << i << " of: [" << kernel_node->DebugString() - << "] to: " << TypeIdLabel((*graph_input_type)[i]); + << "] to: " << TypeIdLabel(graph_input_type[i]); kernel::KernelBuildInfo::KernelBuildInfoBuilder builder; - std::vector outputs_format = {(*graph_input_format)[i]}; - std::vector outputs_device_type = {(*graph_input_type)[i]}; + std::vector outputs_format = {graph_input_format[i]}; + std::vector outputs_device_type = {graph_input_type[i]}; builder.SetOutputsFormat(outputs_format); builder.SetOutputsDeviceType(outputs_device_type); AnfAlgo::SetSelectKernelBuildInfo(builder.Build(), input_list[i].get()); @@ -487,8 +500,8 @@ void SelectGraphKernelInfo(const CNodePtr &kernel_node, const FuncGraphPtr &func std::vector graph_input_format; std::vector graph_input_type; - UpdateGraphKernelInputsKernelInfo(kernel_node, input_list, default_format, use_same_format, &graph_input_format, - &graph_input_type); + UpdateInputsKernelInfo(kernel_node, input_list, default_format, use_same_format, &graph_input_format, + &graph_input_type); auto mng = func_graph->manager(); if (mng == nullptr) { @@ -502,8 +515,10 @@ void SelectGraphKernelInfo(const CNodePtr &kernel_node, const FuncGraphPtr &func kernel::GetValidKernelNodes(func_graph, &node_list, &input_list, &output_list); // update graph input format and dtype use inner ops. - UpdateFormatsAndDtypes(kernel_node, node_list, input_list, mng, default_format, &graph_input_format, - &graph_input_type); + std::vector need_update(AnfAlgo::GetInputTensorNum(kernel_node), false); + CheckFormatsAndDtypes(kernel_node, input_list, mng, default_format, &graph_input_format, &graph_input_type, + &need_update); + UpdateFormatsAndDtypes(kernel_node, node_list, input_list, need_update, graph_input_format, graph_input_type); // set fix_precision for kernel when the me prim has fix_precision attr UpdateKernelInfo(node_list); diff --git a/mindspore/ccsrc/device/ascend/profiling/profiling_engine_impl.cc b/mindspore/ccsrc/device/ascend/profiling/profiling_engine_impl.cc index cbecb3030d..a393409334 100644 --- a/mindspore/ccsrc/device/ascend/profiling/profiling_engine_impl.cc +++ b/mindspore/ccsrc/device/ascend/profiling/profiling_engine_impl.cc @@ -28,6 +28,7 @@ PluginIntf *ProfilingEngineImpl::CreatePlugin() { int ProfilingEngineImpl::ReleasePlugin(PluginIntf *plugin) { if (plugin != nullptr) { delete plugin; + plugin = nullptr; } return 0; } diff --git a/mindspore/ccsrc/device/ascend/profiling/profiling_manager.cc b/mindspore/ccsrc/device/ascend/profiling/profiling_manager.cc index fec1aac685..a2fe5b852d 100644 --- a/mindspore/ccsrc/device/ascend/profiling/profiling_manager.cc +++ b/mindspore/ccsrc/device/ascend/profiling/profiling_manager.cc @@ -15,11 +15,8 @@ */ #include "device/ascend/profiling/profiling_manager.h" - #include #include - -#include #include "securec/include/securec.h" #include "./prof_mgr_core.h" #include "device/ascend/profiling/plugin_impl.h" @@ -30,9 +27,6 @@ #include "utils/convert_utils.h" #include "runtime/base.h" -using std::vector; -using Json = nlohmann::json; - namespace mindspore { namespace device { namespace ascend { @@ -124,35 +118,43 @@ bool ProfilingManager::StartupProfiling(uint32_t device_id) { auto context = MsContext::GetInstance(); MS_EXCEPTION_IF_NULL(context); const string prof_options_str = context->profiling_options(); - vector opts = Split(prof_options_str, ':'); + std::vector opts = Split(prof_options_str, ':'); if (opts.empty()) { MS_LOG(WARNING) << "Profiling is enabled, but profiling option is not set!"; return true; } // current one docker only use one device` - Json p_device; + nlohmann::json p_device; // JOBID auto job_id = GetJobId(); p_device["jobID"] = std::to_string(job_id); // device_id p_device["deviceID"] = std::to_string(device_id); // features:'training_trace', 'task_trace' etc - Json features; - for (vector::size_type i = 0; i < opts.size(); i++) { - Json f; + nlohmann::json features; + for (std::vector::size_type i = 0; i < opts.size(); i++) { + nlohmann::json f; f["name"] = opts[i]; features[i] = f; } p_device["features"] = features; // only one device, but sProfMgrStartUp API require for device list - Json devices; + nlohmann::json devices; devices[0] = p_device; - Json startCfg; + nlohmann::json startCfg; startCfg["startCfg"] = devices; + if (!ProfStartUp(NOT_NULL(&startCfg))) { + MS_LOG(ERROR) << "ProfMgrStartUp failed."; + return false; + } + return true; +} + +bool ProfilingManager::ProfStartUp(NotNull startCfg) { // convert json to string std::stringstream ss; - ss << startCfg; + ss << *startCfg; std::string cfg = ss.str(); MS_LOG(INFO) << "profiling config " << cfg; auto ret = rtProfilerStart(); diff --git a/mindspore/ccsrc/device/ascend/profiling/profiling_manager.h b/mindspore/ccsrc/device/ascend/profiling/profiling_manager.h index c30c6898ea..05b5248996 100644 --- a/mindspore/ccsrc/device/ascend/profiling/profiling_manager.h +++ b/mindspore/ccsrc/device/ascend/profiling/profiling_manager.h @@ -20,18 +20,15 @@ #include #include #include +#include +#include "utils/contract.h" #include "utils/context/ms_context.h" + using std::map; using std::string; - namespace mindspore { namespace device { namespace ascend { -// PROFILING_CUSTOM_LOGID_START 3 -const uint64_t kProfilingFpStartLogId = 1; -const uint64_t kProfilingBpEndLogId = 2; -const uint64_t kProfilingIterEndLogId = 255; - class ProfilingEngineImpl; class ProfilingManager { public: @@ -52,6 +49,7 @@ class ProfilingManager { ~ProfilingManager() { prof_handle_ = nullptr; } private: + bool ProfStartUp(NotNull json); std::shared_ptr engine_0_; uint32_t device_id_; void *prof_handle_; diff --git a/mindspore/ccsrc/device/ascend/profiling/profiling_utils.cc b/mindspore/ccsrc/device/ascend/profiling/profiling_utils.cc index 131a22805d..17ac4c4530 100644 --- a/mindspore/ccsrc/device/ascend/profiling/profiling_utils.cc +++ b/mindspore/ccsrc/device/ascend/profiling/profiling_utils.cc @@ -33,6 +33,10 @@ constexpr char kCustomNode[] = "PROFILING_CUSTOM_"; constexpr char kFpStartNode[] = "PROFILING_FP_START"; constexpr char kBpEndNode[] = "PROFILING_BP_END"; constexpr char kIterEndNode[] = "PROFILING_ITER_END"; +// PROFILING_CUSTOM_LOGID_START 3 +constexpr uint64_t kProfilingFpStartLogId = 1; +constexpr uint64_t kProfilingBpEndLogId = 2; +constexpr uint64_t kProfilingIterEndLogId = 255; std::map> ProfilingUtils::graph_profiling_cnode_; std::map> ProfilingUtils::graph_kernel_name_; std::map>> ProfilingUtils::graph_point_; diff --git a/mindspore/ccsrc/device/ascend/tasksink/runtime_utils.cc b/mindspore/ccsrc/device/ascend/tasksink/runtime_utils.cc index 603dd989e5..3faeefb820 100644 --- a/mindspore/ccsrc/device/ascend/tasksink/runtime_utils.cc +++ b/mindspore/ccsrc/device/ascend/tasksink/runtime_utils.cc @@ -58,9 +58,9 @@ bool RuntimeUtils::HcomDistribute(const std::shared_ptr &task_info if (task_info->hccl_type() == kBroadcastOpName) { // call hcom broadcast interface to run op const string tag_broadcast = kHcomBroadcast + std::to_string(task_counter++) + kUnderline + std::to_string(0); - ret = hcom_broadcast(tag_broadcast.c_str(), reinterpret_cast(task_info->input_data_addr()), - static_cast(task_info->count()), static_cast(task_info->data_type()), - static_cast(task_info->root_id()), hccl_group.c_str(), stream); + ret = hcom_broadcast(tag_broadcast.c_str(), task_info->input_data_addr(), static_cast(task_info->count()), + static_cast(task_info->data_type()), static_cast(task_info->root_id()), + hccl_group.c_str(), stream); if (ret != HCCL_SUCCESS) { MS_LOG(ERROR) << "hcom_broadcast fail, return ret: " << static_cast(ret); return false; @@ -68,9 +68,9 @@ bool RuntimeUtils::HcomDistribute(const std::shared_ptr &task_info } else if (task_info->hccl_type() == kAllGatherOpName) { // call hcom allgather interface to run op const string tag_all_gather = kHcomAllGather + std::to_string(task_counter++) + kUnderline + std::to_string(0); - ret = hcom_all_gather(tag_all_gather.c_str(), reinterpret_cast(task_info->input_data_addr()), - reinterpret_cast(task_info->output_data_addr()), static_cast(task_info->count()), - static_cast(task_info->data_type()), hccl_group.c_str(), stream); + ret = hcom_all_gather(tag_all_gather.c_str(), task_info->input_data_addr(), task_info->output_data_addr(), + static_cast(task_info->count()), static_cast(task_info->data_type()), + hccl_group.c_str(), stream); if (ret != HCCL_SUCCESS) { MS_LOG(ERROR) << "hcom_all_gather fail, return ret: " << ret; return false; @@ -78,9 +78,8 @@ bool RuntimeUtils::HcomDistribute(const std::shared_ptr &task_info } else if (task_info->hccl_type() == kAllReduceOpName) { // call hcom allreduce interface to run op const string tag_all_reduce = kHcomAllReduce + std::to_string(task_counter++) + kUnderline + std::to_string(0); - ret = hcom_all_reduce(tag_all_reduce.c_str(), reinterpret_cast(task_info->input_data_addr()), - reinterpret_cast(task_info->output_data_addr()), static_cast(task_info->count()), - static_cast(task_info->data_type()), + ret = hcom_all_reduce(tag_all_reduce.c_str(), task_info->input_data_addr(), task_info->output_data_addr(), + static_cast(task_info->count()), static_cast(task_info->data_type()), static_cast(task_info->op_type()), hccl_group.c_str(), stream); if (ret != HCCL_SUCCESS) { MS_LOG(ERROR) << "hcom_all_reduce fail, return ret: " << ret; @@ -90,8 +89,7 @@ bool RuntimeUtils::HcomDistribute(const std::shared_ptr &task_info // call hcom reducescatter interface to run op const string tag_reduce_scatter = kHcomReduceScatter + std::to_string(task_counter++) + kUnderline + std::to_string(0); - ret = hcom_reduce_scatter(tag_reduce_scatter.c_str(), reinterpret_cast(task_info->input_data_addr()), - reinterpret_cast(task_info->output_data_addr()), + ret = hcom_reduce_scatter(tag_reduce_scatter.c_str(), task_info->input_data_addr(), task_info->output_data_addr(), static_cast(task_info->count()), static_cast(task_info->data_type()), static_cast(task_info->op_type()), hccl_group.c_str(), stream); if (ret != HCCL_SUCCESS) { diff --git a/mindspore/ccsrc/device/ascend/tasksink/task_generator.cc b/mindspore/ccsrc/device/ascend/tasksink/task_generator.cc index 0cdf751801..e026459ae9 100644 --- a/mindspore/ccsrc/device/ascend/tasksink/task_generator.cc +++ b/mindspore/ccsrc/device/ascend/tasksink/task_generator.cc @@ -40,39 +40,46 @@ bool TaskGenerator::GenTasks(const std::vector &anf_node_list, std::ve return true; } +void TaskGenerator::LaunchAddrCleanAkgKernel(const CNodePtr &anf_node_ptr, AddressPtrList *kernel_inputs) { + MS_EXCEPTION_IF_NULL(anf_node_ptr); + MS_EXCEPTION_IF_NULL(kernel_inputs); + // akg process + // set atomic clean addr + if (AnfAlgo::HasNodeAttr(kAttrAtomicOutputIndexs, anf_node_ptr)) { + auto clean_output_indexs = AnfAlgo::GetNodeAttr>(anf_node_ptr, kAttrAtomicOutputIndexs); + auto graph = anf_node_ptr->func_graph(); + MS_EXCEPTION_IF_NULL(graph); + auto manager = graph->manager(); + MS_EXCEPTION_IF_NULL(manager); + auto node_users = manager->node_users(); + if (node_users[anf_node_ptr].empty()) { + MS_LOG(EXCEPTION) << "Node users of " << anf_node_ptr->ToString() << " is empty."; + } + auto depend_node = node_users[anf_node_ptr].pop().first; + if (!IsPrimitiveCNode(depend_node, prim::kPrimDepend)) { + MS_LOG(EXCEPTION) << "Checking Depend node failed"; + } + if (node_users[depend_node].empty()) { + MS_LOG(EXCEPTION) << "Node users of " << depend_node->ToString() << " is empty."; + } + auto post_node = node_users[depend_node].pop().first; + for (auto index : clean_output_indexs) { + auto device_address = AnfAlgo::GetOutputAddr(post_node, index); + kernel::AddressPtr input = std::make_shared(); + MS_EXCEPTION_IF_NULL(input); + input->addr = device_address->ptr_; + input->size = device_address->size_; + kernel_inputs->push_back(input); + } + MS_LOG(DEBUG) << "AtomicAddClean clean output size: " << clean_output_indexs.size(); + } +} + void TaskGenerator::LaunchAddrCleanKernel(const CNodePtr &anf_node_ptr, AddressPtrList *kernel_inputs) { MS_EXCEPTION_IF_NULL(anf_node_ptr); + MS_EXCEPTION_IF_NULL(kernel_inputs); if (anf_node_ptr->inputs().size() != 2) { - // akg process - // set atomic clean addr - if (AnfAlgo::HasNodeAttr(kAttrAtomicOutputIndexs, anf_node_ptr)) { - auto clean_output_indexs = AnfAlgo::GetNodeAttr>(anf_node_ptr, kAttrAtomicOutputIndexs); - auto graph = anf_node_ptr->func_graph(); - MS_EXCEPTION_IF_NULL(graph); - auto manager = graph->manager(); - MS_EXCEPTION_IF_NULL(manager); - auto node_users = manager->node_users(); - if (node_users[anf_node_ptr].empty()) { - MS_LOG(EXCEPTION) << "Node users of " << anf_node_ptr->ToString() << " is empty."; - } - auto depend_node = node_users[anf_node_ptr].pop().first; - if (!IsPrimitiveCNode(depend_node, prim::kPrimDepend)) { - MS_LOG(EXCEPTION) << "Checking Depend node failed"; - } - if (node_users[depend_node].empty()) { - MS_LOG(EXCEPTION) << "Node users of " << depend_node->ToString() << " is empty."; - } - auto post_node = node_users[depend_node].pop().first; - for (auto index : clean_output_indexs) { - auto device_address = AnfAlgo::GetOutputAddr(post_node, index); - kernel::AddressPtr input = std::make_shared(); - input->addr = device_address->ptr_; - MS_EXCEPTION_IF_NULL(input->addr); - input->size = device_address->size_; - kernel_inputs->push_back(input); - } - MS_LOG(DEBUG) << "AtomicAddClean clean output size: " << clean_output_indexs.size(); - } + LaunchAddrCleanAkgKernel(anf_node_ptr, kernel_inputs); return; } MS_EXCEPTION_IF_NULL(anf_node_ptr->inputs()[1]); diff --git a/mindspore/ccsrc/device/ascend/tasksink/task_generator.h b/mindspore/ccsrc/device/ascend/tasksink/task_generator.h index ffedcd7930..ecd5889b04 100644 --- a/mindspore/ccsrc/device/ascend/tasksink/task_generator.h +++ b/mindspore/ccsrc/device/ascend/tasksink/task_generator.h @@ -48,6 +48,7 @@ class TaskGenerator { private: static void LaunchAddrCleanKernel(const CNodePtr &anf_node_ptr, AddressPtrList *kernel_inputs); + static void LaunchAddrCleanAkgKernel(const CNodePtr &anf_node_ptr, AddressPtrList *kernel_inputs); static bool LaunchKernel(const CNodePtr &anf_node_ptr, uint32_t stream_id, std::vector *task_info_list); static bool LaunchAllKernel(const std::vector &anf_node_list, std::vector *task_info_list, uint32_t graph_id); diff --git a/mindspore/ccsrc/device/cpu/cpu_kernel_runtime.cc b/mindspore/ccsrc/device/cpu/cpu_kernel_runtime.cc index 6725dff524..f46d10ed82 100644 --- a/mindspore/ccsrc/device/cpu/cpu_kernel_runtime.cc +++ b/mindspore/ccsrc/device/cpu/cpu_kernel_runtime.cc @@ -26,6 +26,7 @@ #include "device/cpu/cpu_device_address.h" #include "utils/context/ms_context.h" #include "utils/config_manager.h" +#include "utils/profile.h" #include "common/utils.h" #include "session/anf_runtime_algorithm.h" #include "session/session_basic.h" @@ -78,12 +79,13 @@ void CPUKernelRuntime::AssignValueNodeAddress(session::KernelGraph *kernel_graph std::vector data_shape = tensor->shape(); size_t tensor_size = std::accumulate(data_shape.begin(), data_shape.end(), type_size, std::multiplies()); DeviceAddressPtr address = CreateDeviceAddress(nullptr, tensor_size, kOpFormat_DEFAULT, kNumberTypeFloat32); + MS_EXCEPTION_IF_NULL(address); if (tensor->data_type() == kNumberTypeFloat32 || tensor->data_type() == kNumberTypeInt32) { - address->ptr_ = tensor->data_c(false); + address->ptr_ = tensor->data_c(); } else { address->ptr_ = resource_manager_.MemMalloc(tensor_size); if (!address->SyncHostToDevice(data_shape, LongToSize(tensor->data().nbytes()), tensor->data_type(), - tensor->data_c(false))) { + tensor->data_c())) { MS_LOG(EXCEPTION) << "Value node sync host to device failed!"; } } @@ -140,6 +142,37 @@ DeviceAddressPtr CPUKernelRuntime::CreateDeviceAddress(void *device_ptr, size_t return std::make_shared(device_ptr, device_size, format, type_id); } +tensor::TensorPtr CPUKernelRuntime::CreatTensorForOutput(const CNodePtr &node, size_t index, + std::set *bound_addresses, + std::vector *need_sync_outputs) { + MS_EXCEPTION_IF_NULL(node); + MS_EXCEPTION_IF_NULL(bound_addresses); + MS_EXCEPTION_IF_NULL(need_sync_outputs); + size_t output_size = AnfAlgo::GetOutputTensorNum(node); + if (index >= output_size) { + MS_LOG(EXCEPTION) << "Invalid input index " << index; + } + auto address = AnfAlgo::GetMutableOutputAddr(node, index); + MS_EXCEPTION_IF_NULL(address); + auto shape = AnfAlgo::GetOutputInferShape(node, index); + std::vector temp_shape; + (void)temp_shape.insert(temp_shape.end(), shape.begin(), shape.end()); + TypeId type_id = AnfAlgo::GetOutputInferDataType(node, index); + type_id = GetCPUSupportOutputTypeId(type_id); + tensor::TensorPtr tensor = std::make_shared(type_id, temp_shape); + MS_EXCEPTION_IF_NULL(tensor); + if (bound_addresses->find(address) != bound_addresses->end()) { + tensor->set_device_address(address); + need_sync_outputs->emplace_back(tensor); + } else { + address->ptr_ = tensor->data_c(); + address->ref_count_ = INIT_NODE_REF; + (void)bound_addresses->insert(address); + } + tensor->set_dirty(false); + return tensor; +} + BaseRef CPUKernelRuntime::CreatTensorForOutput(const session::KernelWithIndex &kernel_with_index, const std::unordered_map &input_map, std::set *bound_addresses, @@ -159,29 +192,7 @@ BaseRef CPUKernelRuntime::CreatTensorForOutput(const session::KernelWithIndex &k } return ret; } - size_t output_size = AnfAlgo::GetOutputTensorNum(node); - if (index >= output_size) { - MS_LOG(EXCEPTION) << "Invalid input index " << index; - } - auto address = AnfAlgo::GetMutableOutputAddr(node, index); - MS_EXCEPTION_IF_NULL(address); - auto shape = AnfAlgo::GetOutputInferShape(node, index); - std::vector temp_shape; - (void)temp_shape.insert(temp_shape.end(), shape.begin(), shape.end()); - TypeId type_id = AnfAlgo::GetOutputInferDataType(node, index); - type_id = GetCPUSupportOutputTypeId(type_id); - tensor::TensorPtr tensor = std::make_shared(type_id, temp_shape); - MS_EXCEPTION_IF_NULL(tensor); - if (bound_addresses->find(address) != bound_addresses->end()) { - tensor->set_device_address(address); - need_sync_outputs->emplace_back(tensor); - } else { - address->ptr_ = tensor->data_c(true); - address->ref_count_ = INIT_NODE_REF; - (void)bound_addresses->insert(address); - } - tensor->set_dirty(false); - return tensor; + return CreatTensorForOutput(node, index, bound_addresses, need_sync_outputs); } else if (input_node->isa() || input_node->isa()) { auto iter = input_map.find(input_node.get()); if (iter != input_map.end()) { @@ -219,11 +230,11 @@ void CPUKernelRuntime::BindInputOutput(const session::KernelGraph *kernel_graph, size_t tensor_size = std::accumulate(data_shape.begin(), data_shape.end(), sizeof(float), std::multiplies()); if (tensor->data_type() == kNumberTypeFloat32 || tensor->data_type() == kNumberTypeInt32) { - address->ptr_ = tensor->data_c(false); + address->ptr_ = tensor->data_c(); } else { address->ptr_ = resource_manager_.MemMalloc(tensor_size); if (!address->SyncHostToDevice(data_shape, LongToSize(tensor->data().nbytes()), tensor->data_type(), - tensor->data_c(false))) { + tensor->data_c())) { MS_LOG(EXCEPTION) << "Parameter node sync host to device failed!"; } tensor->set_dirty(true); @@ -245,6 +256,7 @@ void CPUKernelRuntime::BindInputOutput(const session::KernelGraph *kernel_graph, void CPUKernelRuntime::AddRuntimeAddress(DeviceAddress *address, std::vector *input_list) { MS_EXCEPTION_IF_NULL(address); + MS_EXCEPTION_IF_NULL(input_list); kernel::AddressPtr input = std::make_shared(); MS_EXCEPTION_IF_NULL(input); if (address->ptr_ == nullptr) { @@ -270,6 +282,9 @@ bool CPUKernelRuntime::Run(session::KernelGraph *kernel_graph) { auto kernels = kernel_graph->execution_order(); for (const auto &kernel : kernels) { +#ifdef ENABLE_PROFILE + double start_time = GetTime(); +#endif std::vector kernel_inputs; std::vector kernel_workspaces; std::vector kernel_outputs; @@ -297,6 +312,10 @@ bool CPUKernelRuntime::Run(session::KernelGraph *kernel_graph) { if (!ret) { MS_LOG(EXCEPTION) << "Launch kernel failed."; } +#ifdef ENABLE_PROFILE + double cost_time = GetTime() - start_time; + MS_LOG(INFO) << "cpu kernel: " << kernel->fullname_with_scope() << " costs " << cost_time * 1e6 << " us"; +#endif } return true; } diff --git a/mindspore/ccsrc/device/cpu/cpu_kernel_runtime.h b/mindspore/ccsrc/device/cpu/cpu_kernel_runtime.h index 27dcefdba9..354d2922c2 100644 --- a/mindspore/ccsrc/device/cpu/cpu_kernel_runtime.h +++ b/mindspore/ccsrc/device/cpu/cpu_kernel_runtime.h @@ -49,6 +49,10 @@ class CPUKernelRuntime : public KernelRuntime { TypeId type_id) override; private: + tensor::TensorPtr CreatTensorForOutput(const CNodePtr &node, size_t index, + std::set *bound_addresses, + std::vector *need_sync_outputs); + BaseRef CreatTensorForOutput(const session::KernelWithIndex &kernel_with_index, const std::unordered_map &input_map, std::set *bound_addresses, diff --git a/mindspore/ccsrc/device/cpu/cpu_simple_mem_plan.cc b/mindspore/ccsrc/device/cpu/cpu_simple_mem_plan.cc index 3fb76d38db..e6cb6ee53a 100644 --- a/mindspore/ccsrc/device/cpu/cpu_simple_mem_plan.cc +++ b/mindspore/ccsrc/device/cpu/cpu_simple_mem_plan.cc @@ -27,7 +27,12 @@ void CPUSimpleMemPlan::MemPlan(const session::KernelGraph *graph) { MS_EXCEPTION_IF_NULL(kernel); size_t input_num = AnfAlgo::GetInputTensorNum(kernel); for (size_t i = 0; i < input_num; ++i) { - auto address = AnfAlgo::GetPrevNodeOutputAddr(kernel, i); + auto kernel_with_index = AnfAlgo::GetPrevNodeOutput(kernel, i); + MS_EXCEPTION_IF_NULL(kernel_with_index.first); + if (kernel_with_index.first->isa()) { + continue; + } + auto address = AnfAlgo::GetOutputAddr(kernel_with_index.first, kernel_with_index.second, true); MS_EXCEPTION_IF_NULL(address); if (address->ptr_ == nullptr) { total_mem_size += address->size_; @@ -56,7 +61,13 @@ void CPUSimpleMemPlan::MemPlan(const session::KernelGraph *graph) { graph_mem_size_[graph] = total_mem_size; } -size_t CPUSimpleMemPlan::GetGraphMemSize(const session::KernelGraph *graph) { return graph_mem_size_[graph]; } +size_t CPUSimpleMemPlan::GetGraphMemSize(const session::KernelGraph *graph) const { + auto iter = graph_mem_size_.find(graph); + if (iter != graph_mem_size_.end()) { + return iter->second; + } + return 0; +} void CPUSimpleMemPlan::MemAssign(const session::KernelGraph *graph, uint8_t *base_ptr) { MS_EXCEPTION_IF_NULL(graph); @@ -67,7 +78,12 @@ void CPUSimpleMemPlan::MemAssign(const session::KernelGraph *graph, uint8_t *bas MS_EXCEPTION_IF_NULL(kernel); size_t input_num = AnfAlgo::GetInputTensorNum(kernel); for (size_t i = 0; i < input_num; ++i) { - auto address = AnfAlgo::GetPrevNodeMutableOutputAddr(kernel, i); + auto kernel_with_index = AnfAlgo::GetPrevNodeOutput(kernel, i); + MS_EXCEPTION_IF_NULL(kernel_with_index.first); + if (kernel_with_index.first->isa()) { + continue; + } + auto address = AnfAlgo::GetMutableOutputAddr(kernel_with_index.first, kernel_with_index.second, true); MS_EXCEPTION_IF_NULL(address); if (address->ptr_ == nullptr) { address->ptr_ = mem_ptr; diff --git a/mindspore/ccsrc/device/cpu/cpu_simple_mem_plan.h b/mindspore/ccsrc/device/cpu/cpu_simple_mem_plan.h index 5f88966103..7633ef3f45 100644 --- a/mindspore/ccsrc/device/cpu/cpu_simple_mem_plan.h +++ b/mindspore/ccsrc/device/cpu/cpu_simple_mem_plan.h @@ -31,7 +31,7 @@ class CPUSimpleMemPlan { void MemPlan(const session::KernelGraph *graph); void MemAssign(const session::KernelGraph *graph, uint8_t *base_ptr); - size_t GetGraphMemSize(const session::KernelGraph *graph); + size_t GetGraphMemSize(const session::KernelGraph *graph) const; private: std::unordered_map graph_mem_size_; diff --git a/mindspore/ccsrc/device/cpu/mpi/mpi_adapter.cc b/mindspore/ccsrc/device/cpu/mpi/mpi_adapter.cc index 0d49846bf7..9b06c0a40a 100644 --- a/mindspore/ccsrc/device/cpu/mpi/mpi_adapter.cc +++ b/mindspore/ccsrc/device/cpu/mpi/mpi_adapter.cc @@ -13,15 +13,42 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - #include "device/cpu/mpi/mpi_adapter.h" +#ifdef ENABLE_MPI #include -#include "utils/mpi/mpi_config.h" +#include +#include "pybind11/pybind11.h" +#endif // ENABLE_MPI #include "utils/log_adapter.h" namespace mindspore { namespace device { namespace cpu { +std::shared_ptr MPIAdapter::instance_ = nullptr; +std::shared_ptr MPIAdapter::Instance() { + if (instance_ == nullptr) { + MS_LOG(DEBUG) << "Create new mpi adapter instance."; + instance_.reset(new (std::nothrow) MPIAdapter()); + } + return instance_; +} + +#ifdef ENABLE_MPI + +#define RAISE_EXCEPTION(message) \ + { \ + std::ostringstream oss; \ + oss << "[" << __FILE__ << "] [" << __LINE__ << "] " << message; \ + pybind11::pybind11_fail(oss.str()); \ + } + +#define RAISE_EXCEPTION_WITH_PARAM(message, param) \ + { \ + std::ostringstream oss; \ + oss << "[" << __FILE__ << "] [" << __LINE__ << "] " << message << param; \ + pybind11::pybind11_fail(oss.str()); \ + } + namespace { MPI_Op GetMpiOp(const std::string &op_type) { if (op_type == "sum") { @@ -33,7 +60,8 @@ MPI_Op GetMpiOp(const std::string &op_type) { } else if (op_type == "prod") { return MPI_PROD; } - MS_LOG(EXCEPTION) << "unsupport op_type:" << op_type; + + RAISE_EXCEPTION_WITH_PARAM("unsupport op_type: ", op_type); return MPI_SUM; } @@ -46,80 +74,72 @@ int GetScatterIndex(int rankid, const std::vector &ranks_group) { } } if (scatter_index == -1) { - MS_LOG(EXCEPTION) << "process rankid " << rankid << " does not in the input rank group!"; + RAISE_EXCEPTION_WITH_PARAM("local rankid does not in the input rank group!local rank id:", rankid); } return scatter_index; } } // namespace -MPIAdapter::MPIAdapter() : rank_id_(0), rank_size_(0), comm_group_world_(MPI_GROUP_NULL) { Init(); } +MPIAdapter::MPIAdapter() : comm_group_world_(MPI_GROUP_NULL) { Init(); } MPIAdapter::~MPIAdapter() { + int finalized; + MPI_Finalized(&finalized); + if (finalized != 0) { + return; + } + for (auto iter = ranks_group_.begin(); iter != ranks_group_.end(); ++iter) { MPI_Group_free(&iter->second); } + ranks_group_.clear(); if (comm_group_world_ != MPI_GROUP_NULL) { MPI_Group_free(&comm_group_world_); + comm_group_world_ = MPI_GROUP_NULL; } - int finalized; - MPI_Finalized(&finalized); - if (finalized == 0) { - MPI_Finalize(); - } + MPI_Finalize(); } -MPIAdapter &MPIAdapter::Instance() { - static MPIAdapter instance; - return instance; -} - -int MPIAdapter::GetRankId() const { return rank_id_; } - void MPIAdapter::Init() { static bool init = false; if (init) { return; } - auto mpi_config_ptr = MpiConfig::GetInstance(); - MS_EXCEPTION_IF_NULL(mpi_config_ptr); - if (!mpi_config_ptr->enable_mpi()) { - MS_LOG(EXCEPTION) << "MPI is disabled now!Please enable mpi with mpi config first."; - } + int init_flag = 0; if (MPI_Initialized(&init_flag) != MPI_SUCCESS) { - MS_LOG(EXCEPTION) << "Check mpi initialized fail!"; + RAISE_EXCEPTION("Check mpi initialized fail!"); } if (init_flag == 0) { auto ret = MPI_Init(nullptr, nullptr); if (ret != MPI_SUCCESS) { - MS_LOG(EXCEPTION) << "Failed to init mpi!"; + RAISE_EXCEPTION("Failed to init mpi!"); } } MPI_Comm_group(MPI_COMM_WORLD, &comm_group_world_); if (comm_group_world_ == MPI_GROUP_NULL) { - MS_LOG(EXCEPTION) << "comm_group_world_ init fail!"; + RAISE_EXCEPTION("comm_group_world_ init fail!"); } auto ret = MPI_Comm_rank(MPI_COMM_WORLD, &rank_id_); if (ret != MPI_SUCCESS) { - MS_LOG(EXCEPTION) << "Failed to init mpi rank id!"; + RAISE_EXCEPTION("Failed to init mpi rank id!"); } ret = MPI_Comm_size(MPI_COMM_WORLD, &rank_size_); if (ret != MPI_SUCCESS) { - MS_LOG(EXCEPTION) << "Failed to init mpi rank size!rankid:" << rank_id_; + RAISE_EXCEPTION_WITH_PARAM("Failed to init mpi rank size!rankid:", rank_id_) } init = true; } MPI_Group MPIAdapter::AddGroup(const std::vector &ranks) { if (ranks.size() > static_cast(rank_size_) || ranks.empty()) { - MS_LOG(EXCEPTION) << "input rank size: " << ranks.size() << ", max rank size: " << rank_size_; + RAISE_EXCEPTION_WITH_PARAM("input rank size:", ranks.size()); } if (std::find(ranks.begin(), ranks.end(), rank_id_) == ranks.end()) { - MS_LOG(ERROR) << "rankid:" << rank_id_ << " is not in the input group."; - return MPI_GROUP_NULL; + RAISE_EXCEPTION_WITH_PARAM("local rankid does not in the input rank group!local rank id:", rank_id_); } std::lock_guard lock(group_mutex_); auto iter = ranks_group_.find(ranks); @@ -135,29 +155,28 @@ MPI_Group MPIAdapter::AddGroup(const std::vector &ranks) { MPI_Group group = MPI_GROUP_NULL; MPI_Group_incl(comm_group_world_, ranks.size(), ranks_input.data(), &group); if (group == MPI_GROUP_NULL) { - MS_LOG(EXCEPTION) << "create mpi group fail!rankid:" << rank_id_; + RAISE_EXCEPTION_WITH_PARAM("create mpi group fail!rankid:", rank_id_) } ranks_group_[ranks] = group; - MS_LOG(INFO) << "rank:" << rank_id_ << " add group:" << group; return group; } bool MPIAdapter::ReduceScatter(const float *input, float *output, const std::vector &ranks_group, size_t data_num, const std::string &op_type) { if (ranks_group.empty()) { - MS_LOG(ERROR) << "input rank group is empty!"; + RAISE_EXCEPTION("input rank group is empty!"); return false; } auto group = AddGroup(ranks_group); if (group == MPI_GROUP_NULL) { - MS_LOG(EXCEPTION) << "Get mpi group fail!rankid:" << rank_id_; + RAISE_EXCEPTION_WITH_PARAM("Get mpi group fail!rankid:", rank_id_) } MPI_Comm comm; MPI_Comm_create_group(MPI_COMM_WORLD, group, 0, &comm); if (comm == MPI_COMM_NULL) { - MS_LOG(EXCEPTION) << "create mpi comm fail!rankid:" << rank_id_; + RAISE_EXCEPTION_WITH_PARAM("create mpi comm fail!rankid:", rank_id_); } std::vector receive_count(ranks_group.size(), 0); for (size_t i = 0; i < ranks_group.size(); ++i) { @@ -168,13 +187,13 @@ bool MPIAdapter::ReduceScatter(const float *input, float *output, const std::vec auto ret = MPI_Reduce_scatter(input, output, receive_count.data(), MPI_FLOAT, op, comm); bool result = true; if (ret != MPI_SUCCESS) { - MS_LOG(ERROR) << "mpi reduce_scatter fail!ret = " << ret << ", rankid:" << rank_id_; + RAISE_EXCEPTION_WITH_PARAM("mpi reduce_scatter fail!ret = ", ret); result = false; } ret = MPI_Comm_free(&comm); if (ret != MPI_SUCCESS) { - MS_LOG(WARNING) << "mpi comm free fail! ret = " << ret << ", rankid:" << rank_id_; + RAISE_EXCEPTION_WITH_PARAM("mpi comm free fail! ret = ", ret); } return result; } @@ -184,19 +203,18 @@ bool MPIAdapter::ReduceScatterOverwriteInput(float *input, const std::vector &ranks_group, size_t data_num, - const std::string &op_type = kOpTypeSum); - bool ReduceScatterOverwriteInput(float *input, const std::vector &ranks_group, size_t input_data_num, - size_t output_size, const std::string &op_type = kOpTypeSum, - float *output = nullptr); - bool AllGather(const float *input, float *output, const std::vector &ranks_group, size_t data_num); + FUNC_EXPORT static std::shared_ptr Instance(); + FUNC_EXPORT int GetRankId() const { return rank_id_; } + FUNC_EXPORT int GetRankSize() const { return rank_size_; } +#ifdef ENABLE_MPI + FUNC_EXPORT ~MPIAdapter(); + FUNC_EXPORT bool ReduceScatter(const float *input, float *output, const std::vector &ranks_group, + size_t data_num, const std::string &op_type = kOpTypeSum); + FUNC_EXPORT bool ReduceScatterOverwriteInput(float *input, const std::vector &ranks_group, size_t in_data_num, + size_t output_size, const std::string &op_type = kOpTypeSum, + float *output = nullptr); + FUNC_EXPORT bool AllGather(const float *input, float *output, const std::vector &ranks_group, size_t data_num); +#else + FUNC_EXPORT ~MPIAdapter() = default; +#endif // ENABLE_MPI private: +#ifdef ENABLE_MPI MPIAdapter(); void Init(); MPI_Group AddGroup(const std::vector &ranks); - int rank_id_; - int rank_size_; MPI_Group comm_group_world_; // key:ranks group, value: mpi group std::map, MPI_Group> ranks_group_; std::mutex group_mutex_; +#else + MPIAdapter() = default; +#endif // ENABLE_MPI + int rank_id_{-1}; + int rank_size_{0}; + + static std::shared_ptr instance_; }; } // namespace cpu } // namespace device } // namespace mindspore -#endif // ENABLE_MPI #endif // MINDSPORE_CCSRC_DEVICE_CPU_MPI_MPI_ADAPTER_H_ diff --git a/mindspore/ccsrc/device/device_address.h b/mindspore/ccsrc/device/device_address.h index e02d231dd5..0447cc2539 100644 --- a/mindspore/ccsrc/device/device_address.h +++ b/mindspore/ccsrc/device/device_address.h @@ -63,6 +63,7 @@ class DeviceAddress { size_t GetSize() const { return size_; } std::string format() const { return format_; } TypeId type_id() const { return type_id_; } + void set_host_shape(const std::vector &shape) { host_shape_ = shape; } virtual void set_status(DeviceAddressStatus status) {} virtual DeviceAddressStatus status() const { return DeviceAddressStatus::kInDevice; } virtual DeviceAddressType DeviceType() const { return DeviceAddressType::kUnknown; } @@ -77,6 +78,7 @@ class DeviceAddress { string format_{"DefaultFormat"}; TypeId type_id_{kNumberTypeFloat16}; bool from_mem_pool_{false}; + std::vector host_shape_{}; friend class KernelRuntime; friend class MemoryManager; friend class mindspore::device::ascend::tasksink::TaskGenerator; diff --git a/mindspore/ccsrc/device/gpu/gpu_device_address.cc b/mindspore/ccsrc/device/gpu/gpu_device_address.cc index 24097f3637..401eb9f34e 100644 --- a/mindspore/ccsrc/device/gpu/gpu_device_address.cc +++ b/mindspore/ccsrc/device/gpu/gpu_device_address.cc @@ -18,7 +18,6 @@ #include #include "device/gpu/gpu_device_manager.h" #include "utils/log_adapter.h" -#include "utils/context/ms_context.h" #include "device/gpu/gpu_memory_allocator.h" namespace mindspore { @@ -55,8 +54,6 @@ GPUDeviceAddress::~GPUDeviceAddress() { if (ptr_ == nullptr) { return; } - auto ms_context = MsContext::GetInstance(); - MS_EXCEPTION_IF_NULL(ms_context); if (from_mem_pool_) { GPUMemoryAllocator::GetInstance().FreeTensorMem(ptr_); ptr_ = nullptr; diff --git a/mindspore/ccsrc/device/gpu/gpu_kernel_runtime.cc b/mindspore/ccsrc/device/gpu/gpu_kernel_runtime.cc index 8095a503e3..ad0e093d7f 100644 --- a/mindspore/ccsrc/device/gpu/gpu_kernel_runtime.cc +++ b/mindspore/ccsrc/device/gpu/gpu_kernel_runtime.cc @@ -565,7 +565,7 @@ void GPUKernelRuntime::FreeKernelDynamicRes(const mindspore::AnfNodePtr &kernel, MS_EXCEPTION_IF_NULL(mem_reuse_util_ptr); auto cnode = kernel->cast(); MS_EXCEPTION_IF_NULL(cnode); - if (AnfAlgo::GetCNodeName(kernel) == kAllReduceOpName) { + if (AnfAlgo::IsCommunicationOp(kernel)) { return; } // Free the input of kernel by reference count. diff --git a/mindspore/ccsrc/device/kernel_adjust.cc b/mindspore/ccsrc/device/kernel_adjust.cc index 93007764af..fd0a8eb967 100644 --- a/mindspore/ccsrc/device/kernel_adjust.cc +++ b/mindspore/ccsrc/device/kernel_adjust.cc @@ -103,12 +103,13 @@ CNodePtr KernelAdjust::CreateRecvApplyKernel(const std::shared_ptr &kernel_graph_ptr) { - device::ascend::AscendStreamMng &stream_manager = device::ascend::AscendStreamMng::GetInstance(); - stream_manager.Reset(); + device::ascend::AscendResourceMng &resource_manager = device::ascend::AscendResourceMng::GetInstance(); + resource_manager.ResetResource(); if (!NeedInsertSwitch()) { return; } MS_EXCEPTION_IF_NULL(kernel_graph_ptr); + bool eos_mode = ConfigManager::GetInstance().iter_num() == INT32_MAX; ReorderGetNext(kernel_graph_ptr); std::map switch_loop_input; CreateSwitchOpParameters(kernel_graph_ptr, &switch_loop_input); @@ -129,29 +130,34 @@ void KernelAdjust::InsertSwitchLoop(const std::shared_ptr } } - auto orders = kernel_graph_ptr->execution_order(); + const std::vector &orders = kernel_graph_ptr->execution_order(); if (orders.empty()) { MS_LOG(EXCEPTION) << "graph execution order is empty"; } std::vector exec_order; + std::vector getnext_active_streams; + std::vector fpbp_active_streams; + CNodePtr getnext_cnode; + uint32_t eos_done_event_id = UINT32_MAX; // getnext loop process // getnext loop stream switch op CNodePtr getnext_switch_app = CreateStreamSwitchOp(kernel_graph_ptr, switch_loop_input); MS_EXCEPTION_IF_NULL(getnext_switch_app); - uint32_t getnext_switch_stream_id = stream_manager.ApplyNewStream(); + uint32_t getnext_switch_stream_id = resource_manager.ApplyNewStream(); AnfAlgo::SetStreamId(getnext_switch_stream_id, getnext_switch_app.get()); exec_order.push_back(getnext_switch_app); // getnext op - uint32_t getnext_stream_id = stream_manager.ApplyNewStream(); + uint32_t getnext_stream_id = resource_manager.ApplyNewStream(); size_t i = 0; for (; i < orders.size(); i++) { auto node = orders[i]; exec_order.push_back(node); AnfAlgo::SetStreamId(getnext_stream_id, exec_order[exec_order.size() - 1].get()); if (AnfAlgo::GetCNodeName(node) == kGetNextOpName) { + getnext_cnode = node; break; } } @@ -159,25 +165,67 @@ void KernelAdjust::InsertSwitchLoop(const std::shared_ptr // update getnext loop stream switch true_branch_stream attr AnfAlgo::SetNodeAttr(kAttrTrueBranchStream, MakeValue(getnext_stream_id), getnext_switch_app); - // getnext loop send - CNodePtr send = CreateSendApplyKernel(kernel_graph_ptr, kFirstEventId); - AnfAlgo::SetStreamId(getnext_stream_id, send.get()); - exec_order.push_back(send); + // getnext loop fpbp start send + uint32_t fpbp_start_event_id = resource_manager.ApplyNewEvent(); + CNodePtr fpbp_start_send = CreateSendApplyKernel(kernel_graph_ptr, fpbp_start_event_id); + AnfAlgo::SetStreamId(getnext_stream_id, fpbp_start_send.get()); + exec_order.push_back(fpbp_start_send); + + if (eos_mode) { + // getnext loop eos start send + uint32_t eos_start_event_id = resource_manager.ApplyNewEvent(); + CNodePtr eos_start_send = CreateSendApplyKernel(kernel_graph_ptr, eos_start_event_id); + AnfAlgo::SetStreamId(getnext_stream_id, eos_start_send.get()); + exec_order.push_back(eos_start_send); + + // End Of Sequence loop process + // eos loop stream switch + CNodePtr eos_switch_app = CreateStreamSwitchOp(kernel_graph_ptr, switch_loop_input); + MS_EXCEPTION_IF_NULL(eos_switch_app); + uint32_t eos_switch_stream_id = resource_manager.ApplyNewStream(); + AnfAlgo::SetStreamId(eos_switch_stream_id, eos_switch_app.get()); + AnfAlgo::SetNodeAttr(kStreamNeedActivedFirst, MakeValue(true), eos_switch_app); + exec_order.push_back(eos_switch_app); + + // eos loop eos start recv + CNodePtr eos_start_recv = CreateRecvApplyKernel(kernel_graph_ptr, eos_start_event_id); + uint32_t eos_stream_id = resource_manager.ApplyNewStream(); + AnfAlgo::SetStreamId(eos_stream_id, eos_start_recv.get()); + exec_order.push_back(eos_start_recv); + + // update eos loop stream switch true_branch_stream attr + AnfAlgo::SetNodeAttr(kAttrTrueBranchStream, MakeValue(eos_stream_id), eos_switch_app); + + // EndOfSequence op + CNodePtr end_of_sequence_op = CreateEndOfSequenceOP(kernel_graph_ptr, getnext_cnode); + MS_EXCEPTION_IF_NULL(end_of_sequence_op); + AnfAlgo::SetStreamId(eos_stream_id, end_of_sequence_op.get()); + exec_order.push_back(end_of_sequence_op); + + // eos loop eos done send + eos_done_event_id = resource_manager.ApplyNewEvent(); + CNodePtr eos_done_send = CreateSendApplyKernel(kernel_graph_ptr, eos_done_event_id); + AnfAlgo::SetStreamId(eos_stream_id, eos_done_send.get()); + exec_order.push_back(eos_done_send); + + // eos loop stream active + fpbp_active_streams.push_back(eos_switch_stream_id); + } // fpbp loop process // fpbp loop stream switch CNodePtr fpbp_switch_app = CreateStreamSwitchOp(kernel_graph_ptr, switch_loop_input); MS_EXCEPTION_IF_NULL(fpbp_switch_app); - uint32_t fpbp_switch_stream_id = stream_manager.ApplyNewStream(); + uint32_t fpbp_switch_stream_id = resource_manager.ApplyNewStream(); AnfAlgo::SetStreamId(fpbp_switch_stream_id, fpbp_switch_app.get()); AnfAlgo::SetNodeAttr(kStreamNeedActivedFirst, MakeValue(true), fpbp_switch_app); exec_order.push_back(fpbp_switch_app); - // fpbp loop recv - CNodePtr recv = CreateRecvApplyKernel(kernel_graph_ptr, kFirstEventId); - uint32_t fpbp_stream_id = stream_manager.ApplyNewStream(); - AnfAlgo::SetStreamId(fpbp_stream_id, recv.get()); - exec_order.push_back(recv); + // fpbp loop fpbp start recv + CNodePtr fpbp_start_recv = CreateRecvApplyKernel(kernel_graph_ptr, fpbp_start_event_id); + uint32_t fpbp_stream_id = resource_manager.ApplyNewStream(); + AnfAlgo::SetStreamId(fpbp_stream_id, fpbp_start_recv.get()); + exec_order.push_back(fpbp_start_recv); // update fpbp loop stream switch true_branch_stream attr AnfAlgo::SetNodeAttr(kAttrTrueBranchStream, MakeValue(fpbp_stream_id), fpbp_switch_app); @@ -190,40 +238,41 @@ void KernelAdjust::InsertSwitchLoop(const std::shared_ptr // fpbp memcpy std::vector memcpy_list; - std::vector before_list; - std::vector after_list; - bool first_memcpy_found = false; + std::vector other_list; CNodePtr cur_cnode = nullptr; for (size_t idx = i + 1; idx < orders.size(); idx++) { cur_cnode = orders[idx]; if (AnfAlgo::HasNodeAttr(kAttrLabelForInsertStreamActive, cur_cnode)) { memcpy_list.emplace_back(cur_cnode); - first_memcpy_found = true; - } else if (first_memcpy_found) { - after_list.emplace_back(cur_cnode); } else { - before_list.emplace_back(cur_cnode); + other_list.emplace_back(cur_cnode); } } - (void)std::copy(before_list.begin(), before_list.end(), std::back_inserter(exec_order)); + (void)std::copy(memcpy_list.begin(), memcpy_list.end(), std::back_inserter(exec_order)); + // fpbp loop eos done recv + if (eos_mode) { + CNodePtr eos_done_recv = CreateRecvApplyKernel(kernel_graph_ptr, eos_done_event_id); + AnfAlgo::SetStreamId(fpbp_stream_id, eos_done_recv.get()); + exec_order.push_back(eos_done_recv); + } + // stream active to activate getnext loop CNodePtr getnext_active_app = CreateStreamActiveOp(kernel_graph_ptr); MS_EXCEPTION_IF_NULL(getnext_active_app); - std::vector getnext_active_streams = {getnext_switch_stream_id}; + getnext_active_streams.push_back(getnext_switch_stream_id); AnfAlgo::SetNodeAttr(kAttrActiveStreamList, MakeValue>(getnext_active_streams), getnext_active_app); exec_order.push_back(getnext_active_app); // fpbp loop other ops - (void)std::copy(after_list.begin(), after_list.end(), std::back_inserter(exec_order)); + (void)std::copy(other_list.begin(), other_list.end(), std::back_inserter(exec_order)); - // stream active to activate fpbp loop + // stream active to activate fpbp loop and eos loop CNodePtr fpbp_active_app = CreateStreamActiveOp(kernel_graph_ptr); MS_EXCEPTION_IF_NULL(fpbp_active_app); - // specific deal for common ctrl stream policy - std::vector fpbp_active_streams = {fpbp_switch_stream_id}; + fpbp_active_streams.push_back(fpbp_switch_stream_id); AnfAlgo::SetNodeAttr(kAttrActiveStreamList, MakeValue>(fpbp_active_streams), fpbp_active_app); exec_order.push_back(fpbp_active_app); @@ -323,6 +372,55 @@ CNodePtr KernelAdjust::CreateStreamActiveOp(const std::shared_ptr &kernel_graph_ptr, + const CNodePtr &node, size_t output_idx) { + auto idx = NewValueNode(SizeToInt(output_idx)); + MS_EXCEPTION_IF_NULL(idx); + auto imm = std::make_shared(SizeToInt(output_idx)); + auto abstract_scalar = std::make_shared(imm); + idx->set_abstract(abstract_scalar); + CNodePtr tuple_getitem = kernel_graph_ptr->NewCNode({NewValueNode(prim::kPrimTupleGetItem), node, idx}); + MS_EXCEPTION_IF_NULL(tuple_getitem); + tuple_getitem->set_scope(node->scope()); + std::vector origin_shape = AnfAlgo::GetOutputInferShape(node, output_idx); + TypeId origin_type = AnfAlgo::GetOutputInferDataType(node, output_idx); + AnfAlgo::SetOutputInferTypeAndShape({origin_type}, {origin_shape}, tuple_getitem.get()); + return tuple_getitem; +} + +CNodePtr KernelAdjust::CreateEndOfSequenceOP(const std::shared_ptr &kernel_graph_ptr, + const CNodePtr &getnext_cnode) { + MS_EXCEPTION_IF_NULL(kernel_graph_ptr); + kernel::KernelBuildInfo::KernelBuildInfoBuilder selected_kernel_builder; + selected_kernel_builder.SetInputsFormat({kOpFormat_DEFAULT}); + selected_kernel_builder.SetInputsDeviceType({kNumberTypeUInt8}); + + selected_kernel_builder.SetFusionType(kernel::FusionType::OPAQUE); + selected_kernel_builder.SetProcessor(kernel::Processor::AICPU); + selected_kernel_builder.SetKernelType(KernelType::AICPU_KERNEL); + + selected_kernel_builder.SetOutputsFormat({kOpFormat_DEFAULT}); + selected_kernel_builder.SetOutputsDeviceType({kNumberTypeUInt8}); + // EndOfSequence + auto end_of_sequence = std::make_shared(kEndOfSequence); + std::vector inputs; + inputs.push_back(NewValueNode(end_of_sequence)); + // GetNext output 0 is EndOfSequence's input + auto tuple_get_item = CreatTupleGetItemNode(kernel_graph_ptr, getnext_cnode, 0); + inputs.push_back(tuple_get_item); + CNodePtr end_of_sequence_node = kernel_graph_ptr->NewCNode(inputs); + MS_EXCEPTION_IF_NULL(end_of_sequence_node); + AnfAlgo::SetSelectKernelBuildInfo(selected_kernel_builder.Build(), end_of_sequence_node.get()); + std::vector input_names = {"x"}; + ValuePtr input_names_v = MakeValue(input_names); + AnfAlgo::SetNodeAttr("input_names", input_names_v, end_of_sequence_node); + std::vector output_names = {"y"}; + ValuePtr output_names_v = MakeValue(output_names); + AnfAlgo::SetNodeAttr("output_names", output_names_v, end_of_sequence_node); + end_of_sequence_node->set_abstract(tuple_get_item->abstract()); + return end_of_sequence_node; +} + CNodePtr KernelAdjust::CreateStreamAssignAddnOP( const std::shared_ptr &kernel_graph_ptr, const std::map &switch_loop_input) { @@ -390,7 +488,7 @@ bool KernelAdjust::StepLoadCtrlInputs(const std::shared_ptrset_device_address(device_address); if (!device_address->SyncHostToDevice(trans::GetRuntimePaddingShape(pk_node, 0), LongToSize(tensor->data().nbytes()), tensor->data_type(), - tensor->data_c(false))) { + tensor->data_c())) { MS_LOG(INFO) << "SyncHostToDevice failed."; return false; } @@ -407,14 +505,14 @@ void KernelAdjust::LoadSwitchInputs(std::vector *inputs) { tensor::TensorPtr loop_count_tensor = std::make_shared(kInt32->type_id(), shp); MS_EXCEPTION_IF_NULL(loop_count_tensor); int32_t *val = nullptr; - val = static_cast(loop_count_tensor->data_c(true)); + val = static_cast(loop_count_tensor->data_c()); MS_EXCEPTION_IF_NULL(val); *val = 0; inputs->push_back(loop_count_tensor); tensor::TensorPtr iter_loop_tensor = std::make_shared(kInt32->type_id(), shp); MS_EXCEPTION_IF_NULL(iter_loop_tensor); - val = static_cast(iter_loop_tensor->data_c(true)); + val = static_cast(iter_loop_tensor->data_c()); MS_EXCEPTION_IF_NULL(val); *val = SizeToInt(LongToSize(ConfigManager::GetInstance().iter_num())); MS_LOG(INFO) << "iter_loop_tensor = " << *val; @@ -422,14 +520,14 @@ void KernelAdjust::LoadSwitchInputs(std::vector *inputs) { tensor::TensorPtr zero_tensor = std::make_shared(kInt32->type_id(), shp); MS_EXCEPTION_IF_NULL(zero_tensor); - val = static_cast(zero_tensor->data_c(true)); + val = static_cast(zero_tensor->data_c()); MS_EXCEPTION_IF_NULL(val); *val = 0; inputs->push_back(zero_tensor); tensor::TensorPtr one_tensor = std::make_shared(kInt32->type_id(), shp); MS_EXCEPTION_IF_NULL(one_tensor); - val = static_cast(one_tensor->data_c(true)); + val = static_cast(one_tensor->data_c()); MS_EXCEPTION_IF_NULL(val); *val = 1; inputs->push_back(one_tensor); diff --git a/mindspore/ccsrc/device/kernel_adjust.h b/mindspore/ccsrc/device/kernel_adjust.h index 1a7436b396..bf3ba2acb2 100644 --- a/mindspore/ccsrc/device/kernel_adjust.h +++ b/mindspore/ccsrc/device/kernel_adjust.h @@ -38,12 +38,8 @@ constexpr auto kIterLoopParamName = "iter_loop"; constexpr auto kZeroParamName = "zero"; constexpr auto kOneParamName = "one"; constexpr auto kStreamNeedActivedFirst = "stream_need_active_first"; +constexpr uint32_t kSecondStreamSwitchLabel = 2; -const uint32_t kFirstStreamSwitchLabel = 0; -const uint32_t kGetNextLabel = 1; -const uint32_t kSecondStreamSwitchLabel = 2; -const uint32_t kInvalidEventId = UINT32_MAX; -const uint32_t kFirstEventId = kInvalidEventId / 2; namespace device { class KernelAdjust { public: @@ -69,6 +65,10 @@ class KernelAdjust { std::map *switch_loop_input); CNodePtr CreateStreamSwitchOp(const std::shared_ptr &kernel_graph_ptr, const std::map &switch_loop_input); + CNodePtr CreatTupleGetItemNode(const std::shared_ptr &kernel_graph_ptr, const CNodePtr &node, + size_t output_idx); + CNodePtr CreateEndOfSequenceOP(const std::shared_ptr &kernel_graph_ptr, + const CNodePtr &getnext_cnode); CNodePtr CreateStreamAssignAddnOP(const std::shared_ptr &kernel_graph_ptr, const std::map &switch_loop_input); kernel::KernelBuildInfo::KernelBuildInfoBuilder CreateMngKernelBuilder(const std::vector &formats, diff --git a/mindspore/ccsrc/device/kernel_runtime.cc b/mindspore/ccsrc/device/kernel_runtime.cc index cc1e3ab8f3..27cf1dfc92 100644 --- a/mindspore/ccsrc/device/kernel_runtime.cc +++ b/mindspore/ccsrc/device/kernel_runtime.cc @@ -79,6 +79,14 @@ bool KernelRuntime::DumpData(mindspore::session::KernelGraph *graph) { return false; } +// for D to impl +bool KernelRuntime::LoadData(mindspore::session::KernelGraph *graph, Debugger *debugger) { + if (graph != nullptr) { + return true; + } + return false; +} + // for D to impl bool KernelRuntime::GenTask(const session::KernelGraph *graph) { if (graph != nullptr) { @@ -154,7 +162,7 @@ void KernelRuntime::RunOpAssignMemory(const std::vector &inpu UpdateRefNodeOutputMem(graph); } -void KernelRuntime::RunOpClearMemory(session::KernelGraph *graph) { +void KernelRuntime::RunOpClearMemory(const session::KernelGraph *graph) { MS_EXCEPTION_IF_NULL(graph); // clear input parameter memory resource for (const auto &input_node : graph->inputs()) { @@ -250,6 +258,7 @@ void KernelRuntime::RunOpAssignOutputMemory(const AnfNodePtr &kernel) { std::string output_format = AnfAlgo::GetOutputFormat(kernel, i); auto output_type = AnfAlgo::GetOutputDeviceDataType(kernel, i); auto device_address = CreateDeviceAddress(nullptr, output_sizes[i], output_format, output_type); + device_address->set_host_shape(trans::GetRuntimePaddingShape(kernel, i)); MS_EXCEPTION_IF_NULL(device_address); auto ret = mem_manager_->MallocMemFromMemPool(device_address, output_sizes[i]); if (!ret) { @@ -330,7 +339,7 @@ void KernelRuntime::AssignStaticMemoryInput(const session::KernelGraph *graph) { } } -void KernelRuntime::AssignStaticMemoryOutput(const session::KernelGraph *graph) { +void KernelRuntime::AssignStaticMemoryOutput(session::KernelGraph *graph) { MS_EXCEPTION_IF_NULL(graph); auto nodes = AnfAlgo::GetAllOutput(graph->output(), {prim::kPrimTupleGetItem}); std::vector non_communication_op; @@ -341,6 +350,7 @@ void KernelRuntime::AssignStaticMemoryOutput(const session::KernelGraph *graph) if (!item_with_index.first->isa() || !AnfAlgo::IsRealKernel(item_with_index.first)) { continue; } + graph->AddFinalOutputKernel(item_with_index.first); if (AnfAlgo::IsCommunicationOp(item_with_index.first)) { AssignCommunicationNodeMem(kStaticMem, item_with_index.first); } else { @@ -498,7 +508,9 @@ void KernelRuntime::AssignNodeOutputMem(int flag, const AnfNodePtr &node, int in } std::string output_format = AnfAlgo::GetOutputFormat(node, i); auto output_type = AnfAlgo::GetOutputDeviceDataType(node, i); - AnfAlgo::SetOutputAddr(CreateDeviceAddress(ptr, output_sizes[i], output_format, output_type), i, node.get()); + auto device_address = CreateDeviceAddress(ptr, output_sizes[i], output_format, output_type); + device_address->set_host_shape(trans::GetRuntimePaddingShape(node, i)); + AnfAlgo::SetOutputAddr(device_address, i, node.get()); } } @@ -535,7 +547,7 @@ void KernelRuntime::AssignValueNodeTensor(const ValueNodePtr &value_node, const } AnfAlgo::SetOutputAddr(address, output_idx, value_node.get()); if (!address->SyncHostToDevice(trans::GetRuntimePaddingShape(value_node, 0), tensor_size, tensor->data_type(), - tensor->data_c(false))) { + tensor->data_c())) { MS_EXCEPTION(NotExistsError) << "ValueNode SyncHostToDevice fail!" << value_node->DebugString() << "node format is" << AnfAlgo::GetOutputFormat(value_node, output_idx) << "node dtype is " << AnfAlgo::GetOutputInferDataType(value_node, output_idx); diff --git a/mindspore/ccsrc/device/kernel_runtime.h b/mindspore/ccsrc/device/kernel_runtime.h index bfe857f61b..8c6a5eb19b 100644 --- a/mindspore/ccsrc/device/kernel_runtime.h +++ b/mindspore/ccsrc/device/kernel_runtime.h @@ -27,6 +27,9 @@ #ifdef ENABLE_DUMP_E2E #include "debug/e2e_dump.h" #endif +#ifdef ENABLE_DEBUGGER +#include "debug/debugger/debugger.h" +#endif #include "session/kernel_graph.h" #include "session/anf_runtime_algorithm.h" #include "kernel/kernel.h" @@ -34,11 +37,15 @@ #include "device/memory_manager.h" using mindspore::tensor::Tensor; +using std::vector; using TensorPtr = std::shared_ptr; using mindspore::kernel::AddressPtr; using AddressPtrList = std::vector; namespace mindspore { +#ifndef ENABLE_DEBUGGER +class Debugger; +#endif namespace device { class KernelRuntime { public: @@ -47,9 +54,10 @@ class KernelRuntime { virtual bool Init() = 0; virtual void AssignMemory(session::KernelGraph *graph); void RunOpAssignMemory(const std::vector &input_tensors, session::KernelGraph *graph); - void RunOpClearMemory(session::KernelGraph *graph); + void RunOpClearMemory(const session::KernelGraph *graph); virtual bool Run(session::KernelGraph *graph); virtual bool DumpData(session::KernelGraph *graph); + virtual bool LoadData(session::KernelGraph *graph, Debugger *debugger); virtual bool RunTask(const session::KernelGraph *graph); virtual bool GenTask(const session::KernelGraph *graph); bool LaunchKernel(const session::KernelGraph *graph); @@ -87,7 +95,7 @@ class KernelRuntime { #endif private: - void AssignStaticMemoryOutput(const session::KernelGraph *graph); + void AssignStaticMemoryOutput(session::KernelGraph *graph); void GenLaunchArgs(const mindspore::kernel::KernelMod &kernel_mod, const AnfNodePtr &kernel, AddressPtrList *kernel_inputs, AddressPtrList *kernel_workspaces, AddressPtrList *kernel_outputs); bool LaunchKernelMod(const session::KernelGraph &graph); diff --git a/mindspore/ccsrc/device/kernel_runtime_manager.cc b/mindspore/ccsrc/device/kernel_runtime_manager.cc index 0f95f3e79b..29d74762b4 100644 --- a/mindspore/ccsrc/device/kernel_runtime_manager.cc +++ b/mindspore/ccsrc/device/kernel_runtime_manager.cc @@ -56,9 +56,14 @@ KernelRuntime *KernelRuntimeManager::GetSingleKernelRuntime(const std::string &d auto cur_runtime_key = runtime_map_.begin()->first; auto find_pos = cur_runtime_key.rfind('_'); if (find_pos != std::string::npos) { - auto cur_device_id = cur_runtime_key.substr(find_pos + 1); - MS_LOG(EXCEPTION) << "Can't change device id in runtime, already set device id: " << cur_device_id - << ", set device id: " << device_id << " failed"; + if (cur_runtime_key.size() > find_pos + 1) { + auto cur_device_id = cur_runtime_key.substr(find_pos + 1); + MS_LOG(EXCEPTION) << "Can't change device id in runtime, already set device id: " << cur_device_id + << ", set device id: " << device_id << " failed"; + } else { + MS_LOG(EXCEPTION) << "Can't change device id in runtime, current runtime_key size error, set device id: " + << device_id << " failed"; + } } } return GetKernelRuntime(device_name, device_id); diff --git a/mindspore/ccsrc/ir/anf.cc b/mindspore/ccsrc/ir/anf.cc index 3b2402172b..4c1d2bf50d 100644 --- a/mindspore/ccsrc/ir/anf.cc +++ b/mindspore/ccsrc/ir/anf.cc @@ -25,7 +25,7 @@ #include "ir/func_graph.h" #include "ir/primitive_base.h" - +#include "utils/context/ms_context.h" #include "operator/ops.h" namespace mindspore { @@ -179,4 +179,43 @@ std::string get_id(const AnfNodePtr &node) { void reset_id() { node_ids.clear(); } } // namespace id_generator + +std::string GetCNodeTarget(const AnfNodePtr &node) { + auto context_ptr = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(context_ptr); + std::string default_target = context_ptr->device_target(); + if (!node->isa()) { + return default_target; + } + auto cnode = node->cast(); + MS_EXCEPTION_IF_NULL(cnode); + auto attr_input = cnode->input(0); + if (attr_input == nullptr) { + return default_target; + } + auto value_node = attr_input->cast(); + if (value_node == nullptr) { + return default_target; + } + auto value = value_node->value(); + if (value == nullptr) { + return default_target; + } + if (!value->isa()) { + return default_target; + } + auto primitive = value->cast(); + auto att_target = primitive->GetAttr("primitive_target"); + if (att_target != nullptr) { + if (!att_target->isa()) { + MS_LOG(EXCEPTION) << "Only support string CPU|GPU|Ascend for primitive_target"; + } + auto target = GetValue(att_target); + if (kTargetSet.find(target) == kTargetSet.end()) { + MS_LOG(EXCEPTION) << "Only support string CPU|GPU|Ascend for primitive_target"; + } + return target; + } + return default_target; +} } // namespace mindspore diff --git a/mindspore/ccsrc/ir/anf.h b/mindspore/ccsrc/ir/anf.h index 95a018af06..8a44627885 100644 --- a/mindspore/ccsrc/ir/anf.h +++ b/mindspore/ccsrc/ir/anf.h @@ -448,7 +448,7 @@ void reset_id(); } // namespace id_generator using TaggedNodeMap = std::unordered_map; using TaggedGraph = std::pair; - +std::string GetCNodeTarget(const AnfNodePtr &node); } // namespace mindspore #endif // MINDSPORE_CCSRC_IR_ANF_H_ diff --git a/mindspore/ccsrc/ir/dtype.cc b/mindspore/ccsrc/ir/dtype.cc index 5e049c0623..71a78bdcf6 100644 --- a/mindspore/ccsrc/ir/dtype.cc +++ b/mindspore/ccsrc/ir/dtype.cc @@ -94,6 +94,48 @@ bool Slice::operator==(const Type &other) const { std::string Slice::DumpText() const { return ToString(); } +TypePtr UndeterminedType::DeepCopy() const { + MS_EXCEPTION_IF_NULL(element_type_); + if (IsGeneric()) { + return std::make_shared(); + } + return std::make_shared(element_type_->DeepCopy()); +} + +std::string UndeterminedType::ToReprString() const { + if (element_type_ == nullptr) { + return "Undetermined"; + } + return "Undetermined[" + element_type_->ToReprString() + "]"; +} + +std::string UndeterminedType::ToString() const { + if (element_type_ == nullptr) { + return "Undetermined"; + } + return "Undetermined[" + element_type_->ToString() + "]"; +} + +std::string UndeterminedType::DumpText() const { + if (element_type_ == nullptr) { + return "Undetermined"; + } + return "Undetermined[" + element_type_->DumpText() + "]"; +} + +bool UndeterminedType::operator==(const Type &other) const { + if (!IsSameObjectType(*this, other)) { + return false; + } + auto other_elem_type = static_cast(other).element_type_; + if (element_type_ == nullptr && other_elem_type == nullptr) { + return true; + } else if (element_type_ == nullptr || other_elem_type == nullptr) { + return false; + } + return *element_type_ == *other_elem_type; +} + TypePtr TensorType::DeepCopy() const { MS_EXCEPTION_IF_NULL(element_type_); if (IsGeneric()) { @@ -137,6 +179,48 @@ bool TensorType::operator==(const Type &other) const { return *element_type_ == *other_elem_type; } +TypePtr IndexedSlicesType::DeepCopy() const { + MS_EXCEPTION_IF_NULL(element_type_); + if (IsGeneric()) { + return std::make_shared(); + } + return std::make_shared(element_type_->DeepCopy()); +} + +std::string IndexedSlicesType::ToReprString() const { + if (element_type_ == nullptr) { + return "IndexedSlices"; + } + return "IndexedSlices[" + element_type_->ToReprString() + "]"; +} + +std::string IndexedSlicesType::ToString() const { + if (element_type_ == nullptr) { + return "IndexedSlices"; + } + return "IndexedSlices[" + element_type_->ToString() + "]"; +} + +std::string IndexedSlicesType::DumpText() const { + if (element_type_ == nullptr) { + return "IndexedSlices"; + } + return "IndexedSlices[" + element_type_->DumpText() + "]"; +} + +bool IndexedSlicesType::operator==(const Type &other) const { + if (!IsSameObjectType(*this, other)) { + return false; + } + auto other_elem_type = static_cast(other).element_type_; + if (element_type_ == nullptr && other_elem_type == nullptr) { + return true; + } else if (element_type_ == nullptr || other_elem_type == nullptr) { + return false; + } + return *element_type_ == *other_elem_type; +} + Function::Function() : Object(kObjectTypeFunction) { args_ = std::vector(); retval_ = nullptr; diff --git a/mindspore/ccsrc/ir/dtype.h b/mindspore/ccsrc/ir/dtype.h index 104a2ec40a..f10c56e659 100644 --- a/mindspore/ccsrc/ir/dtype.h +++ b/mindspore/ccsrc/ir/dtype.h @@ -108,10 +108,34 @@ class Slice : public Object { }; using SlicePtr = std::shared_ptr; +class UndeterminedType : public Object { + public: + UndeterminedType() : Object(kObjectTypeUndeterminedType) {} + explicit UndeterminedType(const TypePtr &ele) + : Object(kObjectTypeUndeterminedType, kMetaTypeObject, false), element_type_(ele) {} + ~UndeterminedType() override = default; + MS_DECLARE_PARENT(UndeterminedType, Object) + + TypeId generic_type_id() const override { return kObjectTypeUndeterminedType; } + const TypePtr element() const { return element_type_; } + void set_element(const TypePtr &element_type) { element_type_ = element_type; } + + TypePtr DeepCopy() const override; + std::string ToString() const override; + std::string ToReprString() const override; + std::string DumpText() const override; + bool operator==(const Type &other) const override; + + protected: + TypePtr element_type_; +}; +using MetaTensorTypePtr = std::shared_ptr; + class TensorType : public Object { public: - TensorType() : Object(kObjectTypeTensorType) {} - explicit TensorType(const TypePtr &ele) : Object(kObjectTypeTensorType, false), element_type_(ele) {} + TensorType() : Object(kObjectTypeTensorType, kObjectTypeUndeterminedType) {} + explicit TensorType(const TypePtr &ele) + : Object(kObjectTypeTensorType, kObjectTypeUndeterminedType, false), element_type_(ele) {} ~TensorType() override = default; MS_DECLARE_PARENT(TensorType, Object) @@ -130,6 +154,29 @@ class TensorType : public Object { }; using TensorTypePtr = std::shared_ptr; +class IndexedSlicesType : public Object { + public: + IndexedSlicesType() : Object(kObjectTypeIndexedSlicesType, kObjectTypeUndeterminedType) {} + explicit IndexedSlicesType(const TypePtr &ele) + : Object(kObjectTypeIndexedSlicesType, kObjectTypeUndeterminedType, false), element_type_(ele) {} + ~IndexedSlicesType() override = default; + MS_DECLARE_PARENT(IndexedSlicesType, Object) + + TypeId generic_type_id() const override { return kObjectTypeIndexedSlicesType; } + const TypePtr element() const { return element_type_; } + void set_element(const TypePtr &element_type) { element_type_ = element_type; } + + TypePtr DeepCopy() const override; + std::string ToString() const override; + std::string ToReprString() const override; + std::string DumpText() const override; + bool operator==(const Type &other) const override; + + private: + TypePtr element_type_; +}; +using IndexedSlicesTypePtr = std::shared_ptr; + class Function : public Object { public: Function(); @@ -255,6 +302,8 @@ TypePtr StringToType(const std::string &type_name); // Judge whether x is predicate or is a subclass of predicate. bool IsIdentidityOrSubclass(TypePtr const &x, TypePtr const &base_type); +bool IsParentOrChildrenType(TypePtr const &x, TypePtr const &base_type); + // Whether t1 is identity or a subclass of t2. bool IsSubType(TypePtr const &t1, TypePtr const &t2 = nullptr); @@ -275,6 +324,11 @@ extern const TypePtr kTypeExternal; extern const TypePtr kTypeEnv; extern const TypePtr kTypeType; extern const TypePtr kString; +extern const TypePtr kList; +extern const TypePtr kTuple; +extern const TypePtr kDict; +extern const TypePtr kSlice; +extern const TypePtr kKeyword; extern const TypePtr kTensorType; } // namespace mindspore diff --git a/mindspore/ccsrc/ir/dtype/empty.cc b/mindspore/ccsrc/ir/dtype/empty.cc index 5cb3a91806..c6abe9a408 100644 --- a/mindspore/ccsrc/ir/dtype/empty.cc +++ b/mindspore/ccsrc/ir/dtype/empty.cc @@ -18,5 +18,7 @@ namespace mindspore { const TypePtr kTypeNone = std::make_shared(); +const TypePtr kTypeNull = std::make_shared(); +const TypePtr kTypeEllipsis = std::make_shared(); const TypePtr kAnyType = std::make_shared(); } // namespace mindspore diff --git a/mindspore/ccsrc/ir/dtype/empty.h b/mindspore/ccsrc/ir/dtype/empty.h index 76cf8ea0eb..e3b46ec7d9 100644 --- a/mindspore/ccsrc/ir/dtype/empty.h +++ b/mindspore/ccsrc/ir/dtype/empty.h @@ -71,20 +71,22 @@ class TypeNull : public Type { }; using TypeNullPtr = std::shared_ptr; -class Ellipsis : public Type { +class TypeEllipsis : public Type { public: - Ellipsis() : Type(kMetaTypeEllipsis) {} - ~Ellipsis() override {} - MS_DECLARE_PARENT(Ellipsis, Type) + TypeEllipsis() : Type(kMetaTypeEllipsis) {} + ~TypeEllipsis() override {} + MS_DECLARE_PARENT(TypeEllipsis, Type) TypeId generic_type_id() const override { return kMetaTypeEllipsis; } - TypePtr DeepCopy() const override { return std::make_shared(); } + TypePtr DeepCopy() const override { return std::make_shared(); } std::string ToReprString() const override { return "Ellipsis"; } std::string DumpText() const override { return "Ellipsis"; } }; -using EllipsisPtr = std::shared_ptr; +using TypeEllipsisPtr = std::shared_ptr; extern const TypePtr kTypeNone; +extern const TypePtr kTypeNull; +extern const TypePtr kTypeEllipsis; extern const TypePtr kAnyType; } // namespace mindspore diff --git a/mindspore/ccsrc/ir/dtype/type.cc b/mindspore/ccsrc/ir/dtype/type.cc index 5395b59617..754876a366 100644 --- a/mindspore/ccsrc/ir/dtype/type.cc +++ b/mindspore/ccsrc/ir/dtype/type.cc @@ -115,6 +115,10 @@ const char *ObjectIdLabel(const TypeId &v) { return "kObjectTypeKeyword"; case kObjectTypeTensorType: return "kObjectTypeTensorType"; + case kObjectTypeIndexedSlicesType: + return "kObjectTypeIndexedSlicesType"; + case kObjectTypeUndeterminedType: + return "kObjectTypeUndeterminedType"; case kObjectTypeDictionary: return "kObjectTypeDictionary"; case kObjectTypeClass: diff --git a/mindspore/ccsrc/ir/dtype/type.h b/mindspore/ccsrc/ir/dtype/type.h index bfe39af43c..cba0d17fce 100644 --- a/mindspore/ccsrc/ir/dtype/type.h +++ b/mindspore/ccsrc/ir/dtype/type.h @@ -67,6 +67,7 @@ class Type : public Value { virtual bool equal(const TypePtr other) const { return *this == *other; } virtual TypeId object_type() const { return kTypeUnknown; } + virtual TypeId parent_type() const { return kTypeUnknown; } virtual TypeId number_type() const { return kTypeUnknown; } virtual TypePtr DeepCopy() const = 0; virtual TypePtr Clone() const { return DeepCopy(); } @@ -97,13 +98,16 @@ using TypePtrList = std::vector; // class Object : public Type { public: - Object() : Type(kMetaTypeObject), object_type_(kMetaTypeObject) {} + Object() : Type(kMetaTypeObject), object_type_(kMetaTypeObject), parent_type_(kMetaTypeObject) {} explicit Object(const TypeId object_type, bool is_generic = true) - : Type(kMetaTypeObject, is_generic), object_type_(object_type) {} + : Type(kMetaTypeObject, is_generic), object_type_(object_type), parent_type_(kMetaTypeObject) {} + explicit Object(const TypeId object_type, const TypeId parent_type, bool is_generic = true) + : Type(kMetaTypeObject, is_generic), object_type_(object_type), parent_type_(parent_type) {} ~Object() override = default; MS_DECLARE_PARENT(Object, Type) TypeId object_type() const override { return object_type_; } + TypeId parent_type() const override { return parent_type_; } TypeId type_id() const override { return object_type_; } TypeId generic_type_id() const override { return kMetaTypeObject; } bool equal(const TypePtr other) const override; @@ -114,6 +118,7 @@ class Object : public Type { private: const TypeId object_type_; + const TypeId parent_type_; }; std::ostream &operator<<(std::ostream &os, const TypePtrList &types); diff --git a/mindspore/ccsrc/ir/dtype/type_id.h b/mindspore/ccsrc/ir/dtype/type_id.h index 17862ad798..a711779e91 100644 --- a/mindspore/ccsrc/ir/dtype/type_id.h +++ b/mindspore/ccsrc/ir/dtype/type_id.h @@ -50,6 +50,8 @@ enum TypeId : int { kObjectTypeSlice, kObjectTypeKeyword, kObjectTypeTensorType, + kObjectTypeIndexedSlicesType, + kObjectTypeUndeterminedType, kObjectTypeClass, kObjectTypeDictionary, kObjectTypeFunction, diff --git a/mindspore/ccsrc/ir/dtype_extends.cc b/mindspore/ccsrc/ir/dtype_extends.cc index 697803fa46..732872cb4f 100644 --- a/mindspore/ccsrc/ir/dtype_extends.cc +++ b/mindspore/ccsrc/ir/dtype_extends.cc @@ -95,12 +95,30 @@ TypePtr TypeIdToType(TypeId id) { return kAnyType; case kMetaTypeNone: return kTypeNone; + case kMetaTypeNull: + return kTypeNull; + case kMetaTypeEllipsis: + return kTypeEllipsis; case kObjectTypeEnvType: return kTypeEnv; case kObjectTypeRefKey: return kRefKeyType; case kObjectTypeRef: return kRefType; + case kMetaTypeTypeType: + return kTypeType; + case kObjectTypeString: + return kString; + case kObjectTypeList: + return kList; + case kObjectTypeTuple: + return kTuple; + case kObjectTypeDictionary: + return kDict; + case kObjectTypeSlice: + return kSlice; + case kObjectTypeKeyword: + return kKeyword; case kTypeUnknown: return kTypeNone; default: @@ -174,6 +192,40 @@ TypePtr TensorStrToType(const std::string &type_name) { return type; } +TypePtr IndexedSlicesStrToType(const std::string &type_name) { + if (type_name == "IndexedSlices") { + return std::make_shared(); + } + auto start = type_name.find_first_of('[') + 1; + auto end = type_name.find_last_of(']'); + if (start >= type_name.size()) { + return nullptr; + } + auto element_str = type_name.substr(start, end - start); + auto element_type = StringToType(element_str); + if (element_type == nullptr) { + return nullptr; + } + return std::make_shared(element_type); +} + +TypePtr UndeterminedStrToType(const std::string &type_name) { + if (type_name == "Undetermined") { + return std::make_shared(); + } + auto start = type_name.find_first_of('[') + 1; + auto end = type_name.find_last_of(']'); + if (start >= type_name.size()) { + return nullptr; + } + auto element_str = type_name.substr(start, end - start); + auto element_type = StringToType(element_str); + if (element_type == nullptr) { + return nullptr; + } + return std::make_shared(element_type); +} + TypePtr ListStrToType(const std::string &type_name) { TypePtr type = nullptr; if (type_name == "List") { @@ -274,7 +326,7 @@ TypePtr StringToType(const std::string &type_name) { if (type_name.compare("None") == 0) { type = std::make_shared(); } else if (type_name.compare("Ellipsis") == 0) { - type = std::make_shared(); + type = std::make_shared(); } else if (type_name.compare("TypeType") == 0) { type = std::make_shared(); } else if (type_name.compare("SymbolicKeyType") == 0) { @@ -295,6 +347,10 @@ TypePtr StringToType(const std::string &type_name) { type = StringToNumberType(type_name, "Float"); } else if (type_name.compare(0, strlen("Tensor"), "Tensor") == 0) { type = TensorStrToType(type_name); + } else if (type_name.compare(0, strlen("Undetermined"), "Undetermined") == 0) { + type = UndeterminedStrToType(type_name); + } else if (type_name.compare(0, strlen("IndexedSlices"), "IndexedSlices") == 0) { + type = IndexedSlicesStrToType(type_name); } else if (type_name.compare(0, strlen("List"), "List") == 0) { type = ListStrToType(type_name); } else if (type_name.compare(0, strlen("Tuple"), "Tuple") == 0) { @@ -322,6 +378,20 @@ TypePtr StringToType(const std::string &type_name) { return type; } +bool IsParentOrChildrenType(TypePtr const &x, TypePtr const &base_type) { + if (x == nullptr || base_type == nullptr) { + MS_LOG(ERROR) << "Type is nullptr."; + return false; + } + if (base_type->type_id() == kTypeUnknown || x->type_id() == kTypeUnknown) { + return false; + } + if (base_type->type_id() == x->parent_type() || x->type_id() == base_type->parent_type()) { + return true; + } + return false; +} + bool IsIdentidityOrSubclass(TypePtr const &x, TypePtr const &base_type) { if (x == nullptr || base_type == nullptr) { MS_LOG(ERROR) << "Type is nullptr."; @@ -463,6 +533,10 @@ REGISTER_PYBIND_DEFINE( TensorType data(TypeIdToType(TypeId(static_cast(t[0].cast())))); return data; })); + (void)py::class_>(m_sub, "IndexedSlicesType") + .def(py::init()); + (void)py::class_>(m_sub, "UndeterminedType") + .def(py::init()); (void)py::class_>(m_sub, "Function") .def(py::init()) .def(py::init, TypePtr>(), py::arg("args"), py::arg("retval")); @@ -476,12 +550,19 @@ REGISTER_PYBIND_DEFINE( (void)py::class_>(m_sub, "RefType").def(py::init()); (void)py::class_>(m_sub, "TypeAnything").def(py::init()); (void)py::class_>(m_sub, "Slice").def(py::init()); - (void)py::class_>(m_sub, "Ellipsis").def(py::init()); + (void)py::class_>(m_sub, "TypeEllipsis").def(py::init()); })); const TypePtr kTypeExternal = std::make_shared(); const TypePtr kTypeEnv = std::make_shared(); const TypePtr kTypeType = std::make_shared(); const TypePtr kTensorType = std::make_shared(); +const TypePtr kIndexedSlicesType = std::make_shared(); +const TypePtr kUndeterminedType = std::make_shared(); const TypePtr kString = std::make_shared(); +const TypePtr kList = std::make_shared(); +const TypePtr kTuple = std::make_shared(); +const TypePtr kDict = std::make_shared(); +const TypePtr kSlice = std::make_shared(); +const TypePtr kKeyword = std::make_shared(); } // namespace mindspore diff --git a/mindspore/ccsrc/ir/func_graph.cc b/mindspore/ccsrc/ir/func_graph.cc index cdca98fc61..4e01e9003f 100644 --- a/mindspore/ccsrc/ir/func_graph.cc +++ b/mindspore/ccsrc/ir/func_graph.cc @@ -68,9 +68,7 @@ ParameterPtr FuncGraph::add_parameter() { void FuncGraph::add_parameter(const ParameterPtr &p) { if (manager_.lock()) { - std::vector new_params = parameters_; - new_params.push_back(p); - manager_.lock()->SetParameters(shared_from_base(), new_params); + manager_.lock()->AddParameter(shared_from_base(), p); } else { parameters_.push_back(p); } @@ -82,12 +80,8 @@ ParameterPtr FuncGraph::AddWeightParameter(const std::string &name) { p->set_name(name); p->debug_info()->set_name(name); - std::vector new_params = parameters_; - // append parameter - new_params.push_back(p); - if (manager_.lock()) { - manager_.lock()->SetParameters(shared_from_base(), new_params); + manager_.lock()->AddParameter(shared_from_base(), p); } else { parameters_.push_back(p); } @@ -432,7 +426,7 @@ AnfNodePtr FuncGraph::GetDefaultValueByName(const std::string &name) { if (default_value == nullptr) { MS_LOG(EXCEPTION) << "Graph parameter " << name << " not exist"; } - if (IsValueNode(default_value)) { + if (IsValueNode(default_value)) { return nullptr; } return default_value; @@ -440,8 +434,8 @@ AnfNodePtr FuncGraph::GetDefaultValueByName(const std::string &name) { // set the default values void FuncGraph::SetDefaultValues(const std::vector &name_list, const std::vector &value_list) { - auto all_is_null = std::all_of(value_list.begin(), value_list.end(), - [](const AnfNodePtr &node) { return IsValueNode(node); }); + auto all_is_null = + std::all_of(value_list.begin(), value_list.end(), [](const AnfNodePtr &node) { return IsValueNode(node); }); if (value_list.empty()) { all_is_null = true; } @@ -457,7 +451,7 @@ void FuncGraph::ClearDefaultValues() { parameter_default_value_.clear(); } size_t FuncGraph::GetDefaultValueCount() { int null_count = std::count_if(parameter_default_value_.begin(), parameter_default_value_.end(), - [](const std::pair &pair) { return IsValueNode(pair.second); }); + [](const std::pair &pair) { return IsValueNode(pair.second); }); return parameter_default_value_.size() - IntToSize(null_count); } diff --git a/mindspore/ccsrc/ir/func_graph.h b/mindspore/ccsrc/ir/func_graph.h index 5f09dfe6b5..b1be892a53 100644 --- a/mindspore/ccsrc/ir/func_graph.h +++ b/mindspore/ccsrc/ir/func_graph.h @@ -158,6 +158,7 @@ class FuncGraph : public FuncGraphBase { const std::vector ¶meters() const { return parameters_; } virtual ParameterPtr add_parameter(); void add_parameter(const ParameterPtr &p); + void append_parameter(const ParameterPtr &p) { parameters_.push_back(p); } void set_parameters(const std::vector ¶ms) { parameters_ = params; } // add a weight parameter with specific name ParameterPtr AddWeightParameter(const std::string &name); diff --git a/mindspore/ccsrc/ir/manager.cc b/mindspore/ccsrc/ir/manager.cc index 291a752405..cf56500aea 100644 --- a/mindspore/ccsrc/ir/manager.cc +++ b/mindspore/ccsrc/ir/manager.cc @@ -420,6 +420,12 @@ void FuncGraphManager::SetParameters(const FuncGraphPtr &fg, const std::vector &changes, EdgeTupl for (auto &iter : changes) { auto operation = iter.op; auto args = iter.args; - if (operation == Change::kTxSetEdge) { - auto edge = args.cast(); - auto old_node = edge.root_node->input(edge.index); - (*rm_edges)[std::make_pair(edge.root_node, std::make_pair(edge.index, old_node))] += 1; - (*add_edges)[std::make_pair(edge.root_node, std::make_pair(edge.index, edge.new_node))] += 1; - (*rms)[old_node] += 1; - (*adds)[edge.new_node] += 1; - edge.root_node->set_input(edge.index, edge.new_node); - } else if (operation == Change::kTxSetParams) { - auto param = args.cast(); - MS_EXCEPTION_IF_NULL(param.func_graph); - auto old_parameters = param.func_graph->parameters(); - for (auto &p : param.params) { - (*adds)[p] += 1; - } - for (auto &p : old_parameters) { - (*rms)[p] += 1; - } - param.func_graph->set_parameters(param.params); + switch (operation) { + case Change::kTxSetEdge: { + auto edge = args.cast(); + auto old_node = edge.root_node->input(edge.index); + (*rm_edges)[std::make_pair(edge.root_node, std::make_pair(edge.index, old_node))] += 1; + (*add_edges)[std::make_pair(edge.root_node, std::make_pair(edge.index, edge.new_node))] += 1; + (*rms)[old_node] += 1; + (*adds)[edge.new_node] += 1; + edge.root_node->set_input(edge.index, edge.new_node); + } break; + case Change::kTxSetParams: { + auto param = args.cast(); + MS_EXCEPTION_IF_NULL(param.func_graph); + auto old_parameters = param.func_graph->parameters(); + for (auto &p : param.params) { + (*adds)[p] += 1; + } + for (auto &p : old_parameters) { + (*rms)[p] += 1; + } + param.func_graph->set_parameters(param.params); + } break; + case Change::kTxAddParam: { + auto param = args.cast(); + MS_EXCEPTION_IF_NULL(param.func_graph); + (*adds)[param.param] += 1; + auto param_node = param.param->cast(); + param.func_graph->append_parameter(param_node); + } break; + default: + break; } } } @@ -599,6 +617,10 @@ void FuncGraphTransaction::SetParameters(FuncGraphPtr fg, const std::vector { void KeepRoots(const std::vector &roots = {}); void RemoveRoots(); void SetParameters(const FuncGraphPtr &fg, const std::vector ¶meters); + void AddParameter(const FuncGraphPtr &fg, const AnfNodePtr ¶meter); void MaybeDropFuncGraphs(const FuncGraphSet &func_graphs, bool ignore_users = false); bool Replace(const AnfNodePtr &old_node, const AnfNodePtr &new_node); void SetEdge(const AnfNodePtr &node, int index, const AnfNodePtr &value); @@ -400,6 +401,7 @@ class FuncGraphTransaction { // set parameters of a func graph void SetParameters(FuncGraphPtr fg, const std::vector ¶ms); + void AddParameter(FuncGraphPtr fg, const AnfNodePtr ¶m); // replace old_node with new_node bool Replace(const AnfNodePtr &old_node, const AnfNodePtr &new_node); @@ -427,6 +429,18 @@ struct ArgsOfSetParams { } }; +// args for add param +struct ArgsOfAddParam { + FuncGraphPtr func_graph; + AnfNodePtr param; + bool operator==(const ArgsOfAddParam &other) const { return &other == this; } + + friend std::ostream &operator<<(std::ostream &os, const ArgsOfAddParam &) { + os << "[ArgsOfAddParam]"; + return os; + } +}; + // args for set edge struct ArgsOfSetEdge { CNodePtr root_node; @@ -441,7 +455,7 @@ struct ArgsOfSetEdge { }; struct Change { - enum OpName { kTxSetParams, kTxSetEdge }; + enum OpName { kTxSetParams, kTxSetEdge, kTxAddParam }; OpName op; Any args; Change(OpName name, const Any ¶) : op(name), args(para) {} diff --git a/mindspore/ccsrc/ir/meta_tensor.h b/mindspore/ccsrc/ir/meta_tensor.h index d78caf3b5d..a8c07d6992 100644 --- a/mindspore/ccsrc/ir/meta_tensor.h +++ b/mindspore/ccsrc/ir/meta_tensor.h @@ -115,7 +115,7 @@ class MetaTensor : public Value { // order it represents. // // return A const vector which represents the shape of the tensor. - std::vector shape() const { return shape_; } + const std::vector &shape() const { return shape_; } // brief Sets the shape of a tensor. // diff --git a/mindspore/ccsrc/ir/named.cc b/mindspore/ccsrc/ir/named.cc index 0a679e6011..9e1a7968b8 100644 --- a/mindspore/ccsrc/ir/named.cc +++ b/mindspore/ccsrc/ir/named.cc @@ -30,9 +30,9 @@ bool Named::operator==(const Value &other) const { abstract::AbstractBasePtr None::ToAbstract() { return std::make_shared(); } const NamedPtr kNone = std::make_shared(); -abstract::AbstractBasePtr NullObj::ToAbstract() { return std::make_shared(); } -const NamedPtr kNull = std::make_shared(); +abstract::AbstractBasePtr Null::ToAbstract() { return std::make_shared(); } +const NamedPtr kNull = std::make_shared(); -abstract::AbstractBasePtr EllipsisObj::ToAbstract() { return std::make_shared(); } -const NamedPtr kEllipsis = std::make_shared(); +abstract::AbstractBasePtr Ellipsis::ToAbstract() { return std::make_shared(); } +const NamedPtr kEllipsis = std::make_shared(); } // namespace mindspore diff --git a/mindspore/ccsrc/ir/named.h b/mindspore/ccsrc/ir/named.h index fbc7969cab..40e544c129 100644 --- a/mindspore/ccsrc/ir/named.h +++ b/mindspore/ccsrc/ir/named.h @@ -71,20 +71,20 @@ class None : public Named { }; extern const NamedPtr kNone; -class NullObj : public Named { +class Null : public Named { public: - NullObj() : Named("Null") {} - ~NullObj() override = default; - MS_DECLARE_PARENT(NullObj, Named); + Null() : Named("Null") {} + ~Null() override = default; + MS_DECLARE_PARENT(Null, Named); abstract::AbstractBasePtr ToAbstract() override; }; extern const NamedPtr kNull; -class EllipsisObj : public Named { +class Ellipsis : public Named { public: - EllipsisObj() : Named("Ellipsis") {} - ~EllipsisObj() override = default; - MS_DECLARE_PARENT(EllipsisObj, Named); + Ellipsis() : Named("Ellipsis") {} + ~Ellipsis() override = default; + MS_DECLARE_PARENT(Ellipsis, Named); abstract::AbstractBasePtr ToAbstract() override; }; extern const NamedPtr kEllipsis; diff --git a/mindspore/ccsrc/ir/optimizer_caller.h b/mindspore/ccsrc/ir/optimizer_caller.h index bd30454147..036f4ab510 100644 --- a/mindspore/ccsrc/ir/optimizer_caller.h +++ b/mindspore/ccsrc/ir/optimizer_caller.h @@ -17,13 +17,23 @@ #ifndef MINDSPORE_CCSRC_IR_OPTIMIZER_CALLER_H_ #define MINDSPORE_CCSRC_IR_OPTIMIZER_CALLER_H_ +#include + #include "ir/anf.h" -#include "optimizer/opt.h" namespace mindspore { +namespace opt { +class Optimizer; +using OptimizerPtr = std::shared_ptr; +using OptimizerWeakPtr = std::weak_ptr; + +using PredicateFuncType = std::function; +} // namespace opt + class OptimizerCaller { public: virtual AnfNodePtr operator()(const opt::OptimizerPtr &, const AnfNodePtr &) { return nullptr; } }; +using OptimizerCallerPtr = std::shared_ptr; } // namespace mindspore #endif // MINDSPORE_CCSRC_IR_OPTIMIZER_CALLER_H_ diff --git a/mindspore/ccsrc/ir/primitive.cc b/mindspore/ccsrc/ir/primitive.cc index 59497affd5..6ec27c2567 100644 --- a/mindspore/ccsrc/ir/primitive.cc +++ b/mindspore/ccsrc/ir/primitive.cc @@ -86,7 +86,7 @@ void PrimitivePy::AddPyAttr(const py::str &name, const py::object &obj) { } bool converted = parse::ConvertData(obj, &converted_ret); if (!converted) { - MS_LOG(EXCEPTION) << "Attribute convert error with type:" << std::string(py::str(obj)); + MS_LOG(EXCEPTION) << "Attribute convert error with type: " << std::string(py::str(obj)); } (void)this->AddAttr(attr_name, converted_ret); } diff --git a/mindspore/ccsrc/ir/tensor.cc b/mindspore/ccsrc/ir/tensor.cc index e5212e922d..c06ba2a820 100644 --- a/mindspore/ccsrc/ir/tensor.cc +++ b/mindspore/ccsrc/ir/tensor.cc @@ -16,321 +16,263 @@ #include "ir/tensor.h" +#include #include #include -#include #include #include #include +#include #include "device/device_address.h" -#include "pybind_api/api_register.h" -#include "pybind_api/export_flags.h" #include "pipeline/static_analysis/abstract_value.h" namespace mindspore { namespace tensor { -static uint64_t count = 0; -void DataBuf2Contiguous(const py::array &src, py::array *const dest) { - if (dest == nullptr) { - MS_LOG(EXCEPTION) << "Failed to copy data to a contiguous buffer as dest is nullptr!"; - } - Py_buffer pybuf_src; - if (PyObject_GetBuffer(src.ptr(), &pybuf_src, PyBUF_ANY_CONTIGUOUS)) { - MS_LOG(EXCEPTION) << "Failed to get buffer info from the src!"; - } - - if (!PyBuffer_IsContiguous(&pybuf_src, 'C')) { - if (PyBuffer_ToContiguous(dest->request(true).ptr, &pybuf_src, pybuf_src.len, 'C')) { - MS_LOG(EXCEPTION) << "Can't copy numpy.ndarray to a contiguous buffer."; - } - } else { - *dest = src; - } - - PyBuffer_Release(&pybuf_src); +static std::string MakeId() { + // Use atomic to make id generator thread safe. + static std::atomic last_id{1}; + return "T" + std::to_string(last_id.fetch_add(1, std::memory_order_relaxed)); } -Tensor::Tensor(const TypePtr &type_ptr, const py::tuple &shape) { - TypeId data_type = TypeId::kTypeUnknown; - if (type_ptr != nullptr) { - data_type = type_ptr->type_id(); - } - data_type_ = data_type; - shape_.resize(shape.size()); - for (size_t i = 0; i < shape.size(); ++i) { - shape_[i] = py::int_(shape[i]); - } - init(data_type_, shape_, &data_); +static TypeId TypeIdOf(const TypePtr &data_type, TypeId defaultTypeId) { + return data_type ? data_type->type_id() : defaultTypeId; } -Tensor::Tensor(TypeId data_type, const std::vector &shape) { init(data_type, shape, &data_); } - -Tensor::Tensor(const py::array &input, const TypePtr &data_type) { init(input, data_type); } - -Tensor::Tensor(const py::list &input, const TypePtr &data_type) { init(py::array(input), data_type); } - -Tensor::Tensor(const py::tuple &input, const TypePtr &data_type) { init(py::array(input), data_type); } - -Tensor::Tensor(const py::float_ &input, const TypePtr &data_type) { init(py::array(input), data_type); } - -Tensor::Tensor(const py::int_ &input, const TypePtr &data_type) { init(py::array(input), data_type); } - -Tensor::Tensor(const Tensor &tensor, const TypePtr &data_type) - : MetaTensor(tensor), device_address_(tensor.device_address_) { - init(tensor.data_, data_type); - dirty_ = tensor.is_dirty(); - id_ = tensor.id(); +static size_t SizeOf(const std::vector &shape) { + return std::accumulate(shape.begin(), shape.end(), size_t(1), std::multiplies()); } -Tensor &Tensor::operator=(const Tensor &tensor) { - if (this != &tensor) { - MetaTensor::operator=(tensor); - dirty_ = tensor.is_dirty(); - device_address_ = tensor.device_address(); - data_ = tensor.data_; - id_ = tensor.id(); +template +std::vector CopyData(const std::vector &shape, void *data, TypeId data_type) { + const size_t count = SizeOf(shape); + switch (data_type) { + case kNumberTypeBool: + case kNumberTypeUInt8: { + auto buf = static_cast(data); + return std::vector(buf, buf + count); + } + case kNumberTypeInt8: { + auto buf = static_cast(data); + return std::vector(buf, buf + count); + } + case kNumberTypeInt16: { + auto buf = static_cast(data); + return std::vector(buf, buf + count); + } + case kNumberTypeInt32: { + auto buf = static_cast(data); + return std::vector(buf, buf + count); + } + case kNumberTypeInt64: { + auto buf = static_cast(data); + return std::vector(buf, buf + count); + } + case kNumberTypeUInt16: { + auto buf = static_cast(data); + return std::vector(buf, buf + count); + } + case kNumberTypeUInt32: { + auto buf = static_cast(data); + return std::vector(buf, buf + count); + } + case kNumberTypeUInt64: { + auto buf = static_cast(data); + return std::vector(buf, buf + count); + } + case kNumberTypeFloat16: { + auto buf = static_cast(data); + return std::vector(buf, buf + count); + } + case kNumberTypeFloat32: { + const float *buf = static_cast(data); + return std::vector(buf, buf + count); + } + case kNumberTypeFloat64: { + auto buf = static_cast(data); + return std::vector(buf, buf + count); + } + default: + break; } - return *this; -} -Tensor &Tensor::AssignValue(const Tensor &tensor) { - *this = tensor; - return *this; -} - -bool Tensor::operator==(const Tensor &tensor) const { - return (MetaTensor::operator==(tensor) && data_ == tensor.data_); -} - -bool Tensor::ValueEqual(const Tensor &other) const { - auto equal = [&other, this]() -> bool { - auto np = py::module::import("numpy"); - auto equal = np.attr("equal")(data_, other.data_); - auto all_equal = np.attr("all")(equal); - return all_equal.cast(); - }; - return (MetaTensor::operator==(other) && (data_.is(other.data_) || equal())); + MS_LOG(EXCEPTION) << "Cannot construct Tensor because of unsupported data type: " << data_type << "."; } -py::tuple Tensor::GetPyTupleShape() const { - std::vector shape = this->shape(); - py::tuple dims(shape.size()); - for (size_t i = 0; i < dims.size(); ++i) { - dims[i] = py::int_(shape[i]); +template +std::vector CopyData(const std::vector &shape, void *data, size_t data_len) { + size_t size = SizeOf(shape); + if (size * sizeof(T) != data_len) { + MS_LOG(EXCEPTION) << "Incorrect tensor input data length " << data_len << ", expect " << size * sizeof(T) + << " item size " << sizeof(T); } - return dims; + auto buf = static_cast(data); + return {buf, buf + size}; } -int Tensor::DataDim() const { return static_cast(data_.ndim()); } +// Tensor data implementation. +template +class TensorDataImpl : public TensorData { + public: + explicit TensorDataImpl(const std::vector &shape) : ndim_(shape.size()), data_size_(SizeOf(shape)) {} -int Tensor::DataSize() const { return static_cast(data_.size()); } + TensorDataImpl(const std::vector &shape, void *data, size_t data_len) + : ndim_(shape.size()), data_size_(SizeOf(shape)), data_(CopyData(shape, data, data_len)) {} -py::array Tensor::data() const { return data_; } + TensorDataImpl(const std::vector &shape, void *data, TypeId data_type) + : ndim_(shape.size()), data_size_(SizeOf(shape)), data_(CopyData(shape, data, data_type)) {} -int Tensor::data_type_c() const { return static_cast(data_type_); } + template + TensorDataImpl(const std::vector &shape, InputIt first, InputIt last) + : ndim_(shape.size()), data_size_(SizeOf(shape)), data_(first, last) {} -std::vector Tensor::shape_c(void) const { return shape(); } + template + TensorDataImpl(const std::vector &shape, Scalar scalar) + : ndim_(shape.size()), data_size_(SizeOf(shape)), data_({static_cast(scalar)}) {} -void *Tensor::data_c(bool writable) { - // operand of bit operation should be unsigned int. - unsigned int flags = ((unsigned int)data_.flags()) & pybind11::detail::npy_api::NPY_ARRAY_C_CONTIGUOUS_; - bool is_c_contiguous = (flags != 0) ? true : false; - if (!is_c_contiguous) { - py::array data_c; - init(data_type_, shape_, &data_c); - DataBuf2Contiguous(data_, &data_c); - data_ = data_c; - } - return data_.request(writable).ptr; -} + ssize_t size() const override { return static_cast(data_size_); } -TypeId Tensor::GetDataType(const py::buffer_info &buf) const { - TypeId data_type = TypeId::kTypeUnknown; - if (buf.format.compare("e") == 0) { - data_type = TypeId::kNumberTypeFloat16; - } else if (buf.format.compare("f") == 0) { - data_type = TypeId::kNumberTypeFloat32; - } else if (buf.format.compare("d") == 0) { - data_type = TypeId::kNumberTypeFloat64; - } else if (buf.format.compare("B") == 0) { - data_type = TypeId::kNumberTypeUInt8; - } else if (buf.format.compare("H") == 0) { - data_type = TypeId::kNumberTypeUInt16; - } else if (buf.format.compare("I") == 0) { - data_type = TypeId::kNumberTypeUInt32; - } else if (buf.format.compare("L") == 0 || buf.format.compare("Q") == 0) { - data_type = TypeId::kNumberTypeUInt64; - } else if (buf.format.compare("b") == 0) { - data_type = TypeId::kNumberTypeInt8; - } else if (buf.format.compare("h") == 0) { - data_type = TypeId::kNumberTypeInt16; - } else if (buf.format.compare("i") == 0) { - data_type = TypeId::kNumberTypeInt32; - } else if (buf.format.compare("l") == 0 || buf.format.compare("q") == 0) { - data_type = TypeId::kNumberTypeInt64; - } else if (buf.format.compare("?") == 0) { - data_type = TypeId::kNumberTypeBool; - } else { - MS_LOG(WARNING) << "Get unsupported DataType " << buf.format << "."; - } - return data_type; -} + ssize_t itemsize() const override { return static_cast(sizeof(T)); } -void Tensor::init(const py::array &input, const TypePtr &type_ptr) { - TypeId data_type = TypeId::kTypeUnknown; - if (type_ptr != nullptr) { - data_type = type_ptr->type_id(); - } - init(input, data_type); -} + ssize_t nbytes() const override { return size() * itemsize(); } -void Tensor::init(const py::array &input, const TypeId &data_type) { - py::buffer_info buf = input.request(); + ssize_t ndim() const override { return static_cast(ndim_); } - data_type_ = GetDataType(buf); - if (TypeId::kTypeUnknown == data_type && TypeId::kTypeUnknown == data_type_) { - MS_LOG(EXCEPTION) << "Unsupported tensor type!"; + void *data() override { + static std::vector empty_data(1); + if (data_size_ == 0) { + // Prevent null pointer for empty shape. + return empty_data.data(); + } + if (data_.empty()) { + // Lazy allocation. + data_.resize(data_size_); + } + return data_.data(); } - std::vector tm = buf.shape; - size_t len = tm.size(); - std::vector dims(len); - for (size_t i = 0; i < len; ++i) { - dims[i] = static_cast(tm[i]); + bool equals(const TensorData &other) const override { + auto ptr = dynamic_cast *>(&other); + if (ptr) { + return (ptr == this) || ((ndim_ == ptr->ndim_) && (data_size_ == ptr->data_size_) && (data_ == ptr->data_)); + } + return false; } - (void)set_shape(dims); - - if (TypeId::kTypeUnknown != data_type && TypeId::kTypeUnknown != data_type_ && data_type_ != data_type) { - // If user defined data type is not same as GetDataType from the data - bool success = convert_data(input, data_type_, &data_, data_type); - if (success) { - data_type_ = data_type; - } else { - data_type_ = TypeId::kTypeUnknown; - MS_LOG(EXCEPTION) << "Convert data from " << data_type_ << " to " << data_type << " failed!"; + + std::string ToString() const override { + std::ostringstream ss; + ss << '['; + for (auto value : data_) { + ss << value << ','; } - } else { - data_ = input; + ss << ']'; + return ss.str(); } - dirty_ = true; - id_ = std::to_string((uintptr_t)(this)) + std::to_string(count++); -} -void Tensor::init(TypeId data_type, const std::vector &shape, py::array *const data) { - data_type_ = data_type; - shape_ = shape; + private: + size_t ndim_{0}; + size_t data_size_{0}; + std::vector data_; +}; + +template +TensorDataPtr MakeTensorData(TypeId data_type, const std::vector &shape, Args... args) { switch (data_type) { case kNumberTypeBool: - *data = py::array_t(shape); - break; + case kNumberTypeUInt8: + return std::make_shared>(shape, args...); case kNumberTypeInt8: - *data = py::array_t(shape); - break; + return std::make_shared>(shape, args...); case kNumberTypeInt16: - *data = py::array_t(shape); - break; + return std::make_shared>(shape, args...); case kNumberTypeInt32: - *data = py::array_t(shape); - break; + return std::make_shared>(shape, args...); case kNumberTypeInt64: - *data = py::array_t(shape); - break; - case kNumberTypeUInt8: - *data = py::array_t(shape); - break; + return std::make_shared>(shape, args...); case kNumberTypeUInt16: - *data = py::array_t(shape); - break; + return std::make_shared>(shape, args...); case kNumberTypeUInt32: - *data = py::array_t(shape); - break; + return std::make_shared>(shape, args...); case kNumberTypeUInt64: - *data = py::array_t(shape); - break; + return std::make_shared>(shape, args...); case kNumberTypeFloat16: - *data = py::array_t(shape); - break; + return std::make_shared>(shape, args...); case kNumberTypeFloat32: - *data = py::array_t(shape); - break; + return std::make_shared>(shape, args...); case kNumberTypeFloat64: - *data = py::array_t(shape); - break; + return std::make_shared>(shape, args...); default: - MS_LOG(EXCEPTION) << "Cannot construct Tensor because of unsupported data type: " << data_type << "."; break; } - id_ = std::to_string((uintptr_t)(this)) + std::to_string(count++); + MS_LOG(EXCEPTION) << "Cannot construct Tensor because of unsupported data type: " << data_type << "."; } -TypePtr Tensor::SetDtype(const TypePtr type_ptr) { - MS_EXCEPTION_IF_NULL(type_ptr); - (void)set_data_type(type_ptr->type_id()); - return type_ptr; -} +Tensor::Tensor(const Tensor &tensor) + : MetaTensor(tensor), + init_flag_(tensor.init_flag_), + data_(tensor.data_), + dirty_(tensor.dirty_), + id_(tensor.id_), + device_address_(tensor.device_address_) {} -TypeId Tensor::set_data_type(const TypeId data_type) { - if (data_.size() > 0 && data_type_ != data_type) { - bool success = convert_data(data_, data_type_, &data_, data_type); - if (success) { - data_type_ = data_type; - } else { - MS_LOG(EXCEPTION) << "Convert data from " << data_type_ << " to " << data_type << " failed!"; - } - } else if (data_.size() == 0) { - data_type_ = data_type; - } +Tensor::Tensor(const Tensor &tensor, TypeId data_type) + : MetaTensor(data_type, tensor.shape_), + init_flag_(tensor.init_flag_), + data_(MakeTensorData(data_type, tensor.shape_, tensor.data_->data(), tensor.data_type_)), + dirty_(tensor.dirty_), + id_(tensor.id_), + device_address_(tensor.device_address_) {} - return data_type_; -} +Tensor::Tensor(TypeId data_type, const std::vector &shape, TensorDataPtr data) + : MetaTensor(data_type, shape), data_(std::move(data)), id_(MakeId()) {} -bool Tensor::is_init() { return init_flag_; } +Tensor::Tensor(TypeId data_type, const std::vector &shape) + : Tensor(data_type, shape, MakeTensorData(data_type, shape)) {} -void Tensor::set_init_flag(bool flag) { init_flag_ = flag; } +Tensor::Tensor(TypeId data_type, const std::vector &shape, void *data, size_t data_len) + : Tensor(data_type, shape, MakeTensorData(data_type, shape, data, data_len)) {} -bool Tensor::convert_data(const py::array &in, const TypeId in_data_type, py::array *const out, - const TypeId out_data_type) { - if (out == nullptr) { - return false; - } +Tensor::Tensor(TypeId data_type, const std::vector &shape, void *data, TypeId src_data_type) + : Tensor(data_type, shape, MakeTensorData(data_type, shape, data, src_data_type)) {} - bool result = true; - if (TypeId::kTypeUnknown == in_data_type || TypeId::kTypeUnknown == out_data_type) { - result = false; - } else if (in_data_type == out_data_type) { - *out = in; - } else if (TypeId::kNumberTypeFloat64 == out_data_type) { - *out = in.attr("astype").cast()("float64").cast(); - } else if (TypeId::kNumberTypeFloat32 == out_data_type) { - *out = in.attr("astype").cast()("float32").cast(); - } else if (TypeId::kNumberTypeFloat16 == out_data_type) { - *out = in.attr("astype").cast()("float16").cast(); - } else if (TypeId::kNumberTypeInt64 == out_data_type) { - *out = in.attr("astype").cast()("int64").cast(); - } else if (TypeId::kNumberTypeInt32 == out_data_type) { - *out = in.attr("astype").cast()("int32").cast(); - } else if (TypeId::kNumberTypeInt16 == out_data_type) { - *out = in.attr("astype").cast()("int16").cast(); - } else if (TypeId::kNumberTypeInt8 == out_data_type) { - *out = in.attr("astype").cast()("int8").cast(); - } else if (TypeId::kNumberTypeUInt8 == out_data_type) { - *out = in.attr("astype").cast()("uint8").cast(); - } else if (TypeId::kNumberTypeUInt16 == out_data_type) { - *out = in.attr("astype").cast()("uint16").cast(); - } else if (TypeId::kNumberTypeUInt32 == out_data_type) { - *out = in.attr("astype").cast()("uint32").cast(); - } else if (TypeId::kNumberTypeUInt64 == out_data_type) { - *out = in.attr("astype").cast()("uint64").cast(); - } else { - data_type_ = TypeId::kTypeUnknown; - MS_LOG(EXCEPTION) << "Cannot convert from " << TypeIdLabel(in_data_type) << " to " << TypeIdLabel(out_data_type) - << "."; - } +Tensor::Tensor(const std::vector &input, const TypePtr &data_type) + : MetaTensor(TypeIdOf(data_type, kNumberTypeInt32), {static_cast(input.size())}), + data_(MakeTensorData(data_type_, shape_, input.begin(), input.end())), + id_(MakeId()) {} + +Tensor::Tensor(const std::vector &input, const TypePtr &data_type) + : MetaTensor(TypeIdOf(data_type, kNumberTypeFloat32), {static_cast(input.size())}), + data_(MakeTensorData(data_type_, shape_, input.begin(), input.end())), + id_(MakeId()) {} + +Tensor::Tensor(int64_t input, const TypePtr &data_type) + : MetaTensor(TypeIdOf(data_type, kNumberTypeInt32), {}), + data_(MakeTensorData(data_type_, {}, input)), + id_(MakeId()) {} - return result; +Tensor::Tensor(double input, const TypePtr &data_type) + : MetaTensor(TypeIdOf(data_type, kNumberTypeFloat32), {}), + data_(MakeTensorData(data_type_, {}, input)), + id_(MakeId()) {} + +bool Tensor::operator==(const Tensor &tensor) const { + return (&tensor == this || (MetaTensor::operator==(tensor) && data_ == tensor.data_)); } +bool Tensor::ValueEqual(const Tensor &tensor) const { + return (&tensor == this || (MetaTensor::operator==(tensor) && data_->equals(*tensor.data_))); +} +// assgin value to this tensor +Tensor &Tensor::AssignValue(const Tensor &tensor) { + if (this != &tensor) { + MetaTensor::operator=(tensor); + dirty_ = tensor.is_dirty(); + device_address_ = tensor.device_address(); + data_ = tensor.data_; + id_ = tensor.id(); + } + return *this; +} abstract::AbstractBasePtr Tensor::ToAbstract() { auto tens = shared_from_base(); auto dtype = tens->Dtype(); @@ -345,17 +287,17 @@ abstract::AbstractBasePtr Tensor::ToAbstract() { std::string Tensor::GetShapeAndDataTypeInfo() const { std::ostringstream buf; - buf << "Tensor \nshape:[" << shape() << "]" << this->Dtype()->ToString(); + buf << "Tensor shape:[" << shape() << "]" << this->Dtype()->ToString(); return buf.str(); } std::string Tensor::ToString() const { const int small_tensor_size = 30; std::ostringstream buf; - buf << "Tensor \nshape:[" << shape() << "]" << this->Dtype()->ToString(); + buf << "Tensor shape:[" << shape() << "]" << this->Dtype()->ToString(); // only print small tensor if (DataSize() < small_tensor_size) { - buf << "val:" << std::string(py::str(data())); + buf << "val:" << data().ToString(); } return buf.str(); } @@ -365,164 +307,25 @@ std::string Tensor::ToStringRepr() const { auto type_ptr = this->Dtype(); MS_EXCEPTION_IF_NULL(type_ptr); buf << "Tensor shape:[" << shape() << "]" << type_ptr->ToString(); - buf << "\nval:" << std::string(py::str(data())); + buf << "\nval:" << data().ToString(); return buf.str(); } -py::array Tensor::data_sync() { +void Tensor::data_sync() const { if (device_address_ != nullptr) { - if (!device_address_->SyncDeviceToHost(this->shape(), static_cast(this->data().nbytes()), this->data_type(), - this->data_c(true))) { + if (!device_address_->SyncDeviceToHost(shape(), static_cast(data().nbytes()), data_type(), data_c())) { MS_LOG(EXCEPTION) << "SyncDeviceToHost when asnumpy."; } } - return data_; } -REGISTER_PYBIND_DEFINE(Tensor, ([](const py::module *m) { - // dtype should define before Tensor, because Tensor init depend dtype - (void)py::class_>(*m, "Tensor") - .def(py::init(), py::arg("dtype"), py::arg("shape")) - .def(py::init(), py::arg("input"), py::arg("dtype") = nullptr) - .def(py::init(), py::arg("input"), py::arg("dtype") = nullptr) - .def(py::init(), py::arg("input"), py::arg("dtype") = nullptr) - .def(py::init(), py::arg("input"), py::arg("dtype") = nullptr) - .def(py::init(), py::arg("input"), py::arg("dtype") = nullptr) - .def(py::init(), py::arg("input"), py::arg("dtype") = nullptr) - .def_readonly(PYTHON_TENSOR_FLAG, &Tensor::parse_info_) - .def_property_readonly("dtype", &Tensor::Dtype, R"mydelimiter( - Get the tensor's data type. - - Returns: - type, the data type of tensor. - - Examples: - >>> data = mindspore.Tensor(np.ones((2, 1), np.int32)) - >>> data.dtype - Int32 - )mydelimiter") - .def_property_readonly("shape", &Tensor::GetPyTupleShape, R"mydelimiter( - Get the tensor's shape. - - Returns: - tuple[int], the shape of tensor. - - Examples: - >>> data = mindspore.Tensor(np.ones((3, 3))) - >>> data.shape() - (3, 3) - )mydelimiter") - .def("asnumpy", &Tensor::data_sync, R"mydelimiter( - Convert tensor to numpy.ndarray. - - Returns: - numpy.ndarray. - - Examples: - >>> data = mindspore.Tensor(np.ones((2, 3))) - >>> array = data.asnumpy() - >>> array - array([[1., 1., 1.], - [1., 1., 1.]]) - )mydelimiter") - .def("size", &Tensor::DataSize, R"mydelimiter( - Get tensor's data size. - - Returns: - int, the size of tensor. - - Examples: - >>> data = mindspore.Tensor(np.ones((2, 3))) - >>> data.size() - 6 - )mydelimiter") - .def("is_init", &Tensor::is_init, R"mydelimiter( - Get tensor init_flag. - - Returns: - bool, whether the tensor init. - - Examples: - >>> data = mindspore.Tensor(np.ones((2, 3))) - >>> data.is_init() - False - )mydelimiter") - .def("set_init_flag", &Tensor::set_init_flag, R"mydelimiter( - Set tensor init_flag. - - Examples: - >>> data = mindspore.Tensor(np.ones((2, 3))) - >>> data.set_init_flag(True) - )mydelimiter") - .def("dim", &Tensor::DataDim, R"mydelimiter( - Get tensor's data dimension. - - Returns: - int, the dimension of tensor. - - Examples: - >>> data = mindspore.Tensor(np.ones((2, 3))) - >>> data.dim() - 2 - )mydelimiter") - .def("set_dtype", &Tensor::SetDtype, R"mydelimiter( - Set the tensor's data type. - - Arg: - dtype (:class:`mindspore.dtype`): The type of output tensor. - - Examples: - >>> data = mindspore.Tensor(np.ones((1, 2), np.float32)) - >>> data.set_dtype(mindspore.int32) - mindspore.int32 - )mydelimiter") - .def("assign_value", &Tensor::AssignValue, R"mydelimiter( - Assign another tensor value to this. - - Arg: - value (:class:`mindspore.tensor`): The value tensor. - - Examples: - >>> data = mindspore.Tensor(np.ones((1, 2), np.float32)) - >>> data2 = mindspore.Tensor(np.ones((2, 2), np.float32)) - >>> data.assign_value(data2) - >>> data.shape - (2, 2) - )mydelimiter") - .def("__str__", &Tensor::ToString) - .def("__repr__", &Tensor::ToStringRepr) - .def(py::pickle( - [](const Tensor &t) { // __getstate__ - /* Return a tuple that fully encodes the state of the object */ - return py::make_tuple(t.data()); - }, - [](const py::tuple &t) { // __setstate__ - if (t.size() != 1) { - throw std::runtime_error("Invalid state!"); - } - /* Create a new C++ instance */ - Tensor tensor(t[0].cast()); - return tensor; - })); - (void)py::class_>(*m, "MetaTensor") - .def(py::init>(), py::arg("dtype"), py::arg("shape")) - .def(py::pickle( - [](const MetaTensor &t) { // __getstate__ - /* Return a tuple that fully encodes the state of the object */ - return py::make_tuple(static_cast(t.data_type()), t.shape()); - }, - [](const py::tuple &t) { // __setstate__ - if (t.size() != 2) { - throw std::runtime_error("Invalid state!"); - } - /* Create a new C++ instance */ - MetaTensor tensor(TypeId(t[0].cast()), t[1].cast>()); - return tensor; - })) - .def_readonly(PYTHON_META_TENSOR_FLAG, &MetaTensor::parse_info_) - .def_property_readonly("dtype", &MetaTensor::Dtype, "Get the MetaTensor's dtype.") - .def_property_readonly("shape", &MetaTensor::shape, "Get the MetaTensor's shape."); - })); +TypeId Tensor::set_data_type(const TypeId data_type) { + if (data_type != data_type_) { + data_ = MakeTensorData(data_type, shape_, data_->data(), data_type_); + return MetaTensor::set_data_type(data_type); + } + return data_type; +} } // namespace tensor namespace inference { @@ -530,8 +333,6 @@ MSTensor *MSTensor::CreateTensor(TypeId data_type, const std::vector &shape return new Tensor(data_type, shape); } -Tensor::Tensor() { this->tensor_impl_ = std::make_shared(); } - Tensor::Tensor(TypeId data_type, const std::vector &shape) { this->tensor_impl_ = std::make_shared(data_type, shape); } @@ -585,7 +386,8 @@ size_t Tensor::Size() const { void *Tensor::MutableData() const { MS_ASSERT(this->tensor_impl_ != nullptr); - return this->tensor_impl_->data_c(true); + return this->tensor_impl_->data_c(); } + } // namespace inference } // namespace mindspore diff --git a/mindspore/ccsrc/ir/tensor.h b/mindspore/ccsrc/ir/tensor.h index 1ce657143b..5be8a063c1 100644 --- a/mindspore/ccsrc/ir/tensor.h +++ b/mindspore/ccsrc/ir/tensor.h @@ -20,9 +20,7 @@ #include #include #include - -#include "pybind11/numpy.h" -#include "pybind11/pybind11.h" +#include #include "Eigen/Core" #include "device/device_address.h" @@ -30,63 +28,8 @@ #include "include/ms_tensor.h" #include "utils/log_adapter.h" -namespace py = pybind11; - using float16 = Eigen::half; -namespace pybind11 { -namespace detail { -// Similar to enums in `pybind11/numpy.h`. Determined by doing: -// python3 -c 'import numpy as np; print(np.dtype(np.float16).num)' -constexpr int NPY_FLOAT16 = 23; - -template -struct npy_scalar_caster { - PYBIND11_TYPE_CASTER(T, _("PleaseOverride")); - using Array = array_t; - - bool load(handle src, bool convert) { - // Taken from Eigen casters. Permits either scalar dtype or scalar array. - handle type = dtype::of().attr("type"); - if (!convert && !isinstance(src) && !isinstance(src, type)) return false; - - Array tmp = Array::ensure(src); - if (tmp && tmp.size() == 1 && tmp.ndim() == 0) { - this->value = *tmp.data(); - return true; - } - - return false; - } - - static handle cast(T src, return_value_policy, handle) { - Array tmp({1}); - tmp.mutable_at(0) = src; - tmp.resize({}); - - // You could also just return the array if you want a scalar array. - object scalar = tmp[tuple()]; - return scalar.release(); - } -}; - -template <> -struct npy_format_descriptor { - static constexpr auto name = "float16"; - static pybind11::dtype dtype() { - handle ptr = npy_api::get().PyArray_DescrFromType_(NPY_FLOAT16); - return reinterpret_borrow(ptr); - } - virtual ~npy_format_descriptor() {} -}; - -template <> -struct type_caster : public npy_scalar_caster { - static constexpr auto name = "float16"; -}; -} // namespace detail -} // namespace pybind11 - using mindspore::device::DeviceAddress; using DeviceAddressPtr = std::shared_ptr; // brief mindspore namespace. @@ -98,179 +41,198 @@ namespace mindspore { // // A sub namespace in ME to support tensor related definition. namespace tensor { +// Tensor data interface. +class TensorData { + public: + /// Total number of elements. + virtual ssize_t size() const = 0; + /// Byte size of a single element. + virtual ssize_t itemsize() const = 0; + /// Total number of bytes. + virtual ssize_t nbytes() const = 0; + /// Number of dimensions. + virtual ssize_t ndim() const = 0; + /// Data pointer. + virtual void *data() = 0; + /// Is data equals. + virtual bool equals(const TensorData &other) const = 0; + /// To string. + virtual std::string ToString() const = 0; +}; + +using TensorDataPtr = std::shared_ptr; + // Tensor entity class class Tensor : public MetaTensor { public: - Tensor() = default; abstract::AbstractBasePtr ToAbstract() override; - // brief Constructor for Python. + + // brief Create tensor from another tensor, data is shared. + // + // param tensor [Tensor] The input tensor. + explicit Tensor(const Tensor &tensor); + + // brief Create tensor with given data type from another tensor. // - // param type_ptr [TypePty] Data type of the tensor. - // param py_shape [py::tuple] The shape represented by py::tuple of the tensor. - Tensor(const TypePtr &type_ptr, const py::tuple &shape); + // param tensor [Tensor] The input tensor. + // param data_type [TypeId] The new tensor data type. + Tensor(const Tensor &tensor, TypeId data_type); - // brief Constructor for C++. + // brief Create tensor with the given shared tensor data. + // + // param data_type [TypeId] Data type of the tensor. + // param shape The shape represented by std::vector of the tensor. + // param data The shared tensor data. + Tensor(TypeId data_type, const std::vector &shape, TensorDataPtr data); + + // brief Create an all zero tensor. // // param data_type [TypeId] Data type of the tensor. // param shape The shape represented by std::vector of the tensor. Tensor(TypeId data_type, const std::vector &shape); - // brief Constructor for Python. + // brief Create a tensor with input data buffer. // - // param input [py::array] Data value of the tensor. // param data_type [TypeId] Data type of the tensor. - explicit Tensor(const py::array &input, const TypePtr &data_type = nullptr); + // param shape The shape represented by std::vector of the tensor. + // param data The input data to be copied into tensor. + // param data_len The length of data in bytes. + Tensor(TypeId data_type, const std::vector &shape, void *data, size_t data_len); - // brief Constructor + // brief Create a tensor with input data buffer and given source data type. // - // param input [py::list] the data for tensor - // param data_type [TypeId] data type - explicit Tensor(const py::list &input, const TypePtr &data_type = nullptr); + // param data_type [TypeId] Data type of the tensor. + // param shape The shape represented by std::vector of the tensor. + // param data The input data to be copied into tensor. + // param src_data_type The source data type. + Tensor(TypeId data_type, const std::vector &shape, void *data, TypeId src_data_type); - // brief Constructor + // brief Create 1 dimension tensor from an int vector. // - // param input [py::tuple] the data for tensor + // param input [std::vector] the data for tensor // param data_type [TypeId] data type - explicit Tensor(const py::tuple &input, const TypePtr &data_type = nullptr); + explicit Tensor(const std::vector &input, const TypePtr &data_type = nullptr); - // brief Constructor + // brief Create 1 dimension tensor from a float vector. // - // param input [py::float_] the data for tensor + // param input [std::vector] the data for tensor // param data_type [TypeId] data type - explicit Tensor(const py::float_ &input, const TypePtr &data_type = nullptr); + explicit Tensor(const std::vector &input, const TypePtr &data_type = nullptr); - // brief Constructor + // brief Create 0 dimension tensor from an int scalar. // - // param input [py::int_] the data for tensor + // param input [int64] the data for tensor // param data_type [TypeId] data type - explicit Tensor(const py::int_ &input, const TypePtr &data_type = nullptr); + explicit Tensor(int64_t input, const TypePtr &data_type = nullptr); - // brief Constructor + // brief Create 0 dimension tensor from a float scalar. // - // param input [Tensor] the data for tensor + // param input [double] the data for tensor // param data_type [TypeId] data type - Tensor(const Tensor &tensor, const TypePtr &data_type = nullptr); + explicit Tensor(double input, const TypePtr &data_type = nullptr); ~Tensor() override = default; MS_DECLARE_PARENT(Tensor, MetaTensor); - // brief Overloads operator = for Tensor. - // - // The constructed Tensor object has the same type and shape with tensor. - // - // param tensor An existing Tensor object. - Tensor &operator=(const Tensor &tensor); - // brief Compares two Tensor objects. // - // Compare two tensor objects to see if they have same data type, shape and - // data value. + // Compare two tensor objects to see if they have same data type, shape and data address. // // param tensor The Tensor object to be compared. - // return true: If having same type, shape and data, return true, or return false. + // return true: If having same type, shape and data address, return true, or return false. bool operator==(const Tensor &tensor) const; - // It is different from 'operator==' which just compare shape/type/address, it do real value comparison. - bool ValueEqual(const Tensor &other) const; + // It is different from 'operator==' which just compare shape/type/address, + // it do real value comparison. + bool ValueEqual(const Tensor &tensor) const; // assgin value to this tensor Tensor &AssignValue(const Tensor &tensor); bool operator==(const Value &other) const override { if (other.isa()) { - auto other_ = static_cast(other); + auto &other_ = static_cast(other); return *this == other_; - } else { - return false; } + return false; } - py::tuple GetPyTupleShape() const; - // brief Gets tensor's dimension // // return The number of dimensions of the tensor data. - int DataDim() const; + int DataDim() const { return static_cast(data().ndim()); } // brief Getting tensor data size // // return The total number of elements of the tensor data. - int DataSize() const; - - // brief Tensor's data value. - // - // return [py::array] The tensor's data in py::array. - py::array data() const; + int DataSize() const { return static_cast(data().size()); } // brief Get the data type fo the tensor for C++ // // return [int] The tensor's data type will be cast to int to return. - int data_type_c() const; + int data_type_c() const { return static_cast(data_type_); } // brief Get the tensor's shape for C++ // // return [std::vector] - std::vector shape_c(void) const; + std::vector shape_c(void) const { return shape(); } // brief Get Tensor data pointer for c++ type // // param writable true if writable, false if read only // return The pointer to the object - void *data_c(bool writable = false); + void *data_c() { return data().data(); } // brief Get Tensor data byte-size for c++ type // // return byte size of Tensor data - size_t Size() const { return this->data().nbytes(); } + size_t Size() const { return data().nbytes(); } - // brief Get data type from tensor data. + void *data_c() const { return data_->data(); } + + // brief Sync data with device. + void data_sync() const; + + // brief Get the internal data object. // - // param buf The buffer info of the py::array data. - // return The [TypeId] of the tensor data. - TypeId GetDataType(const py::buffer_info &buf) const; + // return The reference to internal data object. + TensorData &data() { return *data_; } - // brief Sets the data type of a tensor. + // brief Get the internal data shared pointer. // - // param data_type The data type of the tensor to be set. + // return The reference to internal data object. + const TensorDataPtr &data_ptr() const { return data_; } + + // brief Get the internal data object. // + // return The reference to internal data object. + const TensorData &data() const { return *data_; } + TypeId set_data_type(const TypeId data_type) override; - TypePtr SetDtype(const TypePtr type_ptr) override; + std::string GetShapeAndDataTypeInfo() const; + std::string ToString() const override; - std::string ToStringRepr() const; - py::array data_; // < Tensor's data value - const bool parse_info_ = true; - bool is_init(); - void set_init_flag(bool flag); - private: - // brief init tensor - // - // param input [py::array] the data for tensor - // param data_type [TypeId] data type - // return true if succeed, false if failed. - void init(const py::array &input, const TypeId &data_type); - void init(const py::array &input, const TypePtr &type_ptr); - bool init_flag_{false}; - // brief init tensor attribute - // - // param data_type [TypeId] Data type of the tensor. - // param shape [py::array] The shape of the tensor. - // return true if succeed, false if failed. - void init(TypeId data_type, const std::vector &shape, py::array *data); + std::string ToStringRepr() const; - bool convert_data(const py::array &in, const TypeId in_data_type, py::array *out, const TypeId out_data_type); + bool is_init() { return init_flag_; } + void set_init_flag(bool flag) { init_flag_ = flag; } - public: bool is_dirty() const { return dirty_; } void set_dirty(const bool dirty) { dirty_ = dirty; } + DeviceAddressPtr device_address() const { return device_address_; } void set_device_address(const DeviceAddressPtr &device_address) { device_address_ = device_address; } - py::array data_sync(); + std::string id() const { return id_; } + const bool parse_info_ = true; + private: + bool init_flag_{false}; + TensorDataPtr data_{nullptr}; bool dirty_{true}; std::string id_{""}; DeviceAddressPtr device_address_{nullptr}; @@ -282,8 +244,6 @@ using TensorPtrList = std::vector>; namespace inference { class Tensor : public MSTensor { public: - Tensor(); - Tensor(TypeId data_type, const std::vector &shape); explicit Tensor(std::shared_ptr tensor_ptr); diff --git a/mindspore/ccsrc/ir/tensor_py.cc b/mindspore/ccsrc/ir/tensor_py.cc new file mode 100644 index 0000000000..11a000cef7 --- /dev/null +++ b/mindspore/ccsrc/ir/tensor_py.cc @@ -0,0 +1,390 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "ir/tensor_py.h" + +#include +#include +#include +#include +#include + +#include "device/device_address.h" +#include "pybind_api/api_register.h" +#include "pybind_api/export_flags.h" +#include "pipeline/static_analysis/abstract_value.h" + +namespace mindspore { +namespace tensor { + +static TypeId GetDataType(const py::buffer_info &buf) { + if (buf.format.size() == 1) { + switch (buf.format.front()) { + case 'e': + case 'f': + case 'd': + switch (buf.itemsize) { + case 2: + return TypeId::kNumberTypeFloat16; + case 4: + return TypeId::kNumberTypeFloat32; + case 8: + return TypeId::kNumberTypeFloat64; + } + break; + case 'b': + case 'h': + case 'i': + case 'l': + case 'q': + switch (buf.itemsize) { + case 1: + return TypeId::kNumberTypeInt8; + case 2: + return TypeId::kNumberTypeInt16; + case 4: + return TypeId::kNumberTypeInt32; + case 8: + return TypeId::kNumberTypeInt64; + } + break; + case 'B': + case 'H': + case 'I': + case 'L': + case 'Q': + switch (buf.itemsize) { + case 1: + return TypeId::kNumberTypeUInt8; + case 2: + return TypeId::kNumberTypeUInt16; + case 4: + return TypeId::kNumberTypeUInt32; + case 8: + return TypeId::kNumberTypeUInt64; + } + break; + case '?': + return TypeId::kNumberTypeBool; + } + } + MS_LOG(WARNING) << "Unsupported DataType format " << buf.format << " item size " << buf.itemsize; + return TypeId::kTypeUnknown; +} + +static std::string GetPyTypeFormat(TypeId data_type) { + switch (data_type) { + case TypeId::kNumberTypeFloat16: + return "e"; + case TypeId::kNumberTypeFloat32: + return py::format_descriptor::format(); + case TypeId::kNumberTypeFloat64: + return py::format_descriptor::format(); + case TypeId::kNumberTypeUInt8: + return py::format_descriptor::format(); + case TypeId::kNumberTypeUInt16: + return py::format_descriptor::format(); + case TypeId::kNumberTypeUInt32: + return py::format_descriptor::format(); + case TypeId::kNumberTypeUInt64: + return py::format_descriptor::format(); + case TypeId::kNumberTypeInt8: + return py::format_descriptor::format(); + case TypeId::kNumberTypeInt16: + return py::format_descriptor::format(); + case TypeId::kNumberTypeInt32: + return py::format_descriptor::format(); + case TypeId::kNumberTypeInt64: + return py::format_descriptor::format(); + case TypeId::kNumberTypeBool: + return py::format_descriptor::format(); + default: + MS_LOG(WARNING) << "Unsupported DataType " << data_type << "."; + return ""; + } +} + +static bool IsCContiguous(const py::array &input) { + auto flags = static_cast(input.flags()); + return (flags & pybind11::detail::npy_api::NPY_ARRAY_C_CONTIGUOUS_) != 0; +} + +TensorPtr TensorPy::MakeTensor(const py::array &input, const TypePtr &type_ptr) { + // Get input buffer info. + py::buffer_info buf = input.request(); + // Check data types. + auto data_type = type_ptr ? type_ptr->type_id() : TypeId::kTypeUnknown; + auto buf_type = GetDataType(buf); + if (buf_type == TypeId::kTypeUnknown && data_type == TypeId::kTypeUnknown) { + MS_LOG(EXCEPTION) << "Unsupported tensor type!"; + } + // Use buf type as data type if type_ptr not set. + if (data_type == TypeId::kTypeUnknown) { + data_type = buf_type; + } + // Convert input array to C contiguous if need. + std::unique_ptr tmp_buf; + if (!IsCContiguous(input)) { + Py_buffer pybuf; + if (PyObject_GetBuffer(input.ptr(), &pybuf, PyBUF_ANY_CONTIGUOUS)) { + MS_LOG(EXCEPTION) << "Failed to get buffer from the input!"; + } + tmp_buf = std::make_unique(pybuf.len); + if (PyBuffer_ToContiguous(tmp_buf.get(), &pybuf, pybuf.len, 'C')) { + MS_LOG(EXCEPTION) << "Can't copy numpy.ndarray to a contiguous buffer."; + } + PyBuffer_Release(&pybuf); + buf.ptr = tmp_buf.get(); + } + // Get tensor shape. + std::vector shape(buf.shape.begin(), buf.shape.end()); + if (data_type == buf_type) { + // Use memory copy if input data type is same as the required type. + return std::make_shared(data_type, shape, buf.ptr, buf.size * buf.itemsize); + } + // Create tensor with data type converted. + return std::make_shared(data_type, shape, buf.ptr, buf_type); +} + +static std::vector GetStrides(const std::vector &shape, ssize_t item_size) { + std::vector strides; + strides.reserve(shape.size()); + const auto ndim = shape.size(); + for (size_t i = 0; i < ndim; ++i) { + auto stride = item_size; + for (size_t j = i + 1; j < ndim; ++j) { + stride *= shape[j]; + } + strides.push_back(stride); + } + return strides; +} + +static py::buffer_info GetPyBufferInfo(const Tensor &tensor) { + std::vector shape(tensor.shape().begin(), tensor.shape().end()); + std::vector strides = GetStrides(shape, tensor.data().itemsize()); + return py::buffer_info{ + tensor.data_c(), tensor.data().itemsize(), GetPyTypeFormat(tensor.data_type()), tensor.DataDim(), shape, strides}; +} + +py::tuple TensorPy::GetPyTupleShape(const Tensor &tensor) { + auto &shape = tensor.shape(); + py::tuple dims(shape.size()); + for (size_t i = 0; i < dims.size(); ++i) { + dims[i] = py::int_(shape[i]); + } + return dims; +} + +py::array TensorPy::SyncAsNumpy(const Tensor &tensor) { + tensor.data_sync(); + auto info = GetPyBufferInfo(tensor); + py::object self = py::cast(&tensor); + return py::array(py::dtype(info), info.shape, info.strides, info.ptr, self); +} + +py::array TensorPy::AsNumpy(const Tensor &tensor) { + auto info = GetPyBufferInfo(tensor); + py::object self = py::cast(&tensor); + return py::array(py::dtype(info), info.shape, info.strides, info.ptr, self); +} + +static std::vector GetShapeFromTuple(const py::tuple &tuple) { + std::vector shape; + const size_t size = tuple.size(); + shape.reserve(tuple.size()); + for (size_t i = 0; i < size; ++i) { + shape.push_back(py::int_(tuple[i])); + } + return shape; +} + +REGISTER_PYBIND_DEFINE(Tensor, ([](const py::module *m) { + // Define python Tensor class. + // dtype should define before Tensor, because Tensor init depend dtype + (void)py::class_>(*m, "Tensor") + .def(py::init([](const Tensor &tensor) { return std::make_shared(tensor); }), + py::arg("input")) + .def(py::init([](const Tensor &tensor, const TypePtr &type_ptr) { + TypeId data_type = type_ptr ? type_ptr->type_id() : kTypeUnknown; + if (data_type == kTypeUnknown || tensor.data_type() == data_type) { + return std::make_shared(tensor); + } + return std::make_shared(tensor, data_type); + }), + py::arg("input"), py::arg("dtype")) + .def(py::init([](const TypePtr &type_ptr, const py::tuple &shape) { + auto data_type = type_ptr ? type_ptr->type_id() : TypeId::kNumberTypeFloat64; + return std::make_shared(data_type, GetShapeFromTuple(shape)); + }), + py::arg("dtype"), py::arg("shape")) + .def(py::init([](const py::array &input, const TypePtr &type_ptr) { + return TensorPy::MakeTensor(input, type_ptr); + }), + py::arg("input"), py::arg("dtype") = nullptr) + .def(py::init([](py::float_ input, const TypePtr &type_ptr) { + return TensorPy::MakeTensor(py::array(input), type_ptr); + }), + py::arg("input"), py::arg("dtype") = nullptr) + .def(py::init([](py::int_ input, const TypePtr &type_ptr) { + return TensorPy::MakeTensor(py::array(input), type_ptr); + }), + py::arg("input"), py::arg("dtype") = nullptr) + .def(py::init([](py::list input, const TypePtr &type_ptr) { + return TensorPy::MakeTensor(py::array(input), type_ptr); + }), + py::arg("input"), py::arg("dtype") = nullptr) + .def(py::init([](py::tuple input, const TypePtr &type_ptr) { + return TensorPy::MakeTensor(py::array(input), type_ptr); + }), + py::arg("input"), py::arg("dtype") = nullptr) + .def_readonly(PYTHON_TENSOR_FLAG, &Tensor::parse_info_) + .def_property_readonly("dtype", &Tensor::Dtype, R"mydelimiter( + Get the tensor's data type. + + Returns: + type, the data type of tensor. + + Examples: + >>> data = mindspore.Tensor(np.ones((2, 1), np.int32)) + >>> data.dtype + Int32 + )mydelimiter") + .def_property_readonly("shape", TensorPy::GetPyTupleShape, R"mydelimiter( + Get the tensor's shape. + + Returns: + tuple[int], the shape of tensor. + + Examples: + >>> data = mindspore.Tensor(np.ones((3, 3))) + >>> data.shape() + (3, 3) + )mydelimiter") + .def("asnumpy", TensorPy::SyncAsNumpy, R"mydelimiter( + Convert tensor to numpy.ndarray. + + Returns: + numpy.ndarray. + + Examples: + >>> data = mindspore.Tensor(np.ones((2, 3))) + >>> array = data.asnumpy() + >>> array + array([[1., 1., 1.], + [1., 1., 1.]]) + )mydelimiter") + .def("size", &Tensor::DataSize, R"mydelimiter( + Get tensor's data size. + + Returns: + int, the size of tensor. + + Examples: + >>> data = mindspore.Tensor(np.ones((2, 3))) + >>> data.size() + 6 + )mydelimiter") + .def("is_init", &Tensor::is_init, R"mydelimiter( + Get tensor init_flag. + + Returns: + bool, whether the tensor init. + + Examples: + >>> data = mindspore.Tensor(np.ones((2, 3))) + >>> data.is_init() + False + )mydelimiter") + .def("set_init_flag", &Tensor::set_init_flag, R"mydelimiter( + Set tensor init_flag. + + Examples: + >>> data = mindspore.Tensor(np.ones((2, 3))) + >>> data.set_init_flag(True) + )mydelimiter") + .def("dim", &Tensor::DataDim, R"mydelimiter( + Get tensor's data dimension. + + Returns: + int, the dimension of tensor. + + Examples: + >>> data = mindspore.Tensor(np.ones((2, 3))) + >>> data.dim() + 2 + )mydelimiter") + .def("assign_value", &Tensor::AssignValue, R"mydelimiter( + Assign another tensor value to this. + + Arg: + value (:class:`mindspore.tensor`): The value tensor. + + Examples: + >>> data = mindspore.Tensor(np.ones((1, 2), np.float32)) + >>> data2 = mindspore.Tensor(np.ones((2, 2), np.float32)) + >>> data.assign_value(data2) + >>> data.shape + (2, 2) + )mydelimiter") + .def("set_dtype", &Tensor::SetDtype, R"mydelimiter( + Set the tensor's data type. + + Arg: + dtype (:class:`mindspore.dtype`): The type of output tensor. + + Examples: + >>> data = mindspore.Tensor(np.ones((1, 2), np.float32)) + >>> data.set_dtype(mindspore.int32) + mindspore.int32 + )mydelimiter") + .def("__str__", &Tensor::ToString) + .def("__repr__", &Tensor::ToStringRepr) + .def(py::pickle( + [](const Tensor &t) { // __getstate__ + /* Return a tuple that fully encodes the state of the object */ + return py::make_tuple(TensorPy::AsNumpy(t)); + }, + [](const py::tuple &t) { // __setstate__ + if (t.size() != 1) { + throw std::runtime_error("Invalid state!"); + } + /* Create a new C++ instance */ + return TensorPy::MakeTensor(t[0].cast()); + })); + // Define python MetaTensor class. + (void)py::class_>(*m, "MetaTensor") + .def(py::init>(), py::arg("dtype"), py::arg("shape")) + .def_readonly(PYTHON_META_TENSOR_FLAG, &MetaTensor::parse_info_) + .def_property_readonly("dtype", &MetaTensor::Dtype, "Get the MetaTensor's dtype.") + .def_property_readonly("shape", &MetaTensor::shape, "Get the MetaTensor's shape.") + .def(py::pickle( + [](const MetaTensor &t) { // __getstate__ + /* Return a tuple that fully encodes the state of the object */ + return py::make_tuple(static_cast(t.data_type()), t.shape()); + }, + [](const py::tuple &t) { // __setstate__ + if (t.size() != 2) { + throw std::runtime_error("Invalid state!"); + } + /* Create a new C++ instance */ + MetaTensor tensor(TypeId(t[0].cast()), t[1].cast>()); + return tensor; + })); + })); + +} // namespace tensor +} // namespace mindspore diff --git a/mindspore/ccsrc/ir/tensor_py.h b/mindspore/ccsrc/ir/tensor_py.h new file mode 100644 index 0000000000..18ee547071 --- /dev/null +++ b/mindspore/ccsrc/ir/tensor_py.h @@ -0,0 +1,114 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_IR_TENSOR_PY_H_ +#define MINDSPORE_CCSRC_IR_TENSOR_PY_H_ + +#include +#include +#include + +#include "pybind11/pybind11.h" +#include "pybind11/numpy.h" + +#include "ir/tensor.h" + +namespace py = pybind11; + +namespace pybind11 { +namespace detail { +// Similar to enums in `pybind11/numpy.h`. Determined by doing: +// python3 -c 'import numpy as np; print(np.dtype(np.float16).num)' +constexpr int NPY_FLOAT16 = 23; + +template +struct npy_scalar_caster { + PYBIND11_TYPE_CASTER(T, _("PleaseOverride")); + using Array = array_t; + + bool load(handle src, bool convert) { + // Taken from Eigen casters. Permits either scalar dtype or scalar array. + handle type = dtype::of().attr("type"); + if (!convert && !isinstance(src) && !isinstance(src, type)) return false; + + Array tmp = Array::ensure(src); + if (tmp && tmp.size() == 1 && tmp.ndim() == 0) { + this->value = *tmp.data(); + return true; + } + + return false; + } + + static handle cast(T src, return_value_policy, handle) { + Array tmp({1}); + tmp.mutable_at(0) = src; + tmp.resize({}); + + // You could also just return the array if you want a scalar array. + object scalar = tmp[tuple()]; + return scalar.release(); + } +}; + +template <> +struct npy_format_descriptor { + static constexpr auto name = "float16"; + static pybind11::dtype dtype() { + handle ptr = npy_api::get().PyArray_DescrFromType_(NPY_FLOAT16); + return reinterpret_borrow(ptr); + } + virtual ~npy_format_descriptor() {} +}; + +template <> +struct type_caster : public npy_scalar_caster { + static constexpr auto name = "float16"; +}; +} // namespace detail +} // namespace pybind11 + +using mindspore::device::DeviceAddress; +using DeviceAddressPtr = std::shared_ptr; +// brief mindspore namespace. +// +// mindspore namespace is the top level namespace of Mindsporeession project. +// Other namespace should be a sub namespace of mindspore namespace in the ME project. +namespace mindspore { +// brief mindspore::tensor namespace +// +// A sub namespace in ME to support tensor related definition. +namespace tensor { +// Tensor python wrapper and adapter class. +class TensorPy { + public: + // brief Create Tensor from a numpy array object. + // + // param input [py::array] Data value of the tensor. + // param data_type [TypeId] Data type of the tensor. + static TensorPtr MakeTensor(const py::array &input, const TypePtr &data_type = nullptr); + + static py::array SyncAsNumpy(const Tensor &tensor); + + static py::array AsNumpy(const Tensor &tensor); + + static py::tuple GetPyTupleShape(const Tensor &tensor); +}; + +} // namespace tensor +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_IR_TENSOR_PY_H_ diff --git a/mindspore/ccsrc/kernel/akg/akg_kernel_attrs_process.cc b/mindspore/ccsrc/kernel/akg/akg_kernel_attrs_process.cc index 3a0cc3eb25..018fbe4f2a 100644 --- a/mindspore/ccsrc/kernel/akg/akg_kernel_attrs_process.cc +++ b/mindspore/ccsrc/kernel/akg/akg_kernel_attrs_process.cc @@ -68,10 +68,10 @@ void SetAkgAttrsForFive2Four(const AnfNodePtr &anf_node) { void SetAkgAttrsForCast(const AnfNodePtr &anf_node) { MS_EXCEPTION_IF_NULL(anf_node); // The x and output are akg op input and output param. - std::vector input_names = {"x"}; + std::vector input_names = {"x", "dst_type"}; std::vector output_names = {"output"}; - AnfAlgo::SetNodeAttr("input_names", MakeValue(input_names), anf_node); - AnfAlgo::SetNodeAttr("output_names", MakeValue(output_names), anf_node); + AnfAlgo::SetNodeAttr(kAttrInputNames, MakeValue(input_names), anf_node); + AnfAlgo::SetNodeAttr(kAttrOutputNames, MakeValue(output_names), anf_node); std::string dst_type; TypeId output_type = AnfAlgo::GetOutputDeviceDataType(anf_node, 0); diff --git a/mindspore/ccsrc/kernel/akg/akg_kernel_build.cc b/mindspore/ccsrc/kernel/akg/akg_kernel_build.cc index 1f88bbb89a..0e8d93d47f 100644 --- a/mindspore/ccsrc/kernel/akg/akg_kernel_build.cc +++ b/mindspore/ccsrc/kernel/akg/akg_kernel_build.cc @@ -276,7 +276,8 @@ bool AkgKernelBuild::CreateInputDescJson(const AnfNodePtr &anf_node, nlohmann::j input_desc_json[kName] = op_input_name; input_desc_json[kTensorName] = "input_" + std::to_string(GetInputTensorIdxInc(anf_node, real_input_index)); auto input_shape = AnfAlgo::GetInputDeviceShape(anf_node, real_input_index); - if (GetInputTensorValue(anf_node, real_input_index, &input_desc_json)) { + if (anf_node->func_graph() != nullptr && anf_node->func_graph()->has_attr(FUNC_GRAPH_ATTR_GRAPH_KERNEL) && + GetInputTensorValue(anf_node, real_input_index, &input_desc_json)) { MS_LOG(WARNING) << "we take input[" << real_input_index << "] of [" << anf_node->DebugString(2) << "] as const tensor, shape: [" << Vector2Str(input_shape) << "], value: " << input_desc_json[kValue]; @@ -618,6 +619,5 @@ size_t AkgKernelBuild::GetOutputTensorIdxInc() { size_t idx = output_tensor_idx_++; return idx; } - } // namespace kernel } // namespace mindspore diff --git a/mindspore/ccsrc/kernel/akg/akg_kernel_build.h b/mindspore/ccsrc/kernel/akg/akg_kernel_build.h index d32bd48ce6..15fa03f45b 100644 --- a/mindspore/ccsrc/kernel/akg/akg_kernel_build.h +++ b/mindspore/ccsrc/kernel/akg/akg_kernel_build.h @@ -70,7 +70,6 @@ void SetTensorName(const std::string &tag, const std::string &new_name, const st nlohmann::json *const node_json); std::string GetTensorName(const nlohmann::json &node_json, const std::string &tag, const std::pair &position); - } // namespace kernel } // namespace mindspore diff --git a/mindspore/ccsrc/kernel/akg/ascend/akg_ascend_kernel_build.cc b/mindspore/ccsrc/kernel/akg/ascend/akg_ascend_kernel_build.cc index 454b8052ab..7200a91ac0 100644 --- a/mindspore/ccsrc/kernel/akg/ascend/akg_ascend_kernel_build.cc +++ b/mindspore/ccsrc/kernel/akg/ascend/akg_ascend_kernel_build.cc @@ -35,7 +35,6 @@ namespace mindspore { namespace kernel { - constexpr int32_t PARALLEL_ARGS_SIZE = 3; constexpr int32_t PROCESS_NUM = 16; constexpr int32_t TIME_OUT = 300; @@ -48,76 +47,9 @@ constexpr auto kOutputDesc = "output_desc"; constexpr auto kTensorName = "tensor_name"; constexpr auto kCompileAkgKernelParallelFunc = "compile_akg_kernel_parallel"; constexpr auto kMultiProcModule = "mindspore._extends.parallel_compile.akg_compiler.multi_process_compiler"; - -bool AkgAscendKernelBuilder::CollectJson(const AnfNodePtr &anf_node) { - MS_EXCEPTION_IF_NULL(anf_node); - std::string op_name = AnfAlgo::GetCNodeName(anf_node); - MS_LOG(INFO) << "AKG start compile, op[" << op_name << "], device[" << AkgKernelBuild::GetProcessor(anf_node) << "]"; - auto it = kAkgKernelAttrsProcessMap.find(op_name); - if (it != kAkgKernelAttrsProcessMap.end()) { - it->second(anf_node); - } - MS_LOG(INFO) << "Akg start compile, op[" << op_name << "], device[" << AkgKernelBuild::GetProcessor(anf_node) << "]"; - nlohmann::json node_json; - if (!GenerateSingleKernelJson(anf_node, op_name, &node_json)) { - MS_LOG(ERROR) << "Op[" << op_name << "] create single kernel json failed."; - } - - kernel_json_ = node_json.dump(); - - if (!GetIOSize(node_json, &input_size_list_, &output_size_list_)) { - MS_LOG(ERROR) << "Cal mem size failed."; - return false; - } - - return true; -} - -bool AkgAscendKernelBuilder::CollectFusedJson(const std::vector &anf_nodes, - const std::vector &input_list, - const std::vector &output_list) { - if (anf_nodes.empty() || input_list.empty()) { - MS_LOG(ERROR) << "Invalid input size, anf_nodes [" << anf_nodes.size() << "], input_list [" << input_list.size() - << "]."; - return false; - } - MS_LOG(INFO) << "anf_nodes [" << output_list.size() << "], input_list [" << anf_nodes.size() << "], output_list [" - << input_list.size() << "]."; - - std::map node_json_map; - - for (auto const &anf_node : anf_nodes) { - MS_EXCEPTION_IF_NULL(anf_node); - std::string op_name = AnfAlgo::GetCNodeName(anf_node); - if (!AnfAlgo::IsRealKernel(anf_node)) { - MS_LOG(ERROR) << "Invalid anf node to build [" << anf_node->fullname_with_scope() << "]."; - return false; - } - auto it = kAkgKernelAttrsProcessMap.find(op_name); - if (it != kAkgKernelAttrsProcessMap.end()) { - it->second(anf_node); - } - - nlohmann::json node_json; - if (!GenerateSingleKernelJson(anf_node, op_name, &node_json)) { - MS_LOG(ERROR) << "Op [" << op_name << "] create single kernel json failed."; - return false; - } - // No need for composite op. - node_json.erase("id"); - node_json.erase("op"); - node_json.erase("composite"); - - auto primitive = AnfAlgo::GetCNodePrimitive(anf_node); - MS_EXCEPTION_IF_NULL(primitive); - - if (primitive->GetAttr("fusion") != nullptr) { - node_json["fusion"] = primitive->GetAttr("fusion")->ToString(); - } - - node_json_map[anf_node] = node_json; - } - +namespace { +void UpdateTensorNameInJson(const std::vector &anf_nodes, + std::map *node_json_map) { for (auto const &anf_node : anf_nodes) { std::vector dyn_input_sizes; auto primitive = AnfAlgo::GetCNodePrimitive(anf_node); @@ -134,11 +66,11 @@ bool AkgAscendKernelBuilder::CollectFusedJson(const std::vector &anf size_t input_tensor_num = is_dynamic_input ? IntToSize(dyn_input_sizes[i]) : 1; for (size_t j = 0; j < input_tensor_num; ++j) { auto tmp_input = GetKernelInput(anf_node, real_input_index); - std::string tensor_name = GetTensorName(node_json_map[anf_node], kInputDesc, std::make_pair(i, j)); - if (node_json_map.find(tmp_input.first) != node_json_map.end()) { + std::string tensor_name = GetTensorName((*node_json_map)[anf_node], kInputDesc, std::make_pair(i, j)); + if (node_json_map->find(tmp_input.first) != node_json_map->end()) { std::string new_tensor_name = - GetTensorName(node_json_map[tmp_input.first], kOutputDesc, std::make_pair(0, tmp_input.second)); - SetTensorName(kInputDesc, new_tensor_name, std::make_pair(i, j), &(node_json_map[anf_node])); + GetTensorName((*node_json_map)[tmp_input.first], kOutputDesc, std::make_pair(0, tmp_input.second)); + SetTensorName(kInputDesc, new_tensor_name, std::make_pair(i, j), &((*node_json_map)[anf_node])); MS_LOG(DEBUG) << "Update [" << real_input_index << "] input [" << tensor_name << "] of [" << anf_node->fullname_with_scope() << "] to [" << tmp_input.second << "] output [" << new_tensor_name << "] of [" << tmp_input.first->fullname_with_scope() << "]."; @@ -150,13 +82,10 @@ bool AkgAscendKernelBuilder::CollectFusedJson(const std::vector &anf } } } +} - nlohmann::json fused_node_json; - std::vector node_json_desc; - std::transform(anf_nodes.begin(), anf_nodes.end(), std::back_inserter(node_json_desc), - [&node_json_map](const AnfNodePtr &anf_node) { return node_json_map[anf_node]; }); - fused_node_json[kOpDesc] = node_json_desc; - +nlohmann::json GetInputsJson(const std::vector &anf_nodes, const std::vector &input_list, + std::map *node_json_map) { nlohmann::json inputs_json; auto input_index = GetInputIndex(anf_nodes, input_list); for (size_t i = 0; i < input_index.size(); ++i) { @@ -164,13 +93,18 @@ bool AkgAscendKernelBuilder::CollectFusedJson(const std::vector &anf auto type_id = AnfAlgo::GetInputDeviceDataType(tmp_input.first, tmp_input.second.first); std::string dtype = TypeId2String(type_id); nlohmann::json input_desc_json; - input_desc_json[kTensorName] = GetTensorName(node_json_map[tmp_input.first], kInputDesc, tmp_input.second); + input_desc_json[kTensorName] = GetTensorName((*node_json_map)[tmp_input.first], kInputDesc, tmp_input.second); input_desc_json[kDataType] = dtype; input_desc_json[kShape] = AnfAlgo::GetInputDeviceShape(tmp_input.first, tmp_input.second.first); inputs_json.emplace_back(std::vector{input_desc_json}); } - fused_node_json[kInputDesc] = inputs_json; + return inputs_json; +} + +nlohmann::json GetOutputsJson(const std::vector &anf_nodes, const std::vector &input_list, + const std::vector &output_list, const nlohmann::json &inputs_json, + std::map *node_json_map) { nlohmann::json outputs_json; auto output_index = GetOutputIndex(anf_nodes, input_list, output_list); for (size_t i = 0; i < output_index.size(); ++i) { @@ -188,7 +122,7 @@ bool AkgAscendKernelBuilder::CollectFusedJson(const std::vector &anf auto type_id = AnfAlgo::GetOutputDeviceDataType(tmp_output.first, tmp_output.second); std::string dtype = TypeId2String(type_id); output_desc_json[kTensorName] = - GetTensorName(node_json_map[tmp_output.first], kOutputDesc, std::make_pair(0, tmp_output.second)); + GetTensorName((*node_json_map)[tmp_output.first], kOutputDesc, std::make_pair(0, tmp_output.second)); output_desc_json[kDataType] = dtype; auto output_shape = AnfAlgo::GetOutputDeviceShape(tmp_output.first, tmp_output.second); if (output_shape.empty()) { @@ -198,7 +132,166 @@ bool AkgAscendKernelBuilder::CollectFusedJson(const std::vector &anf } outputs_json.emplace_back(output_desc_json); } - fused_node_json[kOutputDesc] = outputs_json; + + return outputs_json; +} + +std::pair, std::vector>> PreProcessJsonForBuild( + const std::vector> &build_args) { + // Remove cached nodes, gether unique nodes, and collect repeated nodes which need postprecess. + std::vector jsons; + std::vector> repeat_nodes; + std::unordered_set json_name_set; + for (const auto &[builder, anf_node] : build_args) { + MS_EXCEPTION_IF_NULL(anf_node); + auto json_name = builder.json_name(); + MS_LOG(DEBUG) << "Akg start compile op: " << json_name; + auto cached_kernel_pack = tbe::TbeUtils::SearchCache(json_name, AkgKernelBuild::GetProcessor(anf_node)); + if (cached_kernel_pack != nullptr) { + MS_LOG(DEBUG) << "Use cached kernel, json_name_[" << json_name << "], fullname_with_scope[" + << anf_node->fullname_with_scope() << "]."; + auto kernel_mod_ptr = std::make_shared(cached_kernel_pack); + kernel_mod_ptr->SetInputSizeList(builder.input_size_list()); + kernel_mod_ptr->SetOutputSizeList(builder.output_size_list()); + AnfAlgo::SetKernelMod(kernel_mod_ptr, anf_node.get()); + continue; + } + + if (json_name_set.count(json_name) != 0) { + repeat_nodes.push_back({builder, anf_node}); + continue; + } + json_name_set.insert(json_name); + auto node_json = builder.kernel_json(); + kernel::SaveJsonInfo(json_name, node_json); + jsons.push_back(node_json); + } + + return std::make_pair(jsons, repeat_nodes); +} + +bool PostProcessAfterCompile(const std::vector> &build_args, + const std::vector> &repeat_nodes) { + for (const auto &[builder, anf_node] : build_args) { + auto json_name = builder.json_name(); + auto new_kernel_pack = tbe::TbeUtils::InsertCache(json_name, AkgKernelBuild::GetProcessor(anf_node)); + if (new_kernel_pack == nullptr) { + MS_LOG(ERROR) << "Insert to cache failed, json_name_[" << json_name << "], fullname_with_scope[" + << anf_node->fullname_with_scope() << "]."; + return false; + } + auto kernel_mod_ptr = std::make_shared(new_kernel_pack); + kernel_mod_ptr->SetInputSizeList(builder.input_size_list()); + kernel_mod_ptr->SetOutputSizeList(builder.output_size_list()); + AnfAlgo::SetKernelMod(kernel_mod_ptr, anf_node.get()); + MS_LOG(DEBUG) << "Akg compile " << json_name << " kernel and insert cache successfully!"; + } + + for (const auto &[builder, anf_node] : repeat_nodes) { + auto node_json = builder.kernel_json(); + auto json_name = builder.json_name(); + auto cached_kernel_pack = tbe::TbeUtils::SearchCache(json_name, AkgKernelBuild::GetProcessor(anf_node)); + if (cached_kernel_pack == nullptr) { + return false; + } + MS_LOG(INFO) << "Use just compiled kernel, json_name_[" << json_name << "], fullname_with_scope[" + << anf_node->fullname_with_scope() << "]."; + auto kernel_mod_ptr = std::make_shared(cached_kernel_pack); + kernel_mod_ptr->SetInputSizeList(builder.input_size_list()); + kernel_mod_ptr->SetOutputSizeList(builder.output_size_list()); + AnfAlgo::SetKernelMod(kernel_mod_ptr, anf_node.get()); + } + + return true; +} +} // namespace + +bool AkgAscendKernelBuilder::CollectJson(const AnfNodePtr &anf_node) { + MS_EXCEPTION_IF_NULL(anf_node); + std::string op_name = AnfAlgo::GetCNodeName(anf_node); + MS_LOG(INFO) << "AKG start compile, op[" << op_name << "], device[" << AkgKernelBuild::GetProcessor(anf_node) << "]"; + auto it = kAkgKernelAttrsProcessMap.find(op_name); + if (it != kAkgKernelAttrsProcessMap.end()) { + it->second(anf_node); + } + MS_LOG(INFO) << "Akg start compile, op[" << op_name << "], device[" << AkgKernelBuild::GetProcessor(anf_node) << "]"; + nlohmann::json node_json; + if (!GenerateSingleKernelJson(anf_node, op_name, &node_json)) { + MS_LOG(ERROR) << "Op[" << op_name << "] create single kernel json failed."; + } + + kernel_json_ = node_json.dump(); + + if (!GetIOSize(node_json, &input_size_list_, &output_size_list_)) { + MS_LOG(ERROR) << "Cal mem size failed."; + return false; + } + + return true; +} + +bool AkgAscendKernelBuilder::GenJsonAndPreprocess4Fused(const std::vector &anf_nodes, + std::map *node_json_map) { + for (auto const &anf_node : anf_nodes) { + MS_EXCEPTION_IF_NULL(anf_node); + std::string op_name = AnfAlgo::GetCNodeName(anf_node); + if (!AnfAlgo::IsRealKernel(anf_node)) { + MS_LOG(ERROR) << "Invalid anf node to build [" << anf_node->fullname_with_scope() << "]."; + return false; + } + auto it = kAkgKernelAttrsProcessMap.find(op_name); + if (it != kAkgKernelAttrsProcessMap.end()) { + it->second(anf_node); + } + + nlohmann::json node_json; + if (!GenerateSingleKernelJson(anf_node, op_name, &node_json)) { + MS_LOG(ERROR) << "Op [" << op_name << "] create single kernel json failed."; + return false; + } + // No need for composite op. + node_json.erase("id"); + node_json.erase("op"); + node_json.erase("composite"); + + auto primitive = AnfAlgo::GetCNodePrimitive(anf_node); + MS_EXCEPTION_IF_NULL(primitive); + + if (primitive->GetAttr("fusion") != nullptr) { + node_json["fusion"] = primitive->GetAttr("fusion")->ToString(); + } + + (*node_json_map)[anf_node] = node_json; + } + return true; +} + +bool AkgAscendKernelBuilder::CollectFusedJson(const std::vector &anf_nodes, + const std::vector &input_list, + const std::vector &output_list) { + if (anf_nodes.empty() || input_list.empty()) { + MS_LOG(ERROR) << "Invalid input size, anf_nodes [" << anf_nodes.size() << "], input_list [" << input_list.size() + << "]."; + return false; + } + MS_LOG(INFO) << "anf_nodes [" << output_list.size() << "], input_list [" << anf_nodes.size() << "], output_list [" + << input_list.size() << "]."; + + std::map node_json_map; + if (!GenJsonAndPreprocess4Fused(anf_nodes, &node_json_map)) { + return false; + } + + UpdateTensorNameInJson(anf_nodes, &node_json_map); + + nlohmann::json fused_node_json; + std::vector node_json_desc; + std::transform(anf_nodes.begin(), anf_nodes.end(), std::back_inserter(node_json_desc), + [&node_json_map](const AnfNodePtr &anf_node) { return node_json_map[anf_node]; }); + fused_node_json[kOpDesc] = node_json_desc; + fused_node_json[kInputDesc] = GetInputsJson(anf_nodes, input_list, &node_json_map); + fused_node_json[kOutputDesc] = + GetOutputsJson(anf_nodes, input_list, output_list, fused_node_json[kInputDesc], &node_json_map); size_t hash_id = std::hash()(fused_node_json.dump()); json_name_ = "Fused_"; @@ -243,36 +336,7 @@ void GenParallelCompileFuncArgs(const std::vector &kernel_jsons, Py } bool AkgOpParallelBuild(const std::vector> &build_args) { - // Remove cached nodes, gether unique nodes, and collect repeated nodes which need postprecess. - std::vector jsons; - std::unordered_set json_name_set; - std::vector> repeat_nodes; - for (const auto &[builder, anf_node] : build_args) { - MS_EXCEPTION_IF_NULL(anf_node); - auto json_name = builder.json_name(); - MS_LOG(DEBUG) << "Akg start compile op: " << json_name; - auto cached_kernel_pack = tbe::TbeUtils::SearchCache(json_name, AkgKernelBuild::GetProcessor(anf_node)); - if (cached_kernel_pack != nullptr) { - MS_LOG(DEBUG) << "Use cached kernel, json_name_[" << json_name << "], fullname_with_scope[" - << anf_node->fullname_with_scope() << "]."; - auto kernel_mod_ptr = std::make_shared(cached_kernel_pack); - kernel_mod_ptr->SetInputSizeList(builder.input_size_list()); - kernel_mod_ptr->SetOutputSizeList(builder.output_size_list()); - AnfAlgo::SetKernelMod(kernel_mod_ptr, anf_node.get()); - continue; - } - - if (json_name_set.count(json_name) != 0) { - repeat_nodes.push_back({builder, anf_node}); - continue; - } - json_name_set.insert(json_name); - auto node_json = builder.kernel_json(); - kernel::SaveJsonInfo(json_name, node_json); - jsons.push_back(node_json); - } - - // No nodes need to be compiled! + auto [jsons, repeat_nodes] = PreProcessJsonForBuild(build_args); if (jsons.empty()) { return true; } @@ -307,34 +371,8 @@ bool AkgOpParallelBuild(const std::vectorfullname_with_scope() << "]."; - return false; - } - auto kernel_mod_ptr = std::make_shared(new_kernel_pack); - kernel_mod_ptr->SetInputSizeList(builder.input_size_list()); - kernel_mod_ptr->SetOutputSizeList(builder.output_size_list()); - AnfAlgo::SetKernelMod(kernel_mod_ptr, anf_node.get()); - MS_LOG(DEBUG) << "Akg compile " << json_name << " kernel and insert cache successfully!"; - } - - // Handle repeated nodes. - for (const auto &[builder, anf_node] : repeat_nodes) { - auto node_json = builder.kernel_json(); - auto json_name = builder.json_name(); - auto cached_kernel_pack = tbe::TbeUtils::SearchCache(json_name, AkgKernelBuild::GetProcessor(anf_node)); - if (cached_kernel_pack == nullptr) return false; - MS_LOG(INFO) << "Use just compiled kernel, json_name_[" << json_name << "], fullname_with_scope[" - << anf_node->fullname_with_scope() << "]."; - auto kernel_mod_ptr = std::make_shared(cached_kernel_pack); - kernel_mod_ptr->SetInputSizeList(builder.input_size_list()); - kernel_mod_ptr->SetOutputSizeList(builder.output_size_list()); - AnfAlgo::SetKernelMod(kernel_mod_ptr, anf_node.get()); + if (!PostProcessAfterCompile(build_args, repeat_nodes)) { + return false; } return true; @@ -380,6 +418,5 @@ bool AkgAscendKernelParallelBuild(const std::vector &anf_nodes) { return AkgOpParallelBuild(json_and_node); } - } // namespace kernel } // namespace mindspore diff --git a/mindspore/ccsrc/kernel/akg/ascend/akg_ascend_kernel_build.h b/mindspore/ccsrc/kernel/akg/ascend/akg_ascend_kernel_build.h index 619b583fde..01752911ed 100644 --- a/mindspore/ccsrc/kernel/akg/ascend/akg_ascend_kernel_build.h +++ b/mindspore/ccsrc/kernel/akg/ascend/akg_ascend_kernel_build.h @@ -20,6 +20,7 @@ #include #include #include +#include #include "ir/anf.h" #include "kernel/kernel.h" #include "kernel/akg/akg_kernel_build.h" @@ -40,6 +41,9 @@ class AkgAscendKernelBuilder : public AkgKernelBuild { const std::vector &output_size_list() const { return output_size_list_; } private: + bool GenJsonAndPreprocess4Fused(const std::vector &anf_nodes, + std::map *node_json_map); + std::string kernel_json_; std::vector input_size_list_; std::vector output_size_list_; diff --git a/mindspore/ccsrc/kernel/akg/ascend/akg_ascend_kernel_mod.cc b/mindspore/ccsrc/kernel/akg/ascend/akg_ascend_kernel_mod.cc index 24324f70e0..69fc82aad3 100644 --- a/mindspore/ccsrc/kernel/akg/ascend/akg_ascend_kernel_mod.cc +++ b/mindspore/ccsrc/kernel/akg/ascend/akg_ascend_kernel_mod.cc @@ -53,57 +53,9 @@ const std::vector &AkgKernelMod::GetOutputSizeList() const { return outp const std::vector &AkgKernelMod::GetWorkspaceSizeList() const { return workspace_size_list_; } -void DumpData(const std::vector &inputs, const std::vector &outputs) { - const char *dump_data = getenv("MS_KERNEL_DUMP_DATA"); - if (dump_data) { - int idx = 0; - for (const auto &x : inputs) { - std::vector buf(x->size); - if (RT_ERROR_NONE != rtMemcpy(buf.data(), buf.size(), reinterpret_cast(x->addr), x->size, - RT_MEMCPY_DEVICE_TO_HOST)) { - MS_LOG(WARNING) << "Call runtime rtMemcpy error."; - return; - } - - std::string file_name("input_"); - file_name += std::to_string(idx); - std::ofstream file(file_name, std::ios::binary); - if (file.is_open()) { - (void)file.write(buf.data(), SizeToLong(buf.size())); - file.close(); - idx++; - } else { - MS_LOG(ERROR) << "Open file failed."; - return; - } - } - idx = 0; - for (const auto &x : outputs) { - std::vector buf(x->size); - if (RT_ERROR_NONE != rtMemcpy(buf.data(), buf.size(), reinterpret_cast(x->addr), x->size, - RT_MEMCPY_DEVICE_TO_HOST)) { - MS_LOG(WARNING) << "Call runtime rtMemcpy error."; - return; - } - - std::string file_name("output_"); - file_name += std::to_string(idx); - std::ofstream file(file_name, std::ios::binary); - if (file.is_open()) { - (void)file.write(buf.data(), SizeToLong(buf.size())); - file.close(); - idx++; - } else { - MS_LOG(ERROR) << "Open file failed."; - return; - } - } - } -} - bool AkgKernelMod::Launch(const std::vector &inputs, const std::vector &, const std::vector &outputs, void *stream_ptr) { - if (stream_ptr == 0) { + if (stream_ptr == nullptr) { MS_LOG(ERROR) << "stream_ptr should not be nullptr."; return false; } @@ -135,8 +87,6 @@ bool AkgKernelMod::Launch(const std::vector &inputs, const std::vect return false; } - DumpData(inputs, outputs); - return true; } @@ -147,10 +97,10 @@ std::vector AkgKernelMod::GenTask(const std::vector &in } std::vector args; - uint32_t args_size = 0; + const uint32_t args_size = 0; std::vector sm_desc; void *binary = nullptr; - uint32_t binary_size = 0; + const uint32_t binary_size = 0; std::vector meta_data; std::vector input_data_addrs; std::vector output_data_addrs; diff --git a/mindspore/ccsrc/kernel/common_utils.cc b/mindspore/ccsrc/kernel/common_utils.cc index 868abeb1cc..ab4f59e549 100644 --- a/mindspore/ccsrc/kernel/common_utils.cc +++ b/mindspore/ccsrc/kernel/common_utils.cc @@ -721,19 +721,16 @@ std::vector> GetOutputIndex(const std::vectorDebugString(2) << "] of [" << output->func_graph()->ToString() << "] found no related kernel info."; @@ -744,18 +741,14 @@ std::vector> GetOutputIndex(const std::vector *node_list) { MS_EXCEPTION_IF_NULL(node_list); - MS_EXCEPTION_IF_NULL(func_graph); - std::vector node_lists = TopoSort(func_graph->get_return()); for (auto const &node : node_lists) { if (!AnfAlgo::IsRealKernel(node) || !node->isa()) { continue; } - auto cnode = node->cast(); MS_EXCEPTION_IF_NULL(cnode); - if (IsValueNode(cnode->input(kAnfPrimitiveIndex))) { node_list->push_back(node); } @@ -883,12 +876,13 @@ bool IsWeightBoundary(const AnfNodePtr &node) { return false; } -void MultiThreadCompute(const MultiThreadComputeFunc &func, MultiThreadComputeParams *params, size_t thread_num, +void MultiThreadCompute(const MultiThreadComputeFunc &func, MultiThreadComputeParams *params, size_t total_compute_size) { + const size_t kThreadNum = 24; std::vector threads; - threads.reserve(thread_num); + threads.reserve(kThreadNum); size_t start = 0; - size_t once_compute_size = (total_compute_size + thread_num - 1) / thread_num; + size_t once_compute_size = (total_compute_size + kThreadNum - 1) / kThreadNum; while (start < total_compute_size) { size_t end = (start + once_compute_size) > total_compute_size ? total_compute_size : (start + once_compute_size); threads.emplace_back(std::thread(func, params, start, end)); diff --git a/mindspore/ccsrc/kernel/common_utils.h b/mindspore/ccsrc/kernel/common_utils.h index e25421c57d..e9d72848f6 100644 --- a/mindspore/ccsrc/kernel/common_utils.h +++ b/mindspore/ccsrc/kernel/common_utils.h @@ -128,7 +128,7 @@ void GetValidKernelNodes(const FuncGraphPtr &func_graph, std::vector bool GetInputTensorValue(const AnfNodePtr &anf_node, size_t input_idx, nlohmann::json *const node_json); void GetGraphRealOutput(const FuncGraphPtr &func_graph, std::vector> *node_list); bool IsWeightBoundary(const AnfNodePtr &node); -void MultiThreadCompute(const MultiThreadComputeFunc &func, MultiThreadComputeParams *params, size_t thread_num, +void MultiThreadCompute(const MultiThreadComputeFunc &func, MultiThreadComputeParams *params, size_t total_compute_size); } // namespace kernel } // namespace mindspore diff --git a/mindspore/ccsrc/kernel/cpu/allgather_cpu_kernel.cc b/mindspore/ccsrc/kernel/cpu/allgather_cpu_kernel.cc index abb0c65d27..9cc5126c08 100644 --- a/mindspore/ccsrc/kernel/cpu/allgather_cpu_kernel.cc +++ b/mindspore/ccsrc/kernel/cpu/allgather_cpu_kernel.cc @@ -46,8 +46,9 @@ bool AllGatherCPUKernel::Launch(const std::vector &inputs, auto input_addr = reinterpret_cast(inputs[0]->addr); auto output_addr = reinterpret_cast(outputs[0]->addr); auto input_data_num = inputs[0]->size / sizeof(float); - - return device::cpu::MPIAdapter::Instance().AllGather(input_addr, output_addr, ranks_group_, input_data_num); + auto mpi_instance = device::cpu::MPIAdapter::Instance(); + MS_EXCEPTION_IF_NULL(mpi_instance); + return mpi_instance->AllGather(input_addr, output_addr, ranks_group_, input_data_num); } } // namespace kernel } // namespace mindspore diff --git a/mindspore/ccsrc/kernel/cpu/allgather_cpu_kernel.h b/mindspore/ccsrc/kernel/cpu/allgather_cpu_kernel.h index 94180fa89b..1dddf810ef 100644 --- a/mindspore/ccsrc/kernel/cpu/allgather_cpu_kernel.h +++ b/mindspore/ccsrc/kernel/cpu/allgather_cpu_kernel.h @@ -36,7 +36,7 @@ class AllGatherCPUKernel : public CPUKernel { std::vector ranks_group_; }; -MS_REG_CPU_KERNEL(HostAllGather, KernelAttr().AddInputAttr(kNumberTypeFloat32).AddOutputAttr(kNumberTypeFloat32), +MS_REG_CPU_KERNEL(_HostAllGather, KernelAttr().AddInputAttr(kNumberTypeFloat32).AddOutputAttr(kNumberTypeFloat32), AllGatherCPUKernel); } // namespace kernel } // namespace mindspore diff --git a/mindspore/ccsrc/kernel/cpu/debug_cpu_kernel.cc b/mindspore/ccsrc/kernel/cpu/debug_cpu_kernel.cc new file mode 100644 index 0000000000..a1dcaca3f3 --- /dev/null +++ b/mindspore/ccsrc/kernel/cpu/debug_cpu_kernel.cc @@ -0,0 +1,50 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "kernel/cpu/debug_cpu_kernel.h" +#include "device/cpu/cpu_device_address.h" +#include "common/utils.h" +#ifdef ENABLE_DEBUGGER +#include "debug/debugger/debugger.h" +#endif + +namespace mindspore { +namespace kernel { +void DebugCPUKernel::InitKernel(const CNodePtr &kernel_node) { MS_EXCEPTION_IF_NULL(kernel_node); } + +bool DebugCPUKernel::Launch(const std::vector &inputs, + const std::vector & /*workspace*/, + const std::vector &outputs) { + if (inputs.size() < 1 || outputs.empty()) { + MS_LOG(EXCEPTION) << " input or output empty!"; + } + auto val = reinterpret_cast(inputs[0]->addr); + MS_LOG(DEBUG) << " launch DebugCountCPUKernel val " << *val; + + auto output = reinterpret_cast(outputs[0]->addr); + size_t elem_num = inputs[0]->size / sizeof(int); + for (size_t i = 0; i < elem_num; i++) { + output[i] = val[i]; + } + +#ifdef ENABLE_DEBUGGER + // debugger will suspend execution is neccessary + Debugger::GetInstance()->PostDebugOp(); +#endif + + return true; +} +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/cpu/debug_cpu_kernel.h b/mindspore/ccsrc/kernel/cpu/debug_cpu_kernel.h new file mode 100644 index 0000000000..da9f3286b9 --- /dev/null +++ b/mindspore/ccsrc/kernel/cpu/debug_cpu_kernel.h @@ -0,0 +1,41 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_KERNEL_CPU_DEBUG_CPU_KERNEL_H_ +#define MINDSPORE_CCSRC_KERNEL_CPU_DEBUG_CPU_KERNEL_H_ + +#include +#include +#include "kernel/cpu/cpu_kernel.h" +#include "kernel/cpu/cpu_kernel_factory.h" + +namespace mindspore { +namespace kernel { +class DebugCPUKernel : public CPUKernel { + public: + DebugCPUKernel() = default; + ~DebugCPUKernel() override = default; + + void InitKernel(const CNodePtr &kernel_node) override; + + bool Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs) override; +}; + +MS_REG_CPU_KERNEL(Debug, KernelAttr().AddInputAttr(kNumberTypeFloat32).AddOutputAttr(kNumberTypeInt32), DebugCPUKernel); +} // namespace kernel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_KERNEL_CPU_DEBUG_CPU_KERNEL_H_ diff --git a/mindspore/ccsrc/kernel/cpu/embedding_look_up_comm_grad_cpu_kernel.cc b/mindspore/ccsrc/kernel/cpu/embedding_look_up_comm_grad_cpu_kernel.cc index 837cb647e3..07da3dcc25 100644 --- a/mindspore/ccsrc/kernel/cpu/embedding_look_up_comm_grad_cpu_kernel.cc +++ b/mindspore/ccsrc/kernel/cpu/embedding_look_up_comm_grad_cpu_kernel.cc @@ -50,9 +50,11 @@ bool EmbeddingLookUpCommGradCPUKernel::Launch(const std::vector &rank_group = {0, 1, 2, 3, 4, 5, 6, 7}; size_t input_split_lens = input_size / split_num_ / sizeof(float_t); size_t output_split_lens = output_size / split_num_ / sizeof(float_t); + auto mpi_instance = device::cpu::MPIAdapter::Instance(); + MS_EXCEPTION_IF_NULL(mpi_instance); for (int i = 0; i < split_num_; i++) { - device::cpu::MPIAdapter::Instance().AllGather(input_addr + i * input_split_lens, - output_addr + i * output_split_lens, rank_group, input_split_lens); + mpi_instance->AllGather(input_addr + i * input_split_lens, output_addr + i * output_split_lens, rank_group, + input_split_lens); } #if defined(_WIN32) || defined(_WIN64) auto end_time = std::chrono::steady_clock::now(); diff --git a/mindspore/ccsrc/kernel/cpu/embedding_look_up_cpu_kernel.cc b/mindspore/ccsrc/kernel/cpu/embedding_look_up_cpu_kernel.cc index e91b5d8109..c8c2c667ad 100644 --- a/mindspore/ccsrc/kernel/cpu/embedding_look_up_cpu_kernel.cc +++ b/mindspore/ccsrc/kernel/cpu/embedding_look_up_cpu_kernel.cc @@ -104,19 +104,20 @@ bool EmbeddingLookUpCPUKernel::Launch(const std::vector &inp size_t one_split_lens = gatherv2_out_lens_ / split_num_ / sizeof(float); size_t reduce_scatter_out_lens = one_split_lens / 8; const std::vector &group = {0, 1, 2, 3, 4, 5, 6, 7}; + auto mpi_instance = device::cpu::MPIAdapter::Instance(); + MS_EXCEPTION_IF_NULL(mpi_instance); for (int i = 0; i < split_num_; i++) { - device::cpu::MPIAdapter::Instance().ReduceScatter(reinterpret_cast(gather_v2_out_) + i * one_split_lens, - output_addr + i * reduce_scatter_out_lens, group, - one_split_lens / 8, "sum"); + mpi_instance->ReduceScatter(reinterpret_cast(gather_v2_out_) + i * one_split_lens, + output_addr + i * reduce_scatter_out_lens, group, one_split_lens / 8, "sum"); } } #endif return true; } -void LookUpTable_task(const float *input_addr, float *output_addr, int *indices_addr, size_t indices_lens, size_t num, - size_t dim0, size_t dim1, size_t dim2, int offset, size_t axis, std::vector input_shape, - size_t input_lens) { +void LookUpTable_task(const float *input_addr, float *output_addr, const int *indices_addr, size_t indices_lens, + size_t num, size_t dim0, size_t dim1, size_t dim2, int offset, size_t axis, + std::vector input_shape, size_t input_lens) { size_t lens = num * sizeof(float); for (size_t i = 0; i < indices_lens; ++i) { int indices = indices_addr[i] - offset; @@ -133,7 +134,6 @@ void LookUpTable_task(const float *input_addr, float *output_addr, int *indices_ } else if (axis == 0) { pos = CPUKernelUtils::CalcOffset(input_shape, index, 0, 0, 0); } - if (pos + num <= input_lens) { auto ret = memcpy_s(output_addr, lens, input_addr + pos, lens); if (ret != EOK) { diff --git a/mindspore/ccsrc/kernel/cpu/gather_cpu_kernel.cc b/mindspore/ccsrc/kernel/cpu/gather_cpu_kernel.cc index 9117a533c8..28090817cb 100644 --- a/mindspore/ccsrc/kernel/cpu/gather_cpu_kernel.cc +++ b/mindspore/ccsrc/kernel/cpu/gather_cpu_kernel.cc @@ -21,17 +21,14 @@ namespace mindspore { namespace kernel { void GatherV2CPUKernel::InitKernel(const CNodePtr &kernel_node) { CheckParam(kernel_node); - input_shape_ = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 0); indices_shape_ = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 1); output_shape_ = AnfAlgo::GetOutputInferShape(kernel_node, 0); - axis_ = AnfAlgo::GetNodeAttr(kernel_node, AXIS); if (axis_ < 0) { axis_ = axis_ + SizeToInt(input_shape_.size()); } axis_ += 4 - input_shape_.size(); - CPUKernelUtils::ExpandDimsTo4(&input_shape_); CPUKernelUtils::ExpandDimsTo4(&output_shape_); } @@ -44,7 +41,6 @@ bool GatherV2CPUKernel::Launch(const std::vector &inputs, size_t dim0 = input_shape_[0]; size_t dim1 = input_shape_[1]; size_t dim2 = input_shape_[2]; - if (axis_ == 3) { for (size_t i = 0; i < dim0; ++i) { for (size_t j = 0; j < dim1; ++j) { @@ -66,7 +62,6 @@ bool GatherV2CPUKernel::Launch(const std::vector &inputs, } else if (axis_ == 0) { CopyDataToOutput(inputs, 0, 0, 0, &output_addr, &buff_size); } - return true; } @@ -75,34 +70,43 @@ void GatherV2CPUKernel::CopyDataToOutput(const std::vector & auto input_addr = reinterpret_cast(inputs[0]->addr); auto indices_addr = reinterpret_cast(inputs[1]->addr); size_t elem_num = inputs[1]->size / 4; + size_t num = CPUKernelUtils::GetElementNumOnAxis(input_shape_, axis_); for (size_t i = 0; i < elem_num; ++i) { - size_t index = IntToSize(indices_addr[i]); - size_t pos = 0; - if (axis_ == 3) { - pos = CPUKernelUtils::CalcOffset(input_shape_, dim0, dim1, dim2, index); - } else if (axis_ == 2) { - pos = CPUKernelUtils::CalcOffset(input_shape_, dim0, dim1, index, 0); - } else if (axis_ == 1) { - pos = CPUKernelUtils::CalcOffset(input_shape_, dim0, index, 0, 0); - } else if (axis_ == 0) { - pos = CPUKernelUtils::CalcOffset(input_shape_, index, 0, 0, 0); + if (indices_addr[i] < 0) { + MS_LOG(EXCEPTION) << "The indices value is less than 0."; } - size_t num = CPUKernelUtils::GetElementNumOnAxis(input_shape_, axis_); - auto ret = memcpy_s(*output_addr, *buff_size, input_addr + pos, num * sizeof(float)); - if (ret != EOK) { - MS_LOG(EXCEPTION) << "memcpy failed."; + size_t index = IntToSize(indices_addr[i]); + if (index >= input_shape_[IntToSize(axis_)]) { + auto ret = memset_s(*output_addr, *buff_size, 0., num * sizeof(float)); + if (ret != EOK) { + MS_LOG(EXCEPTION) << "memset failed."; + } + } else { + size_t pos = 0; + if (axis_ == 3) { + pos = CPUKernelUtils::CalcOffset(input_shape_, dim0, dim1, dim2, index); + } else if (axis_ == 2) { + pos = CPUKernelUtils::CalcOffset(input_shape_, dim0, dim1, index, 0); + } else if (axis_ == 1) { + pos = CPUKernelUtils::CalcOffset(input_shape_, dim0, index, 0, 0); + } else if (axis_ == 0) { + pos = CPUKernelUtils::CalcOffset(input_shape_, index, 0, 0, 0); + } + auto ret = memcpy_s(*output_addr, *buff_size, input_addr + pos, num * sizeof(float)); + if (ret != EOK) { + MS_LOG(EXCEPTION) << "memcpy failed."; + } } *output_addr += num; *buff_size -= num * sizeof(float); } -} +} // namespace kernel void GatherV2CPUKernel::CheckParam(const CNodePtr &kernel_node) { auto input_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 0); if (input_shape.size() > 4) { MS_LOG(EXCEPTION) << "Input dims is " << input_shape.size() << ", but GatherV2CPUKernel olny support 4d or lower."; } - size_t input_num = AnfAlgo::GetInputTensorNum(kernel_node); if (input_num != 2) { MS_LOG(EXCEPTION) << "Argument number is " << input_num << ", but GatherV2CPUKernel needs 2."; diff --git a/mindspore/ccsrc/kernel/cpu/reduce_cpu_kernel.cc b/mindspore/ccsrc/kernel/cpu/reduce_cpu_kernel.cc index b12371c933..e56f2af8c7 100644 --- a/mindspore/ccsrc/kernel/cpu/reduce_cpu_kernel.cc +++ b/mindspore/ccsrc/kernel/cpu/reduce_cpu_kernel.cc @@ -55,7 +55,6 @@ void ReduceCPUKernel::InitKernel(const CNodePtr &kernel_node) { } } else if (axis_addr->isa()) { int axis = AnfAlgo::GetNodeAttr(kernel_node, AXIS); - if (axis >= 0 && IntToSize(axis) >= shape_.size()) { MS_LOG(EXCEPTION) << "axis value is oversize."; } diff --git a/mindspore/ccsrc/kernel/cpu/reduce_cpu_kernel.h b/mindspore/ccsrc/kernel/cpu/reduce_cpu_kernel.h index 27d28ba3bd..3317ec72ed 100644 --- a/mindspore/ccsrc/kernel/cpu/reduce_cpu_kernel.h +++ b/mindspore/ccsrc/kernel/cpu/reduce_cpu_kernel.h @@ -46,7 +46,6 @@ MS_REG_CPU_KERNEL(ReduceMax, KernelAttr().AddInputAttr(kNumberTypeFloat32).AddOu ReduceCPUKernel); MS_REG_CPU_KERNEL(ReduceSum, KernelAttr().AddInputAttr(kNumberTypeFloat32).AddOutputAttr(kNumberTypeFloat32), ReduceCPUKernel); - } // namespace kernel } // namespace mindspore #endif // MINDSPORE_CCSRC_KERNEL_CPU_REDUCE_CPU_KERNEL_H_ diff --git a/mindspore/ccsrc/kernel/cpu/reduce_scatter_cpu_kernel.cc b/mindspore/ccsrc/kernel/cpu/reduce_scatter_cpu_kernel.cc index fd8a74eb6b..19a4e907a0 100644 --- a/mindspore/ccsrc/kernel/cpu/reduce_scatter_cpu_kernel.cc +++ b/mindspore/ccsrc/kernel/cpu/reduce_scatter_cpu_kernel.cc @@ -46,9 +46,9 @@ bool ReduceScatterCPUKernel::Launch(const std::vector &input auto input_addr = reinterpret_cast(inputs[0]->addr); auto output_addr = reinterpret_cast(outputs[0]->addr); auto output_data_num = outputs[0]->size / sizeof(float); - - return device::cpu::MPIAdapter::Instance().ReduceScatter(input_addr, output_addr, ranks_group_, output_data_num, - op_type_); + auto mpi_instance = device::cpu::MPIAdapter::Instance(); + MS_EXCEPTION_IF_NULL(mpi_instance); + return mpi_instance->ReduceScatter(input_addr, output_addr, ranks_group_, output_data_num, op_type_); } } // namespace kernel } // namespace mindspore diff --git a/mindspore/ccsrc/kernel/cpu/reduce_scatter_cpu_kernel.h b/mindspore/ccsrc/kernel/cpu/reduce_scatter_cpu_kernel.h index c3bfe571a4..5c6907602a 100644 --- a/mindspore/ccsrc/kernel/cpu/reduce_scatter_cpu_kernel.h +++ b/mindspore/ccsrc/kernel/cpu/reduce_scatter_cpu_kernel.h @@ -37,7 +37,7 @@ class ReduceScatterCPUKernel : public CPUKernel { std::vector ranks_group_; }; -MS_REG_CPU_KERNEL(HostReduceScatter, KernelAttr().AddInputAttr(kNumberTypeFloat32).AddOutputAttr(kNumberTypeFloat32), +MS_REG_CPU_KERNEL(_HostReduceScatter, KernelAttr().AddInputAttr(kNumberTypeFloat32).AddOutputAttr(kNumberTypeFloat32), ReduceScatterCPUKernel); } // namespace kernel } // namespace mindspore diff --git a/mindspore/ccsrc/kernel/cpu/sparse_apply_adam_cpu_kernel.cc b/mindspore/ccsrc/kernel/cpu/sparse_apply_adam_cpu_kernel.cc index 5e2fc79576..ef3db78275 100644 --- a/mindspore/ccsrc/kernel/cpu/sparse_apply_adam_cpu_kernel.cc +++ b/mindspore/ccsrc/kernel/cpu/sparse_apply_adam_cpu_kernel.cc @@ -81,6 +81,7 @@ void SparseApplyAdamCPUKernel::InitInputOutputSize(const CNodePtr &kernel_node) MS_EXCEPTION_IF_NULL(kernel_node); workspace_size_list_.emplace_back(indices_size_ * var_outer_dim_size_ * sizeof(float)); workspace_size_list_.emplace_back(indices_size_ * sizeof(int)); + workspace_size_list_.emplace_back(var_first_dim_size_ * var_outer_dim_size_ * sizeof(float)); } void SparseApplyAdamCPUKernel::InitKernel(const CNodePtr &kernel_node) { @@ -141,6 +142,7 @@ bool SparseApplyAdamCPUKernel::Launch(const std::vector &inp auto indices = reinterpret_cast(inputs[10]->addr); auto new_grad = reinterpret_cast(workspace[0]->addr); auto new_indices = reinterpret_cast(workspace[1]->addr); + auto m_t = reinterpret_cast(workspace[2]->addr); SparseGradient unique_sparse_grad({new_grad, new_indices, indices_size_}); ReduceSparseGradient(SparseGradient({grad, indices, indices_size_}), &unique_sparse_grad, var_first_dim_size_, @@ -153,16 +155,14 @@ bool SparseApplyAdamCPUKernel::Launch(const std::vector &inp input_params.v_ = v; input_params.beta1_ = beta1; input_params.beta2_ = beta2; - const size_t kThreadNum = 16; - MultiThreadCompute(ComputeMomentum, &input_params, kThreadNum, total_dim_size); + MultiThreadCompute(ComputeMomentum, &input_params, total_dim_size); - std::vector m_t(m, m + total_dim_size); - input_params.m_t_ = m_t.data(); + input_params.m_t_ = m_t; input_params.use_nesterov_ = use_nesterov_; input_params.sparse_grad_ = unique_sparse_grad; input_params.var_first_dim_size_ = var_first_dim_size_; input_params.var_outer_dim_size_ = var_outer_dim_size_; - MultiThreadCompute(ComputeAdam, &input_params, kThreadNum, unique_sparse_grad.indices_size_); + MultiThreadCompute(ComputeAdam, &input_params, unique_sparse_grad.indices_size_); if (use_nesterov_) { input_params.m_ = input_params.m_t_; @@ -170,7 +170,7 @@ bool SparseApplyAdamCPUKernel::Launch(const std::vector &inp input_params.var_ = var; input_params.lr_ = lr; input_params.epsilon_ = epsilon; - MultiThreadCompute(ComputeWeight, &input_params, kThreadNum, total_dim_size); + MultiThreadCompute(ComputeWeight, &input_params, total_dim_size); return true; } } // namespace kernel diff --git a/mindspore/ccsrc/kernel/cpu/sparse_apply_ftrl_cpu_kernel.cc b/mindspore/ccsrc/kernel/cpu/sparse_apply_ftrl_cpu_kernel.cc index 005195ea33..0537e746f3 100644 --- a/mindspore/ccsrc/kernel/cpu/sparse_apply_ftrl_cpu_kernel.cc +++ b/mindspore/ccsrc/kernel/cpu/sparse_apply_ftrl_cpu_kernel.cc @@ -29,7 +29,7 @@ void ComputeFtrl(MultiThreadComputeParams *input_params, size_t start, size_t en auto linear = input_params->linear_; auto lr = input_params->lr_; auto l1 = input_params->l1_; - auto l2 = input_params->l2_; + auto l2_plus = 2 * input_params->l2_; auto lr_power = input_params->lr_power_; auto unique_sparse_grad = input_params->sparse_grad_; auto var_first_dim_size = input_params->var_first_dim_size_; @@ -44,21 +44,18 @@ void ComputeFtrl(MultiThreadComputeParams *input_params, size_t start, size_t en for (size_t j = start_index, k = var_outer_dim_size * i; j < end_index; ++j, ++k) { auto summed_grad = unique_sparse_grad.value_[k]; auto accum_new = accum[j] + summed_grad * summed_grad; - if (lr_power == -0.5) { - linear[j] += summed_grad - (std::sqrt(accum_new) - std::sqrt(accum[j])) / lr * var[j]; - } else { - linear[j] += summed_grad - (std::pow(accum_new, -lr_power) - std::pow(accum[j], -lr_power)) / lr * var[j]; - } - auto x = Sign(linear[j]) * l1 - linear[j]; float y; if (lr_power == -0.5) { - y = std::sqrt(accum_new) / lr + 2 * l2; + y = std::sqrt(accum_new); + linear[j] += summed_grad - (y - std::sqrt(accum[j])) / lr * var[j]; } else { - y = std::pow(accum_new, -lr_power) / lr + 2 * l2; + y = std::pow(accum_new, -lr_power); + linear[j] += summed_grad - (y - std::pow(accum[j], -lr_power)) / lr * var[j]; } - auto pre_shrink = x / y; - var[j] = std::fabs(linear[j]) > l1 ? pre_shrink : 0; accum[j] = accum_new; + auto x = Sign(linear[j]) * l1 - linear[j]; + y = y / lr + l2_plus; + var[j] = std::fabs(linear[j]) > l1 ? x / y : 0; } } } @@ -148,8 +145,7 @@ bool SparseApplyFtrlCPUKernel::Launch(const std::vector &inp input_params.sparse_grad_ = unique_sparse_grad; input_params.var_first_dim_size_ = var_first_dim_size_; input_params.var_outer_dim_size_ = var_outer_dim_size_; - const size_t kThreadNum = 16; - MultiThreadCompute(ComputeFtrl, &input_params, kThreadNum, unique_sparse_grad.indices_size_); + MultiThreadCompute(ComputeFtrl, &input_params, unique_sparse_grad.indices_size_); return true; } } // namespace kernel diff --git a/mindspore/ccsrc/kernel/cpu/sparse_apply_ftrl_cpu_kernel.h b/mindspore/ccsrc/kernel/cpu/sparse_apply_ftrl_cpu_kernel.h index b4e5a48109..9e79dc83c7 100644 --- a/mindspore/ccsrc/kernel/cpu/sparse_apply_ftrl_cpu_kernel.h +++ b/mindspore/ccsrc/kernel/cpu/sparse_apply_ftrl_cpu_kernel.h @@ -53,6 +53,18 @@ MS_REG_CPU_KERNEL(SparseApplyFtrl, .AddOutputAttr(kNumberTypeFloat32) .AddOutputAttr(kNumberTypeFloat32), SparseApplyFtrlCPUKernel); + +MS_REG_CPU_KERNEL(SparseApplyFtrlNoReturn, + KernelAttr() + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeInt32) + .AddOutputAttr(kNumberTypeFloat32) + .AddOutputAttr(kNumberTypeFloat32) + .AddOutputAttr(kNumberTypeFloat32), + SparseApplyFtrlCPUKernel); } // namespace kernel } // namespace mindspore diff --git a/mindspore/ccsrc/kernel/cpu/sparse_apply_lazy_adam_cpu_kernel.cc b/mindspore/ccsrc/kernel/cpu/sparse_apply_lazy_adam_cpu_kernel.cc index 2460dc0f27..16cb901b04 100644 --- a/mindspore/ccsrc/kernel/cpu/sparse_apply_lazy_adam_cpu_kernel.cc +++ b/mindspore/ccsrc/kernel/cpu/sparse_apply_lazy_adam_cpu_kernel.cc @@ -139,8 +139,7 @@ bool SparseApplyLazyAdamCPUKernel::Launch(const std::vector input_params.sparse_grad_ = unique_sparse_grad; input_params.var_first_dim_size_ = var_first_dim_size_; input_params.var_outer_dim_size_ = var_outer_dim_size_; - const size_t kThreadNum = 16; - MultiThreadCompute(ComputeLazyAdam, &input_params, kThreadNum, unique_sparse_grad.indices_size_); + MultiThreadCompute(ComputeLazyAdam, &input_params, unique_sparse_grad.indices_size_); return true; } } // namespace kernel diff --git a/mindspore/ccsrc/kernel/cpu/sparse_apply_proximal_adagrad_cpu_kernel.cc b/mindspore/ccsrc/kernel/cpu/sparse_apply_proximal_adagrad_cpu_kernel.cc index 64cb65764f..6069fb708e 100644 --- a/mindspore/ccsrc/kernel/cpu/sparse_apply_proximal_adagrad_cpu_kernel.cc +++ b/mindspore/ccsrc/kernel/cpu/sparse_apply_proximal_adagrad_cpu_kernel.cc @@ -132,8 +132,7 @@ bool SparseApplyProximalAdagradCPUKernel::Launch(const std::vector(inputs, 0); T *output = GetDeviceAddress(outputs, 1); S *index = GetDeviceAddress(outputs, 0); - CalArgmaxWithValue(input_size_ / sizeof(T), input, bound_, outerSize_, innerSize_, index, output, + CalArgmaxWithValue(input, bound_, outerSize_, innerSize_, index, output, reinterpret_cast(stream_ptr)); return true; } diff --git a/mindspore/ccsrc/kernel/gpu/arrays/array_reduce_gpu_kernel.h b/mindspore/ccsrc/kernel/gpu/arrays/array_reduce_gpu_kernel.h index e1f995d648..4a52439305 100644 --- a/mindspore/ccsrc/kernel/gpu/arrays/array_reduce_gpu_kernel.h +++ b/mindspore/ccsrc/kernel/gpu/arrays/array_reduce_gpu_kernel.h @@ -94,7 +94,8 @@ class ArrayReduceGpuKernel : public GpuKernel { } int input_dim_length = SizeToInt(AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 0).size()); - if (AnfAlgo::GetCNodePrimitive(kernel_node)->GetAttr("axis")->isa()) { + if (AnfAlgo::GetCNodePrimitive(kernel_node)->GetAttr("axis")->isa() || + AnfAlgo::GetCNodePrimitive(kernel_node)->GetAttr("axis")->isa()) { auto attr_axis = GetAttr>(kernel_node, "axis"); if (attr_axis.empty()) { axis_.push_back(-1); diff --git a/mindspore/ccsrc/kernel/gpu/cuda_impl/adam_weight_decay_impl.cu b/mindspore/ccsrc/kernel/gpu/cuda_impl/adam_weight_decay_impl.cu new file mode 100644 index 0000000000..dfadaa09d6 --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/cuda_impl/adam_weight_decay_impl.cu @@ -0,0 +1,50 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "adam_weight_decay_impl.cuh" +#include "device/gpu/cuda_common.h" + +template +__global__ void AdamWeightDecayKernel(const int element_num_, const bool need_decay, const float *beta1, + const float *one_sub_beta1, const float *beta2, const float *one_sub_beta2, + const float *epsilon, const float *lr, const float *weight_decay, T *m, T *v, + T *param, T *gradient) { + for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < element_num_; i += blockDim.x * gridDim.x) { + float next_m = beta1[0] * m[i] + one_sub_beta1[0] * gradient[i]; + float next_v = beta2[0] * v[i] + one_sub_beta2[0] * gradient[i] * gradient[i]; + float update = next_m / (sqrt(next_v) + epsilon[0]); + if (need_decay && weight_decay != nullptr) { + update += weight_decay[0] * param[i]; + } + param[i] -= lr[0] * update; + m[i] = next_m; + v[i] = next_v; + } +} + +template +void AdamWeightDecay(const int &element_num_, const bool &need_decay, const float *beta1, const float *one_sub_beta1, + const float *beta2, const float *one_sub_beta2, const float *epsilon, const float *lr, + const float *weight_decay, T *m, T *v, T *param, T *gradient, cudaStream_t stream) { + AdamWeightDecayKernel<<>>( + element_num_, need_decay, beta1, one_sub_beta1, beta2, one_sub_beta2, epsilon, lr, weight_decay, m, v, param, + gradient); +} + +template void AdamWeightDecay(const int &element_num_, const bool &need_decay, const float *beta1, + const float *one_sub_beta1, const float *beta2, const float *one_sub_beta2, + const float *epsilon, const float *lr, const float *weight_decay, float *m, float *v, + float *param, float *gradient, cudaStream_t stream); diff --git a/mindspore/ccsrc/kernel/gpu/cuda_impl/adam_weight_decay_impl.cuh b/mindspore/ccsrc/kernel/gpu/cuda_impl/adam_weight_decay_impl.cuh new file mode 100644 index 0000000000..2addffbf00 --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/cuda_impl/adam_weight_decay_impl.cuh @@ -0,0 +1,24 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMPL_ADAM_WEIGHT_DECAY_H_ +#define MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMPL_ADAM_WEIGHT_DECAY_H_ +template +void AdamWeightDecay(const int &element_num_, const bool &need_decay, const float *beta1, const float *one_sub_beta1, + const float *beta2, const float *one_sub_beta2, const float *epsilon, const float *lr, + const float *weight_decay, T *m, T *v, T *param, T *gradient, cudaStream_t stream); + +#endif // MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMPL_ADAM_WEIGHT_DECAY_H_ diff --git a/mindspore/ccsrc/kernel/gpu/cuda_impl/argmaxwithvalue_impl.cu b/mindspore/ccsrc/kernel/gpu/cuda_impl/argmaxwithvalue_impl.cu index a0687a2768..3313fc6853 100644 --- a/mindspore/ccsrc/kernel/gpu/cuda_impl/argmaxwithvalue_impl.cu +++ b/mindspore/ccsrc/kernel/gpu/cuda_impl/argmaxwithvalue_impl.cu @@ -18,41 +18,39 @@ #include "device/gpu/cuda_common.h" #include "include/cuda_fp16.h" template -__global__ void ArgmaxWithValue(size_t size, const T* input, const int bound, int outerSize, int innerSize, - S* index, T* output) { - for (size_t pos = blockIdx.x * blockDim.x + threadIdx.x; pos < (size); pos += blockDim.x * gridDim.x) { - for (int i = 0; i < outerSize; i++) { - int inputOutterOffset = i * innerSize * bound; - int outputOutterOffset = i * innerSize; - for (int j = 0; j < innerSize; j++) { - auto outputInnerOffset = outputOutterOffset + j; - S idx = 0; - T maxData = input[j + inputOutterOffset]; - for (S c = 0; c < bound; c++) { - int offset = j + c * innerSize; - auto inputData = input[inputOutterOffset + offset]; - idx = inputData > maxData ? c : idx; - maxData = inputData > maxData ? inputData : maxData; - } - output[outputInnerOffset] = maxData; - index[outputInnerOffset] = idx; +__global__ void ArgmaxWithValue(const T* input, const int bound, int outerSize, int innerSize, S* index, + T* output) { + for (size_t pos = blockIdx.x * blockDim.x + threadIdx.x; pos < (outerSize); pos += blockDim.x * gridDim.x) { + int inputOutterOffset = pos * innerSize * bound; + int outputOutterOffset = pos * innerSize; + for (int j = 0; j < innerSize; j++) { + auto outputInnerOffset = outputOutterOffset + j; + S idx = 0; + T maxData = input[j + inputOutterOffset]; + for (S c = 0; c < bound; c++) { + int offset = j + c * innerSize; + auto inputData = input[inputOutterOffset + offset]; + idx = inputData > maxData ? c : idx; + maxData = inputData > maxData ? inputData : maxData; + } + output[outputInnerOffset] = maxData; + index[outputInnerOffset] = idx; } - } } return; } template -void CalArgmaxWithValue(size_t size, const T* input, const int bound_, const int outerSize_, const int innerSize_, +void CalArgmaxWithValue(const T* input, const int bound_, const int outerSize_, const int innerSize_, S* index, T* output, cudaStream_t cuda_stream) { - ArgmaxWithValue<<>>(size, input, bound_, outerSize_, innerSize_, - index, output); + ArgmaxWithValue<<>>(input, bound_, outerSize_, innerSize_, + index, output); return; } -template void CalArgmaxWithValue(size_t size, const float* input, const int bound_, const int outerSize_, +template void CalArgmaxWithValue(const float* input, const int bound_, const int outerSize_, const int innerSize_, int* index, float* output, cudaStream_t cuda_stream); -template void CalArgmaxWithValue(size_t size, const half* input, const int bound_, const int outerSize_, +template void CalArgmaxWithValue(const half* input, const int bound_, const int outerSize_, const int innerSize_, int* index, half* output, cudaStream_t cuda_stream); diff --git a/mindspore/ccsrc/kernel/gpu/cuda_impl/argmaxwithvalue_impl.cuh b/mindspore/ccsrc/kernel/gpu/cuda_impl/argmaxwithvalue_impl.cuh index 0d4f4b62a3..67c061a966 100644 --- a/mindspore/ccsrc/kernel/gpu/cuda_impl/argmaxwithvalue_impl.cuh +++ b/mindspore/ccsrc/kernel/gpu/cuda_impl/argmaxwithvalue_impl.cuh @@ -17,6 +17,6 @@ #ifndef MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMP_ARGMAXWITHVALUE_H_ #define MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMP_ARGMAXWITHVALUE_H_ template -void CalArgmaxWithValue(size_t size, const T* input, const int bound_, const int outerSize_, const int innerSize_, - S* index, T* output, cudaStream_t cuda_stream); +void CalArgmaxWithValue(const T *input, const int bound_, const int outerSize_, const int innerSize_, S *index, + T *output, cudaStream_t cuda_stream); #endif // MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMP_ARGMAXWITHVALUE_H_ diff --git a/mindspore/ccsrc/kernel/gpu/cuda_impl/dropout_impl.cu b/mindspore/ccsrc/kernel/gpu/cuda_impl/dropout_impl.cu index 019d71d740..d65f319ab7 100644 --- a/mindspore/ccsrc/kernel/gpu/cuda_impl/dropout_impl.cu +++ b/mindspore/ccsrc/kernel/gpu/cuda_impl/dropout_impl.cu @@ -17,31 +17,59 @@ #include #include "dropout_impl.cuh" #include "include/cuda_runtime.h" - -__global__ void DropoutForwardKernel(const float *input, float *mask, float *output, size_t num_count, +template +__global__ void DropoutForwardKernel(const T *input, T *mask, T *output, float *mask_f, size_t num_count, float keep_prob) { float scale = 1.f / keep_prob; for (size_t i = blockIdx.x * blockDim.x + threadIdx.x; i < num_count; i += blockDim.x * gridDim.x) { - mask[i] = mask[i] <= keep_prob; - output[i] = scale * input[i] * mask[i]; + mask_f[i] = mask_f[i] <= keep_prob; + output[i] = scale * input[i] * mask_f[i]; + mask[i] = mask_f[i]; } } - -void DropoutForward(const float *input, float *mask, float *output, size_t num_count, float drop_prob, +template <> +__global__ void DropoutForwardKernel(const half *input, half *mask, half *output, float *mask_f, + size_t num_count, float keep_prob) { + half scale = __float2half(1.f / keep_prob); + for (size_t i = blockIdx.x * blockDim.x + threadIdx.x; i < num_count; i += blockDim.x * gridDim.x) { + mask_f[i] = mask_f[i] <= keep_prob; + output[i] = scale * input[i] * __float2half(mask_f[i]); + mask[i] = __float2half(mask_f[i]); + } +} +template +void DropoutForward(const T *input, T *mask, T *output, float *mask_f, size_t num_count, float drop_prob, cudaStream_t cuda_stream) { - DropoutForwardKernel<<>>(input, mask, output, num_count, - drop_prob); + DropoutForwardKernel<<>>(input, mask, output, mask_f, + num_count, drop_prob); } - -__global__ void DropoutBackwardKernel(const float *dy, const float *mask, float *dx, size_t num_count, +template +__global__ void DropoutBackwardKernel(const T *dy, const T *mask, T *dx, size_t num_count, float keep_prob) { float scale = 1.f / keep_prob; for (size_t i = blockIdx.x * blockDim.x + threadIdx.x; i < num_count; i += blockDim.x * gridDim.x) { dx[i] = scale * dy[i] * mask[i]; } } - -void DropoutBackward(const float *dy, const float *mask, float *dx, size_t num_count, float drop_prob, +template <> +__global__ void DropoutBackwardKernel(const half *dy, const half *mask, half *dx, size_t num_count, + float keep_prob) { + half scale = __float2half(1.f / keep_prob); + for (size_t i = blockIdx.x * blockDim.x + threadIdx.x; i < num_count; i += blockDim.x * gridDim.x) { + dx[i] = scale * dy[i] * mask[i]; + } +} +template +void DropoutBackward(const T *dy, const T *mask, T *dx, size_t num_count, float drop_prob, cudaStream_t cuda_stream) { DropoutBackwardKernel<<>>(dy, mask, dx, num_count, drop_prob); } + +template void DropoutForward(const float *input, float *mask, float *output, float *mask_f, + size_t num_count, float drop_prob, cudaStream_t cuda_stream); +template void DropoutForward(const half *input, half *mask, half *output, float *mask_f, + size_t num_count, float drop_prob, cudaStream_t cuda_stream); +template void DropoutBackward(const float *dy, const float *mask, float *dx, size_t num_count, + float drop_prob, cudaStream_t cuda_stream); +template void DropoutBackward(const half *dy, const half *mask, half *dx, size_t num_count, + float drop_prob, cudaStream_t cuda_stream); diff --git a/mindspore/ccsrc/kernel/gpu/cuda_impl/dropout_impl.cuh b/mindspore/ccsrc/kernel/gpu/cuda_impl/dropout_impl.cuh index bd3de6524d..f89d42ce49 100644 --- a/mindspore/ccsrc/kernel/gpu/cuda_impl/dropout_impl.cuh +++ b/mindspore/ccsrc/kernel/gpu/cuda_impl/dropout_impl.cuh @@ -18,9 +18,10 @@ #define MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMPL_DROPOUT_H_ #include "device/gpu/cuda_common.h" -void DropoutForward(const float *input, float *mask, float *output, size_t num_count, float keep_prob, +template +void DropoutForward(const T *input, T *mask, T *output, float *mask_f, size_t num_count, float keep_prob, cudaStream_t cuda_stream); -void DropoutBackward(const float *dy, const float *mask, float *dx, size_t num_count, float keep_prob, - cudaStream_t cuda_stream); +template +void DropoutBackward(const T *dy, const T *mask, T *dx, size_t num_count, float keep_prob, cudaStream_t cuda_stream); #endif // MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMPL_DROPOUT_H_ diff --git a/mindspore/ccsrc/kernel/gpu/cuda_impl/fake_quant_perchannel_impl.cu b/mindspore/ccsrc/kernel/gpu/cuda_impl/fake_quant_perchannel_impl.cu index 75c5eacb25..0e762b7dc4 100644 --- a/mindspore/ccsrc/kernel/gpu/cuda_impl/fake_quant_perchannel_impl.cu +++ b/mindspore/ccsrc/kernel/gpu/cuda_impl/fake_quant_perchannel_impl.cu @@ -20,7 +20,6 @@ #include #include #include "fake_quant_perchannel_impl.cuh" -#include "device/gpu/cuda_common.h" /** * Find the nudge min, max and scale value as output. @@ -34,13 +33,17 @@ * @param channel_num * @return */ -__global__ void NudgeMinMaxPerChannel(const float *input_min, const float *input_max, const float quant_min, - const float quant_max, float *nudge_min, float *nudge_max, float *scale, - int channel_num) { +__global__ void NudgeMinMaxPerChannel(float *input_min, float *input_max, const float quant_min, const float quant_max, + float *nudge_min, float *nudge_max, float *scale, int channel_num, + const bool symmetric) { float zp_from_min = 0.f; float nudge_zp = 0.f; for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < channel_num; i += blockDim.x * gridDim.x) { + if (symmetric) { + input_max[i] = abs(input_min[0]) < input_max[i] ? input_max[i] : -input_min[i]; + input_min[i] = abs(input_min[i]) < input_max[i] ? -input_max[i] : input_min[i]; + } if ((quant_max - quant_min) == 0 || (input_max[i] - input_min[i]) == 0) { scale[i] = 0.f; zp_from_min = 0.f; @@ -62,11 +65,11 @@ __global__ void NudgeMinMaxPerChannel(const float *input_min, const float *input } } -void CalNudgePerChannel(const float *input_min, const float *input_max, const float quant_min, const float quant_max, - float *nudge_min, float *nudge_max, float *scale, const int channel_num, +void CalNudgePerChannel(float *input_min, float *input_max, const float quant_min, const float quant_max, + float *nudge_min, float *nudge_max, float *scale, const int channel_num, const bool symmetric, cudaStream_t cuda_stream) { NudgeMinMaxPerChannel<<>>( - input_min, input_max, quant_min, quant_max, nudge_min, nudge_max, scale, channel_num); + input_min, input_max, quant_min, quant_max, nudge_min, nudge_max, scale, channel_num, symmetric); } /** @@ -80,9 +83,8 @@ void CalNudgePerChannel(const float *input_min, const float *input_max, const fl * @param scale - array * @return */ -__global__ void FakeQuantizePerChannel(const float *input, float *output, const int total_size, const int channel_size, - const float *nudge_min, const float *nudge_max, const float *scale, - bool symmetric) { +__global__ void FakeQuantPerChannel(const float *input, float *output, const int total_size, const int channel_size, + const float *nudge_min, const float *nudge_max, const float *scale) { float input_x = 0.f; int nudge_input = 0; int channel_idx = 0; @@ -106,16 +108,15 @@ __global__ void FakeQuantizePerChannel(const float *input, float *output, const } } -void CalFakeQuantizePerChannel(const float *input, float *output, const int total_size, const int channel_size, - const float *nudge_min, const float *nudge_max, const float *scale, bool symmetric, - cudaStream_t cuda_stream) { - FakeQuantizePerChannel<<>>( - input, output, total_size, channel_size, nudge_min, nudge_max, scale, symmetric); +void CalFakeQuantPerChannel(const float *input, float *output, const int total_size, const int channel_size, + const float *nudge_min, const float *nudge_max, const float *scale, + cudaStream_t cuda_stream) { + FakeQuantPerChannel<<>>(input, output, total_size, channel_size, + nudge_min, nudge_max, scale); } -__global__ void FakeQuantizePerChannelGrad(const float *input, const float *gradient, float *output, - const int total_size, const int channel_size, const float *nudge_min, - const float *nudge_max) { +__global__ void FakeQuantPerChannelGrad(const float *input, const float *gradient, float *output, const int total_size, + const int channel_size, const float *nudge_min, const float *nudge_max) { int channel_idx = 0; int per_channel_num = total_size / channel_size; @@ -129,9 +130,9 @@ __global__ void FakeQuantizePerChannelGrad(const float *input, const float *grad } } -void CalFakeQuantizePerChannelGrad(const float *input, const float *gradient, float *output, const int total_num, - const int channel_num, const float *nudge_min, const float *nudge_max, - cudaStream_t cuda_stream) { - FakeQuantizePerChannelGrad<<>>( - input, gradient, output, total_num, channel_num, nudge_min, nudge_max); +void CalFakeQuantPerChannelGrad(const float *input, const float *gradient, float *output, const int total_num, + const int channel_num, const float *nudge_min, const float *nudge_max, + cudaStream_t cuda_stream) { + FakeQuantPerChannelGrad<<>>(input, gradient, output, total_num, + channel_num, nudge_min, nudge_max); } diff --git a/mindspore/ccsrc/kernel/gpu/cuda_impl/fake_quant_perchannel_impl.cuh b/mindspore/ccsrc/kernel/gpu/cuda_impl/fake_quant_perchannel_impl.cuh index 3dff7156a7..ad2e387b08 100644 --- a/mindspore/ccsrc/kernel/gpu/cuda_impl/fake_quant_perchannel_impl.cuh +++ b/mindspore/ccsrc/kernel/gpu/cuda_impl/fake_quant_perchannel_impl.cuh @@ -14,22 +14,21 @@ * limitations under the License. */ -#ifndef MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMP_FAKEQUANTIZE_H_ -#define MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMP_FAKEQUANTIZE_H_ +#ifndef MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMP_FAKE_QUANT_PERCHANNEL_H_ +#define MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMP_FAKE_QUANT_PERCHANNEL_H_ -void CalNudgePerChannel(const float* input_min, const float* input_max, const float quant_min, const float quant_max, - float* nudge_min, float* nudge_max, float* scale, const int channel_num, - cudaStream_t cuda_stream); +#include "device/gpu/cuda_common.h" -void CalFakeQuantizePerChannel(const float* input, float* output, const int total_num, const int channel_num, - const float* nudge_min, const float* nudge_max, const float* scale, bool symmetric, - cudaStream_t cuda_stream); +void CalNudgePerChannel(float *input_min, float *input_max, const float quant_min, const float quant_max, + float *nudge_min, float *nudge_max, float *scale, const int channel_num, const bool symmetric, + cudaStream_t cuda_stream); -void CalMinMaxPerChannel(float* input, float* input_min, float* input_max, const int total_num, const int channel_num, - const float ema_decay, const bool ema, cudaStream_t cuda_stream); +void CalFakeQuantPerChannel(const float *input, float *output, const int total_num, const int channel_num, + const float *nudge_min, const float *nudge_max, const float *scale, + cudaStream_t cuda_stream); -void CalFakeQuantizePerChannelGrad(const float* input, const float* gradient, float* output, const int total_num, - const int channel_num, const float* nudge_min, const float* nudge_max, - cudaStream_t cuda_stream); +void CalFakeQuantPerChannelGrad(const float *input, const float *gradient, float *output, const int total_num, + const int channel_num, const float *nudge_min, const float *nudge_max, + cudaStream_t cuda_stream); -#endif // MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMP_FAKEQUANTIZE_H_ +#endif // MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMP_FAKE_QUANT_PERCHANNEL_H_ diff --git a/mindspore/ccsrc/kernel/gpu/cuda_impl/fake_quant_perlayer_impl.cu b/mindspore/ccsrc/kernel/gpu/cuda_impl/fake_quant_perlayer_impl.cu index 11a25ba294..f527d33df9 100644 --- a/mindspore/ccsrc/kernel/gpu/cuda_impl/fake_quant_perlayer_impl.cu +++ b/mindspore/ccsrc/kernel/gpu/cuda_impl/fake_quant_perlayer_impl.cu @@ -17,11 +17,10 @@ #include #include #include -#include "device/gpu/cuda_common.h" #include "fake_quant_perlayer_impl.cuh" -__global__ void FakeQuantize(const float *input, float *output, const int size, const float *nudge_min, - const float *nudge_max, const float *scale) { +__global__ void FakeQuantPerLayer(const float *input, float *output, const int size, const float *nudge_min, + const float *nudge_max, const float *scale) { float input_x = 0.f; int nudge_input = 0; @@ -43,8 +42,8 @@ __global__ void FakeQuantize(const float *input, float *output, const int size, return; } -__global__ void FakeQuantizeGrad(const float *input, const float *gradient, float *output, const int size, - const float *nudge_min, const float *nudge_max) { +__global__ void FakeQuantPerLayerGrad(const float *input, const float *gradient, float *output, const int size, + const float *nudge_min, const float *nudge_max) { for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < size; i += blockDim.x * gridDim.x) { if (input[i] < nudge_min[0] || input[i] > nudge_max[0]) { output[i] = 0; @@ -55,12 +54,18 @@ __global__ void FakeQuantizeGrad(const float *input, const float *gradient, floa return; } -__global__ void NudgeMinMax(const float *input_min, const float *input_max, const float quant_min, - const float quant_max, float *nudge_min, float *nudge_max, float *scale) { +__global__ void NudgeMinMaxPerLayer(float *input_min, float *input_max, const float quant_min, const float quant_max, + float *nudge_min, float *nudge_max, float *scale, const bool symmetric) { float zp_from_min = 0.f; scale[0] = 0.f; nudge_max[0] = 0.f; nudge_min[0] = 0.f; + + if (symmetric) { + input_max[0] = abs(input_min[0]) < input_max[0] ? input_max[0] : -input_min[0]; + input_min[0] = abs(input_min[0]) < input_max[0] ? -input_max[0] : input_min[0]; + } + if ((quant_max - quant_min) == 0 || (input_max[0] - input_min[0]) == 0) { scale[0] = 0.f; zp_from_min = 0.f; @@ -83,53 +88,24 @@ __global__ void NudgeMinMax(const float *input_min, const float *input_max, cons return; } -__global__ void UpdateInputMinMaxWithEMA(float *input_min, float *input_max, const float min, const float max, - const float decay) { - input_min[0] = decay * (min) + (1 - decay) * (input_min[0]); - input_min[0] = input_min[0] > 0 ? 0 : input_min[0]; - input_max[0] = decay * (max) + (1 - decay) * (input_max[0]); - input_max[0] = input_max[0] < 0 ? 0 : input_max[0]; - return; -} - -__global__ void UpdateInputMinMax(float *input_min, float *input_max, const float min, const float max) { - input_min[0] = min > 0 ? 0 : min; - input_max[0] = max < 0 ? 0 : max; -} - -void CalFakeQuantize(const float *input, float *output, const int size, const float *nudge_min, const float *nudge_max, - const float *scale, bool symmetric, cudaStream_t cuda_stream) { - FakeQuantize<<>>(input, output, size, nudge_min, nudge_max, scale); - return; -} - -void CalFakeQuantizeGrad(const float *input, const float *gradient, float *output, const int size, - const float *nudge_min, const float *nudge_max, cudaStream_t cuda_stream) { - FakeQuantizeGrad<<>>(input, gradient, output, size, nudge_min, - nudge_max); +void CalFakeQuantPerLayer(const float *input, float *output, const int size, const float *nudge_min, + const float *nudge_max, const float *scale, cudaStream_t cuda_stream) { + FakeQuantPerLayer<<>>(input, output, size, nudge_min, nudge_max, + scale); return; } -void CalNudge(const float *input_min, const float *input_max, const float quant_min, const float quant_max, - float *nudge_min, float *nudge_max, float *scale, cudaStream_t cuda_stream) { - NudgeMinMax<<<1, 1, 0, cuda_stream>>>(input_min, input_max, quant_min, quant_max, nudge_min, nudge_max, scale); +void CalFakeQuantPerLayerGrad(const float *input, const float *gradient, float *output, const int size, + const float *nudge_min, const float *nudge_max, cudaStream_t cuda_stream) { + FakeQuantPerLayerGrad<<>>(input, gradient, output, size, nudge_min, + nudge_max); return; } -void CalMinMax(float *input, float *input_min, float *input_max, const int size, const float ema_decay, const bool ema, - cudaStream_t cuda_stream) { - float minel = 0.f; - float maxel = 0.f; - auto policy = thrust::cuda::par.on(cuda_stream); - thrust::pair, thrust::device_ptr> tuple; - tuple = thrust::minmax_element(policy, thrust::device_pointer_cast(input), thrust::device_pointer_cast(input) + size); - minel = tuple.first[0]; - maxel = tuple.second[0]; - - if (ema) { - UpdateInputMinMaxWithEMA<<<1, 1, 0, cuda_stream>>>(input_min, input_max, minel, maxel, ema_decay); - } else { - UpdateInputMinMax<<<1, 1, 0, cuda_stream>>>(input_min, input_max, minel, maxel); - } +void CalNudgePerLayer(float *input_min, float *input_max, const float quant_min, const float quant_max, + float *nudge_min, float *nudge_max, float *scale, const bool symmetric, + cudaStream_t cuda_stream) { + NudgeMinMaxPerLayer<<<1, 1, 0, cuda_stream>>>(input_min, input_max, quant_min, quant_max, nudge_min, nudge_max, scale, + symmetric); return; } diff --git a/mindspore/ccsrc/kernel/gpu/cuda_impl/fake_quant_perlayer_impl.cuh b/mindspore/ccsrc/kernel/gpu/cuda_impl/fake_quant_perlayer_impl.cuh index 27c39dead1..dda95ed781 100644 --- a/mindspore/ccsrc/kernel/gpu/cuda_impl/fake_quant_perlayer_impl.cuh +++ b/mindspore/ccsrc/kernel/gpu/cuda_impl/fake_quant_perlayer_impl.cuh @@ -14,19 +14,18 @@ * limitations under the License. */ -#ifndef MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMP_FAKEQUANTIZE_H_ -#define MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMP_FAKEQUANTIZE_H_ +#ifndef MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMP_FAKE_QUANT_PERLAYER_H_ +#define MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMP_FAKE_QUANT_PERLAYER_H_ -void CalFakeQuantize(const float *input, float *output, const int size, const float *nudge_min, const float *nudge_max, - const float *scale, bool symmetric, cudaStream_t cuda_stream); +#include "device/gpu/cuda_common.h" -void CalFakeQuantizeGrad(const float *input, const float *gradient, float *output, const int size, - const float *nudge_min, const float *nudge_max, cudaStream_t cuda_stream); +void CalNudgePerLayer(float *input_min, float *input_max, const float quant_min, const float quant_max, + float *nudge_min, float *nudge_max, float *scale, const bool symmetric, cudaStream_t cuda_stream); -void CalNudge(const float *input_min, const float *input_max, const float quant_min, const float quant_max, - float *nudge_min, float *nudge_max, float *scale, cudaStream_t cuda_stream); +void CalFakeQuantPerLayer(const float *input, float *output, const int size, const float *nudge_min, + const float *nudge_max, const float *scale, cudaStream_t cuda_stream); -void CalMinMax(float *input, float *input_min, float *input_max, const int size, const float ema_decay, const bool ema, - cudaStream_t cuda_stream); +void CalFakeQuantPerLayerGrad(const float *input, const float *gradient, float *output, const int size, + const float *nudge_min, const float *nudge_max, cudaStream_t cuda_stream); -#endif // MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMP_FAKEQUANTIZE_H_ +#endif // MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMP_FAKE_QUANT_PERLAYER_H_ diff --git a/mindspore/ccsrc/kernel/gpu/cuda_impl/unary_op_impl.cu b/mindspore/ccsrc/kernel/gpu/cuda_impl/unary_op_impl.cu index 4a8af83aa4..09b347e3d5 100755 --- a/mindspore/ccsrc/kernel/gpu/cuda_impl/unary_op_impl.cu +++ b/mindspore/ccsrc/kernel/gpu/cuda_impl/unary_op_impl.cu @@ -36,6 +36,13 @@ __global__ void LogarithmKernel(T *input, T *output, size_t count) { } return; } +template <> +__global__ void LogarithmKernel(half *input, half *output, size_t count) { + for (size_t i = blockIdx.x * blockDim.x + threadIdx.x; i < (count); i += blockDim.x * gridDim.x) { + output[i] = hlog(input[i]); + } + return; +} template __global__ void NegativeKernel(T *input, T *output, size_t count) { T neg_one = -1; diff --git a/mindspore/ccsrc/kernel/gpu/math/broadcast_gpu_kernel.cc b/mindspore/ccsrc/kernel/gpu/math/broadcast_gpu_kernel.cc index e299946780..96d51b704c 100644 --- a/mindspore/ccsrc/kernel/gpu/math/broadcast_gpu_kernel.cc +++ b/mindspore/ccsrc/kernel/gpu/math/broadcast_gpu_kernel.cc @@ -96,5 +96,8 @@ MS_REG_GPU_KERNEL_TWO( MS_REG_GPU_KERNEL_TWO( Maximum, KernelAttr().AddInputAttr(kNumberTypeInt32).AddInputAttr(kNumberTypeInt32).AddOutputAttr(kNumberTypeInt32), BroadcastOpGpuKernel, int, int) +MS_REG_GPU_KERNEL_TWO( + Mul, KernelAttr().AddInputAttr(kNumberTypeInt32).AddInputAttr(kNumberTypeInt32).AddOutputAttr(kNumberTypeInt32), + BroadcastOpGpuKernel, int, int) } // namespace kernel } // namespace mindspore diff --git a/mindspore/ccsrc/kernel/gpu/nn/dropout_gpu_kernel.cc b/mindspore/ccsrc/kernel/gpu/nn/dropout_gpu_kernel.cc index b84dc628e0..459010e9e9 100644 --- a/mindspore/ccsrc/kernel/gpu/nn/dropout_gpu_kernel.cc +++ b/mindspore/ccsrc/kernel/gpu/nn/dropout_gpu_kernel.cc @@ -15,84 +15,16 @@ */ #include "kernel/gpu/nn/dropout_gpu_kernel.h" -#include "kernel/gpu/cuda_impl/dropout_impl.cuh" namespace mindspore { namespace kernel { -DropoutGpuFwdKernel::DropoutGpuFwdKernel() - : cudnn_handle_(nullptr), - is_null_input_(false), - num_count_(0), - keep_prob_(0.0), - states_init_(false), - mask_generator_(nullptr) {} - -DropoutGpuFwdKernel::~DropoutGpuFwdKernel() { DestroyResource(); } - -const std::vector &DropoutGpuFwdKernel::GetInputSizeList() const { return input_size_list_; } - -const std::vector &DropoutGpuFwdKernel::GetOutputSizeList() const { return output_size_list_; } - -const std::vector &DropoutGpuFwdKernel::GetWorkspaceSizeList() const { return workspace_size_list_; } - -bool DropoutGpuFwdKernel::Init(const CNodePtr &kernel_node) { - InitResource(); - - size_t input_num = AnfAlgo::GetInputTensorNum(kernel_node); - if (input_num != 1) { - MS_LOG(EXCEPTION) << "Argument number is " << input_num << ", but DropoutGpuFwdKernel needs 1."; - } - - auto input_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 0); - is_null_input_ = CHECK_NULL_INPUT(input_shape); - if (is_null_input_) { - InitSizeLists(); - return true; - } - - num_count_ = 1; - for (size_t x : input_shape) { - num_count_ *= x; - } - keep_prob_ = GetValue(AnfAlgo::GetCNodePrimitive(kernel_node)->GetAttr("keep_prob")); - - InitSizeLists(); - return true; -} - -void DropoutGpuFwdKernel::InitResource() { - cudnn_handle_ = device::gpu::GPUDeviceManager::GetInstance().GetCudnnHandle(); -} - -void DropoutGpuFwdKernel::DestroyResource() noexcept {} - -void DropoutGpuFwdKernel::InitSizeLists() { - size_t input_size = num_count_ * sizeof(float); - input_size_list_.push_back(input_size); - output_size_list_.push_back(input_size); // output size: the same with input size - output_size_list_.push_back(input_size); // mask size: the same with input size -} - -bool DropoutGpuFwdKernel::Launch(const std::vector &inputs, const std::vector &, - const std::vector &outputs, void *stream_ptr) { - if (is_null_input_) { - return true; - } - - auto *input = reinterpret_cast(inputs[0]->addr); - auto *output = reinterpret_cast(outputs[0]->addr); - auto *mask = reinterpret_cast(outputs[1]->addr); - - if (!states_init_) { - curandCreateGenerator(&mask_generator_, CURAND_RNG_PSEUDO_DEFAULT); - curandSetPseudoRandomGeneratorSeed(mask_generator_, time(NULL)); - states_init_ = true; - } - - curandGenerateUniform(mask_generator_, mask, num_count_); - DropoutForward(input, mask, output, num_count_, keep_prob_, reinterpret_cast(stream_ptr)); - - return true; -} +MS_REG_GPU_KERNEL_ONE( + Dropout, + KernelAttr().AddInputAttr(kNumberTypeFloat32).AddOutputAttr(kNumberTypeFloat32).AddOutputAttr(kNumberTypeFloat32), + DropoutGpuFwdKernel, float) +MS_REG_GPU_KERNEL_ONE( + Dropout, + KernelAttr().AddInputAttr(kNumberTypeFloat16).AddOutputAttr(kNumberTypeFloat16).AddOutputAttr(kNumberTypeFloat16), + DropoutGpuFwdKernel, half) } // namespace kernel } // namespace mindspore diff --git a/mindspore/ccsrc/kernel/gpu/nn/dropout_gpu_kernel.h b/mindspore/ccsrc/kernel/gpu/nn/dropout_gpu_kernel.h index 81eb78c880..4dfacb7ca1 100644 --- a/mindspore/ccsrc/kernel/gpu/nn/dropout_gpu_kernel.h +++ b/mindspore/ccsrc/kernel/gpu/nn/dropout_gpu_kernel.h @@ -20,35 +20,88 @@ #include #include "kernel/gpu/gpu_kernel.h" #include "kernel/gpu/gpu_kernel_factory.h" +#include "kernel/gpu/cuda_impl/dropout_impl.cuh" #include "include/curand.h" namespace mindspore { namespace kernel { +template class DropoutGpuFwdKernel : public GpuKernel { public: - DropoutGpuFwdKernel(); + DropoutGpuFwdKernel() + : cudnn_handle_(nullptr), + is_null_input_(false), + num_count_(0), + keep_prob_(0.0), + states_init_(false), + mask_generator_(nullptr) {} - ~DropoutGpuFwdKernel() override; + ~DropoutGpuFwdKernel() override = default; - const std::vector &GetInputSizeList() const override; + const std::vector &GetInputSizeList() const override { return input_size_list_; } + const std::vector &GetOutputSizeList() const override { return output_size_list_; } + const std::vector &GetWorkspaceSizeList() const override { return workspace_size_list_; } - const std::vector &GetOutputSizeList() const override; + bool Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs, void *stream_ptr) override { + if (is_null_input_) { + return true; + } - const std::vector &GetWorkspaceSizeList() const override; + T *input = GetDeviceAddress(inputs, 0); + T *output = GetDeviceAddress(outputs, 0); + T *mask = GetDeviceAddress(outputs, 1); + float *mask_f = GetDeviceAddress(workspace, 0); - bool Launch(const std::vector &inputs, const std::vector &workspace, - const std::vector &outputs, void *stream_ptr) override; + if (!states_init_) { + curandCreateGenerator(&mask_generator_, CURAND_RNG_PSEUDO_DEFAULT); + curandSetPseudoRandomGeneratorSeed(mask_generator_, time(NULL)); + states_init_ = true; + } + // curandGen only support float or double for mask. + curandGenerateUniform(mask_generator_, mask_f, num_count_); + DropoutForward(input, mask, output, mask_f, num_count_, keep_prob_, reinterpret_cast(stream_ptr)); + + return true; + } + + bool Init(const CNodePtr &kernel_node) override { + InitResource(); + + size_t input_num = AnfAlgo::GetInputTensorNum(kernel_node); + if (input_num != 1) { + MS_LOG(EXCEPTION) << "Argument number is " << input_num << ", but DropoutGpuFwdKernel needs 1."; + } - bool Init(const CNodePtr &kernel_node) override; + auto input_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 0); + is_null_input_ = CHECK_NULL_INPUT(input_shape); + if (is_null_input_) { + InitSizeLists(); + return true; + } + + num_count_ = 1; + for (size_t x : input_shape) { + num_count_ *= x; + } + keep_prob_ = GetAttr(kernel_node, "keep_prob"); + + InitSizeLists(); + return true; + } protected: - void InitResource() override; + void InitResource() override { cudnn_handle_ = device::gpu::GPUDeviceManager::GetInstance().GetCudnnHandle(); } - void InitSizeLists() override; + void InitSizeLists() override { + size_t input_size = num_count_ * sizeof(T); + input_size_list_.push_back(input_size); + output_size_list_.push_back(input_size); // output size: the same with input size + output_size_list_.push_back(input_size); // mask size: the same with input size + workspace_size_list_.push_back(num_count_ * sizeof(float)); // temp mask_f for curandGen + } private: - void DestroyResource() noexcept; - cudnnHandle_t cudnn_handle_; bool is_null_input_; size_t num_count_; @@ -59,8 +112,6 @@ class DropoutGpuFwdKernel : public GpuKernel { std::vector output_size_list_; std::vector workspace_size_list_; }; - -MS_REG_GPU_KERNEL(Dropout, DropoutGpuFwdKernel) } // namespace kernel } // namespace mindspore diff --git a/mindspore/ccsrc/kernel/gpu/nn/dropout_grad_kernel.cc b/mindspore/ccsrc/kernel/gpu/nn/dropout_grad_kernel.cc index 2194805e92..2fd21c96ee 100644 --- a/mindspore/ccsrc/kernel/gpu/nn/dropout_grad_kernel.cc +++ b/mindspore/ccsrc/kernel/gpu/nn/dropout_grad_kernel.cc @@ -15,76 +15,16 @@ */ #include "kernel/gpu/nn/dropout_grad_kernel.h" -#include "kernel/gpu/cuda_impl/dropout_impl.cuh" namespace mindspore { namespace kernel { -DropoutGradGpuFwdKernel::DropoutGradGpuFwdKernel() - : cudnn_handle_(nullptr), is_null_input_(false), num_count_(0), keep_prob_(0.0) {} - -DropoutGradGpuFwdKernel::~DropoutGradGpuFwdKernel() { DestroyResource(); } - -const std::vector &DropoutGradGpuFwdKernel::GetInputSizeList() const { return input_size_list_; } - -const std::vector &DropoutGradGpuFwdKernel::GetOutputSizeList() const { return output_size_list_; } - -const std::vector &DropoutGradGpuFwdKernel::GetWorkspaceSizeList() const { return workspace_size_list_; } - -bool DropoutGradGpuFwdKernel::Init(const CNodePtr &kernel_node) { - InitResource(); - - size_t input_num = AnfAlgo::GetInputTensorNum(kernel_node); - if (input_num != 2) { - MS_LOG(ERROR) << "Argument number is " << input_num << ", but DropoutGradGpuFwdKernel needs 2."; - return false; - } - - auto input_shape = AnfAlgo::GetOutputInferShape(kernel_node, 0); - is_null_input_ = CHECK_NULL_INPUT(input_shape); - if (is_null_input_) { - InitSizeLists(); - return true; - } - - num_count_ = 1; - for (size_t x : input_shape) { - num_count_ *= x; - } - keep_prob_ = GetValue(AnfAlgo::GetCNodePrimitive(kernel_node)->GetAttr("keep_prob")); - - InitSizeLists(); - return true; -} - -void DropoutGradGpuFwdKernel::InitResource() { - cudnn_handle_ = device::gpu::GPUDeviceManager::GetInstance().GetCudnnHandle(); -} - -void DropoutGradGpuFwdKernel::DestroyResource() noexcept {} - -void DropoutGradGpuFwdKernel::InitSizeLists() { - size_t dy_size = num_count_ * sizeof(float); - size_t mask_size = dy_size; - size_t dx_size = dy_size; - - input_size_list_.push_back(dy_size); - input_size_list_.push_back(mask_size); - output_size_list_.push_back(dx_size); -} - -bool DropoutGradGpuFwdKernel::Launch(const std::vector &inputs, const std::vector &, - const std::vector &outputs, void *stream_ptr) { - if (is_null_input_) { - return true; - } - - auto *dy = reinterpret_cast(inputs[0]->addr); - auto *mask = reinterpret_cast(inputs[1]->addr); - auto *dx = reinterpret_cast(outputs[0]->addr); - - DropoutBackward(dy, mask, dx, num_count_, keep_prob_, reinterpret_cast(stream_ptr)); - - return true; -} +MS_REG_GPU_KERNEL_ONE( + DropoutGrad, + KernelAttr().AddInputAttr(kNumberTypeFloat32).AddInputAttr(kNumberTypeFloat32).AddOutputAttr(kNumberTypeFloat32), + DropoutGradGpuBwdKernel, float) +MS_REG_GPU_KERNEL_ONE( + DropoutGrad, + KernelAttr().AddInputAttr(kNumberTypeFloat16).AddInputAttr(kNumberTypeFloat16).AddOutputAttr(kNumberTypeFloat16), + DropoutGradGpuBwdKernel, half) } // namespace kernel } // namespace mindspore diff --git a/mindspore/ccsrc/kernel/gpu/nn/dropout_grad_kernel.h b/mindspore/ccsrc/kernel/gpu/nn/dropout_grad_kernel.h index 4991b9dad5..e6683e15dd 100644 --- a/mindspore/ccsrc/kernel/gpu/nn/dropout_grad_kernel.h +++ b/mindspore/ccsrc/kernel/gpu/nn/dropout_grad_kernel.h @@ -20,28 +20,72 @@ #include #include "kernel/gpu/gpu_kernel.h" #include "kernel/gpu/gpu_kernel_factory.h" +#include "kernel/gpu/cuda_impl/dropout_impl.cuh" namespace mindspore { namespace kernel { -class DropoutGradGpuFwdKernel : public GpuKernel { +template +class DropoutGradGpuBwdKernel : public GpuKernel { public: - DropoutGradGpuFwdKernel(); - ~DropoutGradGpuFwdKernel() override; + DropoutGradGpuBwdKernel() : cudnn_handle_(nullptr), is_null_input_(false), num_count_(0), keep_prob_(0.0) {} + ~DropoutGradGpuBwdKernel() override = default; - const std::vector &GetInputSizeList() const override; - const std::vector &GetOutputSizeList() const override; - const std::vector &GetWorkspaceSizeList() const override; + const std::vector &GetInputSizeList() const override { return input_size_list_; } + const std::vector &GetOutputSizeList() const override { return output_size_list_; } + const std::vector &GetWorkspaceSizeList() const override { return workspace_size_list_; } bool Launch(const std::vector &inputs, const std::vector &workspace, - const std::vector &outputs, void *stream_ptr) override; - bool Init(const CNodePtr &kernel_node) override; + const std::vector &outputs, void *stream_ptr) override { + if (is_null_input_) { + return true; + } + + T *dy = GetDeviceAddress(inputs, 0); + T *mask = GetDeviceAddress(inputs, 1); + T *dx = GetDeviceAddress(outputs, 0); + + DropoutBackward(dy, mask, dx, num_count_, keep_prob_, reinterpret_cast(stream_ptr)); + + return true; + } + bool Init(const CNodePtr &kernel_node) override { + InitResource(); + + size_t input_num = AnfAlgo::GetInputTensorNum(kernel_node); + if (input_num != 2) { + MS_LOG(ERROR) << "Argument number is " << input_num << ", but DropoutGradGpuBwdKernel needs 2."; + return false; + } + + auto input_shape = AnfAlgo::GetOutputInferShape(kernel_node, 0); + is_null_input_ = CHECK_NULL_INPUT(input_shape); + if (is_null_input_) { + InitSizeLists(); + return true; + } + + num_count_ = 1; + for (size_t x : input_shape) { + num_count_ *= x; + } + keep_prob_ = GetAttr(kernel_node, "keep_prob"); + + InitSizeLists(); + return true; + } protected: - void InitResource() override; - void InitSizeLists() override; + void InitResource() override { cudnn_handle_ = device::gpu::GPUDeviceManager::GetInstance().GetCudnnHandle(); } + void InitSizeLists() override { + size_t dy_size = num_count_ * sizeof(T); + size_t mask_size = dy_size; + size_t dx_size = dy_size; - private: - void DestroyResource() noexcept; + input_size_list_.push_back(dy_size); + input_size_list_.push_back(mask_size); + output_size_list_.push_back(dx_size); + } + private: cudnnHandle_t cudnn_handle_; bool is_null_input_; size_t num_count_; @@ -50,8 +94,6 @@ class DropoutGradGpuFwdKernel : public GpuKernel { std::vector output_size_list_; std::vector workspace_size_list_; }; - -MS_REG_GPU_KERNEL(DropoutGrad, DropoutGradGpuFwdKernel) } // namespace kernel } // namespace mindspore diff --git a/mindspore/ccsrc/kernel/gpu/nn/fused_adam_weight_decay.cc b/mindspore/ccsrc/kernel/gpu/nn/fused_adam_weight_decay.cc new file mode 100644 index 0000000000..77cb7f8608 --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/nn/fused_adam_weight_decay.cc @@ -0,0 +1,52 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/gpu/nn/fused_adam_weight_decay.h" + +namespace mindspore { +namespace kernel { +MS_REG_GPU_KERNEL_ONE(FusedAdamWeightDecay, + KernelAttr() + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddOutputAttr(kNumberTypeFloat32), + FusedAdamWeightDecayGpuKernel, float) +MS_REG_GPU_KERNEL_ONE(FusedAdam, + KernelAttr() + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddOutputAttr(kNumberTypeFloat32), + FusedAdamWeightDecayGpuKernel, float) + +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/gpu/nn/fused_adam_weight_decay.h b/mindspore/ccsrc/kernel/gpu/nn/fused_adam_weight_decay.h new file mode 100644 index 0000000000..f13f6ed59f --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/nn/fused_adam_weight_decay.h @@ -0,0 +1,103 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_GPU_NN_FUSED_ADAM_WEIGHT_DECAY_KERNEL_H_ +#define MINDSPORE_CCSRC_KERNEL_GPU_NN_FUSED_ADAM_WEIGHT_DECAY_KERNEL_H_ + +#include +#include "kernel/gpu/gpu_kernel.h" +#include "kernel/gpu/gpu_kernel_factory.h" +#include "kernel/gpu/kernel_constants.h" +#include "kernel/gpu/cuda_impl/adam_weight_decay_impl.cuh" + +namespace mindspore { +namespace kernel { +template +class FusedAdamWeightDecayGpuKernel : public GpuKernel { + public: + FusedAdamWeightDecayGpuKernel() : element_nums_(0), weight_decay_(false) {} + ~FusedAdamWeightDecayGpuKernel() override = default; + + bool Init(const CNodePtr &kernel_node) override { + auto node_name = AnfAlgo::GetCNodeName(kernel_node); + if (node_name == "AdamWeighDecay") { + weight_decay_ = true; + } + + auto shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 7); + element_nums_ = 1; + for (auto i : shape) { + element_nums_ *= i; + } + + InitSizeLists(); + return true; + } + + const std::vector &GetInputSizeList() const override { return input_size_list_; } + const std::vector &GetOutputSizeList() const override { return output_size_list_; } + const std::vector &GetWorkspaceSizeList() const override { return workspace_size_list_; } + + bool Launch(const std::vector &inputs, const std::vector &, + const std::vector &outputs, void *stream_ptr) override { + float *beta1 = GetDeviceAddress(inputs, 0); + float *one_sub_beta1 = GetDeviceAddress(inputs, 1); + float *beta2 = GetDeviceAddress(inputs, 2); + float *one_sub_beta2 = GetDeviceAddress(inputs, 3); + float *epsilon = GetDeviceAddress(inputs, 4); + float *lr = GetDeviceAddress(inputs, 5); + T *param = GetDeviceAddress(inputs, 6); + T *m = GetDeviceAddress(inputs, 7); + T *v = GetDeviceAddress(inputs, 8); + T *gradient = GetDeviceAddress(inputs, 9); + float *weight_decay = nullptr; + if (weight_decay_) { + weight_decay = GetDeviceAddress(inputs, 10); + } + AdamWeightDecay(element_nums_, true, beta1, one_sub_beta1, beta2, one_sub_beta2, epsilon, lr, weight_decay, m, v, + param, gradient, reinterpret_cast(stream_ptr)); + return true; + } + + protected: + void InitResource() override{}; + void InitSizeLists() override { + input_size_list_.push_back(sizeof(float)); + input_size_list_.push_back(sizeof(float)); + input_size_list_.push_back(sizeof(float)); + input_size_list_.push_back(sizeof(float)); + input_size_list_.push_back(element_nums_ * sizeof(T)); + input_size_list_.push_back(sizeof(float)); + input_size_list_.push_back(sizeof(float)); + input_size_list_.push_back(element_nums_ * sizeof(T)); + if (weight_decay_) { + input_size_list_.push_back(sizeof(float)); + } + output_size_list_.push_back(element_nums_ * sizeof(T)); + } + + private: + std::vector input_size_list_; + std::vector output_size_list_; + std::vector workspace_size_list_; + + int element_nums_; + bool weight_decay_; +}; +} // namespace kernel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_KERNEL_GPU_NN_FUSED_ADAM_WEIGHT_DECAY_KERNEL_H_ diff --git a/mindspore/ccsrc/kernel/gpu/quant/fake_quant_perchannel_gpu_kernel.cc b/mindspore/ccsrc/kernel/gpu/quant/fake_quant_perchannel_gpu_kernel.cc index ffed550fbb..8db6ddd848 100644 --- a/mindspore/ccsrc/kernel/gpu/quant/fake_quant_perchannel_gpu_kernel.cc +++ b/mindspore/ccsrc/kernel/gpu/quant/fake_quant_perchannel_gpu_kernel.cc @@ -102,9 +102,9 @@ void FakeQuantPerChannelGpuKernel::InitSizeLists() { void FakeQuantPerChannelGpuKernel::CalFakeQuantize(float *input, float *output, float *input_min, float *input_max, float *nudge_min, float *nudge_max, float *scale, void *stream_ptr) { CalNudgePerChannel(input_min, input_max, quant_min_, quant_max_, nudge_min, nudge_max, scale, num_channels_, - reinterpret_cast(stream_ptr)); - CalFakeQuantizePerChannel(input, output, input_size_ / sizeof(float), num_channels_, nudge_min, nudge_max, scale, - symmetric_, reinterpret_cast(stream_ptr)); + symmetric_, reinterpret_cast(stream_ptr)); + CalFakeQuantPerChannel(input, output, input_size_ / sizeof(float), num_channels_, nudge_min, nudge_max, scale, + reinterpret_cast(stream_ptr)); } bool FakeQuantPerChannelGpuKernel::Launch(const std::vector &inputs, diff --git a/mindspore/ccsrc/kernel/gpu/quant/fake_quant_perchannel_grad_gpu_kernel.cc b/mindspore/ccsrc/kernel/gpu/quant/fake_quant_perchannel_grad_gpu_kernel.cc index a57516eb2c..5c774c05ed 100644 --- a/mindspore/ccsrc/kernel/gpu/quant/fake_quant_perchannel_grad_gpu_kernel.cc +++ b/mindspore/ccsrc/kernel/gpu/quant/fake_quant_perchannel_grad_gpu_kernel.cc @@ -119,9 +119,9 @@ bool FakeQuantPerChannelGradGpuKernel::Launch(const std::vector &inp int total_size = input_size_ / sizeof(float); if (global_step_ >= quant_delay_) { CalNudgePerChannel(input_min, input_max, quant_min_, quant_max_, nudge_min, nudge_max, scale, num_channels_, - reinterpret_cast(stream_ptr)); - CalFakeQuantizePerChannelGrad(input, gradient, output, total_size, num_channels_, nudge_min, nudge_max, - reinterpret_cast(stream_ptr)); + symmetric_, reinterpret_cast(stream_ptr)); + CalFakeQuantPerChannelGrad(input, gradient, output, total_size, num_channels_, nudge_min, nudge_max, + reinterpret_cast(stream_ptr)); } else { CHECK_CUDA_RET_WITH_ERROR(cudaMemcpyAsync(output, gradient, input_size_, cudaMemcpyDeviceToDevice, reinterpret_cast(stream_ptr)), diff --git a/mindspore/ccsrc/kernel/gpu/quant/fake_quant_perlayer_gpu_kernel.cc b/mindspore/ccsrc/kernel/gpu/quant/fake_quant_perlayer_gpu_kernel.cc index 845fb5b923..44869983eb 100644 --- a/mindspore/ccsrc/kernel/gpu/quant/fake_quant_perlayer_gpu_kernel.cc +++ b/mindspore/ccsrc/kernel/gpu/quant/fake_quant_perlayer_gpu_kernel.cc @@ -117,10 +117,10 @@ bool FakeQuantPerLayerGpuKernel::Launch(const std::vector &inputs, c // control flow for quant_delay if (global_step_ >= quant_delay_) { // real launch - CalNudge(input_min, input_max, quant_min_, quant_max_, nudge_min, nudge_max, scale, - reinterpret_cast(stream_ptr)); - CalFakeQuantize(input, output, quant_num_, nudge_min, nudge_max, scale, symmetric_, - reinterpret_cast(stream_ptr)); + CalNudgePerLayer(input_min, input_max, quant_min_, quant_max_, nudge_min, nudge_max, scale, symmetric_, + reinterpret_cast(stream_ptr)); + CalFakeQuantPerLayer(input, output, quant_num_, nudge_min, nudge_max, scale, + reinterpret_cast(stream_ptr)); } else { CHECK_CUDA_RET_WITH_ERROR(cudaMemcpyAsync(output, input, input_size_, cudaMemcpyDeviceToDevice, reinterpret_cast(stream_ptr)), @@ -129,10 +129,10 @@ bool FakeQuantPerLayerGpuKernel::Launch(const std::vector &inputs, c global_step_++; } else { // real launch - CalNudge(input_min, input_max, quant_min_, quant_max_, nudge_min, nudge_max, scale, - reinterpret_cast(stream_ptr)); - CalFakeQuantize(input, output, quant_num_, nudge_min, nudge_max, scale, symmetric_, - reinterpret_cast(stream_ptr)); + CalNudgePerLayer(input_min, input_max, quant_min_, quant_max_, nudge_min, nudge_max, scale, symmetric_, + reinterpret_cast(stream_ptr)); + CalFakeQuantPerLayer(input, output, quant_num_, nudge_min, nudge_max, scale, + reinterpret_cast(stream_ptr)); } return true; diff --git a/mindspore/ccsrc/kernel/gpu/quant/fake_quant_perlayer_grad_gpu_kernel.cc b/mindspore/ccsrc/kernel/gpu/quant/fake_quant_perlayer_grad_gpu_kernel.cc index 9c6584e239..c8d57b2bb1 100644 --- a/mindspore/ccsrc/kernel/gpu/quant/fake_quant_perlayer_grad_gpu_kernel.cc +++ b/mindspore/ccsrc/kernel/gpu/quant/fake_quant_perlayer_grad_gpu_kernel.cc @@ -115,10 +115,10 @@ bool FakeQuantPerLayerGradGpuKernel::Launch(const std::vector &input } if (global_step_ >= quant_delay_) { - CalNudge(input_min, input_max, quant_min_, quant_max_, nudge_min, nudge_max, scale, - reinterpret_cast(stream_ptr)); - CalFakeQuantizeGrad(input, gradient, output, quant_num_, nudge_min, nudge_max, - reinterpret_cast(stream_ptr)); + CalNudgePerLayer(input_min, input_max, quant_min_, quant_max_, nudge_min, nudge_max, scale, symmetric_, + reinterpret_cast(stream_ptr)); + CalFakeQuantPerLayerGrad(input, gradient, output, quant_num_, nudge_min, nudge_max, + reinterpret_cast(stream_ptr)); } else { CHECK_CUDA_RET_WITH_ERROR(cudaMemcpyAsync(output, gradient, input_size_, cudaMemcpyDeviceToDevice, reinterpret_cast(stream_ptr)), diff --git a/mindspore/ccsrc/kernel/hccl/hcom_util.cc b/mindspore/ccsrc/kernel/hccl/hcom_util.cc index 61a4d43eb5..088dbe59d5 100644 --- a/mindspore/ccsrc/kernel/hccl/hcom_util.cc +++ b/mindspore/ccsrc/kernel/hccl/hcom_util.cc @@ -67,18 +67,17 @@ bool HcomUtil::GetHcomDataType(const AnfNodePtr &anf_node, vector &shape, size_t *size) { MS_EXCEPTION_IF_NULL(size); - int tmp_size = 1; + size_t tmp_size = 1; uint32_t type_size = 4; for (size_t i = 0; i < shape.size(); i++) { - IntMulWithOverflowCheck(tmp_size, SizeToInt(shape[i]), &tmp_size); + tmp_size = SizetMulWithOverflowCheck(tmp_size, shape[i]); } if (!GetHcomTypeSize(data_type, &type_size)) { return false; } - IntMulWithOverflowCheck(tmp_size, UintToInt(type_size), &tmp_size); - *size = IntToSize(tmp_size); + *size = SizetMulWithOverflowCheck(tmp_size, type_size); MS_LOG(INFO) << "size[" << *size << "]"; return true; diff --git a/mindspore/ccsrc/kernel/kash/kernel_pack.cc b/mindspore/ccsrc/kernel/kash/kernel_pack.cc index 79e2ab9dbb..a87441031b 100644 --- a/mindspore/ccsrc/kernel/kash/kernel_pack.cc +++ b/mindspore/ccsrc/kernel/kash/kernel_pack.cc @@ -50,6 +50,8 @@ bool CheckHash(const std::string &json_file, const std::string &bin_file, const } // namespace const std::string KernelPack::Serialize() const { + MS_EXCEPTION_IF_NULL(json_); + MS_EXCEPTION_IF_NULL(kernel_); std::string buffer; (void)buffer.append((const char *)json_, json_->len + sizeof(json_->len)); (void)buffer.append((const char *)kernel_, kernel_->len + sizeof(kernel_->len)); diff --git a/mindspore/ccsrc/kernel/kernel_query.cc b/mindspore/ccsrc/kernel/kernel_query.cc index 5eda847917..4a8ae81afa 100755 --- a/mindspore/ccsrc/kernel/kernel_query.cc +++ b/mindspore/ccsrc/kernel/kernel_query.cc @@ -23,6 +23,7 @@ #include "kernel/tbe/tbe_kernel_select/tbe_kernel_select.h" #include "kernel/akg/akg_kernel_metadata.h" #include "session/anf_runtime_algorithm.h" +#include "utils/context/ms_context.h" namespace mindspore { namespace kernel { @@ -97,6 +98,12 @@ void KernelQuery(const CNodePtr &kernel_node, std::vectorenable_graph_kernel() && IsPrimitiveCNode(kernel_node, prim::kPrimBatchMatMul)) { + kernel_type = KernelType::AKG_KERNEL; + } + switch (kernel_type) { case KernelType::AKG_KERNEL: AkgMetadataInfo(kernel_node, kernel_info_list); diff --git a/mindspore/ccsrc/kernel/oplib/oplib.cc b/mindspore/ccsrc/kernel/oplib/oplib.cc index 35bc407026..e01bbe9162 100644 --- a/mindspore/ccsrc/kernel/oplib/oplib.cc +++ b/mindspore/ccsrc/kernel/oplib/oplib.cc @@ -293,8 +293,10 @@ bool OpLib::GetRefInfo(const std::shared_ptr &op_info) { const auto &output_infos = op_info->outputs_ptr(); const auto &input_infos = op_info->inputs_ptr(); for (size_t out_index = 0; out_index < output_infos.size(); out_index++) { + MS_EXCEPTION_IF_NULL(output_infos[out_index]); const auto &out_name = output_infos[out_index]->name(); for (size_t in_index = 0; in_index < input_infos.size(); in_index++) { + MS_EXCEPTION_IF_NULL(input_infos[in_index]); const auto &in_name = input_infos[in_index]->name(); if (out_name == in_name) { if (op_info->has_ref_index(out_index)) { diff --git a/mindspore/ccsrc/kernel/rts/label_switch.cc b/mindspore/ccsrc/kernel/rts/label_switch.cc index d84407a930..fb1ad1601a 100644 --- a/mindspore/ccsrc/kernel/rts/label_switch.cc +++ b/mindspore/ccsrc/kernel/rts/label_switch.cc @@ -75,8 +75,8 @@ std::vector LabelSwitchKernel::GenTask(const std::vector> LabelSwitchDesc::GetKernelInfo() { std::vector> label_switch_build_info{}; - vector input_format{kOpFormat_DEFAULT, kOpFormat_DEFAULT}; - vector input_type{kNumberTypeUInt32, kNumberTypeBool}; + vector input_format{kOpFormat_DEFAULT}; + vector input_type{kNumberTypeInt32}; if (input_format.size() != input_type.size()) { MS_LOG(EXCEPTION) << "Invalid param num, input_format size " << input_format.size() << " input_type size " << input_type.size(); diff --git a/mindspore/ccsrc/kernel/tbe/tbe_adapter.cc b/mindspore/ccsrc/kernel/tbe/tbe_adapter.cc index b7bad4fff8..c38f48763e 100644 --- a/mindspore/ccsrc/kernel/tbe/tbe_adapter.cc +++ b/mindspore/ccsrc/kernel/tbe/tbe_adapter.cc @@ -70,13 +70,17 @@ static std::map tbe_func_adapter_map = { {"strided_slice", "strided_slice_d"}, {"strided_slice_grad", "strided_slice_grad_d"}, {"sparse_apply_ftrl", "sparse_apply_ftrl_d"}, + {"sparse_apply_ftrl_v2", "sparse_apply_ftrl_v2_d"}, {"apply_ada_max", "apply_ada_max_d"}, {"apply_adadelta", "apply_adadelta_d"}, {"apply_adagrad", "apply_adagrad_d"}, {"apply_adagrad_v2", "apply_adagradv2_d"}, {"sparse_apply_adagrad", "sparse_apply_adagrad_d"}, + {"sparse_apply_adagrad_v2", "sparse_apply_adagrad_v2_d"}, {"apply_proximal_adagrad", "apply_proximal_adagrad_d"}, {"sparse_apply_proximal_adagrad", "sparse_apply_proximal_adagrad_d"}, + {"apply_add_sign", "apply_add_sign_d"}, + {"apply_power_sign", "apply_power_sign_d"}, {"transpose", "transpose_d"}, {"fill", "fill_d"}, {"unsorted_segment_sum", "unsorted_segment_sum_d"}, @@ -107,6 +111,8 @@ static std::map tbe_func_adapter_map = { {"r_oi_align_grad", "roi_align_grad"}, {"i_ou", "iou"}, {"s_gd", "sgd"}, + {"l_rn", "lrn"}, + {"l_rn_grad", "lrn_grad"}, {"l_ars_update", "lars_v2_update"}, {"n_ms_with_mask", "nms_with_mask"}, {"square_sum_all", "square_sum_all"}, diff --git a/mindspore/ccsrc/kernel/tbe/tbe_kernel_build.cc b/mindspore/ccsrc/kernel/tbe/tbe_kernel_build.cc index 76df819043..645a195f5e 100644 --- a/mindspore/ccsrc/kernel/tbe/tbe_kernel_build.cc +++ b/mindspore/ccsrc/kernel/tbe/tbe_kernel_build.cc @@ -189,7 +189,7 @@ bool TbeKernelJsonCreator::GenInputList(const std::shared_ptr &anf_node input_list->emplace_back(input_desc_json); continue; } - MS_LOG(ERROR) << "input num: " << *real_input_index << " is not match op inputs"; + MS_LOG(ERROR) << "Input num: " << *real_input_index << " is not match op inputs"; return false; } if (op_name == "BatchNorm") { @@ -197,7 +197,7 @@ bool TbeKernelJsonCreator::GenInputList(const std::shared_ptr &anf_node auto attr = primitive->GetAttr("is_training"); MS_EXCEPTION_IF_NULL(attr); bool is_training = GetValue(attr); - MS_LOG(INFO) << "op_name" << op_name << ", tensor_name " << input_ptr->name() << ", is_training " + MS_LOG(INFO) << "Op_name" << op_name << ", tensor_name " << input_ptr->name() << ", is_training " << is_training; if (is_training) { (*real_input_index)++; @@ -230,7 +230,7 @@ bool GetInputNameAndRealNum(const std::shared_ptr &anf_node, const std: if (input_ptr->param_type() == kParamDynamic) { if (*dyn_input_index >= dyn_input_sizes.size()) { - MS_LOG(ERROR) << "dyn input index" << *dyn_input_index << "is over dyn input num" << dyn_input_sizes.size(); + MS_LOG(ERROR) << "Dyn input index" << *dyn_input_index << "is over dyn input num" << dyn_input_sizes.size(); return false; } *input_num = IntToSize(dyn_input_sizes[*dyn_input_index]); @@ -314,7 +314,7 @@ bool TbeKernelJsonCreator::GenOutputDescJson( output_obj_num = real_output_num; } else { if (output_idx >= real_output_num) { - MS_LOG(INFO) << "op:" << op_name << ", output" << output_ptr->name() << " is optional, output is none."; + MS_LOG(INFO) << "Op:" << op_name << ", output" << output_ptr->name() << " is optional, output is none."; std::vector output_list; nlohmann::json output_obj; output_obj[kJName] = output_ptr->name(); @@ -389,7 +389,7 @@ bool TbeKernelJsonCreator::GenTbeAttrJson(const std::shared_ptr &anf_no attr_obj[kJValid] = false; } else { if (attr_ptr->param_type() == kParamRequred && creater_type_ == SINGLE_BUILD) { - MS_LOG(EXCEPTION) << "op name: " << op_info->op_name() << " attr: " << attr_name + MS_LOG(EXCEPTION) << "Op name: " << op_info->op_name() << " attr: " << attr_name << " is required, but not set."; } else { attr_obj[kJValid] = false; @@ -451,7 +451,7 @@ void TbeKernelJsonCreator::ParseAttrValue(const std::string &type, const mindspo auto attr_value = GetValue>>(value); (*attr_obj)[kJValue] = attr_value; } else { - MS_LOG(EXCEPTION) << "type: " << type << "not support"; + MS_LOG(EXCEPTION) << "Type: " << type << "not support"; } } @@ -536,7 +536,7 @@ std::string TbeKernelJsonCreator::GetDeviceOutputFormat(const AnfNodePtr &anf_no bool TbeKernelBuild::GetIOSize(const nlohmann::json &kernel_json, std::vector *input_size_list, std::vector *output_size_list) { if (input_size_list == nullptr || output_size_list == nullptr) { - MS_LOG(ERROR) << "input size or output size is nullptr"; + MS_LOG(ERROR) << "Input size or output size is nullptr"; return false; } input_size_list->clear(); @@ -750,7 +750,7 @@ bool TbeKernelBuild::GenFusionDataInputJson(const std::shared_ptr output_desc_list; if (!data_input) { - MS_LOG(INFO) << "data input is optional node"; + MS_LOG(INFO) << "Data input is optional node"; auto name = std::string(kOptional) + std::to_string(*index); (*data_str)[kJName] = name; nlohmann::json output_desc; @@ -766,7 +766,7 @@ bool TbeKernelBuild::GenFusionDataInputJson(const std::shared_ptrfullname_with_scope() << " index:" << real_idx; + MS_LOG(INFO) << "Real name " << real_node->fullname_with_scope() << " index:" << real_idx; // kJOutputDesc nlohmann::json output_desc; GenDescJson(real_node, real_idx, real_idx, &output_desc, fusion_data_type); @@ -842,18 +842,18 @@ bool TbeKernelBuild::GenFusionComputeInputJson(const mindspore::CNodePtr &cnode, auto kernel_idx = AnfAlgo::VisitKernel(input, 0); auto real_node = kernel_idx.first; size_t real_idx = kernel_idx.second; - MS_LOG(INFO) << "real name" << real_node->fullname_with_scope() << "index:" << real_idx; + MS_LOG(INFO) << "Real name" << real_node->fullname_with_scope() << "index:" << real_idx; nlohmann::json input_desc; GenDescJson(real_node, real_idx, real_idx, &input_desc); if (is_dynamic_input) { - MS_LOG(INFO) << "node has dynamic input."; + MS_LOG(INFO) << "Node has dynamic input."; input_desc[kJDynIndex] = (i - 1); } input_desc_list_tmp.emplace_back(input_desc); } size_t optional_num = GetOptionalInput(cnode, is_dynamic_input); if (optional_num > 0) { - MS_LOG(INFO) << "node has optional input."; + MS_LOG(INFO) << "Node has optional input."; for (size_t i = 0; i < optional_num; ++i) { nlohmann::json optional_input_desc; optional_input_desc[kJName] = std::string(kOptional) + std::to_string(*index); @@ -871,7 +871,7 @@ std::vector TbeKernelBuild::GetDescOutputIndex(const std::vector &o std::vector desc_output_index = {}; for (size_t idx = 0; idx < output_used_nums.size(); ++idx) { auto output_use_num_item = output_used_nums[idx]; - MS_LOG(INFO) << "output used num[" << idx << "] = " << output_use_num_item; + MS_LOG(INFO) << "Output used num[" << idx << "] = " << output_use_num_item; desc_output_index.emplace_back(idx); if (output_use_num_item > 1) { desc_output_index.emplace_back(idx); @@ -990,7 +990,7 @@ bool TbeKernelBuild::GetIOSize(const nlohmann::json &fusion_op_list, auto op_output_desces = op[kJOutputDesc]; if (output_node != real_node) { // tuple_get item - MS_LOG(INFO) << "output is a tuple getitem node"; + MS_LOG(INFO) << "Output is a tuple getitem node"; auto output_desc = op_output_desces[real_idx]; if (output_desc[kJShape].empty()) { MS_LOG(INFO) << "Fusion error: output_desc's shape is empty. real_index " << real_idx; diff --git a/mindspore/ccsrc/kernel/tbe/tbe_kernel_parallel_build.cc b/mindspore/ccsrc/kernel/tbe/tbe_kernel_parallel_build.cc index 79e5e0e109..43d492f397 100644 --- a/mindspore/ccsrc/kernel/tbe/tbe_kernel_parallel_build.cc +++ b/mindspore/ccsrc/kernel/tbe/tbe_kernel_parallel_build.cc @@ -77,7 +77,7 @@ bool TbeOpParallelPreBuild(const std::vector &anf_nodes) { return true; } -bool TbeOpParallelBuild(std::vector anf_nodes) { +bool TbeOpParallelBuild(const std::vector &anf_nodes) { auto build_manger = std::make_shared(); MS_EXCEPTION_IF_NULL(build_manger); set processed_kernel; diff --git a/mindspore/ccsrc/kernel/tbe/tbe_kernel_parallel_build.h b/mindspore/ccsrc/kernel/tbe/tbe_kernel_parallel_build.h index c900baf036..637c03bce3 100644 --- a/mindspore/ccsrc/kernel/tbe/tbe_kernel_parallel_build.h +++ b/mindspore/ccsrc/kernel/tbe/tbe_kernel_parallel_build.h @@ -27,7 +27,7 @@ namespace mindspore { namespace kernel { bool TbeOpParallelPreBuild(const std::vector &anf_nodes); -bool TbeOpParallelBuild(std::vector anf_nodes); +bool TbeOpParallelBuild(const std::vector &anf_nodes); struct KernelBuildTaskInfo { AnfNode *node; diff --git a/mindspore/ccsrc/kernel/tbe/tbe_kernel_select/tbe_kernel_broadcast_selecter.cc b/mindspore/ccsrc/kernel/tbe/tbe_kernel_select/tbe_kernel_broadcast_selecter.cc index 9d28af3f3f..8050f02f95 100644 --- a/mindspore/ccsrc/kernel/tbe/tbe_kernel_select/tbe_kernel_broadcast_selecter.cc +++ b/mindspore/ccsrc/kernel/tbe/tbe_kernel_select/tbe_kernel_broadcast_selecter.cc @@ -20,7 +20,6 @@ namespace mindspore { namespace kernel { -constexpr char kDynInputKey[] = "dyn_input_sizes"; constexpr size_t kInputIndex_0 = 0; constexpr size_t kChannelN = 0; constexpr size_t kChannelC = 1; @@ -34,9 +33,9 @@ bool TbeKernelBroadCastSelecter::GetShapeInfo(SupportFormat *support_format) { output_num_ = 0; input_shapes_.clear(); output_shapes_.clear(); - if (AnfAlgo::HasNodeAttr(kDynInputKey, cnode_ptr_)) { + if (AnfAlgo::HasNodeAttr(kAttrDynInputSizes, cnode_ptr_)) { MS_LOG(INFO) << "This broadcast node has dynamic input."; - auto dynamic_size_vec = AnfAlgo::GetNodeAttr>(cnode_ptr_, kDynInputKey); + auto dynamic_size_vec = AnfAlgo::GetNodeAttr>(cnode_ptr_, kAttrDynInputSizes); if (dynamic_size_vec.empty() || dynamic_size_vec[0] < 2) { MS_LOG(EXCEPTION) << "dynamic attr set error, please check."; } diff --git a/mindspore/ccsrc/kernel/tbe/tbe_kernel_select/tbe_kernel_reduce_selecter.cc b/mindspore/ccsrc/kernel/tbe/tbe_kernel_select/tbe_kernel_reduce_selecter.cc index da0466feaa..3f8e5b85c3 100644 --- a/mindspore/ccsrc/kernel/tbe/tbe_kernel_select/tbe_kernel_reduce_selecter.cc +++ b/mindspore/ccsrc/kernel/tbe/tbe_kernel_select/tbe_kernel_reduce_selecter.cc @@ -23,7 +23,6 @@ namespace mindspore { namespace kernel { -constexpr char kKeepDims[] = "keep_dims"; constexpr char kAxis[] = "axis"; constexpr char kTypeInt32[] = "Int32"; constexpr size_t kInputIndex_0 = 0; @@ -148,12 +147,12 @@ void TbeKernelReduceSelecter::GetReduceAttrAxis() { } void TbeKernelReduceSelecter::GetReduceAttrKeepDim() { - if (!AnfAlgo::HasNodeAttr(kKeepDims, cnode_ptr_)) { + if (!AnfAlgo::HasNodeAttr(kAttrKeepDims, cnode_ptr_)) { MS_LOG(INFO) << "This node does't have keep_attr."; keep_dims_ = false; return; } - keep_dims_ = AnfAlgo::GetNodeAttr(cnode_ptr_, kKeepDims); + keep_dims_ = AnfAlgo::GetNodeAttr(cnode_ptr_, kAttrKeepDims); } void TbeKernelReduceSelecter::AssignSupportFormat(const std::string &support_format_str, diff --git a/mindspore/ccsrc/kernel/tbe/tbe_kernel_select/tbe_kernel_select.cc b/mindspore/ccsrc/kernel/tbe/tbe_kernel_select/tbe_kernel_select.cc index 573ad176cf..5ef5d50e9c 100644 --- a/mindspore/ccsrc/kernel/tbe/tbe_kernel_select/tbe_kernel_select.cc +++ b/mindspore/ccsrc/kernel/tbe/tbe_kernel_select/tbe_kernel_select.cc @@ -39,7 +39,6 @@ constexpr auto kDtype = "dtype"; constexpr auto kFormat = "format"; constexpr auto kPrefixInput = "input"; constexpr auto kPrefixOutput = "output"; -constexpr char kDynInputKey[] = "dyn_input_sizes"; constexpr char kParamTypeDynamic[] = "dynamic"; constexpr char kParamTypeRequre[] = "required"; constexpr char kParamTypeOptional[] = "optional"; @@ -87,8 +86,8 @@ void TbeKernelSelect::GetCommonPatternKernelInfo(const OpInfo &op_info) { auto primitive = AnfAlgo::GetCNodePrimitive(cnode_ptr_); MS_EXCEPTION_IF_NULL(primitive); std::vector dyn_input_sizes; - if (primitive->HasAttr(kDynInputKey)) { - dyn_input_sizes = GetValue>(primitive->GetAttr(kDynInputKey)); + if (primitive->HasAttr(kAttrDynInputSizes)) { + dyn_input_sizes = GetValue>(primitive->GetAttr(kAttrDynInputSizes)); } // get real input/output num size_t real_input_tensor_num = AnfAlgo::GetInputTensorNum(cnode_ptr_); diff --git a/mindspore/ccsrc/kernel/tbe/tbe_utils.cc b/mindspore/ccsrc/kernel/tbe/tbe_utils.cc index a930fd3dca..ae7e5cb6d5 100644 --- a/mindspore/ccsrc/kernel/tbe/tbe_utils.cc +++ b/mindspore/ccsrc/kernel/tbe/tbe_utils.cc @@ -59,14 +59,14 @@ void TbeUtils::SaveJsonInfo(const std::string &json_name, const std::string &inf MS_LOG(INFO) << "json file exist, no need to create."; return; } - std::ofstream filewrite; - filewrite.open(path); - if (!filewrite.is_open()) { + std::ofstream file_write; + file_write.open(path); + if (!file_write.is_open()) { return; } - filewrite << info << std::endl; - filewrite.close(); - if (nullptr == realpath(path.c_str(), real_path)) { + file_write << info << std::endl; + file_write.close(); + if (realpath(path.c_str(), real_path) == nullptr) { MS_LOG(INFO) << "dir: " << path << "does not exit."; return; } @@ -144,12 +144,12 @@ uintptr_t KernelManager::GenFuncStub(const mindspore::kernel::KernelPack &kernel auto kernel_json_info = kernel_pack.kernel_json_info(); *block_dim = kernel_json_info.block_dim; - string funcname = kernel_json_info.kernel_name; + string func_name = kernel_json_info.kernel_name; string magic = kernel_json_info.magic; if (!force_reload) { // use the cached object. - auto iter = info_table_.find(funcname); + auto iter = info_table_.find(func_name); if (iter != info_table_.end()) { auto kernelmeta = iter->second; *block_dim = kernelmeta->block_dim_; @@ -157,23 +157,24 @@ uintptr_t KernelManager::GenFuncStub(const mindspore::kernel::KernelPack &kernel } } void *module = nullptr; - if (0 != BinaryRegister((*kernel_pack.GetKernel()), &module, magic)) { + if (BinaryRegister((*kernel_pack.GetKernel()), &module, magic) != 0) { MS_LOG(INFO) << "Call runtime BinaryRegister error."; return 0; } // to diff different funcs. - uintptr_t funcstub = ++kernel_stub_gen_; + uintptr_t func_stub = ++kernel_stub_gen_; if (RT_ERROR_NONE != - rtFunctionRegister(module, reinterpret_cast(funcstub), funcname.c_str(), funcname.c_str(), 0)) { + rtFunctionRegister(module, reinterpret_cast(func_stub), func_name.c_str(), func_name.c_str(), 0)) { MS_LOG(INFO) << "Call runtime rtFunctionRegister error."; return 0; } // cache the registered kernelmeta. - info_table_[funcname] = std::make_shared(KernelMetaInfo{funcstub, *block_dim}); - return funcstub; + info_table_[func_name] = std::make_shared(KernelMetaInfo{func_stub, *block_dim}); + return func_stub; } std::string KernelManager::GetStubFuncName(const KernelPackPtr &kernel_pack) { + MS_EXCEPTION_IF_NULL(kernel_pack); auto kernel_json_info = kernel_pack->kernel_json_info(); return kernel_json_info.kernel_name; } diff --git a/mindspore/ccsrc/onnx/ir_exporter.cc b/mindspore/ccsrc/onnx/ir_exporter.cc index d74233d79a..2f02f483f5 100644 --- a/mindspore/ccsrc/onnx/ir_exporter.cc +++ b/mindspore/ccsrc/onnx/ir_exporter.cc @@ -23,6 +23,7 @@ #include #include +#include "ir/tensor_py.h" #include "ir/param_value_py.h" #include "debug/anf_ir_utils.h" #include "operator/ops.h" @@ -257,7 +258,7 @@ void IrExportBuilder::SetTensorToAttributeProto(const ValuePtr &value, onnx::Att attr_proto->set_type(onnx::AttributeProto_AttributeType_TENSOR); onnx::TensorProto *tensor_proto = attr_proto->mutable_t(); auto data = value->cast(); - tensor_proto->set_raw_data(data->data().request(true).ptr, static_cast(data->data().nbytes())); + tensor_proto->set_raw_data(data->data_c(), static_cast(data->data().nbytes())); auto dtype = data->data_type(); auto shape = data->shape_c(); tensor_proto->set_data_type(GetOnnxDataType(dtype)); diff --git a/mindspore/ccsrc/onnx/onnx_exporter.cc b/mindspore/ccsrc/onnx/onnx_exporter.cc index 2a038bbf1a..65a841246b 100644 --- a/mindspore/ccsrc/onnx/onnx_exporter.cc +++ b/mindspore/ccsrc/onnx/onnx_exporter.cc @@ -27,6 +27,7 @@ #include "proto/onnx.pb.h" #include "operator/ops.h" #include "ir/param_value_py.h" +#include "ir/tensor_py.h" namespace mindspore { enum OpMergeMode { @@ -1190,7 +1191,7 @@ void OnnxExporter::SetNodeAttribute(const ValuePtr &value, onnx::NodeProto *cons attr_proto->set_type(onnx::AttributeProto_AttributeType_TENSOR); onnx::TensorProto *tensor_proto = attr_proto->mutable_t(); auto data = dyn_cast(value); - tensor_proto->set_raw_data(data->data().request(true).ptr, static_cast(data->data().nbytes())); + tensor_proto->set_raw_data(data->data_c(), static_cast(data->data().nbytes())); auto dtype = data->data_type(); auto shape = data->shape_c(); diff --git a/mindspore/ccsrc/operator/composite/map.cc b/mindspore/ccsrc/operator/composite/map.cc index a054da5f4d..2149285323 100644 --- a/mindspore/ccsrc/operator/composite/map.cc +++ b/mindspore/ccsrc/operator/composite/map.cc @@ -181,6 +181,9 @@ AnfNodePtr Map::FullMakeClass(const std::shared_ptr &type, const FuncGrap } AnfNodePtr Map::Make(const FuncGraphPtr &func_graph, const AnfNodePtr &fn_arg, const ArgsPairList &arg_pairs) { + if (arg_pairs.empty()) { + MS_EXCEPTION(TypeError) << "map() must have at least two arguments"; + } bool found = false; TypeId id = kObjectTypeEnd; std::pair pair; diff --git a/mindspore/ccsrc/operator/composite/multitype_funcgraph.cc b/mindspore/ccsrc/operator/composite/multitype_funcgraph.cc index 88b3134508..de6526f642 100644 --- a/mindspore/ccsrc/operator/composite/multitype_funcgraph.cc +++ b/mindspore/ccsrc/operator/composite/multitype_funcgraph.cc @@ -93,15 +93,17 @@ static TypePtr UnwrapRef(const TypePtr &type) { } return type; } -FuncGraphPtr MultitypeFuncGraph::GenerateFromTypes(const TypePtrList &types) { - bool find_fn = false; - py::function py_fn; + +// Return Exact match if exists, else return non ambiguous sub class match +// Return py::none() if matching is ambiguous +const py::function MultitypeFuncGraph::SignMatch(const TypePtrList &types) { + // Exact match for (auto &item : fn_cache_py_) { TypePtrList sign = item.first; if (sign.size() != types.size()) { continue; } - bool match = true; + auto match = true; for (size_t i = 0; i < sign.size(); ++i) { if (!IsIdentidityOrSubclass(UnwrapRef(types[i]), sign[i])) { match = false; @@ -111,13 +113,45 @@ FuncGraphPtr MultitypeFuncGraph::GenerateFromTypes(const TypePtrList &types) { if (!match) { continue; } - find_fn = true; - py_fn = item.second; - break; + return item.second; } + // Try best match + py::function py_fn_subclass; + size_t subclass_match_cnt = 0; + for (auto &item : fn_cache_py_) { + TypePtrList sign = item.first; + if (sign.size() != types.size()) { + continue; + } + auto match = true; + for (size_t i = 0; i < sign.size(); ++i) { + if (!IsIdentidityOrSubclass(UnwrapRef(types[i]), sign[i]) && + !IsParentOrChildrenType(UnwrapRef(types[i]), sign[i])) { + match = false; + break; + } + } + if (!match) { + continue; + } + py_fn_subclass = item.second; + subclass_match_cnt++; + } + if (subclass_match_cnt > 1) { + MS_LOG(EXCEPTION) << "There are more than one prototypes for overload function match by subclass"; + } + if (subclass_match_cnt == 1) { + MS_LOG(DEBUG) << "Found one subclass match"; + return py_fn_subclass; + } + return py::none(); +} + +FuncGraphPtr MultitypeFuncGraph::GenerateFromTypes(const TypePtrList &types) { + auto py_fn = SignMatch(types); std::ostringstream buffer; buffer << types; - if (find_fn) { + if (py_fn != py::none()) { FuncGraphPtr func_graph = parse::ParsePythonCode(py_fn); if (func_graph == nullptr) { MS_LOG(EXCEPTION) << "Fail to parse overload function " << buffer.str(); diff --git a/mindspore/ccsrc/operator/composite/multitype_funcgraph.h b/mindspore/ccsrc/operator/composite/multitype_funcgraph.h index feb38f17ba..ababf21883 100644 --- a/mindspore/ccsrc/operator/composite/multitype_funcgraph.h +++ b/mindspore/ccsrc/operator/composite/multitype_funcgraph.h @@ -54,6 +54,7 @@ class MultitypeFuncGraph : public MetaFuncGraph { } private: + const py::function SignMatch(const TypePtrList &types); std::unordered_map fn_cache_; std::unordered_map fn_cache_py_; }; diff --git a/mindspore/ccsrc/operator/composite/zip_operation.cc b/mindspore/ccsrc/operator/composite/zip_operation.cc index 33e21da044..38f2b51614 100644 --- a/mindspore/ccsrc/operator/composite/zip_operation.cc +++ b/mindspore/ccsrc/operator/composite/zip_operation.cc @@ -18,44 +18,44 @@ #include "operator/composite/zip_operation.h" #include -#include #include "pipeline/static_analysis/abstract_value.h" #include "ir/anf.h" #include "pipeline/static_analysis/dshape.h" -#include "pipeline/static_analysis/param_validator.h" #include "operator/cc_implementations.h" #include "optimizer/opt.h" -#include "utils/symbolic.h" -#include "./common.h" #include "pybind_api/api_register.h" namespace mindspore { // namespace to support composite operators definition namespace prim { using mindspore::abstract::AbstractBase; +using mindspore::abstract::AbstractList; +using mindspore::abstract::AbstractSequeue; +using mindspore::abstract::AbstractSequeuePtr; using mindspore::abstract::AbstractTuple; FuncGraphPtr ZipOperation::GenerateFuncGraph(const AbstractBasePtrList &args_spec_list) { // zip operation: // input: tuple arguments // output: tuple of items of input iterated on every input - if (args_spec_list.size() == 0) { - MS_LOG(EXCEPTION) << "zip arguments input should not be empty"; + if (args_spec_list.empty()) { + MS_LOG(EXCEPTION) << "For 'zip', there is at least one input."; } - auto is_all_tuple = std::all_of(args_spec_list.begin(), args_spec_list.end(), [](const AbstractBasePtr &abs) -> bool { - MS_EXCEPTION_IF_NULL(abs); - return abs->isa(); - }); - if (!is_all_tuple) { - MS_LOG(EXCEPTION) << "zip input args should be tuple"; + auto is_all_sequeue = + std::all_of(args_spec_list.begin(), args_spec_list.end(), [](const AbstractBasePtr &abs) -> bool { + MS_EXCEPTION_IF_NULL(abs); + return abs->isa(); + }); + if (!is_all_sequeue) { + MS_LOG(EXCEPTION) << "For 'zip', all inputs must be sequence."; } - auto min_abs = std::min_element(args_spec_list.begin(), args_spec_list.end(), - [](const AbstractBasePtr &x, const AbstractBasePtr &y) { - return (x->cast()->size() < y->cast()->size()); - }); + auto min_abs = std::min_element( + args_spec_list.begin(), args_spec_list.end(), [](const AbstractBasePtr &x, const AbstractBasePtr &y) { + return (x->cast()->size() < y->cast()->size()); + }); FuncGraphPtr ret_graph = std::make_shared(); ret_graph->set_flag(FUNC_GRAPH_FLAG_CORE, true); for (size_t idx = 0; idx < args_spec_list.size(); idx++) { @@ -65,12 +65,14 @@ FuncGraphPtr ZipOperation::GenerateFuncGraph(const AbstractBasePtrList &args_spe // generate tuple output of ziped arguments input std::vector make_tuple_nodes; make_tuple_nodes.push_back(NewValueNode(prim::kPrimMakeTuple)); - for (size_t idx = 0; idx < (*min_abs)->cast()->size(); idx++) { + for (size_t idx = 0; idx < (*min_abs)->cast()->size(); idx++) { std::vector make_tuple_zip_nodes; make_tuple_zip_nodes.push_back(NewValueNode(prim::kPrimMakeTuple)); + std::string module_name = "mindspore.ops.composite.multitype_ops.getitem_impl"; + ValuePtr op = prim::GetPythonOps("getitem", module_name); for (size_t arg_idx = 0; arg_idx < args_spec_list.size(); arg_idx++) { - std::vector tuple_get_item_nodes{NewValueNode(prim::kPrimTupleGetItem), - ret_graph->parameters()[arg_idx], NewValueNode(SizeToInt(idx))}; + std::vector tuple_get_item_nodes{NewValueNode(op), ret_graph->parameters()[arg_idx], + NewValueNode(SizeToInt(idx))}; auto tuple_get_item_op = ret_graph->NewCNode(tuple_get_item_nodes); make_tuple_zip_nodes.push_back(tuple_get_item_op); } diff --git a/mindspore/ccsrc/operator/ops.cc b/mindspore/ccsrc/operator/ops.cc index f86cbd7fd2..b682847ed7 100755 --- a/mindspore/ccsrc/operator/ops.cc +++ b/mindspore/ccsrc/operator/ops.cc @@ -231,6 +231,7 @@ const PrimitivePtr kPrimFakeBprop = std::make_shared("fake_bprop"); const PrimitivePtr kPrimBpropCut = std::make_shared("bprop_cut"); const PrimitivePtr kPrimFakeQuantPerLayer = std::make_shared("FakeQuantPerLayer"); const PrimitivePtr kPrimFakeQuantPerChannel = std::make_shared("FakeQuantPerChannel"); +const PrimitivePtr kPrimApplyRMSProp = std::make_shared("ApplyRMSProp"); // Other miscellaneous const PrimitivePtr kPrimIdentity = std::make_shared("identity"); @@ -275,5 +276,13 @@ const PrimitivePtr kPrimScalarSummary = std::make_shared("ScalarSumma const PrimitivePtr kPrimImageSummary = std::make_shared("ImageSummary"); const PrimitivePtr kPrimTensorSummary = std::make_shared("TensorSummary"); const PrimitivePtr kPrimHistogramSummary = std::make_shared("HistogramSummary"); +const PrimitivePtr kPrimDebug = std::make_shared("Debug"); + +// IndexedSlices +const PrimitivePtr kPrimMakeIndexedSlices = std::make_shared("MakeIndexedSlices"); +const PrimitivePtr kPrimIndexedSlicesGetValues = std::make_shared("IndexedSlicesGetValues"); +const PrimitivePtr kPrimIndexedSlicesGetIndices = std::make_shared("IndexedSlicesGetIndices"); +const PrimitivePtr kPrimIndexedSlicesGetDenseShape = std::make_shared("IndexedSlicesGetDenseShape"); +const PrimitivePtr kPrimIsIndexedSlices = std::make_shared("IsIndexedSlices"); } // namespace prim } // namespace mindspore diff --git a/mindspore/ccsrc/operator/ops.h b/mindspore/ccsrc/operator/ops.h index 65327cf407..f778013896 100755 --- a/mindspore/ccsrc/operator/ops.h +++ b/mindspore/ccsrc/operator/ops.h @@ -182,9 +182,11 @@ extern const PrimitivePtr kPrimReduceMin; extern const PrimitivePtr kPrimNeg; extern const PrimitivePtr kPrimSub; extern const PrimitivePtr kPrimMul; +extern const PrimitivePtr kPrimRealDiv; extern const PrimitivePtr kPrimMinimum; extern const PrimitivePtr kPrimMaximum; extern const PrimitivePtr kPrimSquare; +extern const PrimitivePtr kPrimSqrt; extern const PrimitivePtr kPrimEqual; extern const PrimitivePtr kPrimLess; extern const PrimitivePtr kPrimLessEqual; @@ -242,6 +244,7 @@ extern const PrimitivePtr kPrimFakeBprop; extern const PrimitivePtr kPrimBpropCut; extern const PrimitivePtr kPrimFakeQuantPerLayer; extern const PrimitivePtr kPrimFakeQuantPerChannel; +extern const PrimitivePtr kPrimApplyRMSProp; // Other Miscellaneous extern const PrimitivePtr kPrimIdentity; @@ -276,6 +279,7 @@ extern const PrimitivePtr kPrimNotInDict; extern const PrimitivePtr kPrimMixedPrecisionCast; extern const PrimitivePtr kPrimIsConsant; extern const PrimitivePtr kPrimEquivFormat; +extern const PrimitivePtr kPrimDebug; // Comm ops extern const PrimitivePtr kPrimAllReduce; @@ -283,6 +287,13 @@ extern const PrimitivePtr kPrimMirror; extern const PrimitivePtr kPrimVirtualDiv; extern const PrimitivePtr kPrimVirtualDataset; +// IndexedSlices +extern const PrimitivePtr kPrimMakeIndexedSlices; +extern const PrimitivePtr kPrimIndexedSlicesGetValues; +extern const PrimitivePtr kPrimIndexedSlicesGetIndices; +extern const PrimitivePtr kPrimIndexedSlicesGetDenseShape; +extern const PrimitivePtr kPrimIsIndexedSlices; + class DoSignaturePrimitive : public Primitive { public: explicit DoSignaturePrimitive(const std::string &name, const ValuePtr &function) diff --git a/mindspore/ccsrc/operator/prim_debug.cc b/mindspore/ccsrc/operator/prim_debug.cc index a9962c6d14..5e6cdcc318 100644 --- a/mindspore/ccsrc/operator/prim_debug.cc +++ b/mindspore/ccsrc/operator/prim_debug.cc @@ -21,5 +21,21 @@ #include "utils/symbolic.h" namespace mindspore { -namespace abstract {} // namespace abstract +namespace abstract { +AbstractBasePtr InferImplDebug(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + // Inputs: a tensor(value) + const std::string op_name = primitive->name(); + + CheckArgsSize(op_name, args_spec_list, 1); + auto tensor_value = CheckArg(op_name, args_spec_list, 0); + + int tensor_rank = SizeToInt(tensor_value->shape()->shape().size()); + if (tensor_rank == 0) { + MS_LOG(EXCEPTION) << op_name << " summary evaluator second arg should be an tensor, but got a scalar, rank is 0"; + } + + return std::make_shared(AbstractBasePtrList({tensor_value->Broaden()})); +} +} // namespace abstract } // namespace mindspore diff --git a/mindspore/ccsrc/operator/prim_others.cc b/mindspore/ccsrc/operator/prim_others.cc index 432b12f83b..ff9ec712bb 100644 --- a/mindspore/ccsrc/operator/prim_others.cc +++ b/mindspore/ccsrc/operator/prim_others.cc @@ -24,6 +24,7 @@ #include "pipeline/static_analysis/prim.h" #include "pipeline/static_analysis/utils.h" #include "utils/symbolic.h" +#include "utils/context/ms_context.h" namespace mindspore { namespace abstract { @@ -118,11 +119,9 @@ const size_t UndeterminedShapeType::fields_num = 6; std::unordered_map g_undetermined_configs; void InitUndeterminedFromEnv(const std::string &sparse_shape_types) { - if (!g_undetermined_configs.empty()) { - return; - } std::string tmp; std::stringstream input(sparse_shape_types); + g_undetermined_configs.clear(); while (std::getline(input, tmp, ';')) { auto config = UndeterminedShapeType(tmp); g_undetermined_configs.insert(std::make_pair(config.param_name(), config)); @@ -145,17 +144,19 @@ AbstractBasePtr InferImplEnvGetItem(const AnalysisEnginePtr &, const PrimitivePt if (!key->sparse_grad().empty()) { // Will be fixed once undetermined type ready - auto sparse_shape_types = common::GetEnv("UNDETERMINED_SPARSE_SHAPE_TYPES"); - if (sparse_shape_types.empty()) { - sparse_shape_types = "sparse_key_w1:2:Int32:2 1 2:Float32:3 1 2;sparse_key_w2:2:Int32:2 1 2:Float32:3 1 2"; + if (g_undetermined_configs.empty()) { + auto sparse_shape_types = common::GetEnv("UNDETERMINED_SPARSE_SHAPE_TYPES"); + MS_LOG(INFO) << "Undetermind sparse shape:" << sparse_shape_types; + if (sparse_shape_types.empty()) { + sparse_shape_types = "sparse_key_w1:2:Int32:2 1 2:Float32:3 1 2;sparse_key_w2:2:Int32:2 1 2:Float32:3 1 2"; + } + InitUndeterminedFromEnv(sparse_shape_types); } - InitUndeterminedFromEnv(sparse_shape_types); auto shape_types = g_undetermined_configs.find(key->sparse_grad()); if (shape_types == g_undetermined_configs.end()) { MS_LOG(EXCEPTION) << "Param " << key->ToString() - << " has sparse_grad, but shape/type is not configured in env UNDETERMINED_SPARSE_SHAPE_TYPES: " - << sparse_shape_types; + << " has sparse_grad, but shape/type is not configured in env UNDETERMINED_SPARSE_SHAPE_TYPES"; } MS_LOG(DEBUG) << "EnvGetItem is sparse_grad " << key->ToString(); AbstractBasePtrList sparse_list; @@ -173,6 +174,13 @@ AbstractBasePtr InferImplEnvGetItem(const AnalysisEnginePtr &, const PrimitivePt return std::make_shared(sparse_list); } + auto context = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(context); + bool enable_sparse_flag = context->enable_sparse_flag(); + if (enable_sparse_flag && key->has_indexed_slices_grad() && dflt->isa()) { + auto dflt_tensor = dflt->cast(); + return std::make_shared(dflt_tensor->element()->Clone(), dflt_tensor->shape()->Clone()); + } if (!key->GetValueTrack()->isa()) { return dflt; } @@ -236,6 +244,7 @@ AbstractBasePtr InferImplMakeRef(const AnalysisEnginePtr &, const PrimitivePtr & } auto ret = std::make_shared(args_spec_list[0], args_spec_list[1], args_spec_list[2]); ret->set_sparse_grad(args_spec_list[2]->sparse_grad()); + ret->set_has_indexed_slices_grad(args_spec_list[2]->has_indexed_slices_grad()); return ret; } @@ -437,5 +446,72 @@ AbstractBasePtr InferImplControlDepend(const AnalysisEnginePtr &, const Primitiv } return std::make_shared(kAnyValue, kBool); } + +AbstractBasePtr InferImplMakeIndexedSlices(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + // Inputs: two tensors and a tuple. + const std::string op_name = primitive->name(); + CheckArgsSize(op_name, args_spec_list, 3); + auto indices = CheckArg(op_name, args_spec_list, 0); + auto values = CheckArg(op_name, args_spec_list, 1); + auto dense_shape = CheckArg(op_name, args_spec_list, 2); + + auto dense_shape_value = dense_shape->BuildValue()->cast(); + MS_EXCEPTION_IF_NULL(dense_shape_value); + auto shp = dense_shape_value->value(); + std::vector dense_shape_vec; + (void)std::transform(std::begin(shp), std::end(shp), std::back_inserter(dense_shape_vec), + [](const ValuePtr &e) -> int { + auto elem = GetValue(e); + return elem; + }); + auto ret = std::make_shared(values->element()->BuildType(), dense_shape_vec); + ret->set_indices(indices); + ret->set_values(values); + ret->set_dense_shape(dense_shape); + return ret; +} + +AbstractBasePtr InferImplIndexedSlicesGetValues(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + // Inputs: two tensors and a tuple. + const std::string op_name = primitive->name(); + CheckArgsSize(op_name, args_spec_list, 1); + auto indexed_slices = CheckArg(op_name, args_spec_list, 0); + MS_EXCEPTION_IF_NULL(indexed_slices->values()); + return indexed_slices->values(); +} + +AbstractBasePtr InferImplIndexedSlicesGetIndices(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + // Inputs: two tensors and a tuple. + const std::string op_name = primitive->name(); + CheckArgsSize(op_name, args_spec_list, 1); + auto indexed_slices = CheckArg(op_name, args_spec_list, 0); + MS_EXCEPTION_IF_NULL(indexed_slices->indices()); + return indexed_slices->indices(); +} + +AbstractBasePtr InferImplIndexedSlicesGetDenseShape(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + // Inputs: two tensors and a tuple. + const std::string op_name = primitive->name(); + CheckArgsSize(op_name, args_spec_list, 1); + auto indexed_slices = CheckArg(op_name, args_spec_list, 0); + MS_EXCEPTION_IF_NULL(indexed_slices->dense_shape()); + return indexed_slices->dense_shape(); +} + +AbstractBasePtr InferImplIsIndexedSlices(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + const std::string op_name = primitive->name(); + CheckArgsSize(op_name, args_spec_list, 1); + bool ret = false; + if (args_spec_list[0]->isa()) { + ret = true; + } + MS_LOG(DEBUG) << "IsIndexedSlices result: " << ret << ", input: " << args_spec_list[0]->ToString(); + return std::make_shared(ret); +} } // namespace abstract } // namespace mindspore diff --git a/mindspore/ccsrc/operator/prim_statement.cc b/mindspore/ccsrc/operator/prim_statement.cc index 5eb8d39996..fc40e511e1 100644 --- a/mindspore/ccsrc/operator/prim_statement.cc +++ b/mindspore/ccsrc/operator/prim_statement.cc @@ -229,6 +229,7 @@ AbstractBasePtr InferImplNotInDict(const AnalysisEnginePtr &, const PrimitivePtr // Inputs: x, t return std::make_shared(!IsInDict(primitive, args_spec_list)); } + AbstractBasePtr InferImplIsConstant(const AnalysisEnginePtr &, const PrimitivePtr &primitive, const AbstractBasePtrList &args_spec_list) { // statement: isconstant(x) diff --git a/mindspore/ccsrc/operator/prim_structures.cc b/mindspore/ccsrc/operator/prim_structures.cc index 33c7a1e209..03c432483a 100644 --- a/mindspore/ccsrc/operator/prim_structures.cc +++ b/mindspore/ccsrc/operator/prim_structures.cc @@ -21,6 +21,9 @@ #include "pipeline/static_analysis/param_validator.h" #include "operator/ops.h" #include "utils/convert_utils.h" +#include "ir/tensor_py.h" + +using mindspore::tensor::TensorPy; namespace mindspore { namespace abstract { @@ -554,7 +557,7 @@ AbstractBasePtr InferImplTuple2Array(const AnalysisEnginePtr &, const PrimitiveP py::tuple data_tuple = ValuePtrToPyData(input->BuildValue()); py::array data = py::array(data_tuple); - auto tensor = std::make_shared(data); + auto tensor = TensorPy::MakeTensor(data); auto ret = tensor->ToAbstract(); ret->set_value(tensor); MS_LOG(DEBUG) << "Tuple2arry result AbstractTensor: " << ret->ToString(); diff --git a/mindspore/ccsrc/optimizer/ad/dfunctor.cc b/mindspore/ccsrc/optimizer/ad/dfunctor.cc index e192f3912e..f9c056a84e 100644 --- a/mindspore/ccsrc/optimizer/ad/dfunctor.cc +++ b/mindspore/ccsrc/optimizer/ad/dfunctor.cc @@ -424,7 +424,6 @@ AnfNodePtr DFunctor::MapToK(const AnfNodePtr &primal) { } auto k_prim = g_k_prims.KPrimitive(value_node, resources_); if (k_prim != nullptr) { - k_prim = BasicClone(k_prim); return NewValueNode(k_prim); } // When failed to find k_prim, try k_meta. diff --git a/mindspore/ccsrc/optimizer/ad/dfunctor.h b/mindspore/ccsrc/optimizer/ad/dfunctor.h index 13a6d5388b..4fa9cf6bb5 100644 --- a/mindspore/ccsrc/optimizer/ad/dfunctor.h +++ b/mindspore/ccsrc/optimizer/ad/dfunctor.h @@ -47,12 +47,12 @@ struct PrimitiveTotalEqual { return false; } - for (auto &attr : attrs1) { - if (!t2->HasAttr(attr.first)) { + for (auto &attr1 : attrs1) { + if (!t2->HasAttr(attr1.first)) { return false; } - if (!(*(attr.second) == *(t2->GetAttr(attr.first)))) { + if (!(*(attr1.second) == *(t2->GetAttr(attr1.first)))) { return false; } } @@ -61,7 +61,7 @@ struct PrimitiveTotalEqual { } }; -using Registry = std::unordered_map; +using Registry = std::unordered_map; class KPrim; extern KPrim g_k_prims; class DFunctor; diff --git a/mindspore/ccsrc/optimizer/ad/kprim.cc b/mindspore/ccsrc/optimizer/ad/kprim.cc index a9883cbf63..4141fb5413 100644 --- a/mindspore/ccsrc/optimizer/ad/kprim.cc +++ b/mindspore/ccsrc/optimizer/ad/kprim.cc @@ -96,34 +96,32 @@ FuncGraphPtr KPrim::KPrimitive(const ValueNodePtr &value_node, const pipeline::R MS_LOG(EXCEPTION) << "Primitive node is not valid."; } - auto prim = value_node->value()->cast(); - MS_EXCEPTION_IF_NULL(prim); - - auto iter = bprop_registry_.find(prim); - if (iter != bprop_registry_.end()) { - return iter->second; - } - - if (prim->Hash() == prim::kPrimSwitchLayer->Hash() && prim->name() == "switch_layer") { + auto prim = GetValueNode(value_node); + if (prim->Hash() == prim::kPrimSwitchLayer->Hash() && prim->name() == prim::kPrimSwitchLayer->name()) { auto fprop = GetFprop(prim); fprop->transforms().emplace("primal", FuncGraphTransform(prim::kPrimSwitchLayer)); - bprop_registry_[prim::kPrimSwitchLayer] = fprop; return fprop; - } - - if (prim->Hash() == prim::kPrimMakeTuple->Hash() && prim->name() == prim::kPrimMakeTuple->name()) { + } else if (prim->Hash() == prim::kPrimMakeTuple->Hash() && prim->name() == prim::kPrimMakeTuple->name()) { return nullptr; } - bool is_faked_bprop = false; FuncGraphPtr bprop_fg = nullptr; - if (prim->Hash() == prim::kPrimHookBackward->Hash() && prim->name() == "HookBackward") { + if (prim->Hash() == prim::kPrimHookBackward->Hash() && prim->name() == prim::kPrimHookBackward->name()) { bprop_fg = BpropCut(value_node, resources); } else { - bprop_fg = GetBprop(prim); + auto iter = bprop_registry_.find(prim); + if (iter != bprop_registry_.end()) { + bprop_fg = iter->second; + } + if (bprop_fg == nullptr) { - bprop_fg = FakeBprop(value_node, resources); - is_faked_bprop = true; + bprop_fg = GetBprop(prim); + if (bprop_fg != nullptr) { + // Set bprop_g graph cache + bprop_registry_[prim] = bprop_fg; + } else { + bprop_fg = FakeBprop(value_node, resources); + } } } @@ -134,11 +132,6 @@ FuncGraphPtr KPrim::KPrimitive(const ValueNodePtr &value_node, const pipeline::R << trace::GetDebugInfo(bprop_fg->debug_info()); } - // To support primitives with variable params, do not cache faked bprop - if (!is_faked_bprop) { - // Set bprop_g graph cache - bprop_registry_[prim] = expanded_fg; - } return expanded_fg; } diff --git a/mindspore/ccsrc/optimizer/clean.cc b/mindspore/ccsrc/optimizer/clean.cc index 6a54597282..bb52273568 100644 --- a/mindspore/ccsrc/optimizer/clean.cc +++ b/mindspore/ccsrc/optimizer/clean.cc @@ -36,6 +36,7 @@ using mindspore::abstract::AbstractJTagged; using mindspore::abstract::AbstractList; using mindspore::abstract::AbstractScalar; using mindspore::abstract::AbstractTuple; +using mindspore::abstract::AbstractUndetermined; static AbstractBasePtr Reabs(const AbstractBasePtr &t) { if (t == nullptr) { @@ -78,7 +79,7 @@ AnfNodePtr ConvertGetAttrToTupleGetItem(const CNodePtr &node) { MS_EXCEPTION_IF_NULL(cons); auto dt = data->abstract(); - if (dt == nullptr) { + if (dt == nullptr || dt->BuildType()->type_id() == kObjectTypeUndeterminedType) { return nullptr; } diff --git a/mindspore/ccsrc/optimizer/cse.cc b/mindspore/ccsrc/optimizer/cse.cc index 1af08ea3e1..0b675cca72 100644 --- a/mindspore/ccsrc/optimizer/cse.cc +++ b/mindspore/ccsrc/optimizer/cse.cc @@ -89,15 +89,28 @@ bool CSE::BuildOrderGroupAndDoReplace(const FuncGraphManagerPtr manager) const { return changed; } - +// The op like print, summary, or the op do not has true output, and always as a depend node input. +static bool HasSideEffect(const AnfNodePtr &node) { + auto prim = GetCNodePrimitive(node); + if (prim == nullptr) { + return false; + } + auto side_effect_v = prim->GetAttr(GRAPH_FLAG_SIDE_EFFECT); + if (side_effect_v != nullptr && side_effect_v->isa()) { + return GetValue(side_effect_v); + } + return false; +} +// If true do not merge the node. bool CSE::CheckRandomEffect(const AnfNodePtr &main, const AnfNodePtr &node) const { bool has_random_effect = false; auto prim_main = GetCNodePrimitive(main); auto prim_node = GetCNodePrimitive(node); - if (prim_main == prim_node) { - return false; - } + // if has random effect, when generate by different op (not same object), do not merge. if (prim_main != nullptr) { + if (prim_main == prim_node) { + return false; + } auto effect_val = prim_main->GetAttr(GRAPH_FLAG_RANDOM_EFFECT); if (effect_val != nullptr && effect_val->isa()) { has_random_effect = GetValue(effect_val); @@ -106,45 +119,58 @@ bool CSE::CheckRandomEffect(const AnfNodePtr &main, const AnfNodePtr &node) cons return has_random_effect; } -bool CSE::CheckReplace(const AnfNodePtr &main, const AnfNodePtr &node) const { +bool CSE::CheckReplace(const AnfNodePtr &main, const AnfNodePtr &node, bool check_side_effect) const { MS_EXCEPTION_IF_NULL(main); MS_EXCEPTION_IF_NULL(node); - bool replace = false; if (main->isa() && node->isa()) { auto main_value = GetValueNode(main); auto node_value = GetValueNode(node); - replace = (AbsOf(main) == AbsOf(node)) && (*main_value == *node_value); + return (AbsOf(main) == AbsOf(node)) && (*main_value == *node_value); } else if (main->isa() && node->isa()) { auto c_main = main->cast(); auto c_node = node->cast(); + // When appsame is true, check if has side effect, do not merge. + if (check_side_effect && HasSideEffect(main)) { + return false; + } const auto &inp1 = c_main->inputs(); const auto &inp2 = c_node->inputs(); - if (inp1.size() == inp2.size()) { - bool appsame = true; - for (size_t j = 0; j < inp1.size(); j++) { - MS_EXCEPTION_IF_NULL(inp1[j]); - MS_EXCEPTION_IF_NULL(inp2[j]); - if (!(*inp1[j] == *inp2[j])) { - // Handle the case of two different Tensor, but with the same value - if (IsValueNode(inp1[j]) && IsValueNode(inp2[j])) { - auto tensor1 = GetValueNode(inp1[j]); - auto tensor2 = GetValueNode(inp2[j]); - if (tensor1->ValueEqual(*tensor2)) { - continue; - } + if (inp1.size() != inp2.size()) { + return false; + } + for (size_t j = 0; j < inp1.size(); j++) { + auto inp1_j = inp1[j]; + auto inp2_j = inp2[j]; + MS_EXCEPTION_IF_NULL(inp1_j); + MS_EXCEPTION_IF_NULL(inp2_j); + if (!(*inp1_j == *inp2_j)) { + // Handle the case of two different Tensor, but with the same value + if (IsValueNode(inp1_j) && IsValueNode(inp2_j)) { + auto tensor1 = GetValueNode(inp1_j); + auto tensor2 = GetValueNode(inp2_j); + if (tensor1->ValueEqual(*tensor2)) { + continue; + } + } else if (HasSideEffect(inp1_j) && HasSideEffect(inp2_j)) { + // When the same side effect node as another two nodes' inputs, we still merge the node. + // Because the node only can be the inputs of `depend`, when the `depend` is duplicated merge the depend the + // node. + if (CheckReplace(inp1_j, inp2_j, false)) { + continue; } - appsame = false; - break; } + return false; } - if (CheckRandomEffect(c_main, c_node)) { - appsame = false; - } - replace = appsame; } + // When appsame is true, check if has random effect do not merge + if (CheckRandomEffect(c_main, c_node)) { + return false; + } + return true; } - return replace; + // a parameter node. + return false; } bool CSE::DoReplace(const FuncGraphManagerPtr manager, const std::vector &order_group, diff --git a/mindspore/ccsrc/optimizer/cse.h b/mindspore/ccsrc/optimizer/cse.h index fd90f61eeb..57163cc5c9 100644 --- a/mindspore/ccsrc/optimizer/cse.h +++ b/mindspore/ccsrc/optimizer/cse.h @@ -41,7 +41,7 @@ class CSE { return chg && report_changes_; } - virtual bool CheckReplace(const AnfNodePtr &main, const AnfNodePtr &node) const; + virtual bool CheckReplace(const AnfNodePtr &main, const AnfNodePtr &node, bool check_side_effect = true) const; virtual bool CheckRandomEffect(const AnfNodePtr &main, const AnfNodePtr &node) const; diff --git a/mindspore/ccsrc/optimizer/irpass.cc b/mindspore/ccsrc/optimizer/irpass.cc index 72177ccb06..166151751f 100644 --- a/mindspore/ccsrc/optimizer/irpass.cc +++ b/mindspore/ccsrc/optimizer/irpass.cc @@ -14,139 +14,160 @@ * limitations under the License. */ -#include "optimizer/irpass.h" - #include -#include "optimizer/irpass/symbol_resolver.h" +#include "optimizer/irpass.h" #include "optimizer/irpass/arithmetic_simplify.h" -#include "optimizer/irpass/special_op_eliminate.h" -#include "optimizer/irpass/item_tuple_eliminate.h" -#include "optimizer/irpass/env_item_eliminate.h" -#include "optimizer/irpass/tile_eliminate.h" -#include "optimizer/irpass/cast_eliminate.h" -#include "optimizer/irpass/reshape_eliminate.h" -#include "optimizer/irpass/transpose_eliminate.h" -#include "optimizer/irpass/reduce_eliminate.h" -#include "optimizer/irpass/partial_eliminate.h" -#include "optimizer/irpass/ref_eliminate.h" -#include "optimizer/irpass/merge_addn.h" #include "optimizer/irpass/branch_culling.h" +#include "optimizer/irpass/cast_eliminate.h" +#include "optimizer/irpass/convert.h" +#include "optimizer/irpass/env_item_eliminate.h" +#include "optimizer/irpass/grad_var_prepare.h" #include "optimizer/irpass/gradient_eliminate.h" -#include "optimizer/irpass/minmax_grad.h" #include "optimizer/irpass/inline.h" -#include "optimizer/irpass/convert.h" -#include "optimizer/irpass/specialize_transform.h" -#include "optimizer/irpass/incorporate_getitem.h" #include "optimizer/irpass/incorporate_call.h" -#include "optimizer/irpass/grad_var_prepare.h" -#include "optimizer/irpass/param_replace.h" +#include "optimizer/irpass/incorporate_getitem.h" +#include "optimizer/irpass/item_tuple_eliminate.h" #include "optimizer/irpass/mark_interface_fusion.h" +#include "optimizer/irpass/merge_addn.h" +#include "optimizer/irpass/minmax_grad.h" +#include "optimizer/irpass/param_replace.h" +#include "optimizer/irpass/partial_eliminate.h" +#include "optimizer/irpass/reduce_eliminate.h" +#include "optimizer/irpass/ref_eliminate.h" +#include "optimizer/irpass/reshape_eliminate.h" +#include "optimizer/irpass/special_op_eliminate.h" +#include "optimizer/irpass/specialize_transform.h" +#include "optimizer/irpass/symbol_resolver.h" +#include "optimizer/irpass/tile_eliminate.h" +#include "optimizer/irpass/transpose_eliminate.h" #include "optimizer/opt.h" +#include "optimizer/irpass/indexed_slices_eliminate.h" namespace mindspore { namespace opt { namespace irpass { OptimizeIRPassLib::OptimizeIRPassLib() { - arithmetic_simplify_ = MakeSubstitution(ArithmeticSimplify(), "arithmetic_simplify", + arithmetic_simplify_ = MakeSubstitution(std::make_shared(), "arithmetic_simplify", {prim::kPrimScalarAdd, prim::kPrimScalarMul, prim::kPrimTensorAdd, prim::kPrimIdentity, prim::kPrimMomentum, prim::kPrimMul, prim::kPrimPow}); + arithmetic_simplify2_ = + MakeSubstitution(std::make_shared(), "arithmetic_simplify2", {prim::kPrimMul}); special_op_eliminate_ = - MakeSubstitution(SpecialOpEliminater(), "special_op_eliminate", + MakeSubstitution(std::make_shared(), "special_op_eliminate", {prim::kPrimInsertGradientOf, prim::kPrimStopGradient, prim::kPrimHookBackward, - prim::kPrimPrintShapeType, prim::kPrimGetRefKey, prim::kPrimMirror, prim::kPrimVirtualDiv}); - zero_like_fill_zero_ = MakeSubstitution(ZeroLikeFillZero(), "zero_like_fill_zero", prim::kPrimZerosLike); - adjust_all_reduce_mul_add_ = MakeSubstitution(AdjustAllReduceMulAdd(), "adjust_all_reduce_mul_add", prim::kPrimAddN); + prim::kPrimPrintShapeType, prim::kPrimGetRefValue, prim::kPrimMirror, prim::kPrimVirtualDiv}); + zero_like_fill_zero_ = + MakeSubstitution(std::make_shared(), "zero_like_fill_zero", prim::kPrimZerosLike); + adjust_all_reduce_mul_add_ = + MakeSubstitution(std::make_shared(), "adjust_all_reduce_mul_add", prim::kPrimAddN); // ops eliminate - item_tuple_eliminate_ = - MakeSubstitution(ItemTupleEliminater(), "item_tuple_eliminate", {prim::kPrimTupleGetItem, prim::kPrimTupleSetItem}); - tile_eliminate_ = MakeSubstitution(TileMultiplyByOne(), "tile_eliminate", prim::kPrimTile); - cast_eliminate_ = MakeSubstitution(CastEliminater(), "cast_eliminate", prim::kPrimCast); - reshape_eliminate_ = MakeSubstitution(ReshapeEliminater(), "reshape_eliminate", prim::kPrimReshape); - transpose_eliminate_ = MakeSubstitution(TransposeSameIOEliminater(), "transpose_eliminate", prim::kPrimTranspose); + item_tuple_eliminate_ = MakeSubstitution(std::make_shared(), "item_tuple_eliminate", + {prim::kPrimTupleGetItem, prim::kPrimTupleSetItem}); + tile_eliminate_ = MakeSubstitution(std::make_shared(), "tile_eliminate", prim::kPrimTile); + cast_eliminate_ = MakeSubstitution(std::make_shared(), "cast_eliminate", prim::kPrimCast); + reshape_eliminate_ = MakeSubstitution(std::make_shared(), "reshape_eliminate", prim::kPrimReshape); + transpose_eliminate_ = + MakeSubstitution(std::make_shared(), "transpose_eliminate", prim::kPrimTranspose); reduce_eliminate_ = MakeSubstitution( - ReduceOneEliminater(), "reduce_eliminate", + std::make_shared(), "reduce_eliminate", {prim::kPrimReduceMean, prim::kPrimReduceAll, prim::kPrimReduceSum, prim::kPrimReduceMax, prim::kPrimReduceMin}); - partial_eliminate_ = MakeSubstitution(PartialEliminater(), "partial_eliminate", IsCNodeDup); - same_eliminate_ = MakeSubstitution(SameEliminater(), "same_eliminate", prim::kPrimSameTypeShape); - check_bprop_eliminate_ = MakeSubstitution(CheckBpropEliminater(), "check_bprop_eliminate", prim::kPrimCheckBprop); - reset_defer_inline_ = MakeSubstitution(ResetDeferInline(), "reset_defer_inline", IsValueNode); - depend_value_elim_ = MakeSubstitution(DependValueElim(), "depend_value_elim", prim::kPrimDepend); + partial_eliminate_ = MakeSubstitution(std::make_shared(), "partial_eliminate", IsCNodeDup); + same_eliminate_ = MakeSubstitution(std::make_shared(), "same_eliminate", prim::kPrimSameTypeShape); + check_bprop_eliminate_ = + MakeSubstitution(std::make_shared(), "check_bprop_eliminate", prim::kPrimCheckBprop); + reset_defer_inline_ = + MakeSubstitution(std::make_shared(), "reset_defer_inline", IsValueNode); + depend_value_elim_ = MakeSubstitution(std::make_shared(), "depend_value_elim", prim::kPrimDepend); // Env Item Eliminate - env_get_item_eliminate_ = MakeSubstitution(EnvGetItemEliminater(), "env_get_item_eliminate", prim::kPrimEnvGetItem); - new_env_get_item_ = MakeSubstitution(NewEnvGetItem(), "new_env_get_item", prim::kPrimEnvGetItem); + env_get_item_eliminate_ = + MakeSubstitution(std::make_shared(), "env_get_item_eliminate", prim::kPrimEnvGetItem); + new_env_get_item_ = MakeSubstitution(std::make_shared(), "new_env_get_item", prim::kPrimEnvGetItem); incorporate_env_getitem_ = - MakeSubstitution(IncorporateEnvGetitem(), "incorporate_env_get_item", prim::kPrimEnvGetItem); - incorporate_env_getitem_switch_ = - MakeSubstitution(IncorporateEnvGetitemSwitch(), "incorporate_env_getitem_switch", prim::kPrimEnvGetItem); + MakeSubstitution(std::make_shared(), "incorporate_env_get_item", prim::kPrimEnvGetItem); + incorporate_env_getitem_switch_ = MakeSubstitution(std::make_shared(), + "incorporate_env_getitem_switch", prim::kPrimEnvGetItem); // Ref eliminate - make_ref_eliminate_ = MakeSubstitution(MakeRefEliminater(), "make_ref_eliminate", prim::kPrimMakeRef); - get_ref_param_eliminate_ = MakeSubstitution(GetRefParamEliminater(), "get_ref_param_eliminate", + make_ref_eliminate_ = + MakeSubstitution(std::make_shared(), "make_ref_eliminate", prim::kPrimMakeRef); + get_ref_param_eliminate_ = MakeSubstitution(std::make_shared(), "get_ref_param_eliminate", {prim::kPrimGetRefValue, prim::kPrimGetRefOrigin}); - get_make_ref_eliminate_ = MakeSubstitution(GetMakeRefEliminater(), "get_make_ref_eliminate", + get_make_ref_eliminate_ = MakeSubstitution(std::make_shared(), "get_make_ref_eliminate", {prim::kPrimGetRefKey, prim::kPrimGetRefValue, prim::kPrimGetRefOrigin}); - replace_refkey_by_param_ = - MakeSubstitution(ReplaceRefkeyByParam(), "replace_refkey_by_param", IsValueNode, opt::FORCE_RENORM); - replace_old_param_ = MakeSubstitution(ReplaceOldParam(), "replace_old_param", IsParam); + replace_refkey_by_param_ = MakeSubstitution(std::make_shared(), "replace_refkey_by_param", + IsValueNode, opt::FORCE_RENORM); + replace_old_param_ = MakeSubstitution(std::make_shared(), "replace_old_param", IsParam); // Gradient transforms - expand_jprim_ = MakeSubstitution(ExpandJPrim(), "expand_jprim", prim::kPrimJ); - minmaximum_grad_ = MakeSubstitution(MinMaximumGrad(), "minmaximum_grad", prim::kPrimTupleGetItem); + expand_jprim_ = MakeSubstitution(std::make_shared(), "expand_jprim", prim::kPrimJ); + minmaximum_grad_ = MakeSubstitution(std::make_shared(), "minmaximum_grad", prim::kPrimTupleGetItem); // branch culling - switch_simplify_ = MakeSubstitution(SwitchSimplify(), "switch_simplify", prim::kPrimSwitch); - float_tuple_getitem_switch_ = - MakeSubstitution(FloatTupleGetItemSwitch(), "float_tuple_getitem_switch", prim::kPrimTupleGetItem); + switch_simplify_ = MakeSubstitution(std::make_shared(), "switch_simplify", prim::kPrimSwitch); + float_tuple_getitem_switch_ = MakeSubstitution(std::make_shared(), + "float_tuple_getitem_switch", prim::kPrimTupleGetItem); float_env_getitem_switch_ = - MakeSubstitution(FloatEnvGetItemSwitch(), "float_env_getitem_switch", prim::kPrimEnvGetItem); - convert_switch_replacement_ = MakeSubstitution(ConvertSwitchReplacement(), "convert_switch_replacement", IsCNodeDup); + MakeSubstitution(std::make_shared(), "float_env_getitem_switch", prim::kPrimEnvGetItem); + convert_switch_replacement_ = + MakeSubstitution(std::make_shared(), "convert_switch_replacement", IsCNodeDup); // Addn - merge_addn_ = MakeSubstitution(MergeAddN(), "merge_addn", prim::kPrimAddN); - addn_zero_filter_ = MakeSubstitution(AddNZeroFilter(), "addn_zero_filter", prim::kPrimAddN); + merge_addn_ = MakeSubstitution(std::make_shared(), "merge_addn", prim::kPrimAddN); + addn_zero_filter_ = MakeSubstitution(std::make_shared(), "addn_zero_filter", prim::kPrimAddN); // inline - inline_ = MakeSubstitution(Inliner(), "inline", IsCNodeGraph); - replace_applicator_ = MakeSubstitution(ReplaceApplicator(), "replace_applicator", IsValueNode); - specialize_transform_ = MakeSubstitution(SpecializeOnGraphArguments(), "specialize_transform", IsCNodeGraph); + inline_ = MakeSubstitution(std::make_shared(), "inline", IsCNodeGraph); + replace_applicator_ = + MakeSubstitution(std::make_shared(), "replace_applicator", IsValueNode); + specialize_transform_ = + MakeSubstitution(std::make_shared(), "specialize_transform", IsCNodeGraph); // Incorporation incorporate_getitem_set_ = - MakeSubstitution(IncorporateGetitemSet(), "incorporate_getitem_set", prim::kPrimTupleGetItem); - incorporate_getitem_from_param_ = - MakeSubstitution(IncorporateGetitemFromParam(), "incorporate_getitem_from_param", IsCNodeGraphKernel); - incorporate_call_ = MakeSubstitution(IncorporateCall(), "incorporate_call", IsCNodeDup); - incorporate_call_switch_ = MakeSubstitution(IncorporateCallSwitch(), "incorporate_call_switch", IsCNodeDup); + MakeSubstitution(std::make_shared(), "incorporate_getitem_set", prim::kPrimTupleGetItem); + incorporate_getitem_from_param_ = MakeSubstitution(std::make_shared(), + "incorporate_getitem_from_param", IsCNodeGraphKernel); + incorporate_call_ = MakeSubstitution(std::make_shared(), "incorporate_call", IsCNodeDup); + incorporate_call_switch_ = + MakeSubstitution(std::make_shared(), "incorporate_call_switch", IsCNodeDup); // Virtual Dataset - virtual_dataset_eliminate_ = - MakeSubstitution(VirtualDatasetEliminater(), "virtual_dataset_eliminate", prim::kPrimVirtualDataset); + virtual_dataset_eliminate_ = MakeSubstitution(std::make_shared(), + "virtual_dataset_eliminate", prim::kPrimVirtualDataset); // Convert - print_tuple_wrapper_ = MakeSubstitution(PrintTupleWrapper(), "print_tuple_wrapper", prim::kPrimPrint); + print_tuple_wrapper_ = + MakeSubstitution(std::make_shared(), "print_tuple_wrapper", prim::kPrimPrint); // Unused parameter eliminate unused_parameter_eliminate_ = - MakeSubstitution(UnusedParasEliminater(), "unused_parameter_eliminate", IsCNodeGraphKernel); - unused_output_eliminate_ = MakeSubstitution(UnusedOutputEliminater(), "unused_output_eliminate", IsCNodeGraphKernel); + MakeSubstitution(std::make_shared(), "unused_parameter_eliminate", IsCNodeGraphKernel); + unused_output_eliminate_ = + MakeSubstitution(std::make_shared(), "unused_output_eliminate", IsCNodeGraphKernel); // AddN eliminate - addn_eliminate_ = MakeSubstitution(AddNEliminater(), "addn_eliminate", IsCNodeGraphKernel); + addn_eliminate_ = MakeSubstitution(std::make_shared(), "addn_eliminate", IsCNodeGraphKernel); // Mark interface fusion - mark_interface_fusion_ = MakeSubstitution(MarkInterfaceFusion(), "mark_interface_fusion", prim::kPrimSelect); + mark_interface_fusion_ = + MakeSubstitution(std::make_shared(), "mark_interface_fusion", prim::kPrimSelect); + + // IndexedSlices Eliminate + indexed_slices_eliminate_ = MakeSubstitution( + std::make_shared(), "indexed_slices_eliminate", + {prim::kPrimIndexedSlicesGetIndices, prim::kPrimIndexedSlicesGetValues, prim::kPrimIndexedSlicesGetDenseShape}); } ResolveIRPassLib::ResolveIRPassLib() { - resolver_resolve_ = MakeSubstitution(ResolverResolve(), "resolver_resolve", prim::kPrimResolve); - resolver_getattr_ = MakeSubstitution(ResolverGetattr(), "resolver_getattr", prim::kPrimGetAttr); + resolver_resolve_ = MakeSubstitution(std::make_shared(), "resolver_resolve", prim::kPrimResolve); + resolver_getattr_ = MakeSubstitution(std::make_shared(), "resolver_getattr", prim::kPrimGetAttr); } InferenceOptPrepareLib::InferenceOptPrepareLib() { - grad_var_prepare_ = MakeSubstitution(GradVarPrepare(), "grad_var_prepare", IsCNode); + grad_var_prepare_ = MakeSubstitution(std::make_shared(), "grad_var_prepare", IsCNode); } } // namespace irpass } // namespace opt diff --git a/mindspore/ccsrc/optimizer/irpass.h b/mindspore/ccsrc/optimizer/irpass.h index 5e1550c883..782eae6124 100644 --- a/mindspore/ccsrc/optimizer/irpass.h +++ b/mindspore/ccsrc/optimizer/irpass.h @@ -33,6 +33,7 @@ class OptimizeIRPassLib { ~OptimizeIRPassLib() = default; SubstitutionPtr arithmetic_simplify_; + SubstitutionPtr arithmetic_simplify2_; SubstitutionPtr special_op_eliminate_; SubstitutionPtr zero_like_fill_zero_; SubstitutionPtr adjust_all_reduce_mul_add_; @@ -103,6 +104,9 @@ class OptimizeIRPassLib { // Fusion SubstitutionPtr mark_interface_fusion_; + + // IndexedSlices Eliminate + SubstitutionPtr indexed_slices_eliminate_; }; // the collection of irpass for resolve action diff --git a/mindspore/ccsrc/optimizer/irpass/arithmetic_simplify.cc b/mindspore/ccsrc/optimizer/irpass/arithmetic_simplify.cc new file mode 100644 index 0000000000..b111a6b67a --- /dev/null +++ b/mindspore/ccsrc/optimizer/irpass/arithmetic_simplify.cc @@ -0,0 +1,680 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include + +#include "optimizer/irpass/arithmetic_simplify.h" +#include "ir/optimizer_caller.h" +#include "ir/visitor.h" +#include "operator/ops.h" +#include "optimizer/irpass.h" +#include "optimizer/irpass/prim_eliminate.h" +#include "optimizer/optimizer.h" + +namespace mindspore { +namespace opt { +namespace irpass { +// {prim::kPrimScalarMul, 0, X}, {prim::kPrimScalarMul, X, 0} +// {prim::kPrimScalarMul, 1, X}, {prim::kPrimScalarMul, X, 1} +AnfNodePtr MultiplyByZeroOrOne::operator()(const OptimizerPtr &, const AnfNodePtr &node) { + Reset(); + AnfVisitor::Match(prim::kPrimScalarMul)(node); + + if (is_zero_) { + return NewValueNode(zero_); + } + if (is_one_) { + return x_; + } + return nullptr; +} + +void MultiplyByZeroOrOne::Visit(const AnfNodePtr &node) { + if (is_one_ || node->isa()) { + x_ = node; + return; + } + + AnfVisitor::Visit(node); + if (!is_one_) { + x_ = node; + } +} + +void MultiplyByZeroOrOne::Visit(const ValueNodePtr &vnode) { + auto value = vnode->value(); + if (*value == *zero_) { + is_zero_ = true; + } else if (*value == *one_) { + is_one_ = true; + } +} + +void MultiplyByZeroOrOne::Reset() { + x_ = nullptr; + is_one_ = false; + is_zero_ = false; +} + +// Support class used for checking if all values of a Tensor are equal `check_value_` +// Supported data types: double, float/float32, int/int32 +bool CheckTensorConstant::IsTensorConstant(const ValuePtr &value) { + if (!value->isa()) { + return false; + } + auto tensor_ptr = dyn_cast(value); + TypeId tensor_type = tensor_ptr->Dtype()->type_id(); + if ((tensor_type == TypeId::kNumberTypeFloat32) || (tensor_type == TypeId::kNumberTypeFloat)) { + float *data2 = reinterpret_cast(tensor_ptr->data_c()); + for (int i = 0; i < tensor_ptr->DataSize(); i++) { + if (fabs(data2[i] - check_value_) > FLT_EPSILON) { + return false; + } + } + return true; + } else if (tensor_type == TypeId::kNumberTypeFloat64) { + double *data2 = reinterpret_cast(tensor_ptr->data_c()); + for (int i = 0; i < tensor_ptr->DataSize(); i++) { + if (fabs(data2[i] - check_value_) > DBL_EPSILON) { + return false; + } + } + return true; + } else if ((tensor_type == TypeId::kNumberTypeInt32) || (tensor_type == TypeId::kNumberTypeInt)) { + int *data2 = reinterpret_cast(tensor_ptr->data_c()); + for (int i = 0; i < tensor_ptr->DataSize(); i++) { + if (data2[i] != check_value_) { + return false; + } + } + return true; + } + // input Data Types is not supported + return false; +} + +bool CheckTensorConstant::IsTensorScalarConstant(const ValuePtr &value) { + if (!value->isa()) { + return false; + } + auto tensor_ptr = dyn_cast(value); + if ((tensor_ptr->DataSize() > 1) || (tensor_ptr->DataDim() > 0)) { + return false; + } + return IsTensorConstant(value); +} + +void *TensorMultiplyBase::GetPointerToTensorData(const AnfNodePtr &node, bool writable) { + if (!node->isa()) { + return nullptr; + } + + auto value = node->cast()->value(); + + if (!value->isa()) { + return nullptr; + } + + tensor::TensorPtr tensor_ptr = dyn_cast(value); + return tensor_ptr->data_c(); +} + +// Make a new tensor (when possible) with the same shape as of `node` +// If x is nullptr then fill new tensor will "0" +// If x is a tensor with empty shape then fill new tensor with the single value of x +// If x is a tensor with same shape as `node` then return x as result +AnfNodePtr TensorMultiplyBase::NewTensorFilledWithData(const AnfNodePtr &node, const AnfNodePtr &x) { + if ((node->abstract() == nullptr) || !node->abstract()->isa()) { + return nullptr; + } + + auto tensor_abstract = node->abstract()->cast(); + TypePtr tensor_type_ptr = tensor_abstract->element()->BuildType(); + std::vector tensor_shape = tensor_abstract->shape()->shape(); + + auto new_tensor_ptr = std::make_shared(tensor_type_ptr->type_id(), tensor_shape); + size_t mem_size = GetTypeByte(tensor_type_ptr) * IntToSize(new_tensor_ptr->ElementsNum()); + char *data = reinterpret_cast(new_tensor_ptr->data_c()); + + if (x == nullptr) { + std::memset(data, 0, mem_size); + auto new_vnode = NewValueNode(new_tensor_ptr); + new_vnode->set_abstract(new_tensor_ptr->ToAbstract()); + return new_vnode; + } + // x is not nullptr + if (x->isa()) { + if ((x->abstract() == nullptr) || !x->abstract()->isa()) { + return nullptr; + } + auto x_abstract = x->abstract()->cast(); + std::vector x_shape = x_abstract->shape()->shape(); + + if (x_shape != tensor_shape) { + return nullptr; + } + return x; + } + + if (!x->isa()) { + return nullptr; + } + auto x_value = x->cast()->value(); + if (!x_value->isa()) { + return nullptr; + } + + auto x_tensor_ptr = dyn_cast(x_value); + + if ((x_tensor_ptr->DataSize() > 1) && (x_tensor_ptr->DataSize() != new_tensor_ptr->DataSize())) { + return nullptr; + } + char *source_data = reinterpret_cast(GetPointerToTensorData(x)); + if (x_tensor_ptr->DataSize() == 1) { + for (int i = 0; i < new_tensor_ptr->ElementsNum(); i++) { + memcpy(data + i * GetTypeByte(tensor_type_ptr), source_data, GetTypeByte(tensor_type_ptr)); + } + } else { + memcpy(data, source_data, mem_size); + } + auto new_vnode = NewValueNode(new_tensor_ptr); + new_vnode->set_abstract(new_tensor_ptr->ToAbstract()); + return new_vnode; +} + +// {prim::kPrimMul, 0, X}, {prim::kPrimMul, X, 0} +AnfNodePtr TensorMultiplyByZero::operator()(const OptimizerPtr &, const AnfNodePtr &node) { + Reset(); + AnfVisitor::Match(prim::kPrimMul)(node); + + if (is_zero_) { + if (x_->func_graph() != node->func_graph()) { + return nullptr; + } + return NewTensorFilledWithData(node); + } + return nullptr; +} + +void TensorMultiplyByZero::Visit(const AnfNodePtr &node) { + if (is_zero_) { + x_ = node; + return; + } + + if (IsParam(node)) { + x_ = node; + return; + } + + if (IsCNode(node)) { + CNodePtr cnode = node->cast(); + if (IsPrimitive(cnode->input(0), prim::kPrimZerosLike)) { + is_zero_ = true; + return; + } + x_ = node; + return; + } + auto value = node->cast()->value(); + if (CheckTensorConstant(0).IsTensorConstant(value)) { + is_zero_ = true; + return; + } + x_ = node; +} + +void TensorMultiplyByZero::Visit(const ValueNodePtr &vnode) { + auto value = vnode->value(); + if (CheckTensorConstant(0).IsTensorConstant(value)) { + is_zero_ = true; + return; + } + x_ = vnode; +} +void TensorMultiplyByZero::Reset() { + x_ = nullptr; + is_zero_ = false; +} + +// {prim::kPrimMul, 1, X}, {prim::kPrimMul, X, 1} +AnfNodePtr TensorMultiplyByOne::operator()(const OptimizerPtr &, const AnfNodePtr &node) { + Reset(); + AnfVisitor::Match(prim::kPrimMul)(node); + + if (is_one_) { + return NewTensorFilledWithData(node, x_); + } + return nullptr; +} + +void TensorMultiplyByOne::Visit(const AnfNodePtr &node) { + if (is_one_) { + x_ = node; + return; + } + + if (IsParam(node) || IsCNode(node)) { + x_ = node; + return; + } + + auto value = node->cast()->value(); + if (CheckTensorConstant(1).IsTensorConstant(value)) { + is_one_ = true; + return; + } + x_ = node; +} + +void TensorMultiplyByOne::Visit(const ValueNodePtr &vnode) { + auto value = vnode->value(); + if (CheckTensorConstant(1).IsTensorConstant(value)) { + is_one_ = true; + return; + } + x_ = vnode; +} +void TensorMultiplyByOne::Reset() { + x_ = nullptr; + is_one_ = false; +} + +// {prim::kPrimScalarAdd, X, 0} +// {prim::kPrimScalarAdd, 0, X} +AnfNodePtr AddByZero::operator()(const OptimizerPtr &, const AnfNodePtr &node) { + Reset(); + AnfVisitor::Match(prim::kPrimScalarAdd)(node); + + if (is_zero_) { + return x_; + } + return nullptr; +} + +void AddByZero::Visit(const AnfNodePtr &node) { + if (node->isa() && + ((*GetValueNode(node) == *zero_) || CheckTensorConstant(0).IsTensorScalarConstant(GetValueNode(node)))) { + is_zero_ = true; + return; + } + + x_ = node; +} + +void AddByZero::Reset() { + x_ = nullptr; + is_zero_ = false; +} + +// {prim::kPrimTensorAdd, {kPrimZerosLike, Y}, X}, +// {prim::kPrimTensorAdd, X, {kPrimZerosLike, Y}} +AnfNodePtr TensorAddByZero::operator()(const OptimizerPtr &, const AnfNodePtr &node) { + Reset(); + AnfVisitor::Match(prim::kPrimTensorAdd)(node); + + if (is_zero_) { + return x_; + } + return nullptr; +} + +void TensorAddByZero::Visit(const AnfNodePtr &node) { + if (node->isa() && CheckTensorConstant(0).IsTensorScalarConstant(GetValueNode(node))) { + is_zero_ = true; + return; + } + + x_ = node; +} + +void TensorAddByZero::Visit(const ValueNodePtr &vnode) { + auto value = vnode->value(); + if (CheckTensorConstant(0).IsTensorConstant(value)) { + is_zero_ = true; + return; + } +} + +void TensorAddByZero::Reset() { + x_ = nullptr; + is_zero_ = false; +} + +// {PrimMomentum, {kPrimZerosLike, X}, Y, Z, Xs} -> {prim::kPrimMakeTuple, Z, Y} +AnfNodePtr OptUpdateZeroTensor::operator()(const OptimizerPtr &, const AnfNodePtr &node) { + if (!IsPrimitiveCNode(node, prim::kPrimMomentum) || node->func_graph() == nullptr) { + return nullptr; + } + + // {PrimMomentum, {...}, Y, Z, Xs} + auto &inputs = node->cast()->inputs(); + if (inputs.size() < 4 || !IsPrimitiveCNode(inputs[1], prim::kPrimZerosLike)) { + return nullptr; + } + auto y = inputs[2]; + auto z = inputs[3]; + + // {kPrimZerosLike, X} + if (inputs[1]->cast()->size() != 2) { + return nullptr; + } + + // {prim::kPrimMakeTuple, Z, Y} + return node->func_graph()->NewCNode({NewValueNode(prim::kPrimMakeTuple), z, y}); +} + +// {prim::kPrimMul, Tensor1, {prim::kPrimMul, Tensor2, {...}}} -> +// {prim::kPrimMul, {...}, {prim::kPrimMul, Tensor1, Tensor2}} +// Support function to multiply two constant tensors: partially support broadcasting shapes +template +void ConstantDuplicateMul::Multiply(void *in_data_1, int in_data_1_size, void *in_data_2, int in_data_2_size, + void **out_data, int out_data_size) { + T *data_1 = reinterpret_cast(in_data_1); + T *data_2 = reinterpret_cast(in_data_2); + T *data_out = new T[out_data_size]; + + if (in_data_1_size == 1) { + for (int i = 0; i < out_data_size; i++) { + data_out[i] = data_1[0]; + } + } else { + for (int i = 0; i < out_data_size; i++) { + data_out[i] = data_1[i]; + } + } + if (in_data_2_size == 1) { + for (int i = 0; i < out_data_size; i++) { + data_out[i] *= data_2[0]; + } + } else { + for (int i = 0; i < out_data_size; i++) { + data_out[i] *= data_2[i]; + } + } + *out_data = reinterpret_cast(data_out); + return; +} + +AnfNodePtr ConstantDuplicateMul::MulConstantTensors(const AnfNodePtr &vnode_1, const AnfNodePtr &vnode_2, + const AnfNodePtr &node_3) { + if (!vnode_1->isa() || !vnode_2->isa() || (vnode_1->abstract() == nullptr) || + (vnode_2->abstract() == nullptr) || (node_3->abstract() == nullptr)) { + return nullptr; + } + + auto value_1 = GetValueNode(vnode_1); + auto value_2 = GetValueNode(vnode_2); + + if (!value_1->isa() || !value_2->isa()) { + return nullptr; + } + + auto tensor_ptr_1 = dyn_cast(value_1); + auto tensor_ptr_2 = dyn_cast(value_2); + + auto tensor_1_abstract = vnode_1->abstract()->cast(); + auto tensor_2_abstract = vnode_1->abstract()->cast(); + auto tensor_3_abstract = node_3->abstract()->cast(); + + TypePtr tensor_1_type_ptr = tensor_1_abstract->element()->BuildType(); + TypePtr tensor_2_type_ptr = tensor_2_abstract->element()->BuildType(); + TypePtr tensor_3_type_ptr = tensor_3_abstract->element()->BuildType(); + + if ((tensor_1_type_ptr->type_id() != tensor_3_type_ptr->type_id()) || + (tensor_2_type_ptr->type_id() != tensor_3_type_ptr->type_id())) { + return nullptr; + } + + std::vector tensor_out_shape = tensor_3_abstract->shape()->shape(); + + int data_out_size = std::accumulate(tensor_out_shape.begin(), tensor_out_shape.end(), 1, std::multiplies()); + + if ((tensor_ptr_1->DataSize() > 1) && (tensor_ptr_1->DataSize() != data_out_size)) { + return nullptr; + } + if ((tensor_ptr_2->DataSize() > 1) && (tensor_ptr_2->DataSize() != data_out_size)) { + return nullptr; + } + + void *data_out; + + if ((tensor_3_type_ptr->type_id() == TypeId::kNumberTypeFloat32) || + (tensor_3_type_ptr->type_id() == TypeId::kNumberTypeFloat)) { + Multiply(tensor_ptr_1->data_c(), tensor_ptr_1->DataSize(), tensor_ptr_2->data_c(), tensor_ptr_2->DataSize(), + &data_out, data_out_size); + } else { + if (tensor_3_type_ptr->type_id() == TypeId::kNumberTypeFloat64) { + Multiply(tensor_ptr_1->data_c(), tensor_ptr_1->DataSize(), tensor_ptr_2->data_c(), + tensor_ptr_2->DataSize(), &data_out, data_out_size); + } else { + if ((tensor_3_type_ptr->type_id() == TypeId::kNumberTypeInt32) || + (tensor_3_type_ptr->type_id() == TypeId::kNumberTypeInt)) { + Multiply(tensor_ptr_1->data_c(), tensor_ptr_1->DataSize(), tensor_ptr_2->data_c(), + tensor_ptr_2->DataSize(), &data_out, data_out_size); + } else { + // Un-support data types + return nullptr; + } + } + } + + auto new_tensor_ptr = std::make_shared(tensor_3_type_ptr->type_id(), tensor_out_shape); + size_t mem_size = GetTypeByte(tensor_3_type_ptr) * IntToSize(new_tensor_ptr->ElementsNum()); + char *data = reinterpret_cast(new_tensor_ptr->data_c()); + memcpy(data, data_out, mem_size); + + auto new_vnode = NewValueNode(new_tensor_ptr); + new_vnode->set_abstract(new_tensor_ptr->ToAbstract()); + return new_vnode; +} + +AnfNodePtr ConstantDuplicateMul::operator()(const OptimizerPtr &, const AnfNodePtr &node) { + Reset(); + // {prim::kPrimMul, Tensor1, {...}} + AnfVisitor::Match(prim::kPrimMul, {IsNode, IsNode})(node); + if (vnode_ == nullptr || c_p_node_ == nullptr) { + return nullptr; + } + + if (!IsCNode(c_p_node_)) { + return nullptr; + } + + auto tensor1 = vnode_; + auto mul = c_p_node_->cast(); + + Reset(); + // {prim::kPrimMul, Tensor2, {...}} + AnfVisitor::Match(prim::kPrimMul, {IsNode, IsNode})(mul); + if (vnode_ == nullptr || c_p_node_ == nullptr) { + return nullptr; + } + auto tensor2 = vnode_; + auto c_p_node = c_p_node_; + + auto PrimMul = GetValueNode(mul->input(0)); + auto fg = node->func_graph(); + + auto new_mul_tensor = MulConstantTensors(tensor1, tensor2, c_p_node); + if (new_mul_tensor == nullptr) { + auto ttmul = NewCNode({NewValueNode(PrimMul), tensor1, tensor2}, fg); + return NewCNode({NewValueNode(PrimMul), c_p_node, ttmul}, fg); + } + return NewCNode({NewValueNode(PrimMul), c_p_node, new_mul_tensor}, fg); +} + +void ConstantDuplicateMul::Visit(const AnfNodePtr &node) { + if (IsValueNode(node)) { + vnode_ = node; + } + + if (IsCNode(node) || IsParam(node)) { + c_p_node_ = node; + } +} + +void ConstantDuplicateMul::Reset() { + vnode_ = nullptr; + c_p_node_ = nullptr; +} + +AnfNodePtr PowerOneEliminate::operator()(const OptimizerPtr &, const AnfNodePtr &node) { + if (!IsPrimitiveCNode(node, prim::kPrimPow) || node->func_graph() == nullptr) { + return nullptr; + } + + auto &inputs = node->cast()->inputs(); + if (!IsValueNode(inputs[2])) { + return nullptr; + } + auto scalar = GetValueNode(inputs[2]); + if (scalar->isa() && GetValue(scalar) == 1.0) { + return inputs[1]; + } else if (scalar->isa() && GetValue(scalar) == 1) { + return inputs[1]; + } + return nullptr; +} + +// grad = AllReduce(grad) / worker_number +// grad = grad + weight * decy +// -> +// grad = grad + weight * decy +// grad = AllReduce(grad) / worker_number +// {prim::kPrimAddN, {prim::kPrimMakeTuple, {prim::kPrimMul, {prim::kPrimAllReduce, X}, Y}, Z}} -> +// {prim::kPrimMul, {prim::kPrimAllReduce, {prim::kPrimAddN,{prim::kPrimMakeTuple, Z, X}}}, Y} +AnfNodePtr AdjustAllReduceMulAdd::operator()(const OptimizerPtr &, const AnfNodePtr &node) { + Reset(); + // {prim::kPrimAddN, Zs} + if (!IsPrimitiveCNode(node, prim::kPrimAddN)) { + return nullptr; + } + auto addn = node->cast(); + if (addn->size() != 2) { + return nullptr; + } + AnfVisitor::Match(prim::kPrimMakeTuple, {IsNode, IsNode})(addn->input(1)); + if (x_ == nullptr || y_ == nullptr || z_ == nullptr || all_reduce_fg_ == nullptr) { + return nullptr; + } + auto addn_maketuple = addn->input(1); + + auto fg = all_reduce_fg_; + // addn inputs cross the graph, make the inputs same as allreduce node. + if (z_->isa() && fg != z_->func_graph()) { + auto cnode_z = z_->cast(); + z_ = NewCNode(cnode_z->inputs(), fg); + } + + auto addn_op_node = addn->input(0); + auto make_tuple_op_node = addn->input(1)->cast()->input(0); + + AnfNodePtr tuple = NewCNode({make_tuple_op_node, z_, x_}, fg); + AnfNodePtr add = NewCNode({addn_op_node, tuple}, fg); + AnfNodePtr all_reduce = NewCNode({all_reduce_, add}, fg); + AnfNodePtr mul = NewCNode({mul_, all_reduce, y_}, fg); + ProcessDependEdge(fg, addn_maketuple, all_reduce); + return mul; +} + +void AdjustAllReduceMulAdd::ProcessDependEdge(const FuncGraphPtr &fg, const AnfNodePtr &addn_maketuple, + const AnfNodePtr &new_node) { + // If has dynamic loss scale. + auto &users_map = fg->manager()->node_users(); + auto it = users_map.find(mul_cnode_); + if (it != users_map.end()) { + auto users = it->second; + for (auto &user_pair : users) { + auto node = user_pair.first; + if (node != addn_maketuple) { + if (IsPrimitiveCNode(node, prim::kPrimMakeTuple)) { + fg->manager()->SetEdge(node, user_pair.second, new_node); + } + } + } + } +} + +void AdjustAllReduceMulAdd::Visit(const AnfNodePtr &node) { + if (level_ == 0) { + level_ = 1; + is_reduce_match_ = false; + // {prim::kPrimMul, {prim::kPrimAllReduce, X}, Y} + AnfVisitor::Match(prim::kPrimMul)(node); + level_ = 0; + if (is_reduce_match_) { + mul_ = node->cast()->input(0); + mul_cnode_ = node->cast(); + y_ = tmp_; + } else { + z_ = node; + } + } + + if (level_ == 1) { + // {prim::kPrimAllReduce, X} + if (IsPrimitiveCNode(node, prim::kPrimAllReduce)) { + auto cnode = node->cast(); + if (cnode->size() > 1) { + all_reduce_ = cnode->input(0); + x_ = cnode->input(1); + is_reduce_match_ = true; + all_reduce_fg_ = cnode->func_graph(); + } + } else { + tmp_ = node; + } + } +} + +void AdjustAllReduceMulAdd::Reset() { + level_ = 0; + is_reduce_match_ = false; + x_ = nullptr; + y_ = nullptr; + z_ = nullptr; + tmp_ = nullptr; + all_reduce_fg_ = nullptr; +} + +AnfNodePtr ArithmeticSimplify::operator()(const OptimizerPtr &optimizer, const AnfNodePtr &node) { + AnfNodePtr new_node; + for (auto &eliminater : eliminaters_) { + new_node = (*eliminater)(optimizer, node); + if (new_node != nullptr) { + return new_node; + } + } + return nullptr; +} + +AnfNodePtr ArithmeticSimplify2::operator()(const OptimizerPtr &optimizer, const AnfNodePtr &node) { + AnfNodePtr new_node; + for (auto &eliminater : eliminaters_) { + new_node = (*eliminater)(optimizer, node); + if (new_node != nullptr) { + return new_node; + } + } + return nullptr; +} +} // namespace irpass +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/optimizer/irpass/arithmetic_simplify.h b/mindspore/ccsrc/optimizer/irpass/arithmetic_simplify.h index 1836a88dbc..f4bdb0d655 100644 --- a/mindspore/ccsrc/optimizer/irpass/arithmetic_simplify.h +++ b/mindspore/ccsrc/optimizer/irpass/arithmetic_simplify.h @@ -17,15 +17,16 @@ #ifndef MINDSPORE_CCSRC_OPTIMIZER_IRPASS_ARITHMETIC_SIMPLIFY_H_ #define MINDSPORE_CCSRC_OPTIMIZER_IRPASS_ARITHMETIC_SIMPLIFY_H_ -#include -#include #include +#include +#include -#include "optimizer/optimizer.h" -#include "optimizer/irpass.h" -#include "optimizer/irpass/prim_eliminate.h" +#include "ir/optimizer_caller.h" #include "ir/visitor.h" #include "operator/ops.h" +#include "optimizer/irpass.h" +#include "optimizer/irpass/prim_eliminate.h" +#include "optimizer/optimizer.h" namespace mindspore { namespace opt { @@ -37,45 +38,11 @@ class MultiplyByZeroOrOne : public AnfVisitor { MultiplyByZeroOrOne() : zero_(MakeValue(0)), one_(MakeValue(1)) {} ~MultiplyByZeroOrOne() override = default; - AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { - Reset(); - AnfVisitor::Match(prim::kPrimScalarMul)(node); - - if (is_zero_) { - return NewValueNode(zero_); - } - if (is_one_) { - return x_; - } - return nullptr; - } + AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override; - void Visit(const AnfNodePtr &node) override { - if (is_one_ || node->isa()) { - x_ = node; - return; - } - - AnfVisitor::Visit(node); - if (!is_one_) { - x_ = node; - } - } - - void Visit(const ValueNodePtr &vnode) override { - auto value = vnode->value(); - if (*value == *zero_) { - is_zero_ = true; - } else if (*value == *one_) { - is_one_ = true; - } - } - - void Reset() { - x_ = nullptr; - is_one_ = false; - is_zero_ = false; - } + void Visit(const AnfNodePtr &node) override; + void Visit(const ValueNodePtr &vnode) override; + void Reset(); private: bool is_zero_{false}, is_one_{false}; @@ -89,208 +56,57 @@ class CheckTensorConstant { public: explicit CheckTensorConstant(int _check_value = 0) : check_value_(_check_value) {} ~CheckTensorConstant() = default; - bool IsTensorConstant(const ValuePtr &value) { - if (!value->isa()) { - return false; - } - auto tensor_ptr = dyn_cast(value); - TypeId tensor_type = tensor_ptr->Dtype()->type_id(); - if ((tensor_type == TypeId::kNumberTypeFloat32) || (tensor_type == TypeId::kNumberTypeFloat)) { - float *data2 = reinterpret_cast(tensor_ptr->data_c()); - for (int i = 0; i < tensor_ptr->DataSize(); i++) { - if (fabs(data2[i] - check_value_) > FLT_EPSILON) { - return false; - } - } - return true; - } else if (tensor_type == TypeId::kNumberTypeFloat64) { - double *data2 = reinterpret_cast(tensor_ptr->data_c()); - for (int i = 0; i < tensor_ptr->DataSize(); i++) { - if (fabs(data2[i] - check_value_) > DBL_EPSILON) { - return false; - } - } - return true; - } else if ((tensor_type == TypeId::kNumberTypeInt32) || (tensor_type == TypeId::kNumberTypeInt)) { - int *data2 = reinterpret_cast(tensor_ptr->data_c()); - for (int i = 0; i < tensor_ptr->DataSize(); i++) { - if (data2[i] != check_value_) { - return false; - } - } - return true; - } - // Un-support Data Types - return false; - } - bool IsTensorScalarConstant(const ValuePtr &value) { - if (!value->isa()) { - return false; - } - auto tensor_ptr = dyn_cast(value); - if ((tensor_ptr->DataSize() > 1) || (tensor_ptr->DataDim() > 0)) { - return false; - } - return IsTensorConstant(value); - } + bool IsTensorConstant(const ValuePtr &value); + bool IsTensorScalarConstant(const ValuePtr &value); private: int check_value_; }; -// {prim::kPrimMul, 0, X}, {prim::kPrimMul, X, 0} -// {prim::kPrimMul, 1, X}, {prim::kPrimMul, X, 1} -class TensorMultiplyByZeroOrOne : public AnfVisitor { - public: - TensorMultiplyByZeroOrOne() : zero_(MakeValue(0)) {} - ~TensorMultiplyByZeroOrOne() override = default; - AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { - Reset(); - AnfVisitor::Match(prim::kPrimMul)(node); - - if (is_zero_) { - if (x_->func_graph() != node->func_graph()) { - return nullptr; - } - return NewTensorFilledWithData(node); - } - if (is_one_) { - return NewTensorFilledWithData(node, x_); - } - return nullptr; - } +class TensorMultiplyBase : public AnfVisitor { + protected: + void *GetPointerToTensorData(const AnfNodePtr &node, bool writable = false); - void Visit(const AnfNodePtr &node) override { - if (is_zero_ || is_one_) { - x_ = node; - return; - } - - if (IsParam(node)) { - x_ = node; - return; - } - - if (IsCNode(node)) { - CNodePtr cnode = node->cast(); - if (IsPrimitive(cnode->input(0), prim::kPrimZerosLike)) { - is_zero_ = true; - return; - } - x_ = node; - return; - } - auto value = node->cast()->value(); - if (CheckTensorConstant(0).IsTensorConstant(value)) { - is_zero_ = true; - return; - } else if (CheckTensorConstant(1).IsTensorConstant(value)) { - is_one_ = true; - return; - } - x_ = node; - } + // Make a new tensor (when possible) with the same shape as of `node` + // If x is nullptr then fill new tensor will "0" + // If x is a tensor with empty shape then fill new tensor with the single value of x + // If x is a tensor with same shape as `node` then return x as result + AnfNodePtr NewTensorFilledWithData(const AnfNodePtr &node, const AnfNodePtr &x = nullptr); - void Visit(const ValueNodePtr &vnode) override { - auto value = vnode->value(); - if (CheckTensorConstant(0).IsTensorConstant(value)) { - is_zero_ = true; - return; - } else if (CheckTensorConstant(1).IsTensorConstant(value)) { - is_one_ = true; - return; - } - x_ = vnode; - } - void Reset() { - x_ = nullptr; - is_one_ = false; - is_zero_ = false; - } + AnfNodePtr x_{nullptr}; +}; - void *GetPointerToTensorData(const AnfNodePtr &node, bool writable = false) { - if (!node->isa()) { - return nullptr; - } +// {prim::kPrimMul, 0, X}, {prim::kPrimMul, X, 0} +class TensorMultiplyByZero : public TensorMultiplyBase { + public: + TensorMultiplyByZero() : zero_(MakeValue(0)) {} + ~TensorMultiplyByZero() override = default; - auto value = node->cast()->value(); + AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override; - if (!value->isa()) { - return nullptr; - } + void Visit(const AnfNodePtr &node) override; + void Visit(const ValueNodePtr &vnode) override; + void Reset(); - tensor::TensorPtr tensor_ptr = dyn_cast(value); - return tensor_ptr->data_c(writable); - } + private: + bool is_zero_{false}; + ValuePtr zero_; +}; - // Make a new tensor (when possible) with the same shape as of `node` - // If x is nullptr then fill new tensor will "0" - // If x is a tensor with empty shape then fill new tensor with the single value of x - // If x is a tensor with same shape as `node` then return x as result - AnfNodePtr NewTensorFilledWithData(const AnfNodePtr &node, const AnfNodePtr &x = nullptr) { - if ((node->abstract() == nullptr) || !node->abstract()->isa()) { - return nullptr; - } - - auto tensor_abstract = node->abstract()->cast(); - TypePtr tensor_type_ptr = tensor_abstract->element()->BuildType(); - std::vector tensor_shape = tensor_abstract->shape()->shape(); - - auto new_tensor_ptr = std::make_shared(tensor_type_ptr->type_id(), tensor_shape); - size_t mem_size = GetTypeByte(tensor_type_ptr) * IntToSize(new_tensor_ptr->ElementsNum()); - char *data = reinterpret_cast(new_tensor_ptr->data_c(true)); - - if (x == nullptr) { - std::memset(data, 0, mem_size); - auto new_vnode = NewValueNode(new_tensor_ptr); - new_vnode->set_abstract(new_tensor_ptr->ToAbstract()); - return new_vnode; - } - // x is not nullptr - if (x->isa()) { - if ((x->abstract() == nullptr) || !x->abstract()->isa()) { - return nullptr; - } - auto x_abstract = x->abstract()->cast(); - std::vector x_shape = x_abstract->shape()->shape(); - - if (x_shape != tensor_shape) { - return nullptr; - } - return x; - } - - if (!x->isa()) { - return nullptr; - } - auto x_value = x->cast()->value(); - if (!x_value->isa()) { - return nullptr; - } - - auto x_tensor_ptr = dyn_cast(x_value); - - if ((x_tensor_ptr->DataSize() > 1) && (x_tensor_ptr->DataSize() != new_tensor_ptr->DataSize())) { - return nullptr; - } - char *source_data = reinterpret_cast(GetPointerToTensorData(x)); - if (x_tensor_ptr->DataSize() == 1) { - for (int i = 0; i < new_tensor_ptr->ElementsNum(); i++) { - memcpy(source_data, data + i * GetTypeByte(tensor_type_ptr), GetTypeByte(tensor_type_ptr)); - } - } else { - memcpy(source_data, data, mem_size); - } - auto new_vnode = NewValueNode(new_tensor_ptr); - new_vnode->set_abstract(new_tensor_ptr->ToAbstract()); - return new_vnode; - } +// {prim::kPrimMul, 1, X}, {prim::kPrimMul, X, 1} +class TensorMultiplyByOne : public TensorMultiplyBase { + public: + TensorMultiplyByOne() {} + ~TensorMultiplyByOne() override = default; + AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override; + + void Visit(const AnfNodePtr &node) override; + void Visit(const ValueNodePtr &vnode) override; + void Reset(); private: - bool is_zero_{false}, is_one_{false}; - ValuePtr zero_; - AnfNodePtr x_{nullptr}; + bool is_one_{false}; }; // {prim::kPrimScalarAdd, X, 0} @@ -300,30 +116,10 @@ class AddByZero : public AnfVisitor { AddByZero() : zero_(MakeValue(0)) {} ~AddByZero() override = default; - AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { - Reset(); - AnfVisitor::Match(prim::kPrimScalarAdd)(node); - - if (is_zero_) { - return x_; - } - return nullptr; - } - - void Visit(const AnfNodePtr &node) override { - if (node->isa() && - ((*GetValueNode(node) == *zero_) || CheckTensorConstant(0).IsTensorScalarConstant(GetValueNode(node)))) { - is_zero_ = true; - return; - } + AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override; - x_ = node; - } - - void Reset() { - x_ = nullptr; - is_zero_ = false; - } + void Visit(const AnfNodePtr &node) override; + void Reset(); private: bool is_zero_{false}; @@ -335,41 +131,11 @@ class AddByZero : public AnfVisitor { // {prim::kPrimTensorAdd, X, {kPrimZerosLike, Y}} class TensorAddByZero : public AnfVisitor { public: - AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { - Reset(); - AnfVisitor::Match(prim::kPrimTensorAdd)(node); - - if (is_zero_) { - return x_; - } - return nullptr; - } - - void Visit(const AnfNodePtr &node) override { - if (IsPrimitive(node, prim::kPrimZerosLike)) { - is_zero_ = true; - return; - } - if (node->isa() && CheckTensorConstant(0).IsTensorScalarConstant(GetValueNode(node))) { - is_zero_ = true; - return; - } - - x_ = node; - } - - void Visit(const ValueNodePtr &vnode) override { - auto value = vnode->value(); - if (CheckTensorConstant(0).IsTensorConstant(value)) { - is_zero_ = true; - return; - } - } + AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override; - void Reset() { - x_ = nullptr; - is_zero_ = false; - } + void Visit(const AnfNodePtr &node) override; + void Visit(const ValueNodePtr &vnode) override; + void Reset(); private: bool is_zero_{false}; @@ -379,27 +145,7 @@ class TensorAddByZero : public AnfVisitor { // {PrimMomentum, {kPrimZerosLike, X}, Y, Z, Xs} -> {prim::kPrimMakeTuple, Z, Y} class OptUpdateZeroTensor : public AnfVisitor { public: - AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { - if (!IsPrimitiveCNode(node, prim::kPrimMomentum) || node->func_graph() == nullptr) { - return nullptr; - } - - // {PrimMomentum, {...}, Y, Z, Xs} - auto &inputs = node->cast()->inputs(); - if (inputs.size() < 4 || !IsPrimitiveCNode(inputs[1], prim::kPrimZerosLike)) { - return nullptr; - } - auto y = inputs[2]; - auto z = inputs[3]; - - // {kPrimZerosLike, X} - if (inputs[1]->cast()->size() != 2) { - return nullptr; - } - - // {prim::kPrimMakeTuple, Z, Y} - return node->func_graph()->NewCNode({NewValueNode(prim::kPrimMakeTuple), z, y}); - } + AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override; }; // {prim::kPrimMul, Tensor1, {orim::kPrimMul, Tensor2, {...}}} -> @@ -409,156 +155,14 @@ class ConstantDuplicateMul : public AnfVisitor { // Support function to multiply two constant tensors: partially support broadcasting shapes template void Multiply(void *in_data_1, int in_data_1_size, void *in_data_2, int in_data_2_size, void **out_data, - int out_data_size) { - T *data_1 = reinterpret_cast(in_data_1); - T *data_2 = reinterpret_cast(in_data_2); - T *data_out = new T[out_data_size]; - - if (in_data_1_size == 1) { - for (int i = 0; i < out_data_size; i++) { - data_out[i] = data_1[0]; - } - } else { - for (int i = 0; i < out_data_size; i++) { - data_out[i] = data_1[i]; - } - } - if (in_data_2_size == 1) { - for (int i = 0; i < out_data_size; i++) { - data_out[i] *= data_2[0]; - } - } else { - for (int i = 0; i < out_data_size; i++) { - data_out[i] *= data_2[i]; - } - } - *out_data = reinterpret_cast(data_out); - return; - } - - AnfNodePtr MulConstantTensors(const AnfNodePtr &vnode_1, const AnfNodePtr &vnode_2, const AnfNodePtr &node_3) { - if (!vnode_1->isa() || !vnode_2->isa() || (vnode_1->abstract() == nullptr) || - (vnode_2->abstract() == nullptr) || (node_3->abstract() == nullptr)) { - return nullptr; - } - - auto value_1 = GetValueNode(vnode_1); - auto value_2 = GetValueNode(vnode_2); - - if (!value_1->isa() || !value_2->isa()) { - return nullptr; - } - - auto tensor_ptr_1 = dyn_cast(value_1); - auto tensor_ptr_2 = dyn_cast(value_2); - - auto tensor_1_abstract = vnode_1->abstract()->cast(); - auto tensor_2_abstract = vnode_1->abstract()->cast(); - auto tensor_3_abstract = node_3->abstract()->cast(); - - TypePtr tensor_1_type_ptr = tensor_1_abstract->element()->BuildType(); - TypePtr tensor_2_type_ptr = tensor_2_abstract->element()->BuildType(); - TypePtr tensor_3_type_ptr = tensor_3_abstract->element()->BuildType(); - - if ((tensor_1_type_ptr->type_id() != tensor_3_type_ptr->type_id()) || - (tensor_2_type_ptr->type_id() != tensor_3_type_ptr->type_id())) { - return nullptr; - } - - std::vector tensor_out_shape = tensor_3_abstract->shape()->shape(); - - int data_out_size = 1; - for (auto it : tensor_out_shape) { - data_out_size *= it; - } - if ((tensor_ptr_1->DataSize() > 1) && (tensor_ptr_1->DataSize() != data_out_size)) { - return nullptr; - } - if ((tensor_ptr_2->DataSize() > 1) && (tensor_ptr_2->DataSize() != data_out_size)) { - return nullptr; - } - - void *data_out; - - if ((tensor_3_type_ptr->type_id() == TypeId::kNumberTypeFloat32) || - (tensor_3_type_ptr->type_id() == TypeId::kNumberTypeFloat)) { - Multiply(tensor_ptr_1->data_c(), tensor_ptr_1->DataSize(), tensor_ptr_2->data_c(), - tensor_ptr_2->DataSize(), &data_out, data_out_size); - } else { - if (tensor_3_type_ptr->type_id() == TypeId::kNumberTypeFloat64) { - Multiply(tensor_ptr_1->data_c(), tensor_ptr_1->DataSize(), tensor_ptr_2->data_c(), - tensor_ptr_2->DataSize(), &data_out, data_out_size); - } else { - if ((tensor_3_type_ptr->type_id() == TypeId::kNumberTypeInt32) || - (tensor_3_type_ptr->type_id() == TypeId::kNumberTypeInt)) { - Multiply(tensor_ptr_1->data_c(), tensor_ptr_1->DataSize(), tensor_ptr_2->data_c(), - tensor_ptr_2->DataSize(), &data_out, data_out_size); - } else { - // Un-support data types - return nullptr; - } - } - } - - auto new_tensor_ptr = std::make_shared(tensor_3_type_ptr->type_id(), tensor_out_shape); - size_t mem_size = GetTypeByte(tensor_3_type_ptr) * IntToSize(new_tensor_ptr->ElementsNum()); - char *data = reinterpret_cast(new_tensor_ptr->data_c(true)); - memcpy(data, data_out, mem_size); - - auto new_vnode = NewValueNode(new_tensor_ptr); - new_vnode->set_abstract(new_tensor_ptr->ToAbstract()); - return new_vnode; - } + int out_data_size); - AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { - Reset(); - // {prim::kPrimMul, Tensor1, {...}} - AnfVisitor::Match(prim::kPrimMul, {IsNode, IsNode})(node); - if (vnode_ == nullptr || c_p_node_ == nullptr) { - return nullptr; - } - - if (!IsCNode(c_p_node_)) { - return nullptr; - } - - auto tensor1 = vnode_; - auto mul = c_p_node_->cast(); - - Reset(); - // {prim::kPrimMul, Tensor2, {...}} - AnfVisitor::Match(prim::kPrimMul, {IsNode, IsNode})(mul); - if (vnode_ == nullptr || c_p_node_ == nullptr) { - return nullptr; - } - auto tensor2 = vnode_; - auto c_p_node = c_p_node_; - - auto PrimMul = GetValueNode(mul->input(0)); - auto fg = node->func_graph(); - - auto new_mul_tensor = MulConstantTensors(tensor1, tensor2, c_p_node); - if (new_mul_tensor == nullptr) { - auto ttmul = NewCNode({NewValueNode(PrimMul), tensor1, tensor2}, fg); - return NewCNode({NewValueNode(PrimMul), c_p_node, ttmul}, fg); - } - return NewCNode({NewValueNode(PrimMul), c_p_node, new_mul_tensor}, fg); - } + AnfNodePtr MulConstantTensors(const AnfNodePtr &vnode_1, const AnfNodePtr &vnode_2, const AnfNodePtr &node_3); - void Visit(const AnfNodePtr &node) override { - if (IsValueNode(node)) { - vnode_ = node; - } + AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override; - if (IsCNode(node) || IsParam(node)) { - c_p_node_ = node; - } - } - - void Reset() { - vnode_ = nullptr; - c_p_node_ = nullptr; - } + void Visit(const AnfNodePtr &node) override; + void Reset(); private: AnfNodePtr vnode_; @@ -567,23 +171,7 @@ class ConstantDuplicateMul : public AnfVisitor { class PowerOneEliminate : public AnfVisitor { public: - AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { - if (!IsPrimitiveCNode(node, prim::kPrimPow) || node->func_graph() == nullptr) { - return nullptr; - } - - auto &inputs = node->cast()->inputs(); - if (!IsValueNode(inputs[2])) { - return nullptr; - } - auto scalar = GetValueNode(inputs[2]); - if (scalar->isa() && GetValue(scalar) == 1.0) { - return inputs[1]; - } else if (scalar->isa() && GetValue(scalar) == 1) { - return inputs[1]; - } - return nullptr; - } + AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override; }; // grad = AllReduce(grad) / worker_number @@ -596,96 +184,11 @@ class PowerOneEliminate : public AnfVisitor { // {prim::kPrimMul, {prim::kPrimAllReduce, {prim::kPrimAddN,{prim::kPrimMakeTuple, Z, X}}}, Y} class AdjustAllReduceMulAdd : public AnfVisitor { public: - AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { - Reset(); - // {prim::kPrimAddN, Zs} - if (!IsPrimitiveCNode(node, prim::kPrimAddN)) { - return nullptr; - } - auto addn = node->cast(); - if (addn->size() != 2) { - return nullptr; - } - AnfVisitor::Match(prim::kPrimMakeTuple, {IsNode, IsNode})(addn->input(1)); - if (x_ == nullptr || y_ == nullptr || z_ == nullptr || all_reduce_fg_ == nullptr) { - return nullptr; - } - auto addn_maketuple = addn->input(1); - - auto fg = all_reduce_fg_; - // addn inputs cross the graph, make the inputs same as allreduce node. - if (z_->isa() && fg != z_->func_graph()) { - auto cnode_z = z_->cast(); - z_ = NewCNode(cnode_z->inputs(), fg); - } - - auto addn_op_node = addn->input(0); - auto make_tuple_op_node = addn->input(1)->cast()->input(0); - - AnfNodePtr tuple = NewCNode({make_tuple_op_node, z_, x_}, fg); - AnfNodePtr add = NewCNode({addn_op_node, tuple}, fg); - AnfNodePtr all_reduce = NewCNode({all_reduce_, add}, fg); - AnfNodePtr mul = NewCNode({mul_, all_reduce, y_}, fg); - ProcessDependEdge(fg, addn_maketuple, all_reduce); - return mul; - } - void ProcessDependEdge(const FuncGraphPtr &fg, const AnfNodePtr &addn_maketuple, const AnfNodePtr &new_node) { - // If has dynamic loss scale. - auto &users_map = fg->manager()->node_users(); - auto it = users_map.find(mul_cnode_); - if (it != users_map.end()) { - auto users = it->second; - for (auto &user_pair : users) { - auto node = user_pair.first; - if (node != addn_maketuple) { - if (IsPrimitiveCNode(node, prim::kPrimMakeTuple)) { - fg->manager()->SetEdge(node, user_pair.second, new_node); - } - } - } - } - } - void Visit(const AnfNodePtr &node) override { - if (level_ == 0) { - level_ = 1; - is_reduce_match_ = false; - // {prim::kPrimMul, {prim::kPrimAllReduce, X}, Y} - AnfVisitor::Match(prim::kPrimMul)(node); - level_ = 0; - if (is_reduce_match_) { - mul_ = node->cast()->input(0); - mul_cnode_ = node->cast(); - y_ = tmp_; - } else { - z_ = node; - } - } - - if (level_ == 1) { - // {prim::kPrimAllReduce, X} - if (IsPrimitiveCNode(node, prim::kPrimAllReduce)) { - auto cnode = node->cast(); - if (cnode->size() > 1) { - all_reduce_ = cnode->input(0); - x_ = cnode->input(1); - is_reduce_match_ = true; - all_reduce_fg_ = cnode->func_graph(); - } - } else { - tmp_ = node; - } - } - } + AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override; - void Reset() { - level_ = 0; - is_reduce_match_ = false; - x_ = nullptr; - y_ = nullptr; - z_ = nullptr; - tmp_ = nullptr; - all_reduce_fg_ = nullptr; - } + void ProcessDependEdge(const FuncGraphPtr &fg, const AnfNodePtr &addn_maketuple, const AnfNodePtr &new_node); + void Visit(const AnfNodePtr &node) override; + void Reset(); private: int level_{0}; @@ -695,19 +198,19 @@ class AdjustAllReduceMulAdd : public AnfVisitor { FuncGraphPtr all_reduce_fg_{nullptr}; }; -class ArithmeticSimplify { +class ArithmeticSimplify : public OptimizerCaller { public: ArithmeticSimplify() - : multiply_by_zero_or_one_(), - tensor_multiply_by_zero_or_one_(), - add_by_zero_(), - tensor_add_by_zero_(), - identity_(prim::kPrimIdentity), - opt_update_zero_tensor_(), - constant_duplicate_mul_(), - power_one_() { + : multiply_by_zero_or_one_(std::make_shared()), + tensor_multiply_by_one_(std::make_shared()), + add_by_zero_(std::make_shared()), + tensor_add_by_zero_(std::make_shared()), + identity_(std::make_shared(prim::kPrimIdentity)), + opt_update_zero_tensor_(std::make_shared()), + constant_duplicate_mul_(std::make_shared()), + power_one_(std::make_shared()) { eliminaters_.emplace_back(multiply_by_zero_or_one_); - eliminaters_.emplace_back(tensor_multiply_by_zero_or_one_); + eliminaters_.emplace_back(tensor_multiply_by_one_); eliminaters_.emplace_back(add_by_zero_); eliminaters_.emplace_back(tensor_add_by_zero_); eliminaters_.emplace_back(identity_); @@ -717,27 +220,38 @@ class ArithmeticSimplify { } ~ArithmeticSimplify() = default; - AnfNodePtr operator()(const OptimizerPtr &optimizer, const AnfNodePtr &node) { - AnfNodePtr new_node; - for (auto &eliminater : eliminaters_) { - new_node = eliminater(optimizer, node); - if (new_node != nullptr) { - return new_node; - } - } - return nullptr; + AnfNodePtr operator()(const OptimizerPtr &optimizer, const AnfNodePtr &node) override; + + private: + OptimizerCallerPtr multiply_by_zero_or_one_; + OptimizerCallerPtr tensor_multiply_by_one_; + OptimizerCallerPtr add_by_zero_; + OptimizerCallerPtr tensor_add_by_zero_; + OptimizerCallerPtr identity_; + OptimizerCallerPtr opt_update_zero_tensor_; + OptimizerCallerPtr constant_duplicate_mul_; + OptimizerCallerPtr power_one_; + + std::vector eliminaters_{}; +}; + +// Arithmetic Simplifications should be done after step_parallel. +// eg: Mul(0, weight) where weight is a parameter will be simplified to a constant tensor +// with shape(weight), but after step_parallel, shape of weight may be changed, so the +// shape of the constant tensor should also be changed. So this pass is seperated from +// ArithmeticSimplify and deferred until step_parallel. +class ArithmeticSimplify2 : public OptimizerCaller { + public: + ArithmeticSimplify2() : tensor_multiply_by_zero_(std::make_shared()) { + eliminaters_.emplace_back(tensor_multiply_by_zero_); } + ~ArithmeticSimplify2() = default; + + AnfNodePtr operator()(const OptimizerPtr &optimizer, const AnfNodePtr &node) override; private: - MultiplyByZeroOrOne multiply_by_zero_or_one_; - TensorMultiplyByZeroOrOne tensor_multiply_by_zero_or_one_; - AddByZero add_by_zero_; - TensorAddByZero tensor_add_by_zero_; - PrimEliminater identity_; - OptUpdateZeroTensor opt_update_zero_tensor_; - ConstantDuplicateMul constant_duplicate_mul_; - PowerOneEliminate power_one_; - std::vector eliminaters_{}; + OptimizerCallerPtr tensor_multiply_by_zero_; + std::vector eliminaters_{}; }; } // namespace irpass } // namespace opt diff --git a/mindspore/ccsrc/optimizer/irpass/branch_culling.cc b/mindspore/ccsrc/optimizer/irpass/branch_culling.cc index 0253cd2b39..726f4a28b0 100644 --- a/mindspore/ccsrc/optimizer/irpass/branch_culling.cc +++ b/mindspore/ccsrc/optimizer/irpass/branch_culling.cc @@ -51,18 +51,31 @@ bool InConvertWhiteList(const AnfNodePtr &node, size_t index) { // node because it is attribute or ge specific reason. // Example : when convert CNode(kPrimReduceSum, x, axis), node of index 2 in CNode->inputs is axis which should not be // converted to switch guarded. - std::vector>> white_list( - {{prim::kPrimApplyMomentum, {1, 2}}, {prim::kPrimMomentum, {2, 3}}, - {prim::kPrimStateSetItem, {1}}, {prim::kPrimTupleGetItem, {2}}, - {prim::kPrimEnvGetItem, {1}}, {prim::kPrimEnvSetItem, {1}}, - {prim::kPrimReduceSum, {2}}, {prim::kPrimReduceMean, {2}}, - {prim::kPrimReduceAll, {2}}, {prim::kPrimCast, {2}}, - {prim::kPrimTranspose, {2}}, {prim::kPrimOneHot, {2}}, - {prim::kPrimGatherV2, {3}}, {prim::kPrimReshape, {2}}, - {prim::kPrimAssign, {1}}, {prim::kPrimAssignAdd, {1}}, - {prim::kPrimAssignSub, {1}}, {prim::kPrimTensorSummary, {1}}, - {prim::kPrimImageSummary, {1}}, {prim::kPrimScalarSummary, {1}}, - {prim::kPrimHistogramSummary, {1}}}); + std::vector>> white_list({{prim::kPrimApplyMomentum, {1, 2}}, + {prim::kPrimMomentum, {2, 3}}, + {prim::kPrimStateSetItem, {1}}, + {prim::kPrimTupleGetItem, {2}}, + {prim::kPrimEnvGetItem, {1}}, + {prim::kPrimEnvSetItem, {1}}, + {prim::kPrimReduceSum, {2}}, + {prim::kPrimReduceMean, {2}}, + {prim::kPrimReduceAll, {2}}, + {prim::kPrimCast, {2}}, + {prim::kPrimTranspose, {2}}, + {prim::kPrimOneHot, {2}}, + {prim::kPrimGatherV2, {3}}, + {prim::kPrimReshape, {2}}, + {prim::kPrimAssign, {1}}, + {prim::kPrimAssignAdd, {1}}, + {prim::kPrimAssignSub, {1}}, + {prim::kPrimTensorSummary, {1}}, + {prim::kPrimImageSummary, {1}}, + {prim::kPrimScalarSummary, {1}}, + {prim::kPrimApplyRMSProp, {6, 7, 8}}, + {prim::kPrimCumSum, {2}}, + {prim::kPrimTile, {2}}, + {prim::kPrimExpandDims, {2}}, + {prim::kPrimHistogramSummary, {1}}}); for (auto &item : white_list) { auto matched = std::any_of(item.second.begin(), item.second.end(), [&item, &node, &index](size_t idx) { return IsPrimitiveCNode(node, item.first) && idx == index; @@ -191,7 +204,7 @@ inline void ResetSharedOp() { tensor::TensorPtr ConstData() { std::vector shp = {1}; tensor::TensorPtr const_data = std::make_shared(kInt32->type_id(), shp); - auto *val = static_cast(const_data->data_c(true)); + auto *val = static_cast(const_data->data_c()); *val = 0; return const_data; } @@ -267,7 +280,7 @@ CNodePtr GenerateSwitchControlDependNode(const FuncGraphPtr &graph, const AnfNod auto PrimSquare = prim::GetPythonOps("square", "mindspore.ops.functional")->cast(); std::vector shp = {1}; tensor::TensorPtr const_data = std::make_shared(kInt32->type_id(), shp); - auto *val = static_cast(const_data->data_c(true)); + auto *val = static_cast(const_data->data_c()); *val = 0; // for the control_depend netoutput node , add two const data to merge the flow ,one for depended node with same // switch the other use the opposite diff --git a/mindspore/ccsrc/optimizer/irpass/cast_eliminate.h b/mindspore/ccsrc/optimizer/irpass/cast_eliminate.h index 734d88cb10..d98d0b677b 100644 --- a/mindspore/ccsrc/optimizer/irpass/cast_eliminate.h +++ b/mindspore/ccsrc/optimizer/irpass/cast_eliminate.h @@ -17,9 +17,9 @@ #ifndef MINDSPORE_CCSRC_OPTIMIZER_IRPASS_CAST_ELIMINATE_H_ #define MINDSPORE_CCSRC_OPTIMIZER_IRPASS_CAST_ELIMINATE_H_ +#include "ir/visitor.h" #include "optimizer/irpass.h" #include "optimizer/optimizer.h" -#include "ir/visitor.h" namespace mindspore { namespace opt { @@ -52,12 +52,12 @@ class TwoCastEliminater : public AnfVisitor { AnfNodePtr x_{nullptr}, t_{nullptr}; }; -class CastEliminater { +class CastEliminater : public OptimizerCaller { public: CastEliminater() : cast_same_type_eliminater_(), two_cast_eliminater_() {} ~CastEliminater() = default; - AnfNodePtr operator()(const OptimizerPtr &optimizer, const AnfNodePtr &node) { + AnfNodePtr operator()(const OptimizerPtr &optimizer, const AnfNodePtr &node) override { auto new_node = cast_same_type_eliminater_(optimizer, node); if (new_node != nullptr) { return new_node; diff --git a/mindspore/ccsrc/optimizer/irpass/env_item_eliminate.h b/mindspore/ccsrc/optimizer/irpass/env_item_eliminate.h index 0f59c69fef..3f100dcaec 100644 --- a/mindspore/ccsrc/optimizer/irpass/env_item_eliminate.h +++ b/mindspore/ccsrc/optimizer/irpass/env_item_eliminate.h @@ -17,18 +17,19 @@ #ifndef MINDSPORE_CCSRC_OPTIMIZER_IRPASS_ENV_ITEM_ELIMINATE_H_ #define MINDSPORE_CCSRC_OPTIMIZER_IRPASS_ENV_ITEM_ELIMINATE_H_ -#include -#include #include -#include #include +#include +#include +#include -#include "optimizer/irpass.h" -#include "optimizer/optimizer.h" -#include "ir/visitor.h" #include "ir/func_graph.h" #include "ir/func_graph_cloner.h" +#include "ir/optimizer_caller.h" +#include "ir/visitor.h" #include "operator/ops.h" +#include "optimizer/irpass.h" +#include "optimizer/optimizer.h" #include "utils/symbolic.h" namespace mindspore { @@ -225,19 +226,22 @@ class EnvGetSetItem : public AnfVisitor { bool is_match_{false}; }; -class EnvGetItemEliminater { +class EnvGetItemEliminater : public OptimizerCaller { public: - EnvGetItemEliminater() : new_env_get_item_(), add_env_get_item_(), env_get_set_item_() { + EnvGetItemEliminater() + : new_env_get_item_(std::make_shared()), + add_env_get_item_(std::make_shared()), + env_get_set_item_(std::make_shared()) { eliminaters_.emplace_back(new_env_get_item_); eliminaters_.emplace_back(add_env_get_item_); eliminaters_.emplace_back(env_get_set_item_); } ~EnvGetItemEliminater() = default; - AnfNodePtr operator()(const OptimizerPtr &optimizer, const AnfNodePtr &node) { + AnfNodePtr operator()(const OptimizerPtr &optimizer, const AnfNodePtr &node) override { AnfNodePtr new_node; for (auto &eliminater : eliminaters_) { - new_node = eliminater(optimizer, node); + new_node = (*eliminater)(optimizer, node); if (new_node != nullptr) { return new_node; } @@ -246,10 +250,8 @@ class EnvGetItemEliminater { } private: - NewEnvGetItem new_env_get_item_; - AddEnvGetItem add_env_get_item_; - EnvGetSetItem env_get_set_item_; - std::vector eliminaters_{}; + OptimizerCallerPtr new_env_get_item_, add_env_get_item_, env_get_set_item_; + std::vector eliminaters_{}; }; // {prim::kPrimEnvGetItem, {G, Xs}, C, Y} diff --git a/mindspore/ccsrc/optimizer/irpass/incorporate_getitem.h b/mindspore/ccsrc/optimizer/irpass/incorporate_getitem.h index 5afee45e95..b6c8fb0e18 100644 --- a/mindspore/ccsrc/optimizer/irpass/incorporate_getitem.h +++ b/mindspore/ccsrc/optimizer/irpass/incorporate_getitem.h @@ -17,18 +17,20 @@ #ifndef MINDSPORE_CCSRC_OPTIMIZER_IRPASS_INCORPORATE_GETITEM_H_ #define MINDSPORE_CCSRC_OPTIMIZER_IRPASS_INCORPORATE_GETITEM_H_ -#include #include -#include #include +#include #include +#include -#include "optimizer/irpass.h" -#include "optimizer/optimizer.h" -#include "ir/visitor.h" #include "ir/func_graph.h" #include "ir/func_graph_cloner.h" +#include "ir/optimizer_caller.h" +#include "ir/visitor.h" #include "operator/ops.h" +#include "optimizer/irpass.h" +#include "optimizer/optimizer.h" + namespace mindspore { namespace opt { namespace irpass { @@ -383,18 +385,20 @@ class IncorporateGetitemSwitch : public AnfVisitor { internal::GetitemTransform getitem_transform_; }; -class IncorporateGetitemSet { +class IncorporateGetitemSet : public OptimizerCaller { public: - IncorporateGetitemSet() : incorporate_getitem_(), incorporate_getitem_switch_() { + IncorporateGetitemSet() + : incorporate_getitem_(std::make_shared()), + incorporate_getitem_switch_(std::make_shared()) { eliminaters_.emplace_back(incorporate_getitem_); eliminaters_.emplace_back(incorporate_getitem_switch_); } ~IncorporateGetitemSet() = default; - AnfNodePtr operator()(const OptimizerPtr &optimizer, const AnfNodePtr &node) { + AnfNodePtr operator()(const OptimizerPtr &optimizer, const AnfNodePtr &node) override { AnfNodePtr new_node; for (auto &eliminater : eliminaters_) { - new_node = eliminater(optimizer, node); + new_node = (*eliminater)(optimizer, node); if (new_node != nullptr) { return new_node; } @@ -403,9 +407,8 @@ class IncorporateGetitemSet { } private: - IncorporateGetitem incorporate_getitem_; - IncorporateGetitemSwitch incorporate_getitem_switch_; - std::vector eliminaters_{}; + OptimizerCallerPtr incorporate_getitem_, incorporate_getitem_switch_; + std::vector eliminaters_{}; }; } // namespace irpass } // namespace opt diff --git a/mindspore/ccsrc/optimizer/irpass/indexed_slices_eliminate.h b/mindspore/ccsrc/optimizer/irpass/indexed_slices_eliminate.h new file mode 100644 index 0000000000..630d567549 --- /dev/null +++ b/mindspore/ccsrc/optimizer/irpass/indexed_slices_eliminate.h @@ -0,0 +1,75 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_OPTIMIZER_IRPASS_INDEXED_SLICES_ELIMINATE_H_ +#define MINDSPORE_CCSRC_OPTIMIZER_IRPASS_INDEXED_SLICES_ELIMINATE_H_ + +#include +#include + +#include "optimizer/irpass.h" +#include "optimizer/optimizer.h" +#include "ir/visitor.h" +#include "operator/ops.h" + +namespace mindspore { +namespace opt { +namespace irpass { +// {prim::kPrimIndexedSlicesGetIndices, {prim::kPrimMakeIndexedSlices, Xs}} +// {prim::kPrimIndexedSlicesGetValues, {prim::kPrimMakeIndexedSlices, Xs}} +// {prim::kPrimIndexedSlicesGetDenseShape, {prim::kPrimMakeIndexedSlices, Xs}} +class IndexedSlicesEliminater : public AnfVisitor { + public: + AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { + Reset(); + AnfVisitor::Match(prim::kPrimIndexedSlicesGetIndices, {IsCNode})(node); + + if (is_match_) { + return tuple_->input(1); + } + AnfVisitor::Match(prim::kPrimIndexedSlicesGetValues, {IsCNode})(node); + + if (is_match_) { + return tuple_->input(2); + } + AnfVisitor::Match(prim::kPrimIndexedSlicesGetDenseShape, {IsCNode})(node); + + if (is_match_) { + return tuple_->input(3); + } + return nullptr; + } + + void Visit(const CNodePtr &cnode) override { + if (IsPrimitiveCNode(cnode, prim::kPrimMakeIndexedSlices)) { + tuple_ = cnode; + is_match_ = true; + } + } + + void Reset() { + tuple_ = nullptr; + is_match_ = false; + } + + private: + bool is_match_{false}; + CNodePtr tuple_{nullptr}; +}; +} // namespace irpass +} // namespace opt +} // namespace mindspore +#endif // MINDSPORE_CCSRC_OPTIMIZER_IRPASS_INDEXED_SLICES_ELIMINATE_H_ diff --git a/mindspore/ccsrc/optimizer/irpass/item_tuple_eliminate.h b/mindspore/ccsrc/optimizer/irpass/item_tuple_eliminate.h index 2693aec1c9..202951a254 100644 --- a/mindspore/ccsrc/optimizer/irpass/item_tuple_eliminate.h +++ b/mindspore/ccsrc/optimizer/irpass/item_tuple_eliminate.h @@ -17,13 +17,15 @@ #ifndef MINDSPORE_CCSRC_OPTIMIZER_IRPASS_ITEM_TUPLE_ELIMINATE_H_ #define MINDSPORE_CCSRC_OPTIMIZER_IRPASS_ITEM_TUPLE_ELIMINATE_H_ -#include #include +#include +#include -#include "optimizer/irpass.h" -#include "optimizer/optimizer.h" +#include "ir/optimizer_caller.h" #include "ir/visitor.h" #include "operator/ops.h" +#include "optimizer/irpass.h" +#include "optimizer/optimizer.h" namespace mindspore { namespace opt { @@ -70,6 +72,45 @@ class GetitemEliminater : public AnfVisitor { CNodePtr tuple_{nullptr}; }; +// (a, b, c, ...)[0] => a +// (a, b, c, ...)[1] => b +// {prim::kPrimTupleGetItem, C1, C} +class GetitemConstEliminater : public AnfVisitor { + public: + AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { + Reset(); + AnfVisitor::Match(prim::kPrimTupleGetItem, {IsVNode, IsVNode})(node); + + if (is_match_) { + return NewValueNode((*tuple_)[id_]); + } + return nullptr; + } + + void Visit(const ValueNodePtr &vnode) override { + if (IsValueNode(vnode)) { + tuple_ = GetValueNode(vnode); + } + if (tuple_ != nullptr && IsValueNode(vnode)) { + id_ = IntToSize(GetValue(vnode->value())); + if (tuple_->size() > id_) { + is_match_ = true; + } + } + } + + void Reset() { + id_ = 0; + tuple_ = nullptr; + is_match_ = false; + } + + private: + bool is_match_{false}; + size_t id_{0}; + ValueTuplePtr tuple_{nullptr}; +}; + // setitem((a, b, c, ...), 0, z) => (z, b, c, ...) // setitem((a, b, c, ...), 1, z) => (a, z, c, ...) // {prim::kPrimTupleSetItem, {prim::kPrimMakeTuple, Xs}, C, Z} @@ -222,21 +263,26 @@ class GetitemDependReorder : public AnfVisitor { AnfNodePtr x_{nullptr}, y_{nullptr}, c_{nullptr}; }; -class ItemTupleEliminater { +class ItemTupleEliminater : public OptimizerCaller { public: ItemTupleEliminater() - : get_item_eliminater_(), set_item_eliminater_(), get_set_item_eliminater_(), get_item_depend_reorder_() { + : get_item_eliminater_(std::make_shared()), + get_item_const_eliminater_(std::make_shared()), + set_item_eliminater_(std::make_shared()), + get_set_item_eliminater_(std::make_shared()), + get_item_depend_reorder_(std::make_shared()) { eliminaters_.emplace_back(get_item_eliminater_); + eliminaters_.emplace_back(get_item_const_eliminater_); eliminaters_.emplace_back(set_item_eliminater_); eliminaters_.emplace_back(get_set_item_eliminater_); eliminaters_.emplace_back(get_item_depend_reorder_); } ~ItemTupleEliminater() = default; - AnfNodePtr operator()(const OptimizerPtr &optimizer, const AnfNodePtr &node) { + AnfNodePtr operator()(const OptimizerPtr &optimizer, const AnfNodePtr &node) override { AnfNodePtr new_node; for (auto &eliminater : eliminaters_) { - new_node = eliminater(optimizer, node); + new_node = (*eliminater)(optimizer, node); if (new_node != nullptr) { return new_node; } @@ -245,11 +291,9 @@ class ItemTupleEliminater { } private: - GetitemEliminater get_item_eliminater_; - SetitemEliminater set_item_eliminater_; - GetSetitemEliminater get_set_item_eliminater_; - GetitemDependReorder get_item_depend_reorder_; - std::vector eliminaters_{}; + OptimizerCallerPtr get_item_eliminater_, get_item_const_eliminater_, set_item_eliminater_, get_set_item_eliminater_, + get_item_depend_reorder_; + std::vector eliminaters_{}; }; } // namespace irpass } // namespace opt diff --git a/mindspore/ccsrc/optimizer/irpass/merge_addn.h b/mindspore/ccsrc/optimizer/irpass/merge_addn.h index 94f9e26c5b..e1e4b8878b 100644 --- a/mindspore/ccsrc/optimizer/irpass/merge_addn.h +++ b/mindspore/ccsrc/optimizer/irpass/merge_addn.h @@ -35,9 +35,6 @@ namespace irpass { // {{PrimAddNClass}, {prim::kPrimMakeTuple, Ys, Xs}} class MergeAddN : public AnfVisitor { public: - MergeAddN() : PrimAddN_(prim::GetPythonOps("AddN", "mindspore.ops.operations")) {} - ~MergeAddN() override = default; - AnfNodePtr operator()(const OptimizerPtr &optimizer, const AnfNodePtr &node) override { Reset(); optimizer_ = optimizer; @@ -47,15 +44,15 @@ class MergeAddN : public AnfVisitor { return nullptr; } - auto fg = node->func_graph(); - // {PrimAddNClass} - auto addn_node = fg->NewCNode({NewValueNode(PrimAddN_)}); + auto cnode = node->cast(); + auto addn = NewValueNode(GetValueNode(cnode->input(0))); // {prim::kPrimMakeTuple, Xs, Ys}, {prim::kPrimMakeTuple, Ys, Xs} (void)args_.insert(args_.begin(), NewValueNode(prim::kPrimMakeTuple)); + auto fg = node->func_graph(); auto make_node = fg->NewCNode(args_); - return fg->NewCNode({addn_node, make_node}); + return fg->NewCNode({addn, make_node}); } void Visit(const CNodePtr &cnode) override { @@ -127,7 +124,6 @@ class MergeAddN : public AnfVisitor { } private: - ValuePtr PrimAddN_; OptimizerPtr optimizer_{nullptr}; std::vector Xs_{}, Ys_{}, args_{}; bool is_inner_{false}, is_outer_{false}, is_match_{false}; @@ -136,9 +132,6 @@ class MergeAddN : public AnfVisitor { // {PrimAddN, {kPrimMakeTuple, Xs}} class AddNZeroFilter : public AnfVisitor { public: - AddNZeroFilter() : PrimAddN_(prim::GetPythonOps("AddN", "mindspore.ops.operations")) {} - ~AddNZeroFilter() override = default; - AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { Reset(); AnfVisitor::Match(prim::kPrimAddN, {IsCNode})(node); @@ -161,8 +154,9 @@ class AddNZeroFilter : public AnfVisitor { return nullptr; } + auto cnode = node->cast(); + auto addn = NewValueNode(GetValueNode(cnode->input(0))); auto fg = node->func_graph(); - auto addn = fg->NewCNode({NewValueNode(PrimAddN_)}); auto make_tuple = fg->NewCNode(filtered_Xs_); return fg->NewCNode({addn, make_tuple}); } @@ -193,7 +187,6 @@ class AddNZeroFilter : public AnfVisitor { } private: - ValuePtr PrimAddN_; std::vector filtered_Xs_{}, Xs_{}; bool has_zero_like_{false}; }; @@ -224,10 +217,9 @@ class AddNEliminater : public AnfVisitor { auto new_fg = TransformableClone(fg, std::make_shared("fg")); mng->AddFuncGraph(new_fg); need_update_ = false; - bool changed = false; + bool changed; do { - changed = false; - changed |= Process(new_fg); + changed = Process(new_fg); } while (changed); if (!need_update_) { @@ -315,7 +307,7 @@ class AddNEliminater : public AnfVisitor { } } - need_update_ |= changed; + need_update_ = need_update_ || changed; return changed; } diff --git a/mindspore/ccsrc/optimizer/irpass/ref_eliminate.h b/mindspore/ccsrc/optimizer/irpass/ref_eliminate.h index 599ee8c339..a9e4659eae 100644 --- a/mindspore/ccsrc/optimizer/irpass/ref_eliminate.h +++ b/mindspore/ccsrc/optimizer/irpass/ref_eliminate.h @@ -19,9 +19,9 @@ #include -#include "optimizer/optimizer.h" -#include "optimizer/irpass.h" #include "ir/pattern_matcher.h" +#include "optimizer/irpass.h" +#include "optimizer/optimizer.h" namespace mindspore { namespace opt { diff --git a/mindspore/ccsrc/optimizer/irpass/reshape_eliminate.h b/mindspore/ccsrc/optimizer/irpass/reshape_eliminate.h index fb43f6ffd8..cafc8b796c 100644 --- a/mindspore/ccsrc/optimizer/irpass/reshape_eliminate.h +++ b/mindspore/ccsrc/optimizer/irpass/reshape_eliminate.h @@ -19,11 +19,12 @@ #include -#include "optimizer/irpass.h" -#include "optimizer/optimizer.h" -#include "ir/visitor.h" #include "ir/func_graph.h" +#include "ir/optimizer_caller.h" +#include "ir/visitor.h" #include "operator/ops.h" +#include "optimizer/irpass.h" +#include "optimizer/optimizer.h" #include "pipeline/static_analysis/dshape.h" namespace mindspore { @@ -124,12 +125,12 @@ class TwoReshapeEliminater : public AnfVisitor { AnfNodePtr x_{nullptr}, shape_{nullptr}; }; -class ReshapeEliminater { +class ReshapeEliminater : public OptimizerCaller { public: ReshapeEliminater() : reshape_same_shape_eliminater_(), two_reshape_eliminater_() {} ~ReshapeEliminater() = default; - AnfNodePtr operator()(const OptimizerPtr &optimizer, const AnfNodePtr &node) { + AnfNodePtr operator()(const OptimizerPtr &optimizer, const AnfNodePtr &node) override { auto new_node = reshape_same_shape_eliminater_(optimizer, node); if (new_node != nullptr) { return new_node; diff --git a/mindspore/ccsrc/optimizer/irpass/special_op_eliminate.h b/mindspore/ccsrc/optimizer/irpass/special_op_eliminate.h index 1dc8fbb344..b6a4e1c852 100644 --- a/mindspore/ccsrc/optimizer/irpass/special_op_eliminate.h +++ b/mindspore/ccsrc/optimizer/irpass/special_op_eliminate.h @@ -18,31 +18,31 @@ #define MINDSPORE_CCSRC_OPTIMIZER_IRPASS_SPECIAL_OP_ELIMINATE_H_ #include -#include -#include #include +#include +#include -#include "optimizer/optimizer.h" -#include "optimizer/irpass.h" #include "ir/optimizer_caller.h" -#include "optimizer/irpass/prim_eliminate.h" +#include "ir/pattern_matcher.h" #include "ir/visitor.h" #include "operator/ops.h" -#include "ir/pattern_matcher.h" +#include "optimizer/irpass.h" +#include "optimizer/irpass/prim_eliminate.h" +#include "optimizer/optimizer.h" namespace mindspore { namespace opt { namespace irpass { -class SpecialOpEliminater { +class SpecialOpEliminater : public OptimizerCaller { public: SpecialOpEliminater() - : insert_gradient_of_(prim::kPrimInsertGradientOf), - stop_gradient_(prim::kPrimStopGradient), - hook_backward_(prim::kPrimHookBackward), - print_shape_type_(prim::kPrimPrintShapeType), - get_ref_value_(prim::kPrimGetRefValue), - mirror_(prim::kPrimMirror), - virtual_div_(prim::kPrimVirtualDiv) { + : insert_gradient_of_(std::make_shared(prim::kPrimInsertGradientOf)), + stop_gradient_(std::make_shared(prim::kPrimStopGradient)), + hook_backward_(std::make_shared(prim::kPrimHookBackward)), + print_shape_type_(std::make_shared(prim::kPrimPrintShapeType)), + get_ref_value_(std::make_shared(prim::kPrimGetRefValue)), + mirror_(std::make_shared(prim::kPrimMirror)), + virtual_div_(std::make_shared(prim::kPrimVirtualDiv)) { eliminaters_.emplace_back(insert_gradient_of_); eliminaters_.emplace_back(stop_gradient_); eliminaters_.emplace_back(hook_backward_); @@ -53,10 +53,10 @@ class SpecialOpEliminater { } ~SpecialOpEliminater() = default; - AnfNodePtr operator()(const OptimizerPtr &optimizer, const AnfNodePtr &node) { + AnfNodePtr operator()(const OptimizerPtr &optimizer, const AnfNodePtr &node) override { AnfNodePtr new_node; for (auto &eliminater : eliminaters_) { - new_node = eliminater(optimizer, node); + new_node = (*eliminater)(optimizer, node); if (new_node != nullptr) { return new_node; } @@ -65,9 +65,9 @@ class SpecialOpEliminater { } private: - PrimEliminater insert_gradient_of_, stop_gradient_, hook_backward_, print_shape_type_, get_ref_value_, mirror_, + OptimizerCallerPtr insert_gradient_of_, stop_gradient_, hook_backward_, print_shape_type_, get_ref_value_, mirror_, virtual_div_; - std::vector eliminaters_{}; + std::vector eliminaters_{}; }; // {PrimVirtualDataset, X} -> X @@ -178,7 +178,7 @@ class ZeroLikeFillZero : public AnfVisitor { tensor::TensorPtr new_tensor_ptr = std::make_shared(tensor_type_ptr->type_id(), tensor_shape); size_t mem_size = GetTypeByte(tensor_type_ptr) * IntToSize(new_tensor_ptr->ElementsNum()); - char *data = reinterpret_cast(new_tensor_ptr->data_c(true)); + char *data = reinterpret_cast(new_tensor_ptr->data_c()); (void)memset_s(data, mem_size, 0, mem_size); auto new_cnode = NewValueNode(new_tensor_ptr); diff --git a/mindspore/ccsrc/optimizer/irpass/specialize_transform.h b/mindspore/ccsrc/optimizer/irpass/specialize_transform.h index 6ac4e40f5e..3db9e7bd51 100644 --- a/mindspore/ccsrc/optimizer/irpass/specialize_transform.h +++ b/mindspore/ccsrc/optimizer/irpass/specialize_transform.h @@ -71,7 +71,7 @@ class SpecializeTransform { continue; } if (value_args[i] != nullptr) { - auto const_tensor = *value_args[i]; + auto &const_tensor = *value_args[i]; auto const_tensor_ptr = std::make_shared(const_tensor); AnfNodePtr arg = NewValueNode(const_tensor_ptr); (void)mng->Replace(params[i], arg); diff --git a/mindspore/ccsrc/optimizer/opt.cc b/mindspore/ccsrc/optimizer/opt.cc index 82fbcc2036..462d08ad3c 100644 --- a/mindspore/ccsrc/optimizer/opt.cc +++ b/mindspore/ccsrc/optimizer/opt.cc @@ -16,28 +16,28 @@ #include "optimizer/opt.h" +#include +#include #include #include -#include -#include +#include #include "ir/anf.h" #include "ir/manager.h" -#include "utils/ordered_set.h" - -#include "utils/log_adapter.h" #include "optimizer/optimizer.h" +#include "utils/log_adapter.h" +#include "utils/ordered_set.h" namespace mindspore { /* namespace to support opt */ namespace opt { -SubstitutionPtr MakeSubstitution(const TransformFuncType &transform, const std::string &name, const PrimitivePtr &prim, +SubstitutionPtr MakeSubstitution(const OptimizerCallerPtr &transform, const std::string &name, const PrimitivePtr &prim, const RenormAction &renorm_action) { auto fn = [prim](const AnfNodePtr &node) -> bool { return IsPrimitiveCNode(node, prim); }; return std::make_shared(transform, name, fn, renorm_action); } -SubstitutionPtr MakeSubstitution(const TransformFuncType &transform, const std::string &name, +SubstitutionPtr MakeSubstitution(const OptimizerCallerPtr &transform, const std::string &name, const std::vector &prims, const RenormAction &renorm_action) { auto fn = [prims](const AnfNodePtr &node) -> bool { if (!node->isa()) { @@ -64,16 +64,16 @@ SubstitutionPtr MakeSubstitution(const TransformFuncType &transform, const std:: return std::make_shared(transform, name, fn, renorm_action); } -SubstitutionPtr MakeSubstitution(const TransformFuncType &transform, const std::string &name, +SubstitutionPtr MakeSubstitution(const OptimizerCallerPtr &transform, const std::string &name, const PredicateFuncType &predicate, const RenormAction &renorm_action) { return std::make_shared(transform, name, predicate, renorm_action); } -AnfNodePtr Substitution::operator()(const OptimizerPtr &optimizer, const AnfNodePtr &node) const { +AnfNodePtr Substitution::operator()(const OptimizerPtr &optimizer, const AnfNodePtr &node) { #ifdef ENABLE_PROFILE double t = GetTime(); #endif - AnfNodePtr result = transform_(optimizer, node); + AnfNodePtr result = (*transform_)(optimizer, node); #ifdef ENABLE_PROFILE if (optimizer != nullptr) { auto time = GetTime(); @@ -192,15 +192,30 @@ bool SubstitutionList::operator()(const FuncGraphPtr &func_graph, const Optimize FuncGraphManagerPtr manager = optimizer->manager(); manager->AddFuncGraph(func_graph); + // for transform status counting + size_t space = 0; + std::unordered_map> status; + if (optimizer->is_on_debug_) { + for (size_t i = 0; i < list_.size(); i++) { + status[list_[i]->name_ + std::to_string(i)] = {}; + } + } + bool loop = false; bool changes = false; do { loop = false; - for (auto const &transform : list_) { - auto change = ApplyTransform(optimizer, func_graph->output(), transform); + for (size_t i = 0; i < list_.size(); i++) { + auto change = ApplyTransform(optimizer, func_graph->output(), list_[i]); changes = changes || change; loop = loop || change; + + // record the status of each transform + if (optimizer->is_on_debug_) { + status[list_[i]->name_ + std::to_string(i)].push_back(change); + space = std::max(list_[i]->name_.size(), space); + } } if (is_once_) { @@ -208,6 +223,23 @@ bool SubstitutionList::operator()(const FuncGraphPtr &func_graph, const Optimize } } while (loop); + // display the status of each transform + if (optimizer->is_on_debug_) { + std::stringstream ss; + ss << std::endl + << "Pass: " << optimizer->name() << "(" << optimizer->CurPass_.counter << ")_" << optimizer->CurPass_.name + << std::endl; + for (size_t i = 0; i < list_.size(); i++) { + auto name = list_[i]->name_; + ss << std::left << std::setw(space + 4) << name << "\t"; + for (auto change : status[name + std::to_string(i)]) { + ss << change << " "; + } + ss << std::endl; + } + MS_LOG(DEBUG) << ss.str(); + } + return changes; } } // namespace opt diff --git a/mindspore/ccsrc/optimizer/opt.h b/mindspore/ccsrc/optimizer/opt.h index fb0bdc58be..6601d969d2 100644 --- a/mindspore/ccsrc/optimizer/opt.h +++ b/mindspore/ccsrc/optimizer/opt.h @@ -17,24 +17,18 @@ #ifndef MINDSPORE_CCSRC_OPTIMIZER_OPT_H_ #define MINDSPORE_CCSRC_OPTIMIZER_OPT_H_ -#include -#include #include +#include +#include #include "ir/anf.h" #include "ir/func_graph.h" +#include "ir/optimizer_caller.h" #include "operator/ops.h" namespace mindspore { /* namespace to support opt */ namespace opt { -class Optimizer; - -using OptimizerPtr = std::shared_ptr; -using OptimizerWeakPtr = std::weak_ptr; - -using PredicateFuncType = std::function; -using TransformFuncType = std::function; // Define the interaction mode between an Optimize pass and Renormalize pass // FORCE_RENORM: if the pass modified the graph then the next Renormalize will be executed @@ -43,26 +37,26 @@ enum RenormAction : int { FORCE_RENORM = 0, CHECK_RENORM }; class Substitution { public: - TransformFuncType transform_{nullptr}; + OptimizerCallerPtr transform_; std::string name_; PredicateFuncType predicate_{nullptr}; // an enum to mark this Substitution relation to renormalize pass RenormAction renorm_action_; - Substitution(const TransformFuncType &transform, const std::string &name, const PredicateFuncType &predicate, + Substitution(const OptimizerCallerPtr &transform, const std::string &name, const PredicateFuncType &predicate, const RenormAction &renorm_action) : transform_(transform), name_(name), predicate_(predicate), renorm_action_(renorm_action) {} ~Substitution() = default; - AnfNodePtr operator()(const OptimizerPtr &optimizer, const AnfNodePtr &node) const; + AnfNodePtr operator()(const OptimizerPtr &optimizer, const AnfNodePtr &node); }; using SubstitutionPtr = std::shared_ptr; -SubstitutionPtr MakeSubstitution(const TransformFuncType &transform, const std::string &name, const PrimitivePtr &prim, +SubstitutionPtr MakeSubstitution(const OptimizerCallerPtr &transform, const std::string &name, const PrimitivePtr &prim, const RenormAction &action_renorm = CHECK_RENORM); -SubstitutionPtr MakeSubstitution(const TransformFuncType &transform, const std::string &name, +SubstitutionPtr MakeSubstitution(const OptimizerCallerPtr &transform, const std::string &name, const std::vector &prims, const RenormAction &action_renorm = CHECK_RENORM); -SubstitutionPtr MakeSubstitution(const TransformFuncType &transform, const std::string &name, +SubstitutionPtr MakeSubstitution(const OptimizerCallerPtr &transform, const std::string &name, const PredicateFuncType &predicate, const RenormAction &action_renorm = CHECK_RENORM); class SubstitutionList { diff --git a/mindspore/ccsrc/optimizer/optimizer.h b/mindspore/ccsrc/optimizer/optimizer.h index 3e77edc1e9..dc423ed314 100644 --- a/mindspore/ccsrc/optimizer/optimizer.h +++ b/mindspore/ccsrc/optimizer/optimizer.h @@ -95,6 +95,7 @@ class Optimizer : public std::enable_shared_from_this { void Init(const OptPassGroupMap &passes, bool run_only_once) { run_only_once_ = run_only_once; is_watch_renormalize_ = false; + is_on_debug_ = IS_OUTPUT_ON(mindspore::DEBUG); for (auto &iter : passes) { const std::string &name = iter.first; @@ -136,7 +137,7 @@ class Optimizer : public std::enable_shared_from_this { return func_graph; } // Optimizer step counter; - int counter = -1; + int counter = 1; bool changes = true; while (changes) { @@ -144,6 +145,7 @@ class Optimizer : public std::enable_shared_from_this { auto run_runc = [&counter, &func_graph, &changes, use_profile, this]() { for (size_t i = 0; i < passes_.size(); ++i) { const OptPass &opt = passes_[i]; + CurPass_ = {counter, pass_names_[i]}; auto opt_func = [&func_graph, &changes, &opt, this]() { if (opt.is_renormalize()) { auto resource_ptr = std::dynamic_pointer_cast(resource_); @@ -173,7 +175,7 @@ class Optimizer : public std::enable_shared_from_this { } }; use_profile ? (WITH(MsProfile::GetProfile()->Step(pass_names_[i])) opt_func) : opt_func(); - if (IS_OUTPUT_ON(mindspore::DEBUG) && MsContext::GetInstance()->save_graphs_flag()) { + if (is_on_debug_ && MsContext::GetInstance()->save_graphs_flag()) { MS_LOG(DEBUG) << "The opt " << name_ << " round " << counter << " OptPass " << pass_names_[i] << " end."; auto fg_name = "opt_substep_" + name_ + "_r" + std::to_string(counter) + "_" + std::to_string(i) + "_" + pass_names_[i]; @@ -184,7 +186,8 @@ class Optimizer : public std::enable_shared_from_this { } } }; - use_profile ? (WITH(MsProfile::GetProfile()->Lap(counter++)) run_runc) : run_runc(); + use_profile ? (WITH(MsProfile::GetProfile()->Lap(counter)) run_runc) : run_runc(); + counter++; if (run_only_once_) { break; @@ -216,6 +219,13 @@ class Optimizer : public std::enable_shared_from_this { bool is_watch_renormalize() { return is_watch_renormalize_; } void set_enable(bool enable) { is_enable_ = enable; } + struct { + int counter; + std::string name; + } CurPass_; + + bool is_on_debug_{false}; + private: const std::string name_; pipeline::ResourceBasePtr resource_; diff --git a/mindspore/ccsrc/optimizer/pass_group.cc b/mindspore/ccsrc/optimizer/pass_group.cc new file mode 100644 index 0000000000..2d1ab07f7d --- /dev/null +++ b/mindspore/ccsrc/optimizer/pass_group.cc @@ -0,0 +1,69 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "optimizer/pass_group.h" + +namespace mindspore { +namespace opt { +namespace python_pass { +void PassGroup::AddPass(const PythonPassPtr &pass) { + if (pass != nullptr) { + passes_.push_back(pass); + } +} + +bool PassGroup::DeletePass(const std::string &pass_name) { + for (auto iter = passes_.begin(); iter != passes_.end(); iter++) { + if ((*iter)->name() == pass_name) { + *iter = nullptr; + passes_.erase(iter); + return true; + } + } + return false; +} + +bool PassGroup::Run(const FuncGraphPtr &func_graph, const std::vector &passes) const { + if (func_graph == nullptr) { + return false; + } + bool changed = false; + for (const auto &pass : passes) { + if (pass != nullptr) { + if (pass->Run(func_graph)) { + changed = true; + } + } + } + return changed; +} + +bool PassGroup::Run(const FuncGraphPtr &func_graph) const { + bool changed = false; + // run all passes + bool change = true; + while (change) { + change = Run(func_graph, passes_); + changed = change || changed; + if (run_only_once_) { + break; + } + } + return changed; +} + +} // namespace python_pass +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/optimizer/pass_group.h b/mindspore/ccsrc/optimizer/pass_group.h new file mode 100644 index 0000000000..895f5a4128 --- /dev/null +++ b/mindspore/ccsrc/optimizer/pass_group.h @@ -0,0 +1,61 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_OPTIMIZER_PASS_GROUP_H_ +#define MINDSPORE_CCSRC_OPTIMIZER_PASS_GROUP_H_ + +#include +#include +#include +#include + +#include "optimizer/py_pass.h" + +namespace mindspore { +namespace opt { +namespace python_pass { +class PassGroup { + public: + explicit PassGroup(const std::string &name = "pass_group", bool run_only_once = false) + : name_(name), passes_{}, run_only_once_(run_only_once) {} + virtual ~PassGroup() = default; + // Add graph pass, the pass object will be freed when pass manager freed. + void AddPass(const PythonPassPtr &pass); + // Delete graph pass before the pass manager is freed. + bool DeletePass(const std::string &pass_name); + // Run passes added in pass manager on the input graph + // @param [inout] graph The graph to be optimized + // @return true, graph changed + // @return false, graph not changed + bool Run(const FuncGraphPtr &func_graph) const; + // Run the given graph passes on the input graph + // @param [inout] graph The graph to be optimized + // @param [in] passes The given graph passes + // @return true, graph changed + // @return false, graph not changed + bool Run(const FuncGraphPtr &func_graph, const std::vector &passes) const; + std::string name() const { return name_; } + + private: + const std::string name_; + std::vector passes_; + bool run_only_once_; +}; +using PassGroupPtr = std::shared_ptr; +} // namespace python_pass +} // namespace opt +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_OPTIMIZER_PASS_GROUP_H_ diff --git a/mindspore/ccsrc/optimizer/py_pass.cc b/mindspore/ccsrc/optimizer/py_pass.cc new file mode 100644 index 0000000000..8ce348b22e --- /dev/null +++ b/mindspore/ccsrc/optimizer/py_pass.cc @@ -0,0 +1,236 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "optimizer/py_pass.h" +#include +#include +#include +#include +#include + +#include "ir/func_graph.h" +#include "ir/manager.h" +#include "pipeline/parse/parse_base.h" +#include "pipeline/resource.h" + +namespace mindspore { +namespace opt { +namespace python_pass { +namespace internal { +std::string GetNodeRepr(AnfNodePtr node) { + if (node != nullptr) { + if (node->isa()) { + std::string repr = "("; + auto const &inputs = node->cast()->inputs(); + for (auto &input : inputs) { + repr += " "; + repr += GetNodeRepr(input); + repr += " "; + } + repr += ")"; + return repr; + } + if (node->isa()) { + return GetValueNode(node)->ToString(); + } + return node->ToString(); + } + return ""; +} + +void ResolveFuncGraph_(const FuncGraphPtr &fg) { + auto manager = Manage(fg, false); + parse::python_adapter::set_use_signature_in_resolve(false); + parse::ResolveAll(manager); +} + +bool Match(const AnfNodePtr &pattern, const AnfNodePtr &node, const NodeEquivPtr &equiv_ptr) { + if (node == nullptr) { + return false; + } + MS_EXCEPTION_IF_NULL(pattern); + if (pattern->isa()) { + if (!node->isa()) { + return false; + } + if (GetNodeRepr(pattern) == GetNodeRepr(node)) { + // add to equiv_ptr + equiv_ptr->insert(std::make_pair(GetValueNode(pattern)->ToString(), node)); + return true; + } + return false; + } else if (pattern->isa()) { + MS_LOG(DEBUG) << pattern->ToString() + "\n"; + // add to equiv_ptr + equiv_ptr->insert(std::make_pair(pattern->ToString(), node)); + return true; + } else if (pattern->isa()) { + // match every single sub ANode + if (!node->isa()) { + return false; + } + auto pattern_inputs = pattern->cast()->inputs(); + auto node_inputs = node->cast()->inputs(); + if (pattern_inputs.size() != node_inputs.size()) { + return false; + } + for (auto p_item = pattern_inputs.begin(), node_item = node_inputs.begin(); p_item != pattern_inputs.end(); + p_item++, node_item++) { + auto res = Match(*p_item, *node_item, equiv_ptr); + if (!res) { + return false; + } + } + return true; + } + MS_LOG(EXCEPTION) << "Unexpected condition, (" + pattern->ToString() + " , " + node->ToString() + ")\n"; +} + +AnfNodePtr BuildTarget(const FuncGraphPtr &func_graph, const AnfNodePtr cur_raw_dst_node_, + const NodeEquivPtr &equiv_ptr) { + if (cur_raw_dst_node_->isa()) { + auto sub_pair = equiv_ptr->find(cur_raw_dst_node_->ToString()); + if (sub_pair != equiv_ptr->end()) { + return sub_pair->second; + } + MS_LOG(EXCEPTION) << "cur_raw_dst_node_ : " + internal::GetNodeRepr(cur_raw_dst_node_) + "\n"; + } else if (cur_raw_dst_node_->isa()) { + // check primitive ValueNode + auto sub_pair = equiv_ptr->find(cur_raw_dst_node_->cast()->value()->ToString()); + if (sub_pair != equiv_ptr->end()) { + return sub_pair->second; + } + return cur_raw_dst_node_; + } else if (cur_raw_dst_node_->isa()) { + std::vector new_inputs; + auto inputs = cur_raw_dst_node_->cast()->inputs(); + for (auto sub_node = inputs.begin(); sub_node != inputs.end(); sub_node++) { + auto subed = internal::BuildTarget(func_graph, *sub_node, equiv_ptr); + new_inputs.push_back(subed); + } + return func_graph->NewCNode(new_inputs); + } + MS_LOG(EXCEPTION) << "Unexpected node type, got : " + internal::GetNodeRepr(cur_raw_dst_node_); +} + +bool isTraversable(const AnfNodePtr &node) { + if (node == nullptr) { + return false; + } + if (node->isa() || node->isa()) { + return true; + } + if (IsValueNode(node) || IsValueNode(node)) { + return true; + } + return false; +} +} // namespace internal + +void PythonPass::Build(const py::function &src, const py::function &dst) { + // 1. get FuncGraph from py::function + auto src_fg_ = parse::ParsePythonCode(src); + auto dst_fg_ = parse::ParsePythonCode(dst); + if (src_fg_ == nullptr || dst_fg_ == nullptr) { + MS_LOG(EXCEPTION) << "Failed to parse python code.\n"; + } + // 2. Resolve + internal::ResolveFuncGraph_(src_fg_); + internal::ResolveFuncGraph_(dst_fg_); + // 3. from FuncGraphPtr to ValueNode + src_node_ = src_fg_->output(); + dst_node_ = dst_fg_->output(); +} + +PythonPass::PythonPass(const std::string &name, const py::function &src, const py::function &dst, bool run_only_once, + bool multigraph) + : name_(name), run_only_once_(run_only_once), multigraph_(multigraph) { + Build(src, dst); +} + +AnfNodePtr PythonPass::Run(const FuncGraphPtr &func_graph, const AnfNodePtr &node) { + auto equiv_ptr = std::make_shared(); + bool is_a_match = internal::Match(src_node_, node, equiv_ptr); + if (is_a_match) { + auto new_node = internal::BuildTarget(func_graph, dst_node_, equiv_ptr); + MS_LOG(DEBUG) << "To be replaced node: " + internal::GetNodeRepr(new_node) + "\n"; + return new_node; + } + return nullptr; +} + +bool PythonPass::Run(const FuncGraphPtr &func_graph) { + MS_EXCEPTION_IF_NULL(func_graph); + FuncGraphManagerPtr manager = func_graph->manager(); + MS_EXCEPTION_IF_NULL(manager); + manager->AddFuncGraph(func_graph); + auto seen = NewSeenGeneration(); + // 1024 is for the initial capacity of deque + std::deque todo(1024); + todo.push_back(func_graph->output()); + bool changes = false; + + auto &all_nodes = manager->all_nodes(); + while (!todo.empty()) { + AnfNodePtr node = todo.front(); + todo.pop_front(); + + // check whether this node has been matched. + if (node == nullptr || node->seen_ == seen || !internal::isTraversable(node) || !all_nodes.contains(node)) { + continue; + } + node->seen_ = seen; + + // select nodes that this transform can be applied. + AnfNodePtr new_node = Run(func_graph, node); + bool change = (new_node != nullptr); + if (new_node != nullptr && new_node != node) { + (void)manager->Replace(node, new_node); + } else if (new_node == nullptr) { + new_node = node; + } + if (run_only_once_) { + return change; + } + + // find success, and add them to todo list + if (IsValueNode(node)) { + todo.push_back(GetValueNode(node)->output()); + } + + if (node->isa()) { + auto &inputs = node->cast()->inputs(); + (void)std::copy(inputs.begin(), inputs.end(), std::back_inserter(todo)); + } + + auto &node_users = manager->node_users(); + if (change && node_users.find(node) != node_users.end()) { + for (auto &use : node_users[node]) { + auto use_node = use.first; + if (use_node == nullptr) { + continue; + } + todo.push_back(use_node); + if (use_node->seen_ == seen) { + use_node->seen_--; + } + } + } + } + return changes; +} +} // namespace python_pass +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/optimizer/py_pass.h b/mindspore/ccsrc/optimizer/py_pass.h new file mode 100644 index 0000000000..b01bf7c942 --- /dev/null +++ b/mindspore/ccsrc/optimizer/py_pass.h @@ -0,0 +1,56 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_OPTIMIZER_PASS_H_ +#define MINDSPORE_CCSRC_OPTIMIZER_PASS_H_ +#include +#include +#include + +#include "ir/anf.h" +#include "pybind_api/api_register.h" +#include "pybind_api/export_flags.h" + +namespace mindspore { +namespace opt { +namespace python_pass { +class PythonPass; +using PythonPassPtr = std::shared_ptr; +using NodeEquiv = std::unordered_map; +using NodeEquivPtr = std::shared_ptr; + +class PythonPass { + public: + explicit PythonPass(const std::string &name, const py::function &src, const py::function &dst, + bool run_only_once = false, bool multigraph = true); + ~PythonPass() = default; + bool Run(const FuncGraphPtr &func_graph); + std::string name() const { return name_; } + AnfNodePtr Run(const FuncGraphPtr &func_graph, const AnfNodePtr &node); + + private: + void Build(const py::function &src, const py::function &dst); + AnfNodePtr src_node_ = nullptr; + AnfNodePtr dst_node_ = nullptr; + const std::string name_; + bool run_only_once_; + bool multigraph_ = true; +}; + +using PythonPassPtr = std::shared_ptr; +} // namespace python_pass +} // namespace opt +} // namespace mindspore +#endif // MINDSPORE_CCSRC_OPTIMIZER_PASS_H_ diff --git a/mindspore/ccsrc/optimizer/py_pass_manager.cc b/mindspore/ccsrc/optimizer/py_pass_manager.cc new file mode 100644 index 0000000000..1c36e93c9a --- /dev/null +++ b/mindspore/ccsrc/optimizer/py_pass_manager.cc @@ -0,0 +1,84 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "optimizer/py_pass_manager.h" + +#include +#include +#include +#include + +#include "ir/manager.h" +#include "optimizer/pass_group.h" + +namespace mindspore { +namespace opt { +namespace python_pass { +PyPassManagerPtr PyPassManager::global_instance = nullptr; +std::unordered_map PyPassManager::phase_to_group_; + +PassGroupPtr PyPassManager::GetPassGroup(Phase phase) { + auto pm = phase_to_group_.find(phase); + if (pm == phase_to_group_.end()) { + return nullptr; + } + return pm->second; +} + +PyPassManagerPtr PyPassManager::GetInstance() { + if (global_instance == nullptr) { + global_instance = std::shared_ptr(new (std::nothrow) PyPassManager()); + } + return global_instance; +} + +PyPassManager::PyPassManager() { + phase_to_group_[Phase::RESOLVE] = std::make_shared(); + phase_to_group_[Phase::OPT] = std::make_shared(); +} + +void PyPassManager::Registe(const std::string &pass_name, const py::function &pattern, const py::function &target, + Phase phase, bool run_only_once, bool multigraph) { + auto cur_pm = GetPassGroup(phase); + MS_EXCEPTION_IF_NULL(cur_pm); + PythonPassPtr new_pass = std::make_shared(pass_name, pattern, target, run_only_once, multigraph); + cur_pm->AddPass(new_pass); +} + +void PyPassManager::Unregiste(const std::string &pass_name, Phase phase) { + auto cur_pm = GetPassGroup(phase); + MS_EXCEPTION_IF_NULL(cur_pm); + if (!cur_pm->DeletePass(pass_name)) { + MS_LOG(WARNING) << "No such pass : " + pass_name + "\n"; + } +} + +void PyPassManager::ClearRes() { + MS_LOG(INFO) << "Clear PyPassManager resources!"; + global_instance = nullptr; + phase_to_group_.clear(); +} + +REGISTER_PYBIND_DEFINE( + PyPassManager_, ([](const py::module *m) { + (void)py::enum_(*m, "phase", py::arithmetic()).value("resolve", Phase::RESOLVE).value("opt", Phase::OPT); + (void)py::class_>(*m, "PyPassManager_") + .def(py::init([]() { return PyPassManager::GetInstance(); })) + .def("registe", &PyPassManager::Registe, "Registe python pass") + .def("unregiste", &PyPassManager::Unregiste, "Delete Python Pass"); + })); +} // namespace python_pass +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/optimizer/py_pass_manager.h b/mindspore/ccsrc/optimizer/py_pass_manager.h new file mode 100644 index 0000000000..eaeefce213 --- /dev/null +++ b/mindspore/ccsrc/optimizer/py_pass_manager.h @@ -0,0 +1,66 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_OPTIMIZER_PY_PASS_MANAGER_H_ +#define MINDSPORE_CCSRC_OPTIMIZER_PY_PASS_MANAGER_H_ + +#include +#include +#include +#include + +#include "ir/anf.h" +#include "ir/func_graph.h" +#include "ir/primitive.h" +#include "utils/graph_utils.h" +#include "common/utils.h" + +#include "pipeline/parse/resolve.h" +#include "optimizer/py_pass.h" +#include "optimizer/pass_group.h" + +namespace mindspore { +namespace opt { +namespace python_pass { +class PyPassManager; +using PyPassManagerPtr = std::shared_ptr; + +enum Phase { RESOLVE, OPT }; + +class PyPassManager { + protected: + PyPassManager(); + static PyPassManagerPtr global_instance; + + public: + // Singletons should not be cloneable and assignable + PyPassManager(const PyPassManager &other) = delete; + void operator=(const PyPassManager &) = delete; + // Access the only global instance + static PyPassManagerPtr GetInstance(); + virtual ~PyPassManager() = default; + void Registe(const std::string &pass_name, const py::function &pattern, const py::function &target, + Phase phase = Phase::RESOLVE, bool run_only_once = false, bool multigraph = true); + void Unregiste(const std::string &pass_name, Phase phase); + PassGroupPtr GetPassGroup(Phase phase); + void ClearRes(); + + private: + static std::unordered_map phase_to_group_; +}; +} // namespace python_pass +} // namespace opt +} // namespace mindspore +#endif // MINDSPORE_CCSRC_OPTIMIZER_PY_PASS_MANAGER_H_ diff --git a/mindspore/ccsrc/parallel/auto_parallel/graph_costmodel.cc b/mindspore/ccsrc/parallel/auto_parallel/graph_costmodel.cc index 043d3a9e23..05be097e6a 100644 --- a/mindspore/ccsrc/parallel/auto_parallel/graph_costmodel.cc +++ b/mindspore/ccsrc/parallel/auto_parallel/graph_costmodel.cc @@ -1406,7 +1406,9 @@ Status CostGraph::InitSelectedStrategy() { int32_t next_index = reshape_info->next_operator_index(); reshape_info->SetOutputLayout((*next_iter)->next_operator()->inputs_tensor_info()[next_index].tensor_layout()); } - return reshape_info->Init(nullptr); + if (reshape_info->Init(nullptr) != SUCCESS) { + return FAILED; + } } } return SUCCESS; diff --git a/mindspore/ccsrc/parallel/auto_parallel/operator_costmodel.cc b/mindspore/ccsrc/parallel/auto_parallel/operator_costmodel.cc index cbf5bc40c8..8ebfdb7d13 100644 --- a/mindspore/ccsrc/parallel/auto_parallel/operator_costmodel.cc +++ b/mindspore/ccsrc/parallel/auto_parallel/operator_costmodel.cc @@ -465,7 +465,7 @@ double ReshapeCost::GetForwardCommCost(const std::vector &inputs, co CheckGlobalDeviceManager(); MS_EXCEPTION_IF_NULL(g_device_manager); RankList dev_list = g_device_manager->GetDeviceListByStageId(stage_id); - TensorRedistribution tensor_redistribution; + TensorRedistribution tensor_redistribution(false, true); if (tensor_redistribution.Init(inputs[0].tensor_layout(), outputs[0].tensor_layout(), dev_list) == FAILED) { MS_LOG(EXCEPTION) << "Failure: tensor_redistribution init failed."; } @@ -503,7 +503,7 @@ double ReshapeCost::GetForwardComputationCost(const std::vector &inp CheckGlobalDeviceManager(); MS_EXCEPTION_IF_NULL(g_device_manager); RankList dev_list = g_device_manager->GetDeviceListByStageId(stage_id); - TensorRedistribution tensor_redistribution; + TensorRedistribution tensor_redistribution(false, true); if (tensor_redistribution.Init(inputs[0].tensor_layout(), outputs[0].tensor_layout(), dev_list) == FAILED) { MS_LOG(EXCEPTION) << "Failure: tensor_redistribution init failed."; } diff --git a/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_generate_strategy.cc b/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_generate_strategy.cc index 19e07aae02..9de71231c0 100644 --- a/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_generate_strategy.cc +++ b/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_generate_strategy.cc @@ -28,10 +28,10 @@ namespace mindspore { namespace parallel { -void GenerateStrategy(std::shared_ptr graph, const std::vector> &ops, - const std::shared_ptr>> eli_list, +void GenerateStrategy(const std::shared_ptr &graph, const std::vector> &ops, + const std::shared_ptr>> &eli_list, const std::vector> &input_tensor_names, - const std::shared_ptr> index_list) { + const std::shared_ptr> &index_list) { MS_EXCEPTION_IF_NULL(graph); MS_EXCEPTION_IF_NULL(eli_list); MS_EXCEPTION_IF_NULL(index_list); @@ -140,19 +140,58 @@ std::vector> PrepareOneHot(const std::shared_ptr &gr const std::vector> &ops, const size_t iter_graph, const size_t iter_ops) { std::vector> strategies = MakeRecSearchStrategy(graph, ops, iter_graph, iter_ops); - strategies[0][0] = strategies[0][1]; - strategies[0][1] = 1; - graph->nodes[iter_graph].tensor_parm.tensor_str.str_h = graph->nodes[iter_graph].tensor_parm.tensor_str.str_w; - graph->nodes[iter_graph].tensor_parm.tensor_str.str_w = 1.0; + + int32_t axis = -1; + auto iter = ops[iter_ops]->attrs().find(AXIS); + if (iter != ops[iter_ops]->attrs().end()) { + MS_EXCEPTION_IF_NULL(iter->second); + if (iter->second->isa()) { + axis = iter->second->cast()->value(); + } else { + MS_LOG(EXCEPTION) << ops[iter_ops]->name() << ": The value of axis is not int."; + } + } + if (axis == -1) { + strategies[0][0] = strategies[0][1]; + strategies[0][1] = 1; + graph->nodes[iter_graph].tensor_parm.tensor_str.str_h = graph->nodes[iter_graph].tensor_parm.tensor_str.str_w; + graph->nodes[iter_graph].tensor_parm.tensor_str.str_w = 1.0; + } + std::vector s_empty = {}; strategies.push_back(s_empty); strategies.push_back(s_empty); return strategies; } -std::vector> PrepareGatherV2(const std::shared_ptr> &s) { +std::vector> PrepareGatherV2(const std::vector> &ops, + const size_t iter_ops, std::vector s) { std::vector> strategies; - strategies.push_back(*s); + + int32_t axis = 0; + auto axis_input = GetValue(ops[iter_ops]->input_value().at(2)); + if (axis_input < 0) { + axis_input += SizeToInt(ops[iter_ops]->inputs_tensor_info()[0].shape().size()); + } + axis = axis_input; + if (axis >= SizeToInt(s.size())) { + MS_LOG(EXCEPTION) << "Failure: GatherV2' axis out of range."; + } + s[axis] = 1; + strategies.push_back(s); + + auto pos = ops[iter_ops]->name().find("Info"); + auto name = ops[iter_ops]->name().substr(0, pos); + if (name == "GatherV2") { + return strategies; + } + + std::vector s_indices; + for (size_t i = 0; i < ops[iter_ops]->inputs_tensor_info()[1].shape().size(); i++) { + s_indices.push_back(1); + } + strategies.push_back(s_indices); + return strategies; } @@ -221,7 +260,7 @@ std::vector> MakeRecSearchStrategy(const std::shared_ptrname() << ": Tensor's output size is unexcepted."; } strategies.push_back(s); } @@ -241,7 +280,7 @@ std::vector> MakeDataParallelStrategy(const std::shared_ptr StrategyPtr origin_strategy = ops[iter_ops]->strategy(); std::vector> strategies; size_t max_device_num = g_device_manager->DeviceNum(); - size_t target_tensor_batch = ops[iter_ops]->outputs_tensor_info()[0].shape()[0]; + size_t target_tensor_batch = ops[iter_ops]->inputs_tensor_info()[0].shape()[0]; for (size_t iter_op_inputs = 0; iter_op_inputs < ops[iter_ops]->inputs_tensor_info().size(); iter_op_inputs++) { if (iter_op_inputs >= origin_strategy->GetInputDim().size()) { MS_LOG(EXCEPTION) << "Failure: Strategy's InputDim out of range."; @@ -256,8 +295,10 @@ std::vector> MakeDataParallelStrategy(const std::shared_ptr } else { s.push_back(1); } + } else if (input_size == 0) { + s = {}; } else { - MS_LOG(ERROR) << "Tensor's shape is unknown."; + MS_LOG(EXCEPTION) << ops[iter_ops]->name() << ": Tensor's shape is unknown."; } } strategies.push_back(s); @@ -304,13 +345,13 @@ std::vector> PrepareStrategy(const std::shared_ptr & } } -void GeneratePartitionedOperatorStrategy(const std::shared_ptr graph, +void GeneratePartitionedOperatorStrategy(const std::shared_ptr &graph, const std::vector> &ops, - const std::shared_ptr> index_list) { + const std::shared_ptr> &index_list) { for (size_t iter_ops = 0; iter_ops < (size_t)index_list->size(); iter_ops++) { std::vector> strategies; size_t iter_graph = index_list->at(iter_ops); - if (iter_graph != SIZE_MAX) { + if (iter_graph != SIZE_MAX && ops[iter_ops]->type() != GET_NEXT) { strategies = PrepareStrategy(graph, ops, iter_graph, iter_ops); } StrategyPtr sp = std::make_shared(0, strategies); @@ -335,7 +376,7 @@ size_t FindIndexOfOperatorIncoming(const std::vector> & return incoming_op_index; } -std::vector CopyIncomingOperatorOutputStrategy(const std::shared_ptr graph, +std::vector CopyIncomingOperatorOutputStrategy(const std::shared_ptr &graph, const std::vector> &ops, const size_t iter_ops, const size_t iter_graph) { std::vector s; @@ -355,7 +396,7 @@ std::vector CopyIncomingOperatorOutputStrategy(const std::shared_ptrnodes[iter_graph].tensor_parm.tensor_str.str_h); s.push_back(1 / graph->nodes[iter_graph].tensor_parm.tensor_str.str_w); } else { - MS_LOG(ERROR) << "Tensor's shape is unknown."; + MS_LOG(EXCEPTION) << ops[iter_ops]->name() << ": Tensor's shape is unknown."; } break; } @@ -365,7 +406,8 @@ std::vector CopyIncomingOperatorOutputStrategy(const std::shared_ptr PrepareIncomingOperatorInputStrategy(const std::vector> &ops, const size_t incoming_op_index) { std::vector s; - if (ops[incoming_op_index]->type() == RESHAPE || ops[incoming_op_index]->type() == GATHERV2) { + if (ops[incoming_op_index]->type() == RESHAPE || ops[incoming_op_index]->type() == GATHERV2 || + ops[incoming_op_index]->type() == TRANSPOSE) { return s; } auto strategy = ops[incoming_op_index]->selected_strategy(); @@ -433,13 +475,23 @@ std::vector ModifyStrategyIfSqueezeIncoming(const std::vector> &ops, const size_t iter_ops) { + bool keepdims = false; + auto keep_dims_iter = ops[iter_ops]->attrs().find(KEEP_DIMS); + if (keep_dims_iter == ops[iter_ops]->attrs().end()) { + MS_LOG(EXCEPTION) << ops[iter_ops]->name() << ": Don't have attr keep_dims."; + } + MS_EXCEPTION_IF_NULL(keep_dims_iter->second); + if (!keep_dims_iter->second->isa()) { + MS_LOG(EXCEPTION) << ops[iter_ops]->name() << ": Keep_dims is not a bool."; + } + keepdims = keep_dims_iter->second->cast()->value(); + return keepdims; +} + std::vector GetDimList(const std::vector> &ops, const size_t iter_ops) { std::vector dim_list; - bool keep_dims; - if (!ops[iter_ops]->attrs().find(KEEP_DIMS)->second->isa()) { - MS_LOG(EXCEPTION) << "Failure: Parameter keep_dims is not a boolean value." << std::endl; - } - keep_dims = ops[iter_ops]->attrs().find(KEEP_DIMS)->second->cast()->value(); + bool keep_dims = GetKeepDims(ops, iter_ops); if (keep_dims != false) { return dim_list; } @@ -485,6 +537,62 @@ std::vector ModifyStrategyIfReduceIncoming(const std::vector GetDimListFromAttrs(const std::vector> &ops, const size_t iter_ops) { + std::vector dim_list; + auto iter = ops[iter_ops]->attrs().find(AXIS); + if (iter == ops[iter_ops]->attrs().end()) { + MS_LOG(EXCEPTION) << ops[iter_ops]->name() << ": Don't have attr axis."; + } + auto input_dim = ops[iter_ops]->inputs_tensor_info()[0].shape().size(); + MS_EXCEPTION_IF_NULL(iter->second); + if (iter->second->isa()) { + auto attr_axis = GetValue>(iter->second); + if (attr_axis.empty()) { + for (size_t i = 0; i < input_dim; ++i) { + dim_list.push_back(SizeToInt(i)); + } + } else { + for (auto &axis : attr_axis) { + axis < 0 ? dim_list.push_back(axis + SizeToInt(input_dim)) : dim_list.push_back(axis); + } + } + } else if (iter->second->isa()) { + int axis = GetValue(iter->second); + axis < 0 ? dim_list.push_back(axis + SizeToInt(input_dim)) : dim_list.push_back(axis); + } else { + MS_LOG(EXCEPTION) << "Axis type is invalid."; + } + return dim_list; +} + +std::vector ModifyStrategyIfArgIncoming(const std::vector> &ops, + const size_t incoming_op_index, std::vector s) { + bool keepdims = GetKeepDims(ops, incoming_op_index); + if (keepdims) { + return s; + } + + std::vector s_Arg; + std::vector axis_list; + for (size_t i = 0; i < s.size(); i++) { + axis_list.push_back(i); + } + + auto dim_list = GetDimListFromAttrs(ops, incoming_op_index); + for (auto axis : dim_list) { + auto it = find(axis_list.begin(), axis_list.end(), axis); + if (it == axis_list.end()) { + MS_LOG(EXCEPTION) << "Failure: Can not find dimension indexes in Axis." << std::endl; + } + axis_list.erase(it); + } + + for (size_t i = 0; i < (size_t)axis_list.size(); i++) { + s_Arg.push_back(s[axis_list[i]]); + } + return s_Arg; +} + std::vector CopyIncomingOperatorInputStrategy(const std::vector> &ops, const size_t iter_ops, const size_t incoming_op_index) { std::vector s; @@ -497,6 +605,9 @@ std::vector CopyIncomingOperatorInputStrategy(const std::vectortype() == REDUCE_MIN || ops[incoming_op_index]->type() == REDUCE_MEAN) { s = ModifyStrategyIfReduceIncoming(ops, incoming_op_index, s); } + if (ops[incoming_op_index]->type() == ARGMAXWITHVALUE || ops[incoming_op_index]->type() == ARGMINWITHVALUE) { + s = ModifyStrategyIfArgIncoming(ops, incoming_op_index, s); + } } return s; } @@ -521,7 +632,7 @@ std::vector> GenerateStrategiesFromStrategy(const std::vect return PrepareBiasAdd(s_ptr); } if (ops[iter_ops]->type() == GATHERV2) { - return PrepareGatherV2(s_ptr); + return PrepareGatherV2(ops, iter_ops, basic_stra); } if (ops[iter_ops]->type() == L2_NORMALIZE) { return PrepareL2Normalize(ops, iter_ops, basic_stra); @@ -551,11 +662,11 @@ std::vector> GenerateStrategiesFromStrategy(const std::vect return stra; } -void GenerateEliminatedOperatorStrategyForward(const std::shared_ptr graph, +void GenerateEliminatedOperatorStrategyForward(const std::shared_ptr &graph, const std::vector> &ops, const std::vector> &input_tensor_names, - const std::shared_ptr> index_list, - const std::shared_ptr> no_stra_op_list) { + const std::shared_ptr> &index_list, + const std::shared_ptr> &no_stra_op_list) { if (no_stra_op_list->size() == 0) { return; } @@ -624,7 +735,8 @@ std::vector CopyOutgoingOperatorInputStrategy(const std::vector s; if (ops[iter_ops]->type() == REDUCE_MAX || ops[iter_ops]->type() == REDUCE_MIN || ops[iter_ops]->type() == REDUCE_SUM || ops[iter_ops]->type() == REDUCE_MEAN || ops[iter_ops]->type() == RESHAPE || - ops[iter_ops]->type() == GATHERV2) { + ops[iter_ops]->type() == GATHERV2 || ops[iter_ops]->type() == TRANSPOSE || + ops[iter_ops]->type() == ARGMAXWITHVALUE || ops[iter_ops]->type() == ARGMINWITHVALUE) { return s; } @@ -656,7 +768,7 @@ std::vector CopyOutgoingOperatorInputStrategy(const std::vector> &ops, const std::vector> &input_tensor_names, - const std::shared_ptr> no_stra_op_list) { + const std::shared_ptr> &no_stra_op_list) { if (no_stra_op_list->size() == 0) { return; } @@ -686,16 +798,16 @@ void GenerateEliminatedOperatorStrategyBackward(const std::vector graph, +void GenerateRemainingOperatorStrategy(const std::shared_ptr &graph, const std::vector> &ops, const std::vector> &input_tensor_names, - const std::shared_ptr> index_list, - const std::shared_ptr> no_stra_op_list) { + const std::shared_ptr> &index_list, + const std::shared_ptr> &no_stra_op_list) { if (no_stra_op_list->size() == 0) { return; } - size_t no_stra_op_list_size; + size_t no_stra_op_list_size = no_stra_op_list->size(); do { no_stra_op_list_size = no_stra_op_list->size(); GenerateEliminatedOperatorStrategyForward(graph, ops, input_tensor_names, index_list, no_stra_op_list); diff --git a/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_generate_strategy.h b/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_generate_strategy.h index c9604b449f..e82efe6798 100644 --- a/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_generate_strategy.h +++ b/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_generate_strategy.h @@ -27,10 +27,10 @@ namespace mindspore { namespace parallel { -void GenerateStrategy(std::shared_ptr graph, const std::vector> &ops, - const std::shared_ptr>> eli_list, +void GenerateStrategy(const std::shared_ptr &graph, const std::vector> &ops, + const std::shared_ptr>> &eli_list, const std::vector> &input_tensor_names, - const std::shared_ptr> index_list); + const std::shared_ptr> &index_list); std::vector> PrepareMatMul(const std::shared_ptr &graph, const std::vector> &ops, const size_t iter_graph, const size_t iter_ops); @@ -38,7 +38,8 @@ std::vector> PrepareBiasAdd(const std::shared_ptr> PrepareOneHot(const std::shared_ptr &graph, const std::vector> &ops, const size_t iter_graph, const size_t iter_ops); -std::vector> PrepareGatherV2(const std::shared_ptr> &s); +std::vector> PrepareGatherV2(const std::vector> &ops, + const size_t iter_ops, std::vector s); std::vector> PrepareL2Normalize(const std::vector> &ops, const size_t iter_ops, std::vector s); std::vector> MakeRecSearchStrategy(const std::shared_ptr &graph, @@ -50,12 +51,12 @@ std::vector> MakeDataParallelStrategy(const std::shared_ptr std::vector> PrepareStrategy(const std::shared_ptr &graph, const std::vector> &ops, const size_t iter_graph, const size_t iter_ops); -void GeneratePartitionedOperatorStrategy(const std::shared_ptr graph, +void GeneratePartitionedOperatorStrategy(const std::shared_ptr &graph, const std::vector> &ops, - const std::shared_ptr> index_list); + const std::shared_ptr> &index_list); size_t FindIndexOfOperatorIncoming(const std::vector> &input_tensor_names, const size_t iter_ops); -std::vector CopyIncomingOperatorOutputStrategy(const std::shared_ptr graph, +std::vector CopyIncomingOperatorOutputStrategy(const std::shared_ptr &graph, const std::vector> &ops, const size_t iter_ops, const size_t iter_graph); std::vector PrepareIncomingOperatorInputStrategy(const std::vector> &ops, @@ -63,19 +64,23 @@ std::vector PrepareIncomingOperatorInputStrategy(const std::vector GetAxisList(const std::vector> &ops, const int iter_ops); std::vector ModifyStrategyIfSqueezeIncoming(const std::vector> &ops, const size_t incoming_op_index, std::vector s); +bool GetKeepDims(const std::vector> &ops, const size_t iter_ops); std::vector GetDimList(const std::vector> &ops, const size_t iter_ops); std::vector ModifyStrategyIfReduceIncoming(const std::vector> &ops, const size_t incoming_op_index, std::vector s); +std::vector GetDimListFromAttrs(const std::vector> &ops, const size_t iter_ops); +std::vector ModifyStrategyIfArgIncoming(const std::vector> &ops, + const size_t incoming_op_index, std::vector s); std::vector CopyIncomingOperatorInputStrategy(const std::vector> &ops, const size_t iter_ops, const size_t incoming_op_index); std::vector> GenerateStrategiesFromStrategy(const std::vector> &ops, const size_t iter_ops, std::vector basic_stra); -void GenerateEliminatedOperatorStrategyForward(std::shared_ptr graph, +void GenerateEliminatedOperatorStrategyForward(const std::shared_ptr &graph, const std::vector> &ops, const std::vector> &input_tensor_names, - const std::shared_ptr> index_list, - const std::shared_ptr> no_stra_op_list); + const std::shared_ptr> &index_list, + const std::shared_ptr> &no_stra_op_list); std::vector ModifyStrategyIfSqueezeOutgoing(const std::vector> &ops, const size_t iter_ops, std::vector s); std::vector CopyOutgoingOperatorInputStrategy(const std::vector> &ops, @@ -83,12 +88,12 @@ std::vector CopyOutgoingOperatorInputStrategy(const std::vector> &ops, const std::vector> &input_tensor_names, - const std::shared_ptr> no_stra_op_list); -void GenerateRemainingOperatorStrategy(const std::shared_ptr graph, + const std::shared_ptr> &no_stra_op_list); +void GenerateRemainingOperatorStrategy(const std::shared_ptr &graph, const std::vector> &ops, const std::vector> &input_tensor_names, - const std::shared_ptr> index_list, - const std::shared_ptr> no_stra_op_list); + const std::shared_ptr> &index_list, + const std::shared_ptr> &no_stra_op_list); } // namespace parallel } // namespace mindspore #endif // PARALLEL_AUTO_PARALLEL_REC_GENERATE_STRATEGY_H_ diff --git a/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_graph.h b/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_graph.h index 647b857e16..9007218d15 100644 --- a/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_graph.h +++ b/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_graph.h @@ -50,7 +50,8 @@ enum OperatorType { kRecCast, kRecReduce, kRecPReLU, - kRecGatherV2 + kRecGatherV2, + kRecArgWithValue }; enum InfoType { kApplication, kConstant }; diff --git a/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_parse_graph.cc b/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_parse_graph.cc index 3e4eafe0a4..c0412e9108 100644 --- a/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_parse_graph.cc +++ b/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_parse_graph.cc @@ -40,7 +40,7 @@ const TensorParam MakeTensor(int n, int c, int h, int w) { return tensor; } -Graph::NodeType MakeNewOperator(std::vector> ops, size_t iter_ops) { +Graph::NodeType MakeNewOperator(const std::vector> &ops, size_t iter_ops) { Graph::NodeType NewOp; NewOp.name = ops[iter_ops]->name(); NewOp.info = InfoType::kApplication; @@ -140,7 +140,7 @@ std::shared_ptr ParseGraph(const std::vector> &input_tensor_names, std::shared_ptr graph) { +void MakeEdge(const std::vector> &input_tensor_names, const std::shared_ptr &graph) { for (size_t iter_i = 0; iter_i < input_tensor_names.size(); iter_i++) { for (size_t iter_j = 1; iter_j < input_tensor_names[iter_i].size(); iter_j++) { size_t head_node_index = GetIndexInInputTensorNames(input_tensor_names, input_tensor_names[iter_i][iter_j]); @@ -163,8 +163,8 @@ size_t GetIndexInInputTensorNames(const std::vector> &i return SIZE_MAX; } -void Eliminate_Aux(const size_t node_index, const std::shared_ptr graph, - const std::shared_ptr>> eli_list) { +void Eliminate_Aux(const size_t node_index, const std::shared_ptr &graph, + const std::shared_ptr>> &eli_list) { std::vector eli; eli.push_back(node_index); for (size_t i = 0; i < (size_t)graph->nodes[node_index].node_out.size(); i++) { @@ -211,18 +211,18 @@ void Eliminate_Aux(const size_t node_index, const std::shared_ptr graph, } } -std::shared_ptr EliminateGraph(const std::shared_ptr graph, - const std::shared_ptr>> eli_list, - const std::shared_ptr> index_list) { +std::shared_ptr EliminateGraph(const std::shared_ptr &graph, + const std::shared_ptr>> &eli_list, + const std::shared_ptr> &index_list) { MS_EXCEPTION_IF_NULL(graph); - const std::set type_list = { - OperatorType::kRecReLU, OperatorType::kRecLog, OperatorType::kRecExp, OperatorType::kRecAdd, - OperatorType::kRecElmWiseOp, OperatorType::kRecBiasAdd, OperatorType::kRecSub, OperatorType::kRecMul, - OperatorType::kRecDiv, OperatorType::kRecSqueeze, OperatorType::kRecReduce, OperatorType::kRecCast, - OperatorType::kRecReshape, OperatorType::kRecGatherV2}; + static const std::set elementwise_type = { + OperatorType::kRecReLU, OperatorType::kRecLog, OperatorType::kRecExp, OperatorType::kRecAdd, + OperatorType::kRecElmWiseOp, OperatorType::kRecBiasAdd, OperatorType::kRecSub, OperatorType::kRecMul, + OperatorType::kRecDiv, OperatorType::kRecSqueeze, OperatorType::kRecReduce, OperatorType::kRecCast, + OperatorType::kRecReshape, OperatorType::kRecGatherV2, OperatorType::kRecArgWithValue}; for (size_t node_index = 0; node_index < (size_t)graph->nodes.size(); node_index++) { auto type = graph->nodes[node_index].apply.op_type; - if (type_list.find(type) != type_list.end()) { + if (elementwise_type.find(type) != elementwise_type.end()) { Eliminate_Aux(node_index, graph, eli_list); } } diff --git a/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_parse_graph.h b/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_parse_graph.h index 536c04cd9f..66fc82b8ce 100644 --- a/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_parse_graph.h +++ b/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_parse_graph.h @@ -47,6 +47,8 @@ const std::map DictOpType{ {REDUCE_MIN, OperatorType::kRecReduce}, {REDUCE_MEAN, OperatorType::kRecReduce}, {GATHERV2, OperatorType::kRecGatherV2}, + {ARGMAXWITHVALUE, OperatorType::kRecArgWithValue}, + {ARGMINWITHVALUE, OperatorType::kRecArgWithValue}, {RELU, OperatorType::kRecReLU}, {"ReLU6", OperatorType::kRecReLU}, @@ -59,6 +61,7 @@ const std::map DictOpType{ {PRELU, OperatorType::kRecPReLU}, + {TRANSPOSE, OperatorType::kRecElmWiseOp}, {L2_NORMALIZE, OperatorType::kRecElmWiseOp}, {TENSOR_ADD, OperatorType::kRecElmWiseOp}, {SUB, OperatorType::kRecElmWiseOp}, @@ -108,7 +111,7 @@ const std::map DictOpType{ const TensorParam MakeTensor(int n, int c, int h, int w); -Graph::NodeType MakeNewOperator(std::vector> ops, size_t iter_ops); +Graph::NodeType MakeNewOperator(const std::vector> &ops, size_t iter_ops); OperatorRec CompleteOperatorInputs(const std::vector> &ops, const size_t iter_ops, Graph::NodeType NewTensor); @@ -119,17 +122,17 @@ TensorParam Complete2DInputs(const std::vector> &o std::shared_ptr ParseGraph(const std::vector> &ops, const std::vector> &input_tensor_names); -void MakeEdge(const std::vector> &input_tensor_names, std::shared_ptr graph); +void MakeEdge(const std::vector> &input_tensor_names, const std::shared_ptr &graph); size_t GetIndexInInputTensorNames(const std::vector> &input_tensor_names, const std::string &input_name); -void Eliminate_Aux(const size_t node_index, const std::shared_ptr graph, - const std::shared_ptr>> eli_list); +void Eliminate_Aux(const size_t node_index, const std::shared_ptr &graph, + const std::shared_ptr>> &eli_list); -std::shared_ptr EliminateGraph(const std::shared_ptr graph, - const std::shared_ptr>> eli_list, - const std::shared_ptr> index_list); +std::shared_ptr EliminateGraph(const std::shared_ptr &graph, + const std::shared_ptr>> &eli_list, + const std::shared_ptr> &index_list); } // namespace parallel } // namespace mindspore #endif // PARALLEL_AUTO_PARALLEL_REC_PARSE_GRAPH_H_ diff --git a/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_partition.cc b/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_partition.cc index 0f6e736d52..d5200f54d8 100644 --- a/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_partition.cc +++ b/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_partition.cc @@ -93,7 +93,7 @@ double GetWeights(const Graph::NodeType &node) { } // Sort all the nodes by their weights -std::vector SortByWeight(const std::shared_ptr graph) { +std::vector SortByWeight(const std::shared_ptr &graph) { MS_EXCEPTION_IF_NULL(graph); std::vector> weight_to_node_index; @@ -124,7 +124,7 @@ std::vector SortByWeight(const std::shared_ptr graph) { // Get optimal strategy to partition the target node StrategyRec PartitionNode(const Graph::NodeType &node, const std::vector> &node_name_to_strategy, - std::shared_ptr graph) { + const std::shared_ptr &graph) { bool enable_conv_chw_partition = false; MS_EXCEPTION_IF_NULL(graph); @@ -191,7 +191,8 @@ StrategyRec PartitionNode(const Graph::NodeType &node, } // Parttion graph into all devices. -Status PartitionForAllDevices(const size_t num_device, const double device_memory, std::shared_ptr graph) { +Status PartitionForAllDevices(const size_t num_device, const double device_memory, + const std::shared_ptr &graph) { if (num_device < 1) { MS_LOG(EXCEPTION) << "ERROR: Number of devices can't be " << num_device << "."; } @@ -261,7 +262,7 @@ Graph::NodeType ApplyStrToTensor(Graph::NodeType Node) { return Node; } -Status DevicesMemoryControl(const size_t num_device, const double device_memory, std::shared_ptr graph) { +Status DevicesMemoryControl(const size_t num_device, const double device_memory, const std::shared_ptr &graph) { MS_EXCEPTION_IF_NULL(graph); if (num_device == 0) { MS_LOG(EXCEPTION) << "Failure: device number is 0."; diff --git a/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_partition.h b/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_partition.h index b2fbeddebd..c98f3317f8 100644 --- a/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_partition.h +++ b/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_partition.h @@ -32,19 +32,19 @@ namespace mindspore { namespace parallel { -std::vector SortByWeight(const std::shared_ptr graph); +std::vector SortByWeight(const std::shared_ptr &graph); double GetWeights(const Graph::NodeType &node); StrategyRec PartitionNode(const Graph::NodeType &node, const std::vector> &node_name_to_strategy, - std::shared_ptr graph); + const std::shared_ptr &graph); -Status PartitionForAllDevices(const size_t num_device, const double device_memory, std::shared_ptr graph); +Status PartitionForAllDevices(const size_t num_device, const double device_memory, const std::shared_ptr &graph); Graph::NodeType ApplyStrToTensor(Graph::NodeType Node); -Status DevicesMemoryControl(const size_t num_device, const double device_memory, std::shared_ptr graph); +Status DevicesMemoryControl(const size_t num_device, const double device_memory, const std::shared_ptr &graph); size_t GetDataTypeSize(const TensorType &type); } // namespace parallel diff --git a/mindspore/ccsrc/parallel/context.cc b/mindspore/ccsrc/parallel/context.cc index 8957dc842c..062d814aa0 100644 --- a/mindspore/ccsrc/parallel/context.cc +++ b/mindspore/ccsrc/parallel/context.cc @@ -62,6 +62,7 @@ void ParallelContext::Reset() { enable_all_reduce_fusion_ = false; strategy_ckpt_load_file_ = ""; strategy_ckpt_save_file_ = ""; + enable_parallel_optimizer_ = false; } void ParallelContext::set_device_num(int32_t device_num) { diff --git a/mindspore/ccsrc/parallel/context.h b/mindspore/ccsrc/parallel/context.h index efa528d179..6a503ca7ed 100644 --- a/mindspore/ccsrc/parallel/context.h +++ b/mindspore/ccsrc/parallel/context.h @@ -100,6 +100,11 @@ class ParallelContext { void set_strategy_ckpt_save_file(const std::string &strategy_ckpt_save_file); std::string strategy_ckpt_save_file() const { return strategy_ckpt_save_file_; } + void set_enable_parallel_optimizer(bool enable_parallel_optimizer) { + enable_parallel_optimizer_ = enable_parallel_optimizer; + } + bool enable_parallel_optimizer() const { return enable_parallel_optimizer_; } + void Reset(); private: @@ -123,6 +128,7 @@ class ParallelContext { std::map> all_reduce_fusion_split_sizes_; std::string strategy_ckpt_load_file_; std::string strategy_ckpt_save_file_; + bool enable_parallel_optimizer_; }; void ParallelParameterContextInit(const FuncGraphPtr &func_graph); diff --git a/mindspore/ccsrc/parallel/node_check.cc b/mindspore/ccsrc/parallel/node_check.cc index 6f30a8ec1c..6b920f82ec 100644 --- a/mindspore/ccsrc/parallel/node_check.cc +++ b/mindspore/ccsrc/parallel/node_check.cc @@ -66,6 +66,7 @@ const std::set BLACK_LIST = {TUPLE_GETITEM, SCALARSUMMARY, IMAGESUMMARY, TENSORSUMMARY, + DEBUG, HISTOGRAMSUMMARY, COL2IMV1, RESOLVE, diff --git a/mindspore/ccsrc/parallel/ops_info/gather_v2_info.cc b/mindspore/ccsrc/parallel/ops_info/gather_v2_info.cc index 2ca5a408b1..078be08128 100644 --- a/mindspore/ccsrc/parallel/ops_info/gather_v2_info.cc +++ b/mindspore/ccsrc/parallel/ops_info/gather_v2_info.cc @@ -210,8 +210,8 @@ OperatorVector CreateSubOp(int32_t sub_value) { OperatorName operator_name = SUB; OperatorAttrs operator_attrs; - py::tuple tuple = py::make_tuple(sub_value); - mindspore::tensor::TensorPtr tensor_ptr = std::make_shared(tuple, kInt32); + std::vector tensor_data = {sub_value}; + mindspore::tensor::TensorPtr tensor_ptr = std::make_shared(tensor_data, kInt32); ValuePtr op_param_value = MakeValue(tensor_ptr); Attr op1_param = std::make_pair("", op_param_value); diff --git a/mindspore/ccsrc/parallel/ops_info/gather_v2_p_info.cc b/mindspore/ccsrc/parallel/ops_info/gather_v2_p_info.cc index 7a16aeafcb..9fb8df0883 100644 --- a/mindspore/ccsrc/parallel/ops_info/gather_v2_p_info.cc +++ b/mindspore/ccsrc/parallel/ops_info/gather_v2_p_info.cc @@ -79,8 +79,8 @@ Status GatherV2PInfo::CheckStrategy(const StrategyPtr &strategy) { auto param_shape = inputs_shape_.at(0); auto param_strategy = strategy->GetInputDim().at(0); auto slice_shape = param_shape.at(param_shape.size() - 1) / param_strategy.at(param_strategy.size() - 1); - if (slice_shape % 8 != 0 && slice_shape != 1) { - MS_LOG(DEBUG) << name_ << ": Last dim of param slice shape need 32Byte aligned."; + if ((target_ != CPU) && (slice_shape % 8 != 0) && (slice_shape != 1)) { + MS_LOG(ERROR) << name_ << ": Last dim of param slice shape need 32Byte aligned."; return FAILED; } diff --git a/mindspore/ccsrc/parallel/ops_info/ops_utils.h b/mindspore/ccsrc/parallel/ops_info/ops_utils.h index 4b8f61bb2e..9cb3c7040a 100644 --- a/mindspore/ccsrc/parallel/ops_info/ops_utils.h +++ b/mindspore/ccsrc/parallel/ops_info/ops_utils.h @@ -145,7 +145,7 @@ constexpr char MIRROR_OPERATOR[] = "_MirrorOperator"; constexpr char STRIDED_SLICE[] = "StridedSlice"; constexpr char ALL_GATHER[] = "AllGather"; constexpr char REDUCE_SCATTER[] = "ReduceScatter"; -constexpr char HOST_REDUCE_SCATTER[] = "HostReduceScatter"; +constexpr char HOST_REDUCE_SCATTER[] = "_HostReduceScatter"; constexpr char EMBEDDING_LOOKUP[] = "EmbeddingLookup"; constexpr char CONCAT[] = "Concat"; constexpr char SOFTMAX_CROSS_ENTROPY_WITH_LOGITS[] = "SoftmaxCrossEntropyWithLogits"; @@ -269,6 +269,7 @@ constexpr char SCALARSUMMARY[] = "ScalarSummary"; constexpr char IMAGESUMMARY[] = "ImageSummary"; constexpr char TENSORSUMMARY[] = "TensorSummary"; constexpr char HISTOGRAMSUMMARY[] = "HistogramSummary"; +constexpr char DEBUG[] = "Debug"; constexpr char BROADCASTGRADIENTARGS[] = "BroadcastGradientArgs"; constexpr char INVERTPERMUTATION[] = "InvertPermutation"; constexpr char CONTROLDEPEND[] = "ControlDepend"; diff --git a/mindspore/ccsrc/parallel/ops_info/reduce_method_info.cc b/mindspore/ccsrc/parallel/ops_info/reduce_method_info.cc index 44eab20588..7304666a77 100644 --- a/mindspore/ccsrc/parallel/ops_info/reduce_method_info.cc +++ b/mindspore/ccsrc/parallel/ops_info/reduce_method_info.cc @@ -204,8 +204,8 @@ ForwardOp CreatReduceMeanForwardOp(const std::vector &forward_group, cons OperatorName operator1_name = REAL_DIV; std::vector device_list = forward_group[0].GetDevicesList(); auto divisor = static_cast(device_list.size()); - py::tuple tuple = py::make_tuple(divisor); - mindspore::tensor::TensorPtr tensor_ptr = std::make_shared(tuple, dtype); + std::vector tensor_data = {divisor}; + mindspore::tensor::TensorPtr tensor_ptr = std::make_shared(tensor_data, dtype); ValuePtr op1_param_value = MakeValue(tensor_ptr); Attr op1_param = std::make_pair("divisor", op1_param_value); OperatorParams operator1_params = {std::make_pair(op1_param, 2)}; diff --git a/mindspore/ccsrc/parallel/ops_info/reshape_info.cc b/mindspore/ccsrc/parallel/ops_info/reshape_info.cc index c470c379ff..57e1a76d0a 100644 --- a/mindspore/ccsrc/parallel/ops_info/reshape_info.cc +++ b/mindspore/ccsrc/parallel/ops_info/reshape_info.cc @@ -133,9 +133,13 @@ Status ReshapeInfo::GetParameterInput() { Status ReshapeInfo::ComputeReplaceOp() { RankList dev_list = global_device_list(); - TensorRedistribution tensor_redistribution(true, true); + TensorRedistribution tensor_redistribution(!is_generating_costs_, true); if (tensor_redistribution.Init(input_layout_, output_layout_, dev_list) == FAILED) { - MS_LOG(ERROR) << name_ << ": tensor_redistribution init failed."; + if (is_generating_costs_) { + MS_LOG(DEBUG) << name_ << ": tensor_redistribution init failed."; + } else { + MS_LOG(ERROR) << name_ << ": tensor_redistribution init failed."; + } return FAILED; } MS_LOG(DEBUG) << name_ << ": input " << input_layout_.ToString(); @@ -143,7 +147,11 @@ Status ReshapeInfo::ComputeReplaceOp() { MS_LOG(DEBUG) << name_ << ": dev_list " << dev_list.size(); RedistributionOpListPtr redistribution_oplist_ptr = tensor_redistribution.InferTensorRedistributionOperatorList(); if (redistribution_oplist_ptr == nullptr) { - MS_LOG(ERROR) << name_ << "InferTensorRedistribution failed."; + if (is_generating_costs_) { + MS_LOG(DEBUG) << name_ << "InferTensorRedistribution failed."; + } else { + MS_LOG(ERROR) << name_ << "InferTensorRedistribution failed."; + } return FAILED; } replace_op_ = redistribution_oplist_ptr->first; @@ -444,6 +452,7 @@ Status ReshapeInfo::GenerateStrategies(int32_t stage_id) { Status ReshapeInfo::GenetateStrategyCosts(const std::vector> &pre_stra_costs, const std::vector> &next_stra_costs, int32_t out_index, int32_t in_index, bool is_prev_param) { + is_generating_costs_ = true; for (auto pre_stra_cost : pre_stra_costs) { std::vector pre_out_tensor_infos; if (is_prev_param) { @@ -488,6 +497,7 @@ Status ReshapeInfo::GenetateStrategyCosts(const std::vector(parse::python_adapter::GetPyObjAttr(param_value->value(), "sparse_grad")); ptr->set_sparse_grad(sparse_grad); + auto has_indexed_slices_grad = + py::cast(parse::python_adapter::GetPyObjAttr(param_value->value(), "has_indexed_slices_grad")); + ptr->set_has_indexed_slices_grad(has_indexed_slices_grad); parallel::ParallelParameterContextRestoreInNoTraining(func_graph, param_node, ptr); args_spec.push_back(ptr); @@ -420,6 +424,25 @@ bool RemoveValueNodeDuplicationsAction(const ResourcePtr &res) { bool ValidateAction(const ResourcePtr &res) { return ValidatePass(res); } +void ActionPyStub(const ResourcePtr &res, opt::python_pass::Phase phase) { + MS_EXCEPTION_IF_NULL(res->manager()); + MS_EXCEPTION_IF_NULL(res->func_graph()); + auto ppm = opt::python_pass::PyPassManager::GetInstance(); + if (!ppm->GetPassGroup(phase)->Run(res->func_graph())) { + MS_LOG(DEBUG) << "No match.\n"; + } +} + +bool ResolveActionPyStub(const ResourcePtr &res) { + ActionPyStub(res, opt::python_pass::Phase::RESOLVE); + return true; +} + +bool OptActionPyStub(const ResourcePtr &res) { + ActionPyStub(res, opt::python_pass::Phase::RESOLVE); + return true; +} + static std::vector CommonPipeline() { std::vector actions; @@ -432,6 +455,8 @@ static std::vector CommonPipeline() { if (!multi_graphs) { actions.emplace_back(std::make_pair("combine_like_graphs", CombineLikeGraphs)); } + // Add resolve-stage python pass stub + actions.emplace_back(std::make_pair("py_resolve", ResolveActionPyStub)); actions.emplace_back(std::make_pair("inference_opt_prepare", InferenceOptPrepareAction)); // Evaluate type and shape, and specialize actions.emplace_back(std::make_pair("abstract_specialize", AbstractSpecializeAction)); @@ -443,6 +468,8 @@ std::vector GePipeline() { auto actions = CommonPipeline(); // optimize actions.emplace_back(std::make_pair("optimize", GeOptimizeAction)); + // Add opt-stage python pass stub + actions.emplace_back(std::make_pair("py_opt", OptActionPyStub)); actions.emplace_back(std::make_pair("remove_value_node_duplications", RemoveValueNodeDuplicationsAction)); actions.emplace_back(std::make_pair("validate", ValidateAction)); return actions; @@ -454,6 +481,9 @@ std::vector VmPipeline() { // optimize actions.emplace_back(std::make_pair("optimize", VmOptimizeAction)); + // Add opt-stage python pass stub + actions.emplace_back(std::make_pair("py_opt", OptActionPyStub)); + actions.emplace_back(std::make_pair("validate", ValidateAction)); // compile the ANF graph diff --git a/mindspore/ccsrc/pipeline/init.cc b/mindspore/ccsrc/pipeline/init.cc index 7025447a29..f28be181dd 100644 --- a/mindspore/ccsrc/pipeline/init.cc +++ b/mindspore/ccsrc/pipeline/init.cc @@ -154,7 +154,9 @@ PYBIND11_MODULE(_c_expression, m) { .def("set_print_file_path", &mindspore::MsContext::set_print_file_path, "Set path to print.") .def("set_enable_graph_kernel", &mindspore::MsContext::set_enable_graph_kernel, "Set the GraphKernel switch to on or off.") - .def("get_enable_graph_kernel", &mindspore::MsContext::enable_graph_kernel, "Get the value of GraphKernel switch."); + .def("get_enable_graph_kernel", &mindspore::MsContext::enable_graph_kernel, "Get the value of GraphKernel switch.") + .def("get_enable_sparse_flag", &mindspore::MsContext::enable_sparse_flag, "Get whether to enable sparse.") + .def("set_enable_sparse_flag", &mindspore::MsContext::set_enable_sparse_flag, "Set whether to enable sparse."); (void)py::class_>(m, "MpiConfig") .def_static("get_instance", &mindspore::MpiConfig::GetInstance, "Get mpi config instance.") @@ -205,6 +207,10 @@ PYBIND11_MODULE(_c_expression, m) { .def("get_strategy_ckpt_save_file", &ParallelContext::strategy_ckpt_save_file, "Get strategy checkpoint save file.") .def("set_full_batch", &ParallelContext::set_full_batch, "Set whether load full batch on each device.") .def("get_full_batch", &ParallelContext::full_batch, "Get whether load full batch on each device.") + .def("set_enable_parallel_optimizer", &ParallelContext::set_enable_parallel_optimizer, + "Set enable/disable parallel optimizer.") + .def("get_enable_parallel_optimizer", &ParallelContext::enable_parallel_optimizer, + "Get enable/disable parallel optimizer.") .def("reset", &ParallelContext::Reset, "Reset auto parallel context."); (void)py::class_>(m, "CostModelContext") diff --git a/mindspore/ccsrc/pipeline/parse/parse.cc b/mindspore/ccsrc/pipeline/parse/parse.cc index 6d5c28c98c..77e865cee9 100644 --- a/mindspore/ccsrc/pipeline/parse/parse.cc +++ b/mindspore/ccsrc/pipeline/parse/parse.cc @@ -234,7 +234,11 @@ FunctionBlockPtr Parser::ParseFunction(const py::object &node, const FunctionBlo current_fg->debug_info()->set_deco_location(GetLocation(deco_list)); } - bool set_flag = ast_->UpdateFuncGraphFlags(current_fg); + bool set_flag = UpdateFuncGraphFlags(ast_->function(), current_fg); + if (ast_->obj() != ast_->function()) { + set_flag = set_flag && UpdateFuncGraphFlags(ast_->obj(), current_fg); + } + if (!set_flag) { MS_LOG(ERROR) << "Set flags failed"; return nullptr; @@ -1044,11 +1048,10 @@ FunctionBlockPtr Parser::ParseFor(const FunctionBlockPtr &block, const py::objec CNodePtr app = body_block->func_graph()->NewCNode({op_next, iter_param}); CNodePtr target_app = body_block->func_graph()->NewCNode({op_getitem, app, NewValueNode(0)}); py::object target_node = python_adapter::GetPyObjAttr(node, "target"); - auto name_id = py::cast(python_adapter::GetPyObjAttr(target_node, "id")); - target_app->debug_info()->set_name(name_id); CNodePtr iter2_app = body_block->func_graph()->NewCNode({op_getitem, app, NewValueNode(1)}); - body_block->WriteVariable(name_id, target_app); + WriteAssignVars(body_block, target_node, target_app); + // link the variable name with the target auto it_info = std::make_shared(target_app->debug_info()); iter_param->debug_info()->set_trace_info(it_info); @@ -1436,17 +1439,17 @@ bool ParseAst::IsClassMember(const py::object &node) { return ret.cast(); } -bool ParseAst::UpdateFuncGraphFlags(const FuncGraphPtr &func_graph) { +bool UpdateFuncGraphFlags(py::object obj, const FuncGraphPtr &func_graph) { if (func_graph == nullptr) { MS_LOG(ERROR) << "FuncGraph is null"; return false; } - if (!py::hasattr(obj_, PYTHON_EXTERN_MINDSPORE_FLAG)) { + if (!py::hasattr(obj, PYTHON_EXTERN_MINDSPORE_FLAG)) { MS_LOG(DEBUG) << "No flags"; return true; } - py::dict flags = python_adapter::GetPyObjAttr(obj_, PYTHON_EXTERN_MINDSPORE_FLAG); + py::dict flags = python_adapter::GetPyObjAttr(obj, PYTHON_EXTERN_MINDSPORE_FLAG); for (auto &item : flags) { if (!py::isinstance(item.first)) { MS_LOG(ERROR) << "Type error in flags dict convert"; @@ -1466,7 +1469,6 @@ bool ParseAst::UpdateFuncGraphFlags(const FuncGraphPtr &func_graph) { return false; } } - return true; } diff --git a/mindspore/ccsrc/pipeline/parse/parse.h b/mindspore/ccsrc/pipeline/parse/parse.h index 0a56ccaed9..19c503c6d0 100644 --- a/mindspore/ccsrc/pipeline/parse/parse.h +++ b/mindspore/ccsrc/pipeline/parse/parse.h @@ -327,9 +327,6 @@ class ParseAst { bool IsClassMember(const py::object &node); - // update the graph flags - bool UpdateFuncGraphFlags(const FuncGraphPtr &func_graph); - private: // save obj,eg: class instance or function py::object obj_; @@ -350,6 +347,9 @@ class ParseAst { int function_line_offset_; }; +// update the graph flags +bool UpdateFuncGraphFlags(py::object obj, const FuncGraphPtr &func_graph); + AnfNodePtr GetMixedPrecisionCastHelp(const FuncGraphPtr &func_graph, const AnfNodePtr ¶m); } // namespace parse diff --git a/mindspore/ccsrc/pipeline/parse/resolve.cc b/mindspore/ccsrc/pipeline/parse/resolve.cc index d5e1f828cc..87c2f78b42 100644 --- a/mindspore/ccsrc/pipeline/parse/resolve.cc +++ b/mindspore/ccsrc/pipeline/parse/resolve.cc @@ -94,7 +94,7 @@ AnfNodePtr ResolveParameterObj(const FuncGraphPtr &func_graph, const py::object auto top_graph = Parser::GetTopFuncGraph(); // if the parameter node has been created , return it AnfNodePtr para_node = nullptr; - for (auto param : top_graph->parameters()) { + for (auto const ¶m : top_graph->parameters()) { auto param_node = dyn_cast(param); if (param_node != nullptr && param_node->name() == param_name) { para_node = param; diff --git a/mindspore/ccsrc/pipeline/pass.cc b/mindspore/ccsrc/pipeline/pass.cc index 94063fb780..f6cfd6362c 100644 --- a/mindspore/ccsrc/pipeline/pass.cc +++ b/mindspore/ccsrc/pipeline/pass.cc @@ -114,6 +114,7 @@ OptPassGroupMap GetOptPassesA(const opt::irpass::OptimizeIRPassLib &irpass) { irpass.depend_value_elim_, }); opt::OptPassConfig a_3 = opt::OptPassConfig({ + irpass.arithmetic_simplify2_, irpass.same_eliminate_, irpass.check_bprop_eliminate_, irpass.replace_applicator_, @@ -155,6 +156,7 @@ OptPassGroupMap GetOptPassesB(const opt::irpass::OptimizeIRPassLib &irpass) { irpass.replace_refkey_by_param_, irpass.make_ref_eliminate_, irpass.get_ref_param_eliminate_, + irpass.indexed_slices_eliminate_, }); OptPassGroupMap map({ {"b_1", b_1}, diff --git a/mindspore/ccsrc/pipeline/pipeline.cc b/mindspore/ccsrc/pipeline/pipeline.cc index bb1f693c6b..6abe198f5a 100644 --- a/mindspore/ccsrc/pipeline/pipeline.cc +++ b/mindspore/ccsrc/pipeline/pipeline.cc @@ -39,6 +39,7 @@ #include "device/kernel_runtime_manager.h" #include "debug/trace.h" #include "pynative/pynative_execute.h" +#include "optimizer/py_pass_manager.h" #if (ENABLE_GE || ENABLE_D) #include "pipeline/pipeline_ge.h" @@ -314,7 +315,7 @@ std::map> ExecutorPy::FetchI auto weight_name = weight_node->cast()->name(); // find the fakequant from input int count = 0; - int max_depth = 5; + const int max_depth = 5; while (!is_quant_cnode(x)) { if (count >= max_depth) { break; @@ -445,7 +446,10 @@ bool ExecutorPy::CompileInner(const py::object &obj, const py::tuple &args, cons std::string backend = MsContext::GetInstance()->backend_policy(); if (use_vm && backend != "ge") { // Create backend and session - resource->results()[kBackend] = compile::CreateBackend(); + auto backend_ptr = compile::CreateBackend(); + // Connect session to debugger + backend_ptr->SetDebugger(); + resource->results()[kBackend] = backend_ptr; p_actions = VmPipeline(); } else { p_actions = GePipeline(); @@ -964,6 +968,7 @@ void ClearResAtexit() { pipeline::ExecutorPy::ClearRes(); pipeline::ReclaimOptimizer(); pynative::PynativeExecutor::GetInstance()->ClearRes(); + opt::python_pass::PyPassManager::GetInstance()->ClearRes(); #ifdef ENABLE_GE transform::DfGraphManager::GetInstance().ClearGraph(); transform::DfGraphConvertor::get_adpt_map().clear(); diff --git a/mindspore/ccsrc/pipeline/pipeline_ge.cc b/mindspore/ccsrc/pipeline/pipeline_ge.cc index ea0ca14c7a..8ec1602315 100644 --- a/mindspore/ccsrc/pipeline/pipeline_ge.cc +++ b/mindspore/ccsrc/pipeline/pipeline_ge.cc @@ -156,11 +156,11 @@ void ConvertObjectToTensors(const py::dict &dict, TensorOrderMap *const tensors) if (py::isinstance(item.second.attr("default_input"))) { // convert float to tensor with shape([1]) tensor = std::make_shared(kNumberTypeFloat32, std::vector({1})); - *(static_cast(tensor->data_c(true))) = py::cast(item.second.attr("default_input")); + *(static_cast(tensor->data_c())) = py::cast(item.second.attr("default_input")); } else if (py::isinstance(item.second.attr("default_input"))) { // convert int to tensor with shape([1]) tensor = std::make_shared(kNumberTypeInt32, std::vector({1})); - *(static_cast(tensor->data_c(true))) = py::cast(item.second.attr("default_input")); + *(static_cast(tensor->data_c())) = py::cast(item.second.attr("default_input")); } else if (py::hasattr(item.second.attr("default_input"), PYTHON_TENSOR_FLAG)) { // cast tensor tensor = py::cast>(item.second.attr("default_input")); @@ -330,7 +330,7 @@ py::object ExtractGeneralCnodeRet(const AbstractBasePtr &cnode_data, const py::t MS_LOG(EXCEPTION) << "The shape of the tensor derived is not Shape, is " << shape->ToString(); } auto shape_me = shape->cast()->shape(); - auto shape_ge = py::cast(data[*count]).shape(); + auto shape_ge = py::cast(data[*count]).shape(); if (shape_ge != shape_me) { MS_LOG(EXCEPTION) << "The shape of the " << *count << "th tensor returned: " << shape_ge << " is not the same as the shape of the tensor derived: " << shape_me; diff --git a/mindspore/ccsrc/pipeline/resource.cc b/mindspore/ccsrc/pipeline/resource.cc index 50ccef2f44..faf1f2015d 100644 --- a/mindspore/ccsrc/pipeline/resource.cc +++ b/mindspore/ccsrc/pipeline/resource.cc @@ -33,148 +33,157 @@ namespace mindspore { namespace pipeline { MethodMap &GetMethodMap() { - static MethodMap method_map = {{kObjectTypeString, - { - {"__bool__", std::string("str_bool")} // C.str_bool - }}, - {kMetaTypeNone, - { - {"__bool__", std::string("none_bool")} // C.none_bool - }}, - {kNumberTypeBool, - { - {"__and__", prim::kPrimBoolAnd}, // P.bool_and - {"__or__", prim::kPrimBoolOr}, // P.bool_or - {"__eq__", prim::kPrimBoolEq}, // P.bool_eq - {"__ne__", std::string("bool_ne")}, // C.bool_ne - {"__bool__", prim::kPrimIdentity} // P.identity - }}, - {kNumberTypeInt, - { - {"__add__", prim::kPrimScalarAdd}, // P.scalar_add - {"__sub__", prim::kPrimScalarSub}, // P.scalar_sub - {"__mul__", prim::kPrimScalarMul}, // P.scalar_mul - {"__floordiv__", std::string("int_floordiv")}, // C.int_floordiv - {"__truediv__", std::string("int_truediv")}, // C.int_truediv - {"__mod__", prim::kPrimScalarMod}, // P.scalar_mod - {"__pow__", prim::kPrimScalarPow}, // P.scalar_pow - {"__floor__", prim::kPrimIdentity}, // P.identity - {"__trunc__", prim::kPrimIdentity}, // P.identity - {"__pos__", prim::kPrimScalarUadd}, // P.scalar_uadd - {"__neg__", prim::kPrimScalarUsub}, // P.scalar_usub - {"__eq__", prim::kPrimScalarEq}, // P.scalar_eq - {"__ne__", prim::kPrimScalarNe}, // P.scalar_ne - {"__lt__", prim::kPrimScalarLt}, // P.scalar_lt - {"__gt__", prim::kPrimScalarGt}, // P.scalar_gt - {"__le__", prim::kPrimScalarLe}, // P.scalar_le - {"__ge__", prim::kPrimScalarGe}, // P.scalar_ge - {"__bool__", std::string("int_bool")}, // C.int_bool - {"__ms_to_array__", prim::kPrimScalarToArray}, // P.scalar_to_array - }}, - {kNumberTypeUInt, - { - {"__add__", prim::kPrimScalarAdd}, // P.scalar_add, - {"__sub__", prim::kPrimScalarSub}, // P.scalar_sub, - {"__mul__", prim::kPrimScalarMul}, // P.scalar_mul, - {"__floordiv__", prim::kPrimScalarDiv}, // P.scalar_div, - {"__truediv__", std::string("int_truediv")}, // C.int_truediv - {"__mod__", prim::kPrimScalarMod}, // P.scalar_mod, - {"__pow__", prim::kPrimScalarPow}, // P.scalar_pow, - {"__floor__", prim::kPrimIdentity}, // P.identity, - {"__trunc__", prim::kPrimIdentity}, // P.identity, - {"__pos__", prim::kPrimScalarUadd}, // P.scalar_uadd, - {"__neg__", prim::kPrimScalarUsub}, // P.scalar_usub, - {"__eq__", prim::kPrimScalarEq}, // P.scalar_eq, - {"__ne__", prim::kPrimScalarNe}, // P.scalar_ne, - {"__lt__", prim::kPrimScalarLt}, // P.scalar_lt, - {"__gt__", prim::kPrimScalarGt}, // P.scalar_gt, - {"__le__", prim::kPrimScalarLe}, // P.scalar_le, - {"__ge__", prim::kPrimScalarGe}, // P.scalar_ge, - {"__bool__", std::string("int_bool")}, // C.int_bool - {"__ms_to_array__", prim::kPrimScalarToArray}, // P.scalar_to_array, - }}, - {kNumberTypeFloat, - { - {"__add__", prim::kPrimScalarAdd}, // P.scalar_add, - {"__sub__", prim::kPrimScalarSub}, // P.scalar_sub, - {"__mul__", prim::kPrimScalarMul}, // P.scalar_mul, - {"__floordiv__", std::string("float_floordiv")}, // C.float_floordiv - {"__truediv__", prim::kPrimScalarDiv}, // P.scalar_div, - {"__mod__", prim::kPrimScalarMod}, // P.scalar_mod, - {"__pow__", prim::kPrimScalarPow}, // P.scalar_pow, - {"__floor__", prim::kPrimScalarFloor}, // P.scalar_floor, - {"__trunc__", prim::kPrimScalarTrunc}, // P.scalar_trunc, - {"__pos__", prim::kPrimScalarUadd}, // P.scalar_uadd, - {"__neg__", prim::kPrimScalarUsub}, // P.scalar_usub, - {"__eq__", prim::kPrimScalarEq}, // P.scalar_eq, - {"__ne__", prim::kPrimScalarNe}, // P.scalar_ne, - {"__lt__", prim::kPrimScalarLt}, // P.scalar_lt, - {"__gt__", prim::kPrimScalarGt}, // P.scalar_gt, - {"__le__", prim::kPrimScalarLe}, // P.scalar_le, - {"__ge__", prim::kPrimScalarGe}, // P.scalar_ge, - {"__bool__", std::string("float_bool")}, // C.float_bool - {"__ms_to_array__", prim::kPrimScalarToArray}, // P.scalar_to_array, - }}, - {kObjectTypeTuple, - { - {"__len__", prim::kPrimTupleLen}, // P.tuple_len, - {"__getitem__", prim::kPrimTupleGetItem}, // P.tuple_getitem, - {"__setitem__", prim::kPrimTupleSetItem}, // P.tuple_setitem, - {"__ms_iter__", prim::kPrimIdentity}, // P.identity, - {"__ms_next__", std::string("tuple_next")}, // C.tuple_next, - {"__ms_hasnext__", std::string("tuple_hasnext")}, // C.tuple_hasnext - {"__bool__", std::string("tuple_bool")} // C.tuple_bool - }}, - {kObjectTypeList, - { - {"__len__", prim::kPrimListLen}, // P.list_len, - {"__getitem__", prim::kPrimListGetItem}, // P.list_getitem, - {"__setitem__", prim::kPrimListSetItem}, // P.list_setitem, - {"__ms_iter__", prim::kPrimIdentity}, // P.identity - {"__ms_next__", std::string("list_next")}, // C.list_next - {"append", std::string("list_append")}, // C.list_next - {"__bool__", std::string("list_bool")}, // C.list_bool - {"__ms_hasnext__", std::string("list_hasnext")}, - }}, - {kObjectTypeDictionary, - { - {"__len__", prim::kPrimDictLen}, // P.dict_len - {"__getitem__", prim::kPrimDictGetItem}, // P.dict_getitem - {"__setitem__", prim::kPrimDictSetItem}, // P.dict_setitem, - {"__bool__", std::string("dict_bool")} // C.dict_bool - }}, - {kObjectTypeTensorType, - { - {"__add__", std::string("add")}, // C.add - {"__sub__", std::string("sub")}, // C.sub - {"__mul__", std::string("mul")}, // C.mul - {"__truediv__", std::string("truediv")}, // C.truediv - {"__floordiv__", std::string("floordiv")}, // C.floordiv - {"__mod__", std::string("mod")}, // C.mod - {"__pow__", std::string("pow_")}, // C.pow - {"__floor__", std::string("array_floor")}, // C.array_floor - {"__trunc__", std::string("array_trunc")}, // C.array_trunc - {"__pos__", std::string("array_uadd")}, // C.array_uadd - {"__neg__", std::string("array_usub")}, // C.array_usub - {"__eq__", std::string("eq")}, // C.eq - {"__ne__", std::string("ne")}, // C.ne - {"__lt__", std::string("lt")}, // C.lt - {"__gt__", std::string("gt")}, // C.gt - {"__le__", std::string("le")}, // C.le - {"__ge__", std::string("ge")}, // C.ge - {"__matmul__", prim::kPrimDot}, // P.dot, - {"__len__", prim::kPrimArrayLen}, // P.array_len, - {"__getitem__", prim::kPrimArrayGetItem}, // P.array_getitem, - {"__setitem__", prim::kPrimArraySetItem}, // P.array_setitem, - {"__ms_iter__", std::string("array_iter")}, // C.array_iter - {"__ms_to_array__", prim::kPrimIdentity}, // P.identity, - {"item", prim::kPrimArrayToScalar}, // P.array_to_scalar, - {"transpose", std::string("transpose")}, // P.transpose - {"__bool__", std::string("tensor_bool")}, // C.tensor_bool - }}, - {kObjectTypeJTagged, {}}, - {kObjectTypeSymbolicKeyType, {}}, - {kObjectTypeEnvType, {}}}; + static MethodMap method_map = { + {kObjectTypeString, + { + {"__bool__", std::string("str_bool")} // C.str_bool + }}, + {kMetaTypeNone, + { + {"__bool__", std::string("none_bool")} // C.none_bool + }}, + {kNumberTypeBool, + { + {"__and__", prim::kPrimBoolAnd}, // P.bool_and + {"__or__", prim::kPrimBoolOr}, // P.bool_or + {"__eq__", prim::kPrimBoolEq}, // P.bool_eq + {"__ne__", std::string("bool_ne")}, // C.bool_ne + {"__bool__", prim::kPrimIdentity} // P.identity + }}, + {kNumberTypeInt, + { + {"__add__", prim::kPrimScalarAdd}, // P.scalar_add + {"__sub__", prim::kPrimScalarSub}, // P.scalar_sub + {"__mul__", prim::kPrimScalarMul}, // P.scalar_mul + {"__floordiv__", std::string("int_floordiv")}, // C.int_floordiv + {"__truediv__", std::string("int_truediv")}, // C.int_truediv + {"__mod__", prim::kPrimScalarMod}, // P.scalar_mod + {"__pow__", prim::kPrimScalarPow}, // P.scalar_pow + {"__floor__", prim::kPrimIdentity}, // P.identity + {"__trunc__", prim::kPrimIdentity}, // P.identity + {"__pos__", prim::kPrimScalarUadd}, // P.scalar_uadd + {"__neg__", prim::kPrimScalarUsub}, // P.scalar_usub + {"__eq__", prim::kPrimScalarEq}, // P.scalar_eq + {"__ne__", prim::kPrimScalarNe}, // P.scalar_ne + {"__lt__", prim::kPrimScalarLt}, // P.scalar_lt + {"__gt__", prim::kPrimScalarGt}, // P.scalar_gt + {"__le__", prim::kPrimScalarLe}, // P.scalar_le + {"__ge__", prim::kPrimScalarGe}, // P.scalar_ge + {"__bool__", std::string("int_bool")}, // C.int_bool + {"__ms_to_array__", prim::kPrimScalarToArray}, // P.scalar_to_array + }}, + {kNumberTypeUInt, + { + {"__add__", prim::kPrimScalarAdd}, // P.scalar_add, + {"__sub__", prim::kPrimScalarSub}, // P.scalar_sub, + {"__mul__", prim::kPrimScalarMul}, // P.scalar_mul, + {"__floordiv__", prim::kPrimScalarDiv}, // P.scalar_div, + {"__truediv__", std::string("int_truediv")}, // C.int_truediv + {"__mod__", prim::kPrimScalarMod}, // P.scalar_mod, + {"__pow__", prim::kPrimScalarPow}, // P.scalar_pow, + {"__floor__", prim::kPrimIdentity}, // P.identity, + {"__trunc__", prim::kPrimIdentity}, // P.identity, + {"__pos__", prim::kPrimScalarUadd}, // P.scalar_uadd, + {"__neg__", prim::kPrimScalarUsub}, // P.scalar_usub, + {"__eq__", prim::kPrimScalarEq}, // P.scalar_eq, + {"__ne__", prim::kPrimScalarNe}, // P.scalar_ne, + {"__lt__", prim::kPrimScalarLt}, // P.scalar_lt, + {"__gt__", prim::kPrimScalarGt}, // P.scalar_gt, + {"__le__", prim::kPrimScalarLe}, // P.scalar_le, + {"__ge__", prim::kPrimScalarGe}, // P.scalar_ge, + {"__bool__", std::string("int_bool")}, // C.int_bool + {"__ms_to_array__", prim::kPrimScalarToArray}, // P.scalar_to_array, + }}, + {kNumberTypeFloat, + { + {"__add__", prim::kPrimScalarAdd}, // P.scalar_add, + {"__sub__", prim::kPrimScalarSub}, // P.scalar_sub, + {"__mul__", prim::kPrimScalarMul}, // P.scalar_mul, + {"__floordiv__", std::string("float_floordiv")}, // C.float_floordiv + {"__truediv__", prim::kPrimScalarDiv}, // P.scalar_div, + {"__mod__", prim::kPrimScalarMod}, // P.scalar_mod, + {"__pow__", prim::kPrimScalarPow}, // P.scalar_pow, + {"__floor__", prim::kPrimScalarFloor}, // P.scalar_floor, + {"__trunc__", prim::kPrimScalarTrunc}, // P.scalar_trunc, + {"__pos__", prim::kPrimScalarUadd}, // P.scalar_uadd, + {"__neg__", prim::kPrimScalarUsub}, // P.scalar_usub, + {"__eq__", prim::kPrimScalarEq}, // P.scalar_eq, + {"__ne__", prim::kPrimScalarNe}, // P.scalar_ne, + {"__lt__", prim::kPrimScalarLt}, // P.scalar_lt, + {"__gt__", prim::kPrimScalarGt}, // P.scalar_gt, + {"__le__", prim::kPrimScalarLe}, // P.scalar_le, + {"__ge__", prim::kPrimScalarGe}, // P.scalar_ge, + {"__bool__", std::string("float_bool")}, // C.float_bool + {"__ms_to_array__", prim::kPrimScalarToArray}, // P.scalar_to_array, + }}, + {kObjectTypeTuple, + { + {"__len__", prim::kPrimTupleLen}, // P.tuple_len, + {"__getitem__", prim::kPrimTupleGetItem}, // P.tuple_getitem, + {"__setitem__", prim::kPrimTupleSetItem}, // P.tuple_setitem, + {"__ms_iter__", prim::kPrimIdentity}, // P.identity, + {"__ms_next__", std::string("tuple_next")}, // C.tuple_next, + {"__ms_hasnext__", std::string("tuple_hasnext")}, // C.tuple_hasnext + {"__bool__", std::string("tuple_bool")} // C.tuple_bool + }}, + {kObjectTypeList, + { + {"__len__", prim::kPrimListLen}, // P.list_len, + {"__getitem__", prim::kPrimListGetItem}, // P.list_getitem, + {"__setitem__", prim::kPrimListSetItem}, // P.list_setitem, + {"__ms_iter__", prim::kPrimIdentity}, // P.identity + {"__ms_next__", std::string("list_next")}, // C.list_next + {"append", std::string("list_append")}, // C.list_next + {"__bool__", std::string("list_bool")}, // C.list_bool + {"__ms_hasnext__", std::string("list_hasnext")}, + }}, + {kObjectTypeDictionary, + { + {"__len__", prim::kPrimDictLen}, // P.dict_len + {"__getitem__", prim::kPrimDictGetItem}, // P.dict_getitem + {"__setitem__", prim::kPrimDictSetItem}, // P.dict_setitem, + {"__bool__", std::string("dict_bool")} // C.dict_bool + }}, + {kObjectTypeTensorType, + { + {"__add__", std::string("add")}, // C.add + {"__sub__", std::string("sub")}, // C.sub + {"__mul__", std::string("mul")}, // C.mul + {"__truediv__", std::string("truediv")}, // C.truediv + {"__floordiv__", std::string("floordiv")}, // C.floordiv + {"__mod__", std::string("mod")}, // C.mod + {"__pow__", std::string("pow_")}, // C.pow + {"__floor__", std::string("array_floor")}, // C.array_floor + {"__trunc__", std::string("array_trunc")}, // C.array_trunc + {"__pos__", std::string("array_uadd")}, // C.array_uadd + {"__neg__", std::string("array_usub")}, // C.array_usub + {"__eq__", std::string("eq")}, // C.eq + {"__ne__", std::string("ne")}, // C.ne + {"__lt__", std::string("lt")}, // C.lt + {"__gt__", std::string("gt")}, // C.gt + {"__le__", std::string("le")}, // C.le + {"__ge__", std::string("ge")}, // C.ge + {"__matmul__", prim::kPrimDot}, // P.dot, + {"__len__", prim::kPrimArrayLen}, // P.array_len, + {"__getitem__", prim::kPrimArrayGetItem}, // P.array_getitem, + {"__setitem__", prim::kPrimArraySetItem}, // P.array_setitem, + {"__ms_iter__", std::string("array_iter")}, // C.array_iter + {"__ms_to_array__", prim::kPrimIdentity}, // P.identity, + {"item", prim::kPrimArrayToScalar}, // P.array_to_scalar, + {"transpose", std::string("transpose")}, // P.transpose + {"__bool__", std::string("tensor_bool")}, // C.tensor_bool + {"is_indexed_slices", prim::kPrimIsIndexedSlices}, // F.is_indexed_slices + }}, + {kObjectTypeIndexedSlicesType, + { + {"is_indexed_slices", prim::kPrimIsIndexedSlices}, // F.is_indexed_slices + {"values", prim::kPrimIndexedSlicesGetValues}, // F.indexed_slices_get_values + {"indices", prim::kPrimIndexedSlicesGetIndices}, // F.indexed_slices_get_indices + {"dense_shape", prim::kPrimIndexedSlicesGetDenseShape}, // F.indexed_slices_get_dense_shape + }}, + {kObjectTypeJTagged, {}}, + {kObjectTypeSymbolicKeyType, {}}, + {kObjectTypeEnvType, {}}}; return method_map; } diff --git a/mindspore/ccsrc/pipeline/static_analysis/abstract_value.cc b/mindspore/ccsrc/pipeline/static_analysis/abstract_value.cc index f23c6e31c4..b59545e5ae 100644 --- a/mindspore/ccsrc/pipeline/static_analysis/abstract_value.cc +++ b/mindspore/ccsrc/pipeline/static_analysis/abstract_value.cc @@ -30,6 +30,10 @@ bool AbstractBase::operator==(const AbstractBase &other) const { if (tid() != other.tid()) { return false; } + if (BuildType()->type_id() == kObjectTypeUndeterminedType && + other.BuildType()->type_id() == kObjectTypeUndeterminedType) { + return true; + } if (value_ == nullptr || other.value_ == nullptr) { MS_LOG(EXCEPTION) << "If value_ is nullptr, AbstractBase::operator== should not be called. this: " << this->ToString() << ", other: " << other.ToString(); @@ -65,7 +69,7 @@ std::string AbstractBase::ToString() const { MS_EXCEPTION_IF_NULL(shape_); buffer << type_name() << "(" << "Type: " << type_->ToString() << " Value: " << value << " Shape: " << shape_->ToString() - << " sparse_grad: " << sparse_grad_ << ")"; + << " sparse_grad: " << sparse_grad_ << " has_indexed_slices_grad: " << has_indexed_slices_grad_ << ")"; return buffer.str(); } @@ -76,6 +80,7 @@ AbstractBasePtr AbstractScalar::Join(const AbstractBasePtr &other) { if (*this == *other) { auto ret = shared_from_base(); ret->set_sparse_grad(sparse_grad()); + ret->set_has_indexed_slices_grad(has_indexed_slices_grad()); return ret; } auto value_self = GetValueTrack(); @@ -85,10 +90,12 @@ AbstractBasePtr AbstractScalar::Join(const AbstractBasePtr &other) { if (res_value == value_self) { auto ret = shared_from_base(); ret->set_sparse_grad(sparse_grad()); + ret->set_has_indexed_slices_grad(has_indexed_slices_grad()); return ret; } auto ret = std::make_shared(res_value, res_type); ret->set_sparse_grad(sparse_grad()); + ret->set_has_indexed_slices_grad(has_indexed_slices_grad()); return ret; } @@ -409,6 +416,14 @@ std::size_t AbstractSlice::hash() const { return hash_combine({tid(), start_->hash(), stop_->hash(), step_->hash()}); } +ShapePtr AbstractUndetermined::shape() const { + auto shp = dyn_cast(GetShapeTrack()); + if (shp == nullptr) { + MS_LOG(EXCEPTION) << "Tensor should have a shape."; + } + return shp; +} + TypePtr AbstractTensor::BuildType() const { MS_EXCEPTION_IF_NULL(element_); TypePtr element_type = element_->BuildType(); @@ -425,14 +440,27 @@ BaseShapePtr AbstractTensor::BuildShape() const { } AbstractBasePtr AbstractTensor::Join(const AbstractBasePtr &other) { + if (other->BuildType()->type_id() == kObjectTypeUndeterminedType) { + auto other_tensor = dyn_cast(other); + auto element = element_->Join(other_tensor->element()); + auto shape = ShapeJoin(this->shape(), other_tensor->shape()); + auto ret = std::make_shared(element, shape); + return ret; + } auto other_tensor = dyn_cast(other); if (other_tensor == nullptr) { MS_LOG(EXCEPTION) << "Join failed as type mismatch, this: " << ToString() << ", other: " << other->ToString(); } + if (*this == *other) { + if (sparse_grad() == other->sparse_grad()) { + return shared_from_base(); + } + } auto element = element_->Join(other_tensor->element_); auto shape = ShapeJoin(this->shape(), other_tensor->shape()); auto ret = std::make_shared(element, shape); ret->set_sparse_grad(sparse_grad()); + ret->set_has_indexed_slices_grad(has_indexed_slices_grad()); return ret; } @@ -474,6 +502,7 @@ AbstractBasePtr AbstractTensor::Clone() const { clone->set_shape(shp->Clone()); clone->set_value(GetValueTrack()); clone->set_sparse_grad(sparse_grad()); + clone->set_has_indexed_slices_grad(has_indexed_slices_grad()); return clone; } @@ -484,6 +513,7 @@ AbstractBasePtr AbstractTensor::Broaden() const { broaden->set_shape(shp->Clone()); broaden->set_value(kAnyValue); broaden->set_sparse_grad(sparse_grad()); + broaden->set_has_indexed_slices_grad(has_indexed_slices_grad()); return broaden; } @@ -495,17 +525,10 @@ AbstractBasePtr AbstractTensor::BroadenWithShape() const { broaden->set_shape(shp); broaden->set_value(kAnyValue); broaden->set_sparse_grad(sparse_grad()); + broaden->set_has_indexed_slices_grad(has_indexed_slices_grad()); return broaden; } -ShapePtr AbstractTensor::shape() const { - auto shp = dyn_cast(GetShapeTrack()); - if (shp == nullptr) { - MS_LOG(EXCEPTION) << "Tensor should have a shape."; - } - return shp; -} - std::string AbstractTensor::ToString() const { std::ostringstream buffer; BaseShapePtr shape_track = GetShapeTrack(); @@ -516,7 +539,7 @@ std::string AbstractTensor::ToString() const { buffer << type_name() << "(" << "shape: " << shape_track->ToString() << ", element: " << element_->ToString() << ", value_ptr: " << value_track << ", value: " << value_track->ToString() << " sparse_grad " << sparse_grad() - << ")"; + << " has_indexed_slices_grad " << has_indexed_slices_grad() << ")"; return buffer.str(); } @@ -812,6 +835,21 @@ bool AbstractRef::operator==(const AbstractBase &other) const { return false; } +AbstractBasePtr AbstractRef::Join(const AbstractBasePtr &other) { + auto other_ref = other->cast(); + if (other_ref == nullptr) { + MS_LOG(EXCEPTION) << "Join failed as type mismatch, this: " << ToString() << ", other: " << other->ToString(); + } + if (*this == *other) { + return shared_from_base(); + } + auto ref_key = ref_key_->Join(other_ref->ref_key_); + auto ref = ref_->Join(other_ref->ref()); + auto ref_origin = ref_origin_->Join(other_ref->ref_origin_); + + return std::make_shared(ref_key, ref, ref_origin); +} + std::string AbstractRef::ToString() const { std::ostringstream buffer; buffer << type_name() << "(" @@ -1019,5 +1057,64 @@ std::size_t AbstractBasePtrListHasher::operator()(const AbstractBasePtrList &arg bool AbstractBasePtrListEqual::operator()(const AbstractBasePtrList &lhs, const AbstractBasePtrList &rhs) const { return AbstractBasePtrListDeepEqual(lhs, rhs); } + +// IndexedSlices +TypePtr AbstractIndexedSlices::BuildType() const { + MS_EXCEPTION_IF_NULL(element()); + TypePtr element_type = element()->BuildType(); + return std::make_shared(element_type); +} + +AbstractBasePtr AbstractIndexedSlices::Clone() const { + MS_EXCEPTION_IF_NULL(element()); + auto clone = std::make_shared(element()->Clone()); + ShapePtr shp = shape(); + clone->set_shape(shp->Clone()); + clone->set_value(GetValueTrack()); + clone->set_indices(indices_->Clone()->cast()); + clone->set_values(values_->Clone()->cast()); + clone->set_dense_shape(dense_shape_->Clone()->cast()); + return clone; +} + +AbstractBasePtr AbstractIndexedSlices::Broaden() const { + MS_EXCEPTION_IF_NULL(element()); + auto broaden = std::make_shared(element()->Broaden()); + auto shp = shape(); + broaden->set_shape(shp->Clone()); + broaden->set_value(kAnyValue); + broaden->set_indices(indices_->Clone()->cast()); + broaden->set_values(values_->Clone()->cast()); + broaden->set_dense_shape(dense_shape_->Clone()->cast()); + return broaden; +} + +AbstractBasePtr AbstractIndexedSlices::BroadenWithShape() const { + MS_EXCEPTION_IF_NULL(element()); + auto broaden = std::make_shared(element()->Broaden()); + auto shp = shape()->Clone(); + shp->Broaden(); + broaden->set_shape(shp); + broaden->set_value(kAnyValue); + broaden->set_indices(indices_->Clone()->cast()); + broaden->set_values(values_->Clone()->cast()); + broaden->set_dense_shape(dense_shape_->Clone()->cast()); + return broaden; +} + +std::string AbstractIndexedSlices::ToString() const { + std::ostringstream buffer; + BaseShapePtr shape_track = GetShapeTrack(); + MS_EXCEPTION_IF_NULL(shape_track); + MS_EXCEPTION_IF_NULL(element()); + auto value_track = GetValueTrack(); + MS_EXCEPTION_IF_NULL(value_track); + buffer << type_name() << "(" + << "shape: " << shape_track->ToString() << ", element: " << element()->ToString() + << ", value_ptr: " << value_track << ", value: " << value_track->ToString() << ")" + << ", indices: " << indices_->ToString() << ", values" << values_->ToString() + << ", dense_shape: " << dense_shape_->ToString(); + return buffer.str(); +} } // namespace abstract } // namespace mindspore diff --git a/mindspore/ccsrc/pipeline/static_analysis/abstract_value.h b/mindspore/ccsrc/pipeline/static_analysis/abstract_value.h index f3375d22d6..3981a6eb23 100644 --- a/mindspore/ccsrc/pipeline/static_analysis/abstract_value.h +++ b/mindspore/ccsrc/pipeline/static_analysis/abstract_value.h @@ -44,7 +44,7 @@ class AbstractBase : public Base { public: explicit AbstractBase(const ValuePtr &value = nullptr, const TypePtr &type = kAnyType, const BaseShapePtr &shape = kNoShape) - : value_(value), type_(type), shape_(shape), sparse_grad_("") {} + : value_(value), type_(type), shape_(shape), sparse_grad_(""), has_indexed_slices_grad_(false) {} ~AbstractBase() override = default; MS_DECLARE_PARENT(AbstractBase, Base) @@ -54,12 +54,16 @@ class AbstractBase : public Base { virtual bool operator==(const AbstractBase &other) const; void set_value(const ValuePtr &value) { value_ = value; } void set_sparse_grad(const std::string &sparse_grad) { sparse_grad_ = sparse_grad; } + void set_has_indexed_slices_grad(const bool &has_indexed_slices_grad) { + has_indexed_slices_grad_ = has_indexed_slices_grad; + } void set_type(const TypePtr &type) { type_ = type; } void set_shape(const BaseShapePtr &shape) { shape_ = shape; } void set_value_desc(const std::string &desc) { value_desc_ = desc; } const std::string &value_desc() const { return value_desc_; } ValuePtr GetValueTrack() const { return value_; } const std::string &sparse_grad() const { return sparse_grad_; } + const bool &has_indexed_slices_grad() const { return has_indexed_slices_grad_; } TypePtr GetTypeTrack() const { return type_; } BaseShapePtr GetShapeTrack() const { return shape_; } @@ -88,6 +92,7 @@ class AbstractBase : public Base { BaseShapePtr shape_; std::string value_desc_; // store initial value description for error report std::string sparse_grad_; + bool has_indexed_slices_grad_; }; class AbstractScalar : public AbstractBase { @@ -231,35 +236,49 @@ class AbstractKeywordArg : public AbstractBase { }; using AbstractKeywordArgPtr = std::shared_ptr; -class AbstractTensor : public AbstractBase { +class AbstractUndetermined : public AbstractBase { public: + // shape and type are all unknown + AbstractUndetermined() : AbstractBase(kAnyValue) {} // only element_ and value, shape track are valid member, type track are unknown. - explicit AbstractTensor(const AbstractBasePtr &element, const BaseShapePtr &shape = std::make_shared()) + explicit AbstractUndetermined(const AbstractBasePtr &element, const BaseShapePtr &shape = std::make_shared()) : AbstractBase(kAnyValue), element_(element) { if (element == nullptr) { MS_LOG(EXCEPTION) << "element is nullptr"; } - if (element->isa()) { + if (element->isa()) { MS_LOG(EXCEPTION) << "element type error"; } set_shape(shape); } - AbstractTensor(const TypePtr &element_type, const std::vector &shape) + AbstractUndetermined(const TypePtr &element_type, const std::vector &shape) : AbstractBase(kAnyValue), element_(std::make_shared(kAnyValue, element_type)) { if (element_type == nullptr) { MS_LOG(EXCEPTION) << "element_type is nullptr"; } set_shape(std::make_shared(shape)); } - explicit AbstractTensor(const tensor::TensorPtr &tensor) - : AbstractBase(tensor), element_(std::make_shared(kAnyValue, tensor->Dtype())) { - if (tensor == nullptr) { - MS_LOG(EXCEPTION) << "tensor is nullptr"; - } - set_shape(std::make_shared(tensor->shape())); - } + ~AbstractUndetermined() override = default; + MS_DECLARE_PARENT(AbstractUndetermined, AbstractBase) + TypePtr BuildType() const override { return std::make_shared(); } + AbstractBasePtr Clone() const override { return std::make_shared(); } + const AbstractBasePtr element() const { return element_; } + ShapePtr shape() const; + + protected: + AbstractBasePtr element_; +}; + +class AbstractTensor : public AbstractUndetermined { + public: + // only element_ and value, shape track are valid member, type track are unknown. + explicit AbstractTensor(const AbstractBasePtr &element, const BaseShapePtr &shape = std::make_shared()) + : AbstractUndetermined(element, shape) {} + AbstractTensor(const TypePtr &element_type, const std::vector &shape) + : AbstractUndetermined(element_type, shape) {} + explicit AbstractTensor(const tensor::TensorPtr &tensor) : AbstractUndetermined(tensor->Dtype(), tensor->shape()) {} ~AbstractTensor() override = default; - MS_DECLARE_PARENT(AbstractTensor, AbstractBase) + MS_DECLARE_PARENT(AbstractTensor, AbstractUndetermined) TypePtr BuildType() const override; BaseShapePtr BuildShape() const override; @@ -271,9 +290,7 @@ class AbstractTensor : public AbstractBase { bool operator==(const AbstractTensor &other) const; bool operator==(const AbstractBase &other) const override; - ShapePtr shape() const; std::string ToString() const override; - const AbstractBasePtr element() const { return element_; } std::size_t hash() const override { auto value = GetValueTrack(); auto hash_sum = hash_combine(tid(), element_->hash()); @@ -285,9 +302,6 @@ class AbstractTensor : public AbstractBase { } return hash_sum; } - - private: - AbstractBasePtr element_; }; using AbstractTensorPtr = std::shared_ptr; using AbstractTensorPtrList = std::vector; @@ -515,11 +529,11 @@ using AbstractNullPtr = std::shared_ptr; class AbstractEllipsis : public AbstractBase { public: - AbstractEllipsis() : AbstractBase(kEllipsis) { set_type(std::make_shared()); } + AbstractEllipsis() : AbstractBase(kEllipsis) { set_type(std::make_shared()); } ~AbstractEllipsis() override = default; MS_DECLARE_PARENT(AbstractEllipsis, AbstractBase) - TypePtr BuildType() const override { return std::make_shared(); } + TypePtr BuildType() const override { return std::make_shared(); } bool operator==(const AbstractEllipsis &other) const; bool operator==(const AbstractBase &other) const override; AbstractBasePtr Clone() const override { return std::make_shared(); } @@ -564,6 +578,7 @@ class AbstractRef : public AbstractBase { AbstractBasePtr Broaden() const override { return std::make_shared(ref_key_->Broaden(), ref_->Broaden(), ref_origin_->Broaden()); } + AbstractBasePtr Join(const AbstractBasePtr &other) override; std::size_t hash() const override { return ref_key_->hash() ^ ref_->hash() ^ ref_origin_->hash() ^ (std::hash{}(this->tid()) << 1); } @@ -585,6 +600,35 @@ struct AbstractBasePtrListEqual { std::size_t AbstractBasePtrListHash(const AbstractBasePtrList &args_spec_list); bool AbstractBasePtrListDeepEqual(const AbstractBasePtrList &lhs, const AbstractBasePtrList &rhs); + +// IndexedSlices +class AbstractIndexedSlices : public AbstractUndetermined { + public: + explicit AbstractIndexedSlices(const AbstractBasePtr &element, const BaseShapePtr &shape = std::make_shared()) + : AbstractUndetermined(element, shape) {} + AbstractIndexedSlices(const TypePtr &element_type, const std::vector &shape) + : AbstractUndetermined(element_type, shape) {} + ~AbstractIndexedSlices() override = default; + MS_DECLARE_PARENT(AbstractIndexedSlices, AbstractUndetermined) + + const AbstractTensorPtr indices() const { return indices_; } + const AbstractTensorPtr values() const { return values_; } + const AbstractTuplePtr dense_shape() const { return dense_shape_; } + void set_indices(const AbstractTensorPtr &indices) { indices_ = indices; } + void set_values(const AbstractTensorPtr &values) { values_ = values; } + void set_dense_shape(const AbstractTuplePtr &dense_shape) { dense_shape_ = dense_shape; } + TypePtr BuildType() const override; + AbstractBasePtr Clone() const override; + AbstractBasePtr Broaden() const override; + AbstractBasePtr BroadenWithShape() const; + + std::string ToString() const override; + + private: + AbstractTensorPtr indices_; + AbstractTensorPtr values_; + AbstractTuplePtr dense_shape_; +}; } // namespace abstract } // namespace mindspore #endif // PIPELINE_STATIC_ANALYSIS_ABSTRACT_VALUE_H_ diff --git a/mindspore/ccsrc/pipeline/static_analysis/evaluator.cc b/mindspore/ccsrc/pipeline/static_analysis/evaluator.cc index c9b1ce4f93..34ecfc8980 100644 --- a/mindspore/ccsrc/pipeline/static_analysis/evaluator.cc +++ b/mindspore/ccsrc/pipeline/static_analysis/evaluator.cc @@ -166,6 +166,7 @@ AbstractBasePtrList FuncGraphEvaluator::BroadenUndeterminedArgs(const AbstractBa // If there is loop variant, all arguments need to be broaden to avoid wrong constant propagation. if (!(joined_args_spec_list == args_spec_list)) { func_graph_->set_flag(FUNC_GRAPH_FLAG_IGNORE_VALUES, true); + MS_LOG(DEBUG) << "Set " << func_graph_->ToString() << " with IGNORE_VALUES flag."; } return joined_args_spec_list; } @@ -179,6 +180,7 @@ AbstractBasePtrList FuncGraphEvaluator::BroadenUndeterminedArgs(const AbstractBa if (!(joined_args_spec_list == args_spec_list)) { trace_.push_back(joined_args_spec_list); func_graph_->set_flag(FUNC_GRAPH_FLAG_IGNORE_VALUES, true); + MS_LOG(DEBUG) << "Set " << func_graph_->ToString() << " with IGNORE_VALUES flag."; } MS_LOG(DEBUG) << "Joined eval args: " << ::mindspore::ToString(joined_args_spec_list); return joined_args_spec_list; diff --git a/mindspore/ccsrc/pipeline/static_analysis/evaluator.h b/mindspore/ccsrc/pipeline/static_analysis/evaluator.h index c7a004ac44..f6430eda84 100644 --- a/mindspore/ccsrc/pipeline/static_analysis/evaluator.h +++ b/mindspore/ccsrc/pipeline/static_analysis/evaluator.h @@ -58,6 +58,20 @@ class Evaluator : public Base { return args_spec_list; } + virtual EvalResultPtr AbstractEval(const AbstractBasePtrList &args_spec_list) { + auto is_abstract = std::any_of(args_spec_list.begin(), args_spec_list.end(), [](auto &arg) { + if (arg->BuildType()->type_id() == kObjectTypeUndeterminedType) { + return true; + } + return false; + }); + if (is_abstract) { + MS_LOG(DEBUG) << "Eval " << identifier_ << " return abstract result"; + return std::make_shared(std::make_shared(), std::make_shared()); + } + return nullptr; + } + std::string ToString() const override { return identifier_; } virtual AnfNodePtr bound_node() const { return bound_node_.lock(); } diff --git a/mindspore/ccsrc/pipeline/static_analysis/param_validator.h b/mindspore/ccsrc/pipeline/static_analysis/param_validator.h index ecb9529a58..daa436d66d 100644 --- a/mindspore/ccsrc/pipeline/static_analysis/param_validator.h +++ b/mindspore/ccsrc/pipeline/static_analysis/param_validator.h @@ -66,6 +66,8 @@ ABSTRACT_REPORT_NAME_TRAITS(Function) ABSTRACT_REPORT_NAME_TRAITS(Type) ABSTRACT_REPORT_NAME_TRAITS(KeywordArg) ABSTRACT_REPORT_NAME_TRAITS(Class) +ABSTRACT_REPORT_NAME_TRAITS(IndexedSlices) +ABSTRACT_REPORT_NAME_TRAITS(Sequeue) template std::shared_ptr CheckArg(const std::string &op, const AbstractBasePtrList &args_spec_list, size_t index) { diff --git a/mindspore/ccsrc/pipeline/static_analysis/prim.cc b/mindspore/ccsrc/pipeline/static_analysis/prim.cc index 82b8395933..99dc085989 100644 --- a/mindspore/ccsrc/pipeline/static_analysis/prim.cc +++ b/mindspore/ccsrc/pipeline/static_analysis/prim.cc @@ -36,6 +36,7 @@ #include "pipeline/parse/resolve.h" #include "ir/tensor.h" #include "utils/convert_utils.h" +#include "utils/context/ms_context.h" #include "pipeline/parse/data_converter.h" #include "pipeline/static_analysis/param_validator.h" #include "common/utils.h" @@ -130,6 +131,14 @@ PrimitiveEvalImplMap &GetPrimitiveToEvalImplMap() { {prim::kPrimDepend, {InferImplDepend, true}}, {prim::kPrimBroadcastGradientArgs, {InferImplBroadcastGradientArgs, false}}, {prim::kPrimControlDepend, {InferImplControlDepend, true}}, + // Debug + {prim::kPrimDebug, {InferImplDebug, true}}, + // IndexedSlices + {prim::kPrimMakeIndexedSlices, {InferImplMakeIndexedSlices, true}}, + {prim::kPrimIndexedSlicesGetValues, {InferImplIndexedSlicesGetValues, true}}, + {prim::kPrimIndexedSlicesGetIndices, {InferImplIndexedSlicesGetIndices, true}}, + {prim::kPrimIndexedSlicesGetDenseShape, {InferImplIndexedSlicesGetDenseShape, true}}, + {prim::kPrimIsIndexedSlices, {InferImplIsIndexedSlices, true}}, }; return prim_eval_implement_map; } @@ -137,6 +146,16 @@ PrimitiveEvalImplMap &GetPrimitiveToEvalImplMap() { using mindspore::parse::PyObjectWrapper; EvalResultPtr StandardPrimEvaluator::EvalPrim(const AnalysisEnginePtr &engine, const AbstractBasePtrList &args) { + auto context = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(context); + bool enable_sparse_flag = context->enable_sparse_flag(); + if (enable_sparse_flag && prim_ != prim::kPrimMakeTuple && prim_ != prim::kPrimSwitch) { + auto ret_abstract = AbstractEval(args); + if (ret_abstract != nullptr) { + MS_LOG(DEBUG) << "StandardPrimEvaluator eval Undetermined"; + return ret_abstract; + } + } prim_->BeginRecordAddAttr(); AbstractBasePtr abs_base = eval_impl_(engine, prim_, args); prim_->EndRecordAddAttr(); @@ -483,6 +502,16 @@ AbstractBasePtr PyInferRes2Abstract(const PrimitivePyPtr &prim_py, const py::dic } // end anonymous namespace EvalResultPtr PythonPrimEvaluator::EvalPrim(const AnalysisEnginePtr &, const AbstractBasePtrList &args) { + auto context = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(context); + bool enable_sparse_flag = context->enable_sparse_flag(); + if (enable_sparse_flag) { + auto ret_abstract = AbstractEval(args); + if (ret_abstract != nullptr) { + MS_LOG(DEBUG) << "PythonPrimEvaluator eval Undetermined"; + return ret_abstract; + } + } MS_LOG(DEBUG) << "Eval for:" << prim_py_->ToString(); const auto &iter = cache_->find(args); @@ -510,6 +539,16 @@ EvalResultPtr PythonPrimEvaluator::EvalPrim(const AnalysisEnginePtr &, const Abs } EvalResultPtr UniformPrimEvaluator::EvalPrim(const AnalysisEnginePtr &, const AbstractBasePtrList &args) { + auto context = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(context); + bool enable_sparse_flag = context->enable_sparse_flag(); + if (enable_sparse_flag) { + auto ret_abstract = AbstractEval(args); + if (ret_abstract != nullptr) { + MS_LOG(DEBUG) << "UniformPrimEvaluator eval Undetermined"; + return ret_abstract; + } + } // if func_desc_.retval type is super class of parameter type, then make the retval type as parameter type. if (nargs_ != args.size()) { MS_LOG(ERROR) << "UniformPrimEvaluator expect " << nargs_ << " args, but got " << args.size() << " inputs"; @@ -869,6 +908,7 @@ class RefToEmbedEvaluator : public SymbolicPrimEvaluator { auto ref_value = ref_abs->ref(); MS_EXCEPTION_IF_NULL(ref_value); ret->set_sparse_grad(ref_value->sparse_grad()); + ret->set_has_indexed_slices_grad(ref_value->has_indexed_slices_grad()); return std::make_shared(ret, std::make_shared()); } @@ -884,6 +924,7 @@ class RefToEmbedEvaluator : public SymbolicPrimEvaluator { std::shared_ptr key = std::make_shared(node, x); std::shared_ptr abs_scalar = std::make_shared(key, type); abs_scalar->set_sparse_grad(x->sparse_grad()); + abs_scalar->set_has_indexed_slices_grad(x->has_indexed_slices_grad()); return std::make_shared(abs_scalar, std::make_shared()); } }; @@ -895,6 +936,16 @@ class GetAttrEvaluator : public TransitionPrimEvaluator { MS_DECLARE_PARENT(GetAttrEvaluator, TransitionPrimEvaluator); EvalResultPtr EvalPrim(const AnalysisEnginePtr &engine, const AbstractBasePtrList &args_spec_list, const ConfigPtr &in_conf0, const AnfNodeConfigPtr &out_conf) override { + auto context = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(context); + bool enable_sparse_flag = context->enable_sparse_flag(); + if (enable_sparse_flag) { + auto ret_abstract = AbstractEval(args_spec_list); + if (ret_abstract != nullptr) { + MS_LOG(DEBUG) << "GetAttrEvaluator eval Undetermined"; + return ret_abstract; + } + } // Inputs: data, item if (args_spec_list.size() != 2) { MS_LOG(EXCEPTION) << "Expected args_spec_list size = 2, but has size:" << args_spec_list.size(); diff --git a/mindspore/ccsrc/pipeline/static_analysis/prim.h b/mindspore/ccsrc/pipeline/static_analysis/prim.h index 5b910f8194..1346dba2a2 100644 --- a/mindspore/ccsrc/pipeline/static_analysis/prim.h +++ b/mindspore/ccsrc/pipeline/static_analysis/prim.h @@ -346,6 +346,21 @@ AbstractBasePtr InferImplBroadcastGradientArgs(const AnalysisEnginePtr &, const const AbstractBasePtrList &args_spec_list); AbstractBasePtr InferImplControlDepend(const AnalysisEnginePtr &, const PrimitivePtr &primitive, const AbstractBasePtrList &args_spec_list); + +AbstractBasePtr InferImplDebug(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +void InitUndeterminedFromEnv(const std::string &sparse_shape_types); + +AbstractBasePtr InferImplMakeIndexedSlices(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplIndexedSlicesGetValues(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplIndexedSlicesGetIndices(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplIndexedSlicesGetDenseShape(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplIsIndexedSlices(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); } // namespace abstract } // namespace mindspore diff --git a/mindspore/ccsrc/pipeline/static_analysis/static_analysis.cc b/mindspore/ccsrc/pipeline/static_analysis/static_analysis.cc index 9da148d2a7..5416576680 100644 --- a/mindspore/ccsrc/pipeline/static_analysis/static_analysis.cc +++ b/mindspore/ccsrc/pipeline/static_analysis/static_analysis.cc @@ -228,6 +228,10 @@ EvalResultPtr AnalysisEngine::EvalCNode(const CNodePtr &cnode, const AnfNodeConf MS_LOG(EXCEPTION) << "func_conf.GetEvaluatedValue() return null, func_conf: " << func_conf->ToString() << " NodeInfo: " << trace::GetDebugInfo(cnode->debug_info()); } + if (maybe_func->BuildType()->type_id() == kObjectTypeUndeterminedType) { + MS_LOG(DEBUG) << "EvalCNode eval Undetermined"; + return std::make_shared(maybe_func->Clone(), std::make_shared()); + } AbstractFunctionPtr func = dyn_cast(maybe_func); if (func == nullptr) { MS_LOG(EXCEPTION) << "func_conf.GetEvaluatedValue() return not AbstractFunction: " << maybe_func->ToString() diff --git a/mindspore/ccsrc/pipeline/validator.cc b/mindspore/ccsrc/pipeline/validator.cc index 4866d43b93..bbca3c8721 100644 --- a/mindspore/ccsrc/pipeline/validator.cc +++ b/mindspore/ccsrc/pipeline/validator.cc @@ -32,6 +32,7 @@ using mindspore::abstract::AbstractBase; using mindspore::abstract::AbstractClass; using mindspore::abstract::AbstractError; using mindspore::abstract::AbstractFunction; +using mindspore::abstract::AbstractIndexedSlices; using mindspore::abstract::AbstractJTagged; using mindspore::abstract::AbstractList; using mindspore::abstract::AbstractScalar; @@ -93,7 +94,8 @@ void ValidateAbstract(const AnfNodePtr &node) { } if (ptrBase->isa() || ptrBase->isa() || ptrBase->isa() || - ptrBase->isa() || ptrBase->isa() || ptrBase->isa()) { + ptrBase->isa() || ptrBase->isa() || ptrBase->isa() || + ptrBase->isa()) { return; } diff --git a/mindspore/ccsrc/pre_activate/ascend/ascend_backend_optimization.cc b/mindspore/ccsrc/pre_activate/ascend/ascend_backend_optimization.cc index 981e2255f3..ff864401b1 100644 --- a/mindspore/ccsrc/pre_activate/ascend/ascend_backend_optimization.cc +++ b/mindspore/ccsrc/pre_activate/ascend/ascend_backend_optimization.cc @@ -238,16 +238,11 @@ void AscendBackendIRFusionOptimization(const std::shared_ptr(); auto ir_fusion_pm = std::make_shared("ir_fusion_pm"); - if (context_ptr->execution_mode() == kPynativeMode) { - ir_fusion_pm->AddPass(std::make_shared()); - ir_fusion_pm->AddPass(std::make_shared()); - } else { - ir_fusion_pm->AddPass(std::make_shared()); - ir_fusion_pm->AddPass(std::make_shared()); - ir_fusion_pm->AddPass(std::make_shared()); - ir_fusion_pm->AddPass(std::make_shared()); - ir_fusion_pm->AddPass(std::make_shared()); - } + ir_fusion_pm->AddPass(std::make_shared()); + ir_fusion_pm->AddPass(std::make_shared()); + ir_fusion_pm->AddPass(std::make_shared()); + ir_fusion_pm->AddPass(std::make_shared()); + ir_fusion_pm->AddPass(std::make_shared()); ir_fusion_pm->AddPass(std::make_shared()); if (context_ptr->ir_fusion_flag()) { AddAscendBackendOptionalIRFusion(ir_fusion_pm.get()); @@ -287,8 +282,11 @@ void RunOpAscendBackendIRFusionOptimization(const std::shared_ptr(); auto ir_fusion_pm = std::make_shared("ir_fusion_pm"); - ir_fusion_pm->AddPass(std::make_shared()); + ir_fusion_pm->AddPass(std::make_shared()); ir_fusion_pm->AddPass(std::make_shared()); + ir_fusion_pm->AddPass(std::make_shared()); + ir_fusion_pm->AddPass(std::make_shared()); + ir_fusion_pm->AddPass(std::make_shared()); ir_fusion_pm->AddPass(std::make_shared()); ir_fusion_pm->AddPass(std::make_shared()); ir_fusion_pm->AddPass(std::make_shared()); diff --git a/mindspore/ccsrc/pre_activate/ascend/format_type/insert_trans_op.cc b/mindspore/ccsrc/pre_activate/ascend/format_type/insert_trans_op.cc index 953f464431..3f77c68f86 100644 --- a/mindspore/ccsrc/pre_activate/ascend/format_type/insert_trans_op.cc +++ b/mindspore/ccsrc/pre_activate/ascend/format_type/insert_trans_op.cc @@ -46,6 +46,11 @@ const AnfNodePtr InsertTransOp::Process(const FuncGraphPtr &func_graph, const An if (node == nullptr || !AnfAlgo::IsRealKernel(node)) { return nullptr; } + AnfNodePtr front_node; + auto kernel_graph = func_graph->cast>(); + if (kernel_graph != nullptr && kernel_graph->IsInternalOutput(node)) { + front_node = kernel_graph->GetFrontNodeByInternalOutput(node); + } AnfAlgo::SetNodeAttr(kAttrVisited, MakeValue(true), node); MS_LOG(DEBUG) << "====process op: " << node->DebugString(); AnfNodePtr new_node = InsertTransOpForInput(func_graph, node, kernel_select_); @@ -56,7 +61,12 @@ const AnfNodePtr InsertTransOp::Process(const FuncGraphPtr &func_graph, const An return new_node; } } - return InsertTransOpForOutput(func_graph, new_node, kernel_select_); + auto final_node = InsertTransOpForOutput(func_graph, new_node, kernel_select_); + if (kernel_graph != nullptr && front_node != nullptr) { + auto old_node = kernel_graph->GetInternalOutputByFrontNode(front_node); + kernel_graph->ReplaceInternalOutput(old_node, final_node); + } + return final_node; } } // namespace opt } // namespace mindspore diff --git a/mindspore/ccsrc/pre_activate/ascend/ir_fission/topk_split.cc b/mindspore/ccsrc/pre_activate/ascend/ir_fission/topk_split.cc index 1cace41fc4..c8477353f9 100644 --- a/mindspore/ccsrc/pre_activate/ascend/ir_fission/topk_split.cc +++ b/mindspore/ccsrc/pre_activate/ascend/ir_fission/topk_split.cc @@ -44,7 +44,7 @@ tensor::TensorPtr CreateTensor(const AnfNodePtr &node) { indices_tensor->set_device_info(device_info); // 2 set value of tensor - auto data_ptr = indices_tensor->data_c(true); + auto data_ptr = indices_tensor->data_c(); MS_EXCEPTION_IF_NULL(data_ptr); std::vector half_data; for (size_t i = 0; i < last_dim; ++i) { diff --git a/mindspore/ccsrc/pre_activate/ascend/ir_fusion/reshape_transpose_fusion.cc b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/reshape_transpose_fusion.cc index 45bf640abe..9b13002798 100644 --- a/mindspore/ccsrc/pre_activate/ascend/ir_fusion/reshape_transpose_fusion.cc +++ b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/reshape_transpose_fusion.cc @@ -51,8 +51,8 @@ const AnfNodePtr ReshapeTransposeFusion::Process(const FuncGraphPtr &func_graph, auto reshape_cnode = CheckAnfNodeIfCNodeAndInputSize(transpose_cnode->input(1), kBackendReshapeInputNum); MS_EXCEPTION_IF_NULL(reshape_cnode); std::vector reshape_input0_shape = AnfAlgo::GetPrevNodeOutputInferShape(reshape_cnode, 0); - std::vector transpose_input0_shape = AnfAlgo::GetPrevNodeOutputInferShape(transpose_cnode, 0); - if (!CheckShapeDimInfo(reshape_input0_shape) || !CheckShapeDimInfo(transpose_input0_shape)) { + std::vector transpose_output0_shape = AnfAlgo::GetOutputInferShape(transpose_cnode, 0); + if (!CheckShapeDimInfo(reshape_input0_shape) || !CheckShapeDimInfo(transpose_output0_shape)) { return nullptr; } auto prim = std::make_shared(kConfusionTransposeDOpName); diff --git a/mindspore/ccsrc/pre_activate/common/helper.cc b/mindspore/ccsrc/pre_activate/common/helper.cc index 1c2ade201c..e1db0ed6ed 100644 --- a/mindspore/ccsrc/pre_activate/common/helper.cc +++ b/mindspore/ccsrc/pre_activate/common/helper.cc @@ -348,7 +348,7 @@ tensor::TensorPtr CreateTensorWithValueTuple(const ValueTuplePtr &value_tuple_pt MS_EXCEPTION_IF_NULL(tensor); tensor::DeviceInfo device_info{kOpFormat_DEFAULT, type_ptr}; tensor->set_device_info(device_info); - auto data_ptr = tensor->data_c(true); + auto data_ptr = tensor->data_c(); MS_EXCEPTION_IF_NULL(data_ptr); auto elem_num = values.size() * data_length; auto ret_code = memcpy_s(data_ptr, static_cast(tensor->data().nbytes()), values.data(), elem_num); diff --git a/mindspore/ccsrc/pre_activate/gpu/adam_fusion.cc b/mindspore/ccsrc/pre_activate/gpu/adam_fusion.cc new file mode 100644 index 0000000000..8111ee429d --- /dev/null +++ b/mindspore/ccsrc/pre_activate/gpu/adam_fusion.cc @@ -0,0 +1,112 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "pre_activate/gpu/adam_fusion.h" + +#include +#include +#include + +#include "session/anf_runtime_algorithm.h" +#include "ir/primitive.h" +#include "utils/utils.h" +#include "pre_activate/common/helper.h" + +namespace mindspore { +namespace opt { +namespace { +kernel::KernelBuildInfoPtr GenerateKernelBuildInfo(CNodePtr node) { + std::vector inputs_format; + std::vector outputs_format; + std::vector inputs_type; + std::vector outputs_type; + kernel::KernelBuildInfo::KernelBuildInfoBuilder builder; + + for (size_t input_index = 0; input_index < AnfAlgo::GetInputTensorNum(node); ++input_index) { + inputs_type.push_back(AnfAlgo::GetPrevNodeOutputInferDataType(node, input_index)); + inputs_format.push_back(kOpFormat_DEFAULT); + } + for (size_t output_index = 0; output_index < AnfAlgo::GetOutputTensorNum(node); ++output_index) { + outputs_type.push_back(AnfAlgo::GetOutputInferDataType(node, output_index)); + outputs_format.push_back(kOpFormat_DEFAULT); + } + builder.SetInputsDeviceType(inputs_type); + builder.SetInputsFormat(inputs_format); + builder.SetOutputsDeviceType(outputs_type); + builder.SetOutputsFormat(outputs_format); + return builder.Build(); +} +} // namespace + +const BaseRef AdamFusion::DefinePattern() const { + VectorRef next_m = VectorRef({prim::kPrimTensorAdd, VectorRef({prim::kPrimMul, beta1_, m_}), + VectorRef({prim::kPrimMul, one_sub_beta1_, gradient_})}); + VectorRef next_v = + VectorRef({prim::kPrimTensorAdd, VectorRef({prim::kPrimMul, beta2_, v_}), + VectorRef({prim::kPrimMul, one_sub_beta2_, VectorRef({prim::kPrimSquare, gradient_})})}); + VectorRef update = VectorRef( + {prim::kPrimRealDiv, next_m, VectorRef({prim::kPrimTensorAdd, eps_, VectorRef({prim::kPrimSqrt, next_v})})}); + VectorRef update_with_lr = VectorRef({prim::kPrimMul, lr_, update}); + VectorRef next_param = VectorRef({prim::kPrimSub, param_, update_with_lr}); + VectorRef depend1 = VectorRef({prim::kPrimDepend, next_v, VectorRef({prim::kPrimAssign, param_, next_param})}); + VectorRef depend2 = VectorRef({prim::kPrimDepend, depend1, VectorRef({prim::kPrimAssign, m_, next_m})}); + VectorRef depend3 = VectorRef({prim::kPrimDepend, depend2, VectorRef({prim::kPrimAssign, v_, depend2})}); + return depend3; +} + +const AnfNodePtr AdamFusion::Process(const FuncGraphPtr &graph, const AnfNodePtr &node, const EquivPtr &equiv) const { + MS_EXCEPTION_IF_NULL(graph); + MS_EXCEPTION_IF_NULL(node); + MS_EXCEPTION_IF_NULL(equiv); + auto beta1_input = utils::cast((*equiv)[beta1_]); + auto one_sub_beta1_input = utils::cast((*equiv)[one_sub_beta1_]); + auto beta2_input = utils::cast((*equiv)[beta2_]); + auto one_sub_beta2_input = utils::cast((*equiv)[one_sub_beta2_]); + auto eps_input = utils::cast((*equiv)[eps_]); + auto lr_input = utils::cast((*equiv)[lr_]); + auto param_input = utils::cast((*equiv)[param_]); + auto m_input = utils::cast((*equiv)[m_]); + auto v_input = utils::cast((*equiv)[v_]); + auto gradient_input = utils::cast((*equiv)[gradient_]); + MS_EXCEPTION_IF_NULL(beta1_input); + MS_EXCEPTION_IF_NULL(one_sub_beta1_input); + MS_EXCEPTION_IF_NULL(beta2_input); + MS_EXCEPTION_IF_NULL(one_sub_beta2_input); + MS_EXCEPTION_IF_NULL(eps_input); + MS_EXCEPTION_IF_NULL(lr_input); + MS_EXCEPTION_IF_NULL(param_input); + MS_EXCEPTION_IF_NULL(m_input); + MS_EXCEPTION_IF_NULL(v_input); + MS_EXCEPTION_IF_NULL(gradient_input); + + auto prim = std::make_shared(kFusedAdamName); + MS_EXCEPTION_IF_NULL(prim); + std::vector inputs = { + NewValueNode(prim), beta1_input, one_sub_beta1_input, beta2_input, one_sub_beta2_input, + eps_input, lr_input, param_input, m_input, v_input, + gradient_input}; + auto adam = graph->NewCNode(inputs); + MS_EXCEPTION_IF_NULL(adam); + auto types = {AnfAlgo::GetOutputInferDataType(node, 0)}; + auto shapes = {AnfAlgo::GetOutputInferShape(node, 0)}; + AnfAlgo::SetOutputInferTypeAndShape(types, shapes, adam.get()); + adam->set_scope(node->scope()); + + auto build_info = GenerateKernelBuildInfo(adam); + AnfAlgo::SetSelectKernelBuildInfo(build_info, adam.get()); + return adam; +} +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/pre_activate/gpu/adam_fusion.h b/mindspore/ccsrc/pre_activate/gpu/adam_fusion.h new file mode 100644 index 0000000000..d8c10a0986 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/gpu/adam_fusion.h @@ -0,0 +1,56 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_PRE_ACTIVATE_GPU_IR_FUSION_ADAM_FUSION_H_ +#define MINDSPORE_CCSRC_PRE_ACTIVATE_GPU_IR_FUSION_ADAM_FUSION_H_ + +#include +#include "pre_activate/common/optimizer.h" + +namespace mindspore { +namespace opt { +class AdamFusion : public PatternProcessPass { + public: + explicit AdamFusion(bool multigraph = true) : PatternProcessPass("adam_fusion", multigraph) { + beta1_ = std::make_shared(); + one_sub_beta1_ = std::make_shared(); + beta2_ = std::make_shared(); + one_sub_beta2_ = std::make_shared(); + eps_ = std::make_shared(); + lr_ = std::make_shared(); + param_ = std::make_shared(); + m_ = std::make_shared(); + v_ = std::make_shared(); + gradient_ = std::make_shared(); + } + ~AdamFusion() override = default; + const BaseRef DefinePattern() const override; + const AnfNodePtr Process(const FuncGraphPtr &, const AnfNodePtr &, const EquivPtr &) const override; + + private: + VarPtr beta1_; + VarPtr one_sub_beta1_; + VarPtr beta2_; + VarPtr one_sub_beta2_; + VarPtr eps_; + VarPtr lr_; + VarPtr param_; + VarPtr m_; + VarPtr v_; + VarPtr gradient_; +}; +} // namespace opt +} // namespace mindspore +#endif // MINDSPORE_CCSRC_PRE_ACTIVATE_GPU_IR_FUSION_ADAM_FUSION_H_ diff --git a/mindspore/ccsrc/pre_activate/gpu/adam_weight_decay_fusion.cc b/mindspore/ccsrc/pre_activate/gpu/adam_weight_decay_fusion.cc new file mode 100644 index 0000000000..c950cbd56f --- /dev/null +++ b/mindspore/ccsrc/pre_activate/gpu/adam_weight_decay_fusion.cc @@ -0,0 +1,117 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "pre_activate/gpu/adam_weight_decay_fusion.h" + +#include +#include +#include + +#include "session/anf_runtime_algorithm.h" +#include "ir/primitive.h" +#include "utils/utils.h" +#include "pre_activate/common/helper.h" + +namespace mindspore { +namespace opt { +namespace { +kernel::KernelBuildInfoPtr GenerateKernelBuildInfo(CNodePtr node) { + std::vector inputs_format; + std::vector outputs_format; + std::vector inputs_type; + std::vector outputs_type; + kernel::KernelBuildInfo::KernelBuildInfoBuilder builder; + + for (size_t input_index = 0; input_index < AnfAlgo::GetInputTensorNum(node); ++input_index) { + inputs_type.push_back(AnfAlgo::GetPrevNodeOutputInferDataType(node, input_index)); + inputs_format.push_back(kOpFormat_DEFAULT); + } + for (size_t output_index = 0; output_index < AnfAlgo::GetOutputTensorNum(node); ++output_index) { + outputs_type.push_back(AnfAlgo::GetOutputInferDataType(node, output_index)); + outputs_format.push_back(kOpFormat_DEFAULT); + } + builder.SetInputsDeviceType(inputs_type); + builder.SetInputsFormat(inputs_format); + builder.SetOutputsDeviceType(outputs_type); + builder.SetOutputsFormat(outputs_format); + return builder.Build(); +} +} // namespace + +const BaseRef AdamWeightDecayFusion::DefinePattern() const { + VectorRef next_m = VectorRef({prim::kPrimTensorAdd, VectorRef({prim::kPrimMul, beta1_, m_}), + VectorRef({prim::kPrimMul, one_sub_beta1_, gradient_})}); + VectorRef next_v = + VectorRef({prim::kPrimTensorAdd, VectorRef({prim::kPrimMul, beta2_, v_}), + VectorRef({prim::kPrimMul, one_sub_beta2_, VectorRef({prim::kPrimSquare, gradient_})})}); + VectorRef update = VectorRef( + {prim::kPrimRealDiv, next_m, VectorRef({prim::kPrimTensorAdd, eps_, VectorRef({prim::kPrimSqrt, next_v})})}); + VectorRef new_update = VectorRef({prim::kPrimTensorAdd, VectorRef({prim::kPrimMul, weight_decay_, param_}), update}); + + VectorRef update_with_lr = VectorRef({prim::kPrimMul, lr_, new_update}); + VectorRef next_param = VectorRef({prim::kPrimSub, param_, update_with_lr}); + VectorRef depend1 = VectorRef({prim::kPrimDepend, next_v, VectorRef({prim::kPrimAssign, param_, next_param})}); + VectorRef depend2 = VectorRef({prim::kPrimDepend, depend1, VectorRef({prim::kPrimAssign, m_, next_m})}); + VectorRef depend3 = VectorRef({prim::kPrimDepend, depend2, VectorRef({prim::kPrimAssign, v_, depend2})}); + return depend3; +} + +const AnfNodePtr AdamWeightDecayFusion::Process(const FuncGraphPtr &graph, const AnfNodePtr &node, + const EquivPtr &equiv) const { + MS_EXCEPTION_IF_NULL(graph); + MS_EXCEPTION_IF_NULL(node); + MS_EXCEPTION_IF_NULL(equiv); + auto beta1_input = utils::cast((*equiv)[beta1_]); + auto one_sub_beta1_input = utils::cast((*equiv)[one_sub_beta1_]); + auto beta2_input = utils::cast((*equiv)[beta2_]); + auto one_sub_beta2_input = utils::cast((*equiv)[one_sub_beta2_]); + auto eps_input = utils::cast((*equiv)[eps_]); + auto lr_input = utils::cast((*equiv)[lr_]); + auto weight_decay_input = utils::cast((*equiv)[weight_decay_]); + auto param_input = utils::cast((*equiv)[param_]); + auto m_input = utils::cast((*equiv)[m_]); + auto v_input = utils::cast((*equiv)[v_]); + auto gradient_input = utils::cast((*equiv)[gradient_]); + MS_EXCEPTION_IF_NULL(beta1_input); + MS_EXCEPTION_IF_NULL(one_sub_beta1_input); + MS_EXCEPTION_IF_NULL(beta2_input); + MS_EXCEPTION_IF_NULL(one_sub_beta2_input); + MS_EXCEPTION_IF_NULL(eps_input); + MS_EXCEPTION_IF_NULL(lr_input); + MS_EXCEPTION_IF_NULL(weight_decay_input); + MS_EXCEPTION_IF_NULL(param_input); + MS_EXCEPTION_IF_NULL(m_input); + MS_EXCEPTION_IF_NULL(v_input); + MS_EXCEPTION_IF_NULL(gradient_input); + + auto prim = std::make_shared(kFusedAdamWeightDecayName); + MS_EXCEPTION_IF_NULL(prim); + std::vector inputs = { + NewValueNode(prim), beta1_input, one_sub_beta1_input, beta2_input, one_sub_beta2_input, + eps_input, lr_input, param_input, m_input, v_input, + gradient_input, weight_decay_input}; + auto adam_weight_decay = graph->NewCNode(inputs); + MS_EXCEPTION_IF_NULL(adam_weight_decay); + auto types = {AnfAlgo::GetOutputInferDataType(node, 0)}; + auto shapes = {AnfAlgo::GetOutputInferShape(node, 0)}; + AnfAlgo::SetOutputInferTypeAndShape(types, shapes, adam_weight_decay.get()); + adam_weight_decay->set_scope(node->scope()); + + auto build_info = GenerateKernelBuildInfo(adam_weight_decay); + AnfAlgo::SetSelectKernelBuildInfo(build_info, adam_weight_decay.get()); + return adam_weight_decay; +} +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/pre_activate/gpu/adam_weight_decay_fusion.h b/mindspore/ccsrc/pre_activate/gpu/adam_weight_decay_fusion.h new file mode 100644 index 0000000000..0ada5756e3 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/gpu/adam_weight_decay_fusion.h @@ -0,0 +1,58 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_PRE_ACTIVATE_GPU_IR_FUSION_ADAM_WEIGHT_DECAY_FUSION_H_ +#define MINDSPORE_CCSRC_PRE_ACTIVATE_GPU_IR_FUSION_ADAM_WEIGHT_DECAY_FUSION_H_ + +#include +#include "pre_activate/common/optimizer.h" + +namespace mindspore { +namespace opt { +class AdamWeightDecayFusion : public PatternProcessPass { + public: + explicit AdamWeightDecayFusion(bool multigraph = true) : PatternProcessPass("adam_weight_decay_fusion", multigraph) { + beta1_ = std::make_shared(); + one_sub_beta1_ = std::make_shared(); + beta2_ = std::make_shared(); + one_sub_beta2_ = std::make_shared(); + eps_ = std::make_shared(); + lr_ = std::make_shared(); + weight_decay_ = std::make_shared(); + param_ = std::make_shared(); + m_ = std::make_shared(); + v_ = std::make_shared(); + gradient_ = std::make_shared(); + } + ~AdamWeightDecayFusion() override = default; + const BaseRef DefinePattern() const override; + const AnfNodePtr Process(const FuncGraphPtr &, const AnfNodePtr &, const EquivPtr &) const override; + + private: + VarPtr beta1_; + VarPtr one_sub_beta1_; + VarPtr beta2_; + VarPtr one_sub_beta2_; + VarPtr eps_; + VarPtr lr_; + VarPtr weight_decay_; + VarPtr param_; + VarPtr m_; + VarPtr v_; + VarPtr gradient_; +}; +} // namespace opt +} // namespace mindspore +#endif // MINDSPORE_CCSRC_PRE_ACTIVATE_GPU_IR_FUSION_ADAM_WEIGHT_DECAY_FUSION_H_ diff --git a/mindspore/ccsrc/pre_activate/mem_reuse/mem_reuse.cc b/mindspore/ccsrc/pre_activate/mem_reuse/mem_reuse.cc index 2927b1204f..d550b77bba 100644 --- a/mindspore/ccsrc/pre_activate/mem_reuse/mem_reuse.cc +++ b/mindspore/ccsrc/pre_activate/mem_reuse/mem_reuse.cc @@ -103,6 +103,7 @@ bool MemReuseUtil::InitDynamicWorkspaceKernelRef() { bool MemReuseUtil::InitDynamicKernelRef(const KernelGraph *graph) { MS_EXCEPTION_IF_NULL(graph); graph_ = graph; + is_all_nop_node_ = opt::IsAllNopNode(graph); if (!InitDynamicOutputKernelRef()) { MS_LOG(INFO) << "InitDynamicOutputKernelRef fail"; return false; @@ -229,7 +230,14 @@ KernelRefCountPtr MemReuseUtil::GetKernelInputRef(const CNodePtr &kernel, size_t } auto input_node = kernel->input(input_idx + 1); // Graph may be all nop nodes and not remove nop node, so this can not skip nop node. - auto kernel_input = AnfAlgo::VisitKernelWithReturnType(input_node, 0, false); + session::KernelWithIndex kernel_input; + if (is_all_nop_node_) { + // The graph does not remove the nop node. + kernel_input = AnfAlgo::VisitKernelWithReturnType(input_node, 0, false); + } else { + // The graph removes the nop node. + kernel_input = AnfAlgo::VisitKernelWithReturnType(input_node, 0, true); + } if (IsPrimitive(kernel_input.first, prim::kPrimMakeTuple)) { MS_LOG(EXCEPTION) << "Input node [" << input_node->DebugString() << "]'s input " << input_idx << " is MakeTuple"; } @@ -272,7 +280,14 @@ void MemReuseUtil::SetKernelDefInputs() { // set the inputs of this kernel_def auto input_node = AnfAlgo::GetInputNode(kernel, i); // Graph may be all nop nodes and not remove nop node, so this can not skip nop node. - auto input = AnfAlgo::VisitKernelWithReturnType(input_node, 0, false); + session::KernelWithIndex input; + if (is_all_nop_node_) { + // The graph does not remove the nop node. + input = AnfAlgo::VisitKernelWithReturnType(input_node, 0, false); + } else { + // The graph removes the nop node. + input = AnfAlgo::VisitKernelWithReturnType(input_node, 0, true); + } if (IsPrimitive(input.first, prim::kPrimMakeTuple)) { MS_LOG(EXCEPTION) << "Input node [" << input_node->DebugString() << "]'s input " << i << " is MakeTuple"; } @@ -333,11 +348,10 @@ void MemReuseUtil::SetSummaryNodesRefCount() { } void MemReuseUtil::SetGraphOutputRefCount() { - auto is_all_nop_node = opt::IsAllNopNode(graph_); auto nodes = AnfAlgo::GetAllOutput(graph_->output(), {prim::kPrimTupleGetItem}); for (const auto &node : nodes) { session::KernelWithIndex kernel_input; - if (is_all_nop_node) { + if (is_all_nop_node_) { // The graph does not remove the nop node. kernel_input = AnfAlgo::VisitKernelWithReturnType(node, 0, false); } else { diff --git a/mindspore/ccsrc/pre_activate/mem_reuse/mem_reuse.h b/mindspore/ccsrc/pre_activate/mem_reuse/mem_reuse.h index c7a129f1e9..37281a7128 100644 --- a/mindspore/ccsrc/pre_activate/mem_reuse/mem_reuse.h +++ b/mindspore/ccsrc/pre_activate/mem_reuse/mem_reuse.h @@ -42,7 +42,7 @@ class MemReuseUtil { KernelRefCountPtrList total_refs_list_; KernelRefCountPtrList total_wk_ref_list_; KernelRefs kernel_workspace_refs_; - MemReuseUtil() : util_index_(kInitIndex), graph_(nullptr) {} + MemReuseUtil() : util_index_(kInitIndex), graph_(nullptr), is_all_nop_node_(false) {} ~MemReuseUtil() { if (graph_ != nullptr) { graph_ = nullptr; @@ -87,6 +87,7 @@ class MemReuseUtil { private: int util_index_; const KernelGraph *graph_; + bool is_all_nop_node_; KernelRefCountPtrList ref_list_; KernelDefPtrMaps kernel_def_ptr_list_; KernelRefCountPtrList last_ref_list_; diff --git a/mindspore/ccsrc/pre_activate/mem_reuse/mem_swap_manager.cc b/mindspore/ccsrc/pre_activate/mem_reuse/mem_swap_manager.cc index d81364edfb..14073bfbc9 100644 --- a/mindspore/ccsrc/pre_activate/mem_reuse/mem_swap_manager.cc +++ b/mindspore/ccsrc/pre_activate/mem_reuse/mem_swap_manager.cc @@ -24,7 +24,15 @@ namespace device { namespace memswap { void MemSwapManager::Init(const mindspore::session::KernelGraph *kernel_graph) { MS_EXCEPTION_IF_NULL(kernel_graph); - execution_order_ = kernel_graph->execution_order(); + graph_manager_ = kernel_graph->manager(); + MS_EXCEPTION_IF_NULL(graph_manager_); + auto &kernels = kernel_graph->execution_order(); + for (const auto &kernel : kernels) { + if (AnfAlgo::IsRealCNodeKernel(kernel) && (!opt::IsNopNode(kernel))) { + execution_order_.push_back(kernel); + } + } + size_t kernel_index = 0; for (const auto &kernel : execution_order_) { // parse topo order of kernel @@ -41,7 +49,7 @@ void MemSwapManager::Init(const mindspore::session::KernelGraph *kernel_graph) { } // parse topo order of user kernel - SaveUserKernelTopoOrder(kernel_graph); + SaveUserKernelTopoOrder(); sort(ordered_tensors_.begin(), ordered_tensors_.end(), [](const TensorInfo &a, const TensorInfo &b) { return a.tensor_size_ > b.tensor_size_; }); @@ -62,11 +70,22 @@ void MemSwapManager::Init(const mindspore::session::KernelGraph *kernel_graph) { mem_copy_manager_->Init(); } -void MemSwapManager::SaveUserKernelTopoOrder(const mindspore::session::KernelGraph *kernel_graph) { - MS_EXCEPTION_IF_NULL(kernel_graph); - FuncGraphManagerPtr manager = kernel_graph->manager(); - MS_EXCEPTION_IF_NULL(manager); - NodeUsersMap user_map = manager->node_users(); +bool MemSwapManager::IsCommunicationRelevantOp(const AnfNodePtr &kernel) const { + MS_EXCEPTION_IF_NULL(kernel); + NodeUsersMap &user_map = graph_manager_->node_users(); + auto iter = user_map.find(kernel); + bool adjacent_with_communication_op = false; + if (iter != user_map.end()) { + AnfNodeIndexSet node_set = iter->second; + adjacent_with_communication_op = std::any_of( + node_set.begin(), node_set.end(), + [](const std::pair &node_pair) { return AnfAlgo::IsCommunicationOp(node_pair.first); }); + } + return (AnfAlgo::IsCommunicationOp(kernel)) || adjacent_with_communication_op; +} + +void MemSwapManager::SaveUserKernelTopoOrder() { + NodeUsersMap &user_map = graph_manager_->node_users(); for (const auto &kernel : execution_order_) { auto iter = user_map.find(kernel); if (iter == user_map.end()) { @@ -76,13 +95,16 @@ void MemSwapManager::SaveUserKernelTopoOrder(const mindspore::session::KernelGra auto &kernel_exec_info = SearchKernelExecutionInfo(kernel); for (auto &node_pair : node_set) { auto user_kernel = node_pair.first; - if (!AnfAlgo::IsRealCNodeKernel(user_kernel)) { + if (!AnfAlgo::IsRealCNodeKernel(user_kernel) || opt::IsNopNode(user_kernel)) { continue; } size_t user_kernel_topo_sort = SearchKernelExecutionInfo(user_kernel).topo_order_; auto kernel_with_index = AnfAlgo::GetPrevNodeOutput(user_kernel, node_pair.second - 1); auto &output_idx = kernel_with_index.second; + if (kernel_with_index.first.get() != kernel.get()) { + MS_LOG(EXCEPTION) << "Save user kernel topo order failed for op[" << AnfAlgo::GetCNodeName(kernel) << "]"; + } kernel_exec_info.node_users_map_[output_idx].push_back(user_kernel_topo_sort); } for (auto &node_user_pair : kernel_exec_info.node_users_map_) { @@ -100,6 +122,9 @@ void MemSwapManager::AddSwapInfo() { size_t output_idx = tensor.output_idx_; const AnfNodePtr &kernel = tensor.kernel_; + if (IsCommunicationRelevantOp(kernel)) { + continue; + } auto &kernel_exec_info = SearchKernelExecutionInfo(kernel); auto &node_users_map = kernel_exec_info.node_users_map_; @@ -178,7 +203,7 @@ bool MemSwapManager::RetreatSwapInfo() { while (tensor_size_threshold_idx_ < ordered_tensors_.size() - 1) { ++tensor_size_threshold_idx_; - if (tensor_size_threshold_idx_ > ordered_tensors_[tensor_size_threshold_idx_].tensor_size_) { + if (tensor_size_threshold_ > ordered_tensors_[tensor_size_threshold_idx_].tensor_size_) { tensor_size_threshold_ = ordered_tensors_[tensor_size_threshold_idx_].tensor_size_; break; } diff --git a/mindspore/ccsrc/pre_activate/mem_reuse/mem_swap_manager.h b/mindspore/ccsrc/pre_activate/mem_reuse/mem_swap_manager.h index 7e2823d27c..1969dadb54 100644 --- a/mindspore/ccsrc/pre_activate/mem_reuse/mem_swap_manager.h +++ b/mindspore/ccsrc/pre_activate/mem_reuse/mem_swap_manager.h @@ -91,7 +91,7 @@ class MemSwapManager { void ResetSwapInfo(); - void SaveUserKernelTopoOrder(const mindspore::session::KernelGraph *kernel_graph); + void SaveUserKernelTopoOrder(); void AddKernelTriggerSwap(const AnfNodePtr &kernel, bool trigger_swap); @@ -99,6 +99,8 @@ class MemSwapManager { void AddKernelMemSwapInfo(const AnfNodePtr &kernel, const MemSwapInfo &mem_swap_info); + bool IsCommunicationRelevantOp(const AnfNodePtr &kernel) const; + std::vector execution_order_; std::vector ordered_tensors_; std::unordered_map kernel_execution_info_; @@ -113,7 +115,8 @@ class MemSwapManager { size_t tensor_size_num_; size_t distance_threshold_; - MemCopyManagerPtr mem_copy_manager_; + MemCopyManagerPtr mem_copy_manager_{nullptr}; + FuncGraphManagerPtr graph_manager_{nullptr}; bool mem_swap_initialized_{false}; bool swap_info_already_set_{false}; bool trigger_swap_{false}; diff --git a/mindspore/ccsrc/pre_activate/pass/common_subexpression_elimination.cc b/mindspore/ccsrc/pre_activate/pass/common_subexpression_elimination.cc index 9af50eac33..297a167aa8 100644 --- a/mindspore/ccsrc/pre_activate/pass/common_subexpression_elimination.cc +++ b/mindspore/ccsrc/pre_activate/pass/common_subexpression_elimination.cc @@ -35,7 +35,7 @@ bool CheckEqualKernelBuildInfo(const AnfNodePtr &main, const AnfNodePtr &node) { } } // namespace -bool BackendCSE::CheckReplace(const AnfNodePtr &main, const AnfNodePtr &node) const { +bool BackendCSE::CheckReplace(const AnfNodePtr &main, const AnfNodePtr &node, bool) const { MS_EXCEPTION_IF_NULL(main); MS_EXCEPTION_IF_NULL(node); diff --git a/mindspore/ccsrc/pre_activate/pass/common_subexpression_elimination.h b/mindspore/ccsrc/pre_activate/pass/common_subexpression_elimination.h index 8e1768ea99..18f433ab95 100644 --- a/mindspore/ccsrc/pre_activate/pass/common_subexpression_elimination.h +++ b/mindspore/ccsrc/pre_activate/pass/common_subexpression_elimination.h @@ -31,7 +31,7 @@ class BackendCSE : public CSE { public: BackendCSE() = default; ~BackendCSE() override = default; - bool CheckReplace(const AnfNodePtr &main, const AnfNodePtr &node) const override; + bool CheckReplace(const AnfNodePtr &main, const AnfNodePtr &node, bool check_side_effect = true) const override; }; } // namespace opt } // namespace mindspore diff --git a/mindspore/ccsrc/pre_activate/pass/fuse_graph_kernel.cc b/mindspore/ccsrc/pre_activate/pass/fuse_graph_kernel.cc index 591b210335..0e287587a2 100644 --- a/mindspore/ccsrc/pre_activate/pass/fuse_graph_kernel.cc +++ b/mindspore/ccsrc/pre_activate/pass/fuse_graph_kernel.cc @@ -452,7 +452,7 @@ void ReplaceNewFuseCNode(const std::shared_ptr &kernel_gra } } -AnfNodePtrList EliminateMakeTuple(FuncGraphPtr *fg, FuncGraphManagerPtr *mng) { +AnfNodePtrList EliminateMakeTuple(const FuncGraphPtr *fg, FuncGraphManagerPtr *mng) { AnfNodePtrList outs; auto out_node = (*fg)->output(); if (IsPrimitiveCNode(out_node, prim::kPrimMakeTuple)) { diff --git a/mindspore/ccsrc/predict/converter/kernel2ms.cc b/mindspore/ccsrc/predict/converter/kernel2ms.cc index 902efac720..1b1277aade 100644 --- a/mindspore/ccsrc/predict/converter/kernel2ms.cc +++ b/mindspore/ccsrc/predict/converter/kernel2ms.cc @@ -538,7 +538,7 @@ bool Kernel2Ms::KernelInput2MS(const std::vector &input_tensors) { auto match_idx = match_to_rel_idxs[j]; auto real_tensor = input_tensors[match_idx]; auto real_size = LongToSize(real_tensor->data().nbytes()); - auto real_data = real_tensor->data_c(false); + auto real_data = real_tensor->data_c(); MS_EXCEPTION_IF_NULL(real_data); if (sub_ms_graph_->allTensors[cache_idx] != nullptr) { sub_ms_graph_->allTensors[cache_idx]->data.resize(real_size); diff --git a/mindspore/ccsrc/pybind_api/export_flags.cc b/mindspore/ccsrc/pybind_api/export_flags.cc index 83392784f3..253e271e52 100644 --- a/mindspore/ccsrc/pybind_api/export_flags.cc +++ b/mindspore/ccsrc/pybind_api/export_flags.cc @@ -33,5 +33,6 @@ const char GRAPH_FLAG_LOOP_CAN_UNROLL[] = "loop_can_unroll"; const char GRAPH_FLAG_HAS_EFFECT[] = "has_effect"; const char GRAPH_FLAG_EFFECT_PATIAL_ORDER[] = "_effect_patial_order"; const char GRAPH_FLAG_RANDOM_EFFECT[] = "_random_effect"; +const char GRAPH_FLAG_SIDE_EFFECT[] = "_side_effect"; } // namespace mindspore diff --git a/mindspore/ccsrc/pybind_api/export_flags.h b/mindspore/ccsrc/pybind_api/export_flags.h index 74c27ff35d..6ea584e66d 100644 --- a/mindspore/ccsrc/pybind_api/export_flags.h +++ b/mindspore/ccsrc/pybind_api/export_flags.h @@ -34,7 +34,7 @@ extern const char GRAPH_FLAG_LOOP_CAN_UNROLL[]; extern const char GRAPH_FLAG_HAS_EFFECT[]; extern const char GRAPH_FLAG_EFFECT_PATIAL_ORDER[]; extern const char GRAPH_FLAG_RANDOM_EFFECT[]; - +extern const char GRAPH_FLAG_SIDE_EFFECT[]; } // namespace mindspore #endif // PYBIND_API_EXPORT_FLAGS_H_ diff --git a/mindspore/ccsrc/pynative/pynative_execute.cc b/mindspore/ccsrc/pynative/pynative_execute.cc index 75653ff5d2..f477bfbdcd 100644 --- a/mindspore/ccsrc/pynative/pynative_execute.cc +++ b/mindspore/ccsrc/pynative/pynative_execute.cc @@ -22,6 +22,8 @@ #include #include +#include "debug/trace.h" +#include "ir/tensor_py.h" #include "ir/param_value_py.h" #include "utils/any.h" #include "utils/utils.h" @@ -51,6 +53,8 @@ #include "pynative/pynative_execute_ge.h" #endif +using mindspore::tensor::TensorPy; + const char SINGLE_OP_GRAPH[] = "single_op_graph"; // primitive unable to infer value for constant input in PyNative mode const std::set vm_operators = {"make_ref", "HookBackward", "stop_gradient"}; @@ -63,6 +67,42 @@ PynativeExecutorPtr PynativeExecutor::executor_ = nullptr; std::mutex PynativeExecutor::instance_lock_; ResourcePtr PynativeExecutor::resource_; +template +void PynativeExecutorTry(PynativeExecutor *const executor, void (PynativeExecutor::*method)(Args...), Args &&... args) { + try { + (executor->*method)(args...); + } catch (const py::error_already_set &ex) { + // print function call stack info before release + std::ostringstream oss; + trace::TraceGraphEval(); + trace::GetEvalStackInfo(oss); + // call py::print to output function call stack to STDOUT, in case of output the log to file, the user can see + // these info from screen, no need to open log file to find these info + py::print(oss.str()); + MS_LOG(ERROR) << oss.str(); + PynativeExecutor::GetInstance()->Clean(); + // re-throw this exception to Python interpreter to handle it + throw(py::error_already_set(ex)); + } catch (const py::type_error &ex) { + PynativeExecutor::GetInstance()->Clean(); + throw py::type_error(ex); + } catch (const py::value_error &ex) { + PynativeExecutor::GetInstance()->Clean(); + throw py::value_error(ex); + } catch (const py::index_error &ex) { + PynativeExecutor::GetInstance()->Clean(); + throw py::index_error(ex); + } catch (const std::exception &ex) { + PynativeExecutor::GetInstance()->Clean(); + // re-throw this exception to Python interpreter to handle it + throw(std::runtime_error(ex.what())); + } catch (...) { + PynativeExecutor::GetInstance()->Clean(); + std::string exName(abi::__cxa_current_exception_type()->name()); + MS_LOG(EXCEPTION) << "Error occurred when compile graph. Exception name: " << exName; + } +} + inline ValuePtr PyAttrValue(const py::object &obj) { ValuePtr converted_ret = parse::data_converter::PyDataToValue(obj); if (!converted_ret) { @@ -141,7 +181,7 @@ std::map GetDstType(const py::tuple &py_args, } py::tuple ConvertInputs(const PrimitivePyPtr &prim, const py::list &args, py::tuple *const out_args, - py::list *out_args_list) { + py::list *const out_args_list) { auto &py_args = *out_args; py::tuple input_mask(args.size()); for (size_t i = 0; i < args.size(); ++i) { @@ -171,7 +211,8 @@ py::tuple ConvertInputs(const PrimitivePyPtr &prim, const py::list &args, py::tu py_args[i] = std::make_shared(py::cast(py_args[i]), tensor_ptr->Dtype()); (*out_args_list)[i] = py_args[i]; } else { - py_args[i] = std::make_shared(py::cast(py_args[i]), tensor_ptr->Dtype()); + double arg_value = py::cast(py_args[i]); + py_args[i] = std::make_shared(arg_value, tensor_ptr->Dtype()); (*out_args_list)[i] = py_args[i]; } continue; @@ -185,11 +226,8 @@ void PynativeInfer(const PrimitivePyPtr &prim, const py::list &py_args, OpExecIn AbstractBasePtrList args_spec_list; for (size_t i = 0; i < size; i++) { ValuePtr input_value = PyAttrValue(py_args[i]); - if (!py::hasattr(prim->GetPyObj(), "const_value") && input_value->isa()) { - args_spec_list.emplace_back(abstract::FromValueInside(input_value, true)); - } else { - args_spec_list.emplace_back(abstract::FromValueInside(input_value, false)); - } + args_spec_list.emplace_back(abstract::FromValueInside( + input_value, !py::hasattr(prim->GetPyObj(), "const_value") && input_value->isa())); } AbstractBasePtr infer_res = EvalOnePrim(prim, args_spec_list)->abstract(); op_exec_info->abstract = infer_res; @@ -261,8 +299,10 @@ py::object RunOpInVM(const OpExecInfoPtr &op_exec_info, PynativeStatusCode *stat if (py::hasattr(input, "__parameter__")) { result[i] = py::getattr(input, "data"); } else { - auto tensor = py::cast(op_inputs[i]); - auto new_tensor = std::make_shared(tensor->data()); + auto tensor = py::cast(input); + auto new_tensor = std::make_shared(tensor->data_type(), tensor->shape(), tensor->data_ptr()); + new_tensor->set_device_address(tensor->device_address()); + new_tensor->set_dirty(tensor->is_dirty()); result[i] = new_tensor; } } @@ -366,13 +406,14 @@ void ConvertPyObjectToTensor(const py::object &input_object, const PrimitivePtr if (py::isinstance(input_object)) { tensor_ptr = py::cast(input_object); } else if (py::isinstance(input_object)) { - tensor_ptr = std::make_shared(py::cast(input_object), kFloat32); + double input_value = py::cast(input_object); + tensor_ptr = std::make_shared(input_value, kFloat32); *tensor_mask = kValueNodeTensorMask; } else if (py::isinstance(input_object)) { tensor_ptr = std::make_shared(py::cast(input_object), kInt32); *tensor_mask = kValueNodeTensorMask; } else if (py::isinstance(input_object)) { - tensor_ptr = std::make_shared(py::cast(input_object), nullptr); + tensor_ptr = TensorPy::MakeTensor(py::cast(input_object), nullptr); } else if (py::isinstance(input_object)) { auto list_inputs = py::cast(input_object); py::tuple tuple_inputs(list_inputs.size()); @@ -468,7 +509,7 @@ py::object RunOpInMs(const OpExecInfoPtr &op_exec_info, PynativeStatusCode *stat return result; } -py::object RunOpWithBackendPolicy(MsBackendPolicy backend_policy, const OpExecInfoPtr op_exec_info, +py::object RunOpWithBackendPolicy(MsBackendPolicy backend_policy, const OpExecInfoPtr &op_exec_info, PynativeStatusCode *const status) { MS_EXCEPTION_IF_NULL(status); py::object result; @@ -506,7 +547,7 @@ py::object RunOpWithBackendPolicy(MsBackendPolicy backend_policy, const OpExecIn } AnfNodePtr PynativeExecutor::MakeCNode(const OpExecInfoPtr &op_exec_info, const py::args &args, const py::tuple &out) { - if (!grad_flag_ || graph_info_map_.size() == 0) { + if (!grad_flag_ || graph_info_map_.empty()) { return nullptr; } std::vector inputs; @@ -559,7 +600,7 @@ AnfNodePtr PynativeExecutor::GetObjNode(const py::object &obj) { return node; } -py::tuple RunOp(const OpExecInfoPtr &op_exec_info, const py::args &args) { +py::tuple RunOpInner(const OpExecInfoPtr &op_exec_info, const py::args &args) { MS_LOG(INFO) << "RunOp start, op name is: " << op_exec_info->op_name; mindspore::parse::python_adapter::set_python_env_flag(true); MsBackendPolicy backend_policy; @@ -598,7 +639,7 @@ py::tuple RunOp(const OpExecInfoPtr &op_exec_info, const py::args &args) { return result; } -py::tuple RunOp(const py::args &args) { +py::tuple RunOpInner(const py::args &args) { MS_LOG(DEBUG) << "RunOp start" << args.size(); py::list args_input = args[PY_INPUTS]; @@ -618,7 +659,42 @@ py::tuple RunOp(const py::args &args) { return value_ret; } } - return RunOp(op_exec_info, args_input); + return RunOpInner(op_exec_info, args_input); +} + +py::tuple RunOp(const py::args &args) { + try { + return RunOpInner(args); + } catch (const py::error_already_set &ex) { + // print function call stack info before release + std::ostringstream oss; + trace::TraceGraphEval(); + trace::GetEvalStackInfo(oss); + // call py::print to output function call stack to STDOUT, in case of output the log to file, the user can see + // these info from screen, no need to open log file to find these info + py::print(oss.str()); + MS_LOG(ERROR) << oss.str(); + PynativeExecutor::GetInstance()->Clean(); + // re-throw this exception to Python interpreter to handle it + throw(py::error_already_set(ex)); + } catch (const py::type_error &ex) { + PynativeExecutor::GetInstance()->Clean(); + throw py::type_error(ex); + } catch (const py::value_error &ex) { + PynativeExecutor::GetInstance()->Clean(); + throw py::value_error(ex); + } catch (const py::index_error &ex) { + PynativeExecutor::GetInstance()->Clean(); + throw py::index_error(ex); + } catch (const std::exception &ex) { + PynativeExecutor::GetInstance()->Clean(); + // re-throw this exception to Python interpreter to handle it + throw(std::runtime_error(ex.what())); + } catch (...) { + PynativeExecutor::GetInstance()->Clean(); + std::string exName(abi::__cxa_current_exception_type()->name()); + MS_LOG(EXCEPTION) << "Error occurred when compile graph. Exception name: " << exName; + } } void ClearPyNativeSession() { session = nullptr; } @@ -627,7 +703,7 @@ PynativeExecutor::~PynativeExecutor() { ClearRes(); } PynativeExecutor::PynativeExecutor() { grad_flag_ = false; } -void PynativeExecutor::NewGraph(const py::object &cell, const py::args &args) { +void PynativeExecutor::NewGraphInner(const py::object &cell, const py::args &args) { auto cell_id = GetId(cell); if (cell_graph_map_.count(cell_id) != 0) { MS_LOG(DEBUG) << "Newgraph already compiled"; @@ -674,7 +750,7 @@ AnfNodePtr PynativeExecutor::GetInput(const py::object &obj, const py::object &o if (py::isinstance(name_attr)) { MS_LOG(EXCEPTION) << "Parameter object should have name attribute"; } - std::string param_name = py::cast(name_attr); + auto param_name = py::cast(name_attr); if (graph_info_map_[df_builder_].param_map.count(obj_id) == 0) { auto free_param = df_builder_->add_parameter(); free_param->set_name(param_name); @@ -748,7 +824,7 @@ void PynativeExecutor::Popp() { graph_p_.pop(); } -void PynativeExecutor::EndGraph(const py::object &cell, const py::object &out, const py::args &args) { +void PynativeExecutor::EndGraphInner(const py::object &cell, const py::object &out, const py::args &args) { auto cell_id = GetId(cell); if (cell_graph_map_.count(cell_id) != 0) { MS_LOG(DEBUG) << "Endgraph already compiled"; @@ -887,8 +963,8 @@ abstract::AbstractBasePtrList PynativeExecutor::GetArgsSpec(const py::args &args return args_spec; } -void PynativeExecutor::GradNet(const GradOperationPtr &grad, const py::object &cell, const py::object &weights, - const py::args &args) { +void PynativeExecutor::GradNetInner(const GradOperationPtr &grad, const py::object &cell, const py::object &weights, + const py::args &args) { MS_LOG(INFO) << "GradNet start" << args.size(); std::size_t size = args.size(); @@ -934,8 +1010,10 @@ void PynativeExecutor::GradNet(const GradOperationPtr &grad, const py::object &c } void PynativeExecutor::Clear(const std::string &flag) { - if (flag == "resource") { + if (!flag.empty()) { MS_LOG(INFO) << "Clear res"; + (void)graph_map_.erase(flag); + (void)cell_graph_map_.erase(flag); Clean(); // Maybe exit in the pynative runing op, so need reset pynative flag. auto ms_context = MsContext::GetInstance(); @@ -944,6 +1022,7 @@ void PynativeExecutor::Clear(const std::string &flag) { } return; } + MS_LOG(INFO) << "Clear"; top_g_ = nullptr; curr_g_ = nullptr; @@ -1005,6 +1084,19 @@ FuncGraphPtr PynativeExecutor::GradGraph(FuncGraphPtr g, const GradOperationPtr return df_builder_; } +void PynativeExecutor::NewGraph(const py::object &cell, const py::args &args) { + PynativeExecutorTry(this, &PynativeExecutor::NewGraphInner, cell, args); +} + +void PynativeExecutor::EndGraph(const py::object &cell, const py::object &out, const py::args &args) { + PynativeExecutorTry(this, &PynativeExecutor::EndGraphInner, cell, out, args); +} + +void PynativeExecutor::GradNet(const GradOperationPtr &grad, const py::object &cell, const py::object &weights, + const py::args &args) { + PynativeExecutorTry(this, &PynativeExecutor::GradNetInner, grad, cell, weights, args); +} + REGISTER_PYBIND_DEFINE(PynativeExecutor_, ([](const py::module *m) { (void)py::class_>(*m, "PynativeExecutor_") .def_static("get_instance", &PynativeExecutor::GetInstance, "PynativeExecutor get_instance.") diff --git a/mindspore/ccsrc/pynative/pynative_execute.h b/mindspore/ccsrc/pynative/pynative_execute.h index 310cf0cb1e..83cbea88d4 100644 --- a/mindspore/ccsrc/pynative/pynative_execute.h +++ b/mindspore/ccsrc/pynative/pynative_execute.h @@ -26,6 +26,7 @@ #include #include "pybind11/pybind11.h" +#include "pybind11/numpy.h" #include "pynative/base.h" #include "utils/context/ms_context.h" @@ -45,7 +46,7 @@ py::object RunOpInVM(const OpExecInfoPtr &op_exec_info, PynativeStatusCode *stat py::tuple RunOp(const py::args &args); py::tuple ConvertInputs(const PrimitivePyPtr &prim, const py::list &py_args, py::tuple *const out_args, - py::list *out_args_list); + py::list *const out_args_list); void ClearPyNativeSession(); @@ -67,11 +68,15 @@ class PynativeExecutor : public std::enable_shared_from_this { return executor_; } void NewGraph(const py::object &cell, const py::args &args); + void NewGraphInner(const py::object &cell, const py::args &args); void EndGraph(const py::object &cell, const py::object &out, const py::args &args); + void EndGraphInner(const py::object &cell, const py::object &out, const py::args &args); void EndGraphByOutId(const std::string &out_id, const py::object &cell, const py::object &out, const py::args &args); std::vector GetWeightsArgs(const py::object &weights); abstract::AbstractBasePtrList GetArgsSpec(const py::args &args); void GradNet(const GradOperationPtr &grad, const py::object &cell, const py::object &weights, const py::args &args); + void GradNetInner(const GradOperationPtr &grad, const py::object &cell, const py::object &weights, + const py::args &args); void Clear(const std::string &flag = ""); void Clean(); void ClearRes(); diff --git a/mindspore/ccsrc/pynative/pynative_execute_ge.cc b/mindspore/ccsrc/pynative/pynative_execute_ge.cc index 7357bdd710..8e10468236 100644 --- a/mindspore/ccsrc/pynative/pynative_execute_ge.cc +++ b/mindspore/ccsrc/pynative/pynative_execute_ge.cc @@ -28,9 +28,12 @@ #include "pipeline/parse/data_converter.h" #include "pipeline/static_analysis/prim.h" #include "session/session_factory.h" +#include "ir/tensor_py.h" const char SINGLE_OP_GRAPH[] = "single_op_graph"; +using mindspore::tensor::TensorPy; + namespace mindspore { namespace pynative { using MeTensor = mindspore::tensor::Tensor; @@ -56,15 +59,15 @@ MeTensorPtr ConvertPyObjToTensor(const py::object &obj) { if (py::isinstance(obj)) { me_tensor_ptr = py::cast(obj); } else if (py::isinstance(obj)) { - me_tensor_ptr = std::make_shared(py::cast(obj), nullptr); + me_tensor_ptr = TensorPy::MakeTensor(py::array(py::cast(obj)), nullptr); } else if (py::isinstance(obj)) { - me_tensor_ptr = std::make_shared(py::cast(obj), nullptr); + me_tensor_ptr = TensorPy::MakeTensor(py::array(py::cast(obj)), nullptr); } else if (py::isinstance(obj)) { - me_tensor_ptr = std::make_shared(py::cast(obj), nullptr); + me_tensor_ptr = TensorPy::MakeTensor(py::array(py::cast(obj)), nullptr); } else if (py::isinstance(obj)) { - me_tensor_ptr = std::make_shared(py::cast(obj), nullptr); + me_tensor_ptr = TensorPy::MakeTensor(py::array(py::cast(obj)), nullptr); } else if (py::isinstance(obj)) { - me_tensor_ptr = std::make_shared(py::cast(obj), nullptr); + me_tensor_ptr = TensorPy::MakeTensor(py::cast(obj), nullptr); } else { MS_LOG(EXCEPTION) << "Run op inputs type is invalid!"; } diff --git a/mindspore/ccsrc/session/anf_runtime_algorithm.cc b/mindspore/ccsrc/session/anf_runtime_algorithm.cc index 5db7dbc324..81ad02e787 100644 --- a/mindspore/ccsrc/session/anf_runtime_algorithm.cc +++ b/mindspore/ccsrc/session/anf_runtime_algorithm.cc @@ -291,7 +291,7 @@ bool AnfRuntimeAlgorithm::HasNodeAttr(const std::string &key, const CNodePtr &no // graph kernel cnode. auto fg = AnfAlgo::GetCNodeFuncGraphPtr(node); MS_EXCEPTION_IF_NULL(fg); - return fg->has_flag(key); + return fg->has_attr(key); } size_t AnfRuntimeAlgorithm::GetInputTensorNum(const AnfNodePtr &node) { @@ -303,7 +303,7 @@ size_t AnfRuntimeAlgorithm::GetInputTensorNum(const AnfNodePtr &node) { MS_EXCEPTION_IF_NULL(cnode); size_t input_num = cnode->inputs().size(); if (input_num == 0) { - MS_LOG(EXCEPTION) << "cnode inputs size can't be zero"; + MS_LOG(EXCEPTION) << "Cnode inputs size can't be zero"; } // exclude intputs[0],which is value_node storing attr,inputs left are real input return input_num - 1; @@ -694,7 +694,7 @@ void AnfRuntimeAlgorithm::SetOutputInferTypeAndShape(const std::vector & MS_LOG(EXCEPTION) << "Types size " << types.size() << "should be same with shapes size " << shapes.size(); } if (shapes.empty()) { - MS_LOG(EXCEPTION) << "Illegal empty output_types_shapes"; + node->set_abstract(std::make_shared()); } else if (shapes.size() == 1) { // single output handle std::vector shape_int; @@ -994,10 +994,10 @@ FuncGraphPtr AnfRuntimeAlgorithm::GetValueNodeFuncGraph(const AnfNodePtr &node) } std::vector AnfRuntimeAlgorithm::GetCallNodeKernelGraph(const CNodePtr &call_node) { + MS_EXCEPTION_IF_NULL(call_node); if (!AnfAlgo::CheckPrimitiveType(call_node, std::make_shared("call"))) { - MS_LOG(EXCEPTION) << "anf node: " << call_node->DebugString() << "is not a call node."; + MS_LOG(EXCEPTION) << "Anf node: " << call_node->DebugString() << "is not a call node."; } - MS_EXCEPTION_IF_NULL(call_node); auto input1 = call_node->input(1); MS_EXCEPTION_IF_NULL(input1); if (input1->isa()) { @@ -1009,9 +1009,12 @@ std::vector AnfRuntimeAlgorithm::GetCallNodeKernelGraph(const CN } else if (input1->isa() && AnfAlgo::CheckPrimitiveType(input1, prim::kPrimSwitch)) { auto switch_node = input1->cast(); MS_EXCEPTION_IF_NULL(switch_node); - auto get_switch_kernel_graph = [&](size_t input_index) -> KernelGraphPtr { + auto get_switch_kernel_graph = [switch_node](size_t input_index) -> KernelGraphPtr { auto partial = switch_node->input(input_index); MS_EXCEPTION_IF_NULL(partial); + if (IsValueNode(partial)) { + return GetValueNode(partial); + } auto partial_cnode = partial->cast(); MS_EXCEPTION_IF_NULL(partial_cnode); auto graph_node = partial_cnode->input(1); @@ -1031,7 +1034,7 @@ std::vector AnfRuntimeAlgorithm::GetCallNodeKernelGraph(const CN bool AnfRuntimeAlgorithm::IsSwitchCall(const CNodePtr &call_node) { MS_EXCEPTION_IF_NULL(call_node); if (!CheckPrimitiveType(call_node, prim::kPrimCall)) { - MS_LOG(EXCEPTION) << "call node should be a 'call', but is a " << call_node->DebugString(); + MS_LOG(EXCEPTION) << "Call node should be a 'call', but is a " << call_node->DebugString(); } auto input1 = call_node->input(1); if (input1->isa()) { diff --git a/mindspore/ccsrc/session/anf_runtime_algorithm.h b/mindspore/ccsrc/session/anf_runtime_algorithm.h index c46f0b5955..8205619793 100644 --- a/mindspore/ccsrc/session/anf_runtime_algorithm.h +++ b/mindspore/ccsrc/session/anf_runtime_algorithm.h @@ -68,9 +68,14 @@ class AnfRuntimeAlgorithm { std::string node_debug_log = node->DebugString(); MS_LOG(EXCEPTION) << "Only cnode has attr, but this anf is " << node_debug_log.c_str(); } - auto primitive = GetCNodePrimitive(node); - MS_EXCEPTION_IF_NULL(primitive); - return GetValue(primitive->GetAttr(key)); + // single op cnode. + if (auto primitive = GetCNodePrimitive(node); primitive != nullptr) { + return GetValue(primitive->GetAttr(key)); + } + // graph kernel cnode. + auto fg = GetCNodeFuncGraphPtr(node); + MS_EXCEPTION_IF_NULL(fg); + return GetValue(fg->get_attr(key)); } static bool IsTupleOutput(const AnfNodePtr &anf); // set attr of anf node @@ -194,7 +199,6 @@ class AnfRuntimeAlgorithm { static bool IsScalarInput(const CNodePtr &cnode, size_t index); static bool IsScalarOutput(const CNodePtr &cnode, size_t index); static void ReorderExecList(NotNull *> node_list); - static bool IsWhileTrueGraph(const KernelGraphPtr &child_graph); // get fix output precision of cnode. static TypeId GetCNodeOutputPrecision(const AnfNodePtr &node); // get fix output precision from prev node, input_idx is the input index of current node related to prev node. diff --git a/mindspore/ccsrc/session/ascend_control_parser.cc b/mindspore/ccsrc/session/ascend_control_parser.cc index 868b968d9e..0c97116c6e 100644 --- a/mindspore/ccsrc/session/ascend_control_parser.cc +++ b/mindspore/ccsrc/session/ascend_control_parser.cc @@ -14,11 +14,12 @@ * limitations under the License. */ +#include "session/ascend_control_parser.h" #include #include -#include "session/ascend_control_parser.h" #include "session/anf_runtime_algorithm.h" #include "utils/union_find_set.h" +#include "device/ascend/ascend_label_assign.h" static constexpr size_t kCNodePrim = 0; static constexpr size_t kCNodeCallArg = 1; @@ -33,6 +34,29 @@ static constexpr size_t kCNodeSwitchLayerLength = 3; namespace mindspore { namespace session { +static CNodePtr GetJumpNode(NotNull parent_graph, NotNull child_graph) { + auto &nodes = parent_graph->execution_order(); + CNodePtr last_jump_node = nullptr; + for (auto &node : nodes) { + if (IsPrimitiveCNode(node, prim::kPrimLabelGoto)) { + if (child_graph->get_start_label() == node->input(kCNodeCallArg)) { + return node; + } + last_jump_node = node; + } else if (IsPrimitiveCNode(node, prim::kPrimLabelSwitch)) { + if (child_graph->get_start_label() == node->input(kCNodeSwitchFalse) || + child_graph->get_start_label() == node->input(kCNodeSwitchTrue)) { + return node; + } + last_jump_node = node; + } + } + if (last_jump_node == nullptr) { + MS_LOG(EXCEPTION) << "Cannot find jump node from " << parent_graph->ToString() << " to " << child_graph->ToString(); + } + return last_jump_node; +} + static void InitUnionFindSet(NotNull kg, const NotNull *> union_find_set, const NotNull *> memo) { if (memo->find(kg.get()) != memo->end()) { @@ -75,6 +99,9 @@ static void UnionParentParameter(NotNull kg, const NotNullisa()) { continue; } + if (kg->unreuse_args().find(arg) != kg->unreuse_args().end()) { + continue; + } union_find_set->Union(arg, para); } } @@ -96,7 +123,7 @@ static void RecursiveReplaceNode(NotNull kg, NotNull const std::set ¶meter_reuse_set, const NotNull *> memo) { if (parameter_reuse_set.empty()) { - MS_LOG(EXCEPTION) << "parameter_reuse_set is empty."; + MS_LOG(EXCEPTION) << "Parameter_reuse_set is empty."; } if (memo->find(kg.get()) != memo->end()) { return; @@ -118,24 +145,28 @@ static void RecursiveReplaceNode(NotNull kg, NotNull } } +static AnfNodePtr GetMainParameter(NotNull root_kg, const AnfNodePtr key, + const std::set ¶meter_reuse_set) { + AnfNodePtr main_parameter = key; + std::set root_inputs_set; + const auto &root_inputs_vector = root_kg->inputs(); + root_inputs_set.insert(root_inputs_vector.begin(), root_inputs_vector.end()); + for (auto &node : parameter_reuse_set) { + if (root_inputs_set.find(node) != root_inputs_set.end()) { + main_parameter = node; + break; + } + } + return main_parameter; +} + static void ReuseParameter(NotNull root_kg, NotNull *> parameter_set) { auto parameter_reuse_sets = parameter_set->GetSets(); for (auto &[key, parameter_reuse_set] : parameter_reuse_sets) { if (parameter_reuse_set.size() <= 1) { continue; } - - AnfNodePtr main_parameter = key; - std::set root_inputs_set; - const auto &root_inputs_vector = root_kg->inputs(); - root_inputs_set.insert(root_inputs_vector.begin(), root_inputs_vector.end()); - for (auto &node : parameter_reuse_set) { - if (root_inputs_set.find(node) != root_inputs_set.end()) { - main_parameter = node; - break; - } - } - + auto main_parameter = GetMainParameter(root_kg, key, parameter_reuse_set); std::set memo; RecursiveReplaceNode(root_kg, NOT_NULL(main_parameter), parameter_reuse_set, NOT_NULL(&memo)); } @@ -153,20 +184,23 @@ CNodePtr GetNextRealKernel(const std::vector &list, size_t start) { void AscendControlParser::LinkGraph(NotNull kg) { std::set memo; (void)ProcessKernelGraph(kg, nullptr, nullptr, NOT_NULL(&memo)); + device::ascend::AscendLabelAssign::GetInstance().AssignLabel(kg); std::map graph_id_map; for (auto &g : memo) { + MS_EXCEPTION_IF_NULL(g); if (graph_id_map.find(g->graph_id()) != graph_id_map.end()) { MS_LOG(EXCEPTION) << "Two graph has same graph id " << g->graph_id() << ", graph: " << graph_id_map[g->graph_id()]->ToString() << " " << g->ToString(); } graph_id_map[g->graph_id()] = g; } + + // Insert Assign + ChildGraphDataAssign(graph_id_map); // Make UnionFindSet UnionFindSet parameter_set = MakeUnionFindSet(kg); // Reuse Parameter ReuseParameter(kg, NOT_NULL(¶meter_set)); - // Insert Assign - ChildGraphDataAssign(graph_id_map); } void AscendControlParser::ExecutorValidate(NotNull root_graph) { @@ -177,6 +211,7 @@ void AscendControlParser::ExecutorValidate(NotNull root_graph) { void AscendControlParser::ChildGraphDataAssign(const std::map &graph_id_map) { for (auto &iter : graph_id_map) { auto &kg = iter.second; + MS_LOG(INFO) << "Data assign graph:" << kg->graph_id(); MS_EXCEPTION_IF_NULL(kg); std::set> memo; const std::vector>> &real_inputs = kg->real_inputs(); @@ -190,8 +225,14 @@ void AscendControlParser::ChildGraphDataAssign(const std::mapisa()) { + auto unreuse_args_map = kg->unreuse_args(); + auto unreuse_arg_iter = unreuse_args_map.find(arg); + if (unreuse_arg_iter == unreuse_args_map.end()) { + MS_EXCEPTION_IF_NULL(arg); MS_EXCEPTION_IF_NULL(parameter); + if (!arg->isa()) { + MS_LOG(EXCEPTION) << "Reused arg must be parameter, arg:" << arg->DebugString() << "."; + } MS_LOG(DEBUG) << "Parameter should be reused, no need insert assign, parameter: " << parameter->DebugString() << ", arg:" << arg->DebugString(); continue; @@ -200,12 +241,28 @@ void AscendControlParser::ChildGraphDataAssign(const std::mapsecond), NOT_NULL(arg), NOT_NULL(parameter)); + InsertMultipleAssignToGraph(NOT_NULL(target_graph_iter->second), NOT_NULL(kg), NOT_NULL(arg), + NOT_NULL(parameter)); } } + kg->SetExecOrderByDefault(); } } +NotNull AscendControlParser::GetStartLabel(NotNull kg, const CNodePtr &last_node, + const CNodePtr &last_label) { + CNodePtr start_label; + if (last_node != nullptr && last_label != nullptr) { + start_label = kg->NewCNode({std::make_shared(std::make_shared(kLabelSetOpName))}); + MS_LOG(INFO) << "Insert start label " << start_label->DebugString() << " to " << kg->ToString(); + kg->set_start_label(start_label); + } else { + // no goto node will jump to start label of root graph, so return a fake label + start_label = std::make_shared(std::vector(), FuncGraphPtr(nullptr)); + } + return NOT_NULL(start_label); +} + NotNull AscendControlParser::ProcessKernelGraph(NotNull kg, const CNodePtr &last_node, const CNodePtr &last_label, const NotNull *> memo) { @@ -225,28 +282,22 @@ NotNull AscendControlParser::ProcessKernelGraph(NotNullSetExecOrderByDefault(); const std::vector &nodes = kg->execution_order(); // 4. insert first_label - CNodePtr start_label; - if (last_node != nullptr && last_label != nullptr) { - start_label = kg->NewCNode({std::make_shared(std::make_shared(kLabelSetOpName))}); - MS_LOG(INFO) << "Insert start label " << start_label->DebugString() << " to " << kg->ToString(); - kg->set_start_label(start_label); - } else { - // no goto node will jump to start label of root graph, so return a fake label - start_label = std::make_shared(std::vector(), FuncGraphPtr(nullptr)); - } + CNodePtr start_label = GetStartLabel(kg, last_node, last_label); // 5. traverse for (size_t i = 0; i < nodes.size(); ++i) { auto &cnode = nodes[i]; + MS_EXCEPTION_IF_NULL(cnode); if (cnode->size() < kCNodePrim + 1) { MS_LOG(EXCEPTION) << "Inputs of apply node is empty"; } AnfNodePtr fn = cnode->input(kAnfPrimitiveIndex); if (!IsPrimitive(fn, prim::kPrimCall) || cnode->size() < kCNodeCallArg + 1) { - MS_LOG(DEBUG) << "continue node " << cnode->DebugString(); + MS_LOG(DEBUG) << "Continue node " << cnode->DebugString(); continue; } AnfNodePtr arg = cnode->input(kFirstDataInputIndex); + MS_EXCEPTION_IF_NULL(arg); if (IsValueNode(arg)) { RecurseCall(kg, NOT_NULL(cnode), GetNextRealKernel(nodes, i + 1), memo); } else if (!arg->isa()) { @@ -263,7 +314,7 @@ NotNull AscendControlParser::ProcessKernelGraph(NotNullSetExecOrderByDefault(); MS_LOG(INFO) << "End KernelGraph process: " << kg->ToString(); return NOT_NULL(start_label); } @@ -293,6 +344,7 @@ void AscendControlParser::LinkParentGraph(NotNull kg, const CNod if (from_graph_call_node != nullptr && last_label != nullptr) { auto label_goto = kg->NewCNode({std::make_shared(std::make_shared(kLabelGotoOpName)), last_label}); + MS_EXCEPTION_IF_NULL(label_goto); MS_LOG(INFO) << "Insert end goto " << label_goto->DebugString() << " to " << kg->ToString(); kg->set_end_goto(label_goto); } @@ -327,7 +379,6 @@ void AscendControlParser::RecurseCall(NotNull kg, NotNullset_inputs(new_inputs); cur_node->set_abstract(nullptr); MS_LOG(INFO) << "Succeed processing call func " << cur_node->DebugString(); @@ -342,6 +393,7 @@ void AscendControlParser::RecurseSwitch(NotNull kg, NotNullNewCNode({std::make_shared(std::make_shared(kLabelSetOpName))}); + MS_EXCEPTION_IF_NULL(back_label); MS_LOG(INFO) << "Insert back label " << back_label->DebugString() << " to " << kg->ToString() << " switch node " << cur_node->DebugString(); // 2 add depend relationship @@ -351,20 +403,21 @@ void AscendControlParser::RecurseSwitch(NotNull kg, NotNull &origin_switch_inputs = cur_node->inputs(); + if (kCNodeSwitchCond >= origin_switch_inputs.size()) { + MS_LOG(EXCEPTION) << "The size of origin_switch_inputs is not more than " << kCNodeSwitchCond; + } std::vector new_switch_inputs = { std::make_shared(std::make_shared(kLabelSwitchOpName)), origin_switch_inputs[kCNodeSwitchCond]}; for (size_t i = kCNodeSwitchCond + 1; i < kCNodeSwitchLength; ++i) { // 3.1 branch kernel graph and args - KernelGraphPtr branch_fg; - std::tie(std::ignore, branch_fg) = ParsePartial(NOT_NULL(origin_switch_inputs[i])); + KernelGraphPtr branch_fg = ParsePartial(NOT_NULL(origin_switch_inputs[i])); // 3.2 recurse sub graph CNodePtr branch_label = ProcessKernelGraph(NOT_NULL(branch_fg), cur_node, back_label, memo); new_switch_inputs.push_back(branch_label); } std::swap(new_switch_inputs[kCNodeSwitchTrue], new_switch_inputs[kCNodeSwitchFalse]); - new_switch_inputs.insert(new_switch_inputs.end(), origin_switch_inputs.begin(), origin_switch_inputs.end()); cur_node->set_inputs(new_switch_inputs); cur_node->set_abstract(nullptr); MS_LOG(INFO) << "Succeed processing switch func " << cur_node->DebugString(); @@ -402,8 +455,7 @@ void AscendControlParser::RecurseSwitchLayer(NotNull kg, NotNull origin_switch_inputs[kCNodeSwitchCond]}; for (size_t i = 0; i < branch_partial.size(); ++i) { // 3.1 branch kernel graph and args - KernelGraphPtr branch_fg; - std::tie(std::ignore, branch_fg) = ParsePartial(NOT_NULL(origin_switch_inputs[i])); + KernelGraphPtr branch_fg = ParsePartial(NOT_NULL(origin_switch_inputs[i])); // 3.2 recurse sub graph CNodePtr branch_label = ProcessKernelGraph(NOT_NULL(branch_fg), cur_node, back_label, memo); new_switch_inputs.push_back(branch_label); @@ -414,8 +466,11 @@ void AscendControlParser::RecurseSwitchLayer(NotNull kg, NotNull MS_LOG(INFO) << "Succeed processing switch layer " << cur_node->DebugString(); } -std::tuple AscendControlParser::ParsePartial(NotNull node) { +KernelGraphPtr AscendControlParser::ParsePartial(NotNull node) { if (!node.get()->isa()) { + if (IsValueNode(node)) { + return GetValueNode(node); + } MS_LOG(EXCEPTION) << "Switch branches must be partial, node: " << node->DebugString(); } // 2.1 branch kernel graph and args @@ -430,10 +485,11 @@ std::tuple AscendControlParser::ParsePartial(NotNull(partial_inputs[kCNodePartialFunc]); - return {partial_cnode, branch_kg}; + return branch_kg; } -void AscendControlParser::InsertMultipleAssignToGraph(NotNull kg, NotNull from, +void AscendControlParser::InsertMultipleAssignToGraph(NotNull from_graph, + NotNull to_graph, NotNull from, NotNull to) { std::vector from_outputs = AnfAlgo::GetAllOutput(from, {prim::kPrimTupleGetItem}); std::vector to_outputs = AnfAlgo::GetAllOutput(to, {prim::kPrimTupleGetItem}); @@ -443,18 +499,34 @@ void AscendControlParser::InsertMultipleAssignToGraph(NotNull kg << to_outputs.size() << "]"; } for (size_t i = 0; i < from_outputs.size(); i++) { - InsertAssignToGraph(kg, NOT_NULL(from_outputs[i]), NOT_NULL(to_outputs[i])); + auto assign_node = InsertAssignToGraph(from_graph, NOT_NULL(from_outputs[i]), NOT_NULL(to_outputs[i])); + if (assign_node != nullptr) { + auto jump_node = GetJumpNode(from_graph, to_graph); + const auto &from_graph_exe_order = from_graph->execution_order(); + auto jump_node_iter = std::find(from_graph_exe_order.begin(), from_graph_exe_order.end(), jump_node); + if (jump_node_iter == from_graph_exe_order.end()) { + MS_EXCEPTION_IF_NULL(jump_node); + MS_LOG(EXCEPTION) << "Can't find node:" << jump_node->DebugString() << " in graph:" << from_graph->graph_id(); + } + // insert assign between jump_node -1 and jump_node + if (jump_node_iter != from_graph_exe_order.begin()) { + InsertControlDependToGraph(from_graph, NOT_NULL(*(jump_node_iter - 1)), NOT_NULL(assign_node)); + } + if (jump_node != nullptr) { + InsertControlDependToGraph(from_graph, NOT_NULL(assign_node), NOT_NULL(jump_node)); + } + } } } -void AscendControlParser::InsertAssignToGraph(NotNull kg, NotNull from, - NotNull to) { +AnfNodePtr AscendControlParser::InsertAssignToGraph(NotNull kg, NotNull from, + NotNull to) { if (AnfAlgo::OutputAddrExist(from, 0) && AnfAlgo::OutputAddrExist(to, 0) && AnfAlgo::GetOutputAddr(from, 0) == AnfAlgo::GetOutputAddr(to, 0)) { - return; + return nullptr; } if (from.get() == to.get()) { - return; + return nullptr; } MS_LOG(INFO) << "Insert assign to graph " << kg->ToString() << " from " << from->DebugString() << " to " << to->DebugString(); @@ -464,8 +536,7 @@ void AscendControlParser::InsertAssignToGraph(NotNull kg, NotNul auto assign_node = kg->NewCNode(inputs); MS_EXCEPTION_IF_NULL(assign_node); assign_node->set_abstract(to->abstract()); - // append the assign at the end of from graph - InsertDependToGraph(kg, NOT_NULL(assign_node)); + return assign_node; } std::vector AscendControlParser::RecurseGraph(NotNull graph, @@ -489,7 +560,6 @@ std::vector AscendControlParser::RecurseGraph(NotNull std::vector execution_order; uint32_t child_order_index = 0; - for (auto &node : cnodes) { execution_order.push_back(node); if (node == graph->get_end_goto()) { @@ -538,6 +608,8 @@ bool AscendControlParser::CheckLabelIndex(uint32_t order_index, uint32_t label_i auto start_label_set = child_graph->get_start_label(); uint32_t start_label_set_index = AnfAlgo::GetNodeAttr(start_label_set, kAttrLabelIndex); if (label_index != start_label_set_index) { + MS_EXCEPTION_IF_NULL(cur_label); + MS_EXCEPTION_IF_NULL(start_label_set); MS_LOG(WARNING) << cur_label->DebugString() << " index " << label_index << " but " << start_label_set->DebugString() << " index " << start_label_set_index << " current child graph order : " << order_index; return false; @@ -563,7 +635,7 @@ void AscendControlParser::UpdateChildGraphOrder(NotNull kg) { } } for (size_t i = 0; i < child_graph_order.size(); i++) { - MS_LOG(INFO) << "child graph[" << i << "][id:" << child_graph_order[i]->graph_id() << "]"; + MS_LOG(INFO) << "Child graph[" << i << "][id:" << child_graph_order[i]->graph_id() << "]"; } kg->set_child_graph_order(child_graph_order); } diff --git a/mindspore/ccsrc/session/ascend_control_parser.h b/mindspore/ccsrc/session/ascend_control_parser.h index 73d68449b3..7530f2019e 100644 --- a/mindspore/ccsrc/session/ascend_control_parser.h +++ b/mindspore/ccsrc/session/ascend_control_parser.h @@ -23,6 +23,7 @@ #include "session/kernel_graph.h" #include "utils/base_ref.h" #include "utils/contract.h" +#include "utils/union_find_set.h" namespace mindspore { namespace session { @@ -38,6 +39,8 @@ class AscendControlParser { static void UpdateChildGraphOrder(NotNull kg); private: + static NotNull GetStartLabel(NotNull kg, const CNodePtr &last_node, + const CNodePtr &last_label); static NotNull ProcessKernelGraph(NotNull kg, const CNodePtr &last_node, const CNodePtr &last_label, const NotNull *> memo); @@ -50,10 +53,11 @@ class AscendControlParser { static void LinkParentGraph(NotNull kg, const CNodePtr &from_graph_call_node, const CNodePtr &last_label); - static std::tuple ParsePartial(NotNull node); + static KernelGraphPtr ParsePartial(NotNull node); - static void InsertMultipleAssignToGraph(NotNull kg, NotNull from, NotNull to); - static void InsertAssignToGraph(NotNull kg, NotNull from, NotNull to); + static void InsertMultipleAssignToGraph(NotNull from_graph, NotNull to_graph, + NotNull from, NotNull to); + static AnfNodePtr InsertAssignToGraph(NotNull kg, NotNull from, NotNull to); // root graph order static bool CheckLabelIndex(uint32_t order_index, uint32_t label_index, const CNodePtr &cnode, diff --git a/mindspore/ccsrc/session/ascend_inference_session.cc b/mindspore/ccsrc/session/ascend_inference_session.cc index ff53874502..aef7738d0b 100644 --- a/mindspore/ccsrc/session/ascend_inference_session.cc +++ b/mindspore/ccsrc/session/ascend_inference_session.cc @@ -16,6 +16,7 @@ #include "session/ascend_inference_session.h" #include "operator/ops.h" #include "ir/tensor.h" +#include "ir/tensor_py.h" #include "ir/anf.h" #include "ir/param_value_py.h" #include "device/kernel_runtime.h" @@ -26,16 +27,73 @@ #include "utils/config_manager.h" #include "utils/base_ref_extends.h" +using mindspore::tensor::TensorPy; + namespace mindspore { namespace session { +namespace { +std::set weight_infos; +static TypeId GetDataType(const py::buffer_info &buf) { + if (buf.format.size() == 1) { + switch (buf.format.front()) { + case 'e': + case 'f': + case 'd': + switch (buf.itemsize) { + case 2: + return TypeId::kNumberTypeFloat16; + case 4: + return TypeId::kNumberTypeFloat32; + case 8: + return TypeId::kNumberTypeFloat64; + } + break; + case 'b': + case 'h': + case 'i': + case 'l': + case 'q': + switch (buf.itemsize) { + case 1: + return TypeId::kNumberTypeInt8; + case 2: + return TypeId::kNumberTypeInt16; + case 4: + return TypeId::kNumberTypeInt32; + case 8: + return TypeId::kNumberTypeInt64; + } + break; + case 'B': + case 'H': + case 'I': + case 'L': + case 'Q': + switch (buf.itemsize) { + case 1: + return TypeId::kNumberTypeUInt8; + case 2: + return TypeId::kNumberTypeUInt16; + case 4: + return TypeId::kNumberTypeUInt32; + case 8: + return TypeId::kNumberTypeUInt64; + } + break; + case '?': + return TypeId::kNumberTypeBool; + } + } + MS_LOG(WARNING) << "Unsupported DataType format " << buf.format << " item size " << buf.itemsize; + return TypeId::kTypeUnknown; +} +} // namespace void AscendInferenceSession::LoadInputData(const std::shared_ptr &kernel_graph, const std::vector &inputs_const) const { MS_EXCEPTION_IF_NULL(kernel_graph); std::vector inputs(inputs_const); auto input_nodes = kernel_graph->inputs(); - auto ms_context = MsContext::GetInstance(); - MS_EXCEPTION_IF_NULL(ms_context); size_t no_weight_input = 0; for (size_t i = 0; i < input_nodes.size(); ++i) { tensor::TensorPtr tensor = nullptr; @@ -45,45 +103,32 @@ void AscendInferenceSession::LoadInputData(const std::shared_ptr &k } auto pk_node = input_nodes[i]->cast(); MS_EXCEPTION_IF_NULL(pk_node); + auto device_address = AnfAlgo::GetMutableOutputAddr(pk_node, 0); + MS_EXCEPTION_IF_NULL(device_address); if (AnfAlgo::IsParameterWeight(pk_node)) { + if (weight_infos.count(pk_node) != 0) { + continue; + } auto param_value = std::dynamic_pointer_cast(pk_node->default_param()); MS_EXCEPTION_IF_NULL(param_value); auto py_param = param_value->value(); MS_EXCEPTION_IF_NULL(py_param); py::array py_array = py_param.cast(); - tensor = std::make_shared(py_array); + py::buffer_info buf = py_array.request(); + auto buf_type = GetDataType(buf); + if (!device_address->SyncHostToDevice(trans::GetRuntimePaddingShape(pk_node, 0), + LongToSize(buf.size * buf.itemsize), buf_type, buf.ptr)) { + MS_LOG(EXCEPTION) << "SyncHostToDevice failed."; + } + weight_infos.insert(pk_node); } else { tensor = inputs[no_weight_input++]; - } - MS_EXCEPTION_IF_NULL(tensor); - if (AnfAlgo::OutputAddrExist(pk_node, 0)) { - auto device_address = AnfAlgo::GetMutableOutputAddr(pk_node, 0); - bool need_sync = false; - if (ms_context->enable_pynative_infer()) { - if (tensor->device_address().get() == nullptr || tensor->device_address() != device_address) { - need_sync = true; - } - } else { - if (tensor->is_dirty()) { - need_sync = true; - } else if (tensor->device_address() != device_address) { - (void)tensor->data_sync(); - need_sync = true; - } - } - if (need_sync) { - if (ms_context->execution_mode() == kPynativeMode || AnfAlgo::IsParameterWeight(pk_node)) { - tensor->set_device_address(device_address); - } - MS_EXCEPTION_IF_NULL(device_address); - if (!device_address->SyncHostToDevice(trans::GetRuntimePaddingShape(pk_node, 0), - LongToSize(tensor->data().nbytes()), tensor->data_type(), - tensor->data_c(false))) { - MS_LOG(EXCEPTION) << "SyncHostToDevice failed."; - } + if (!device_address->SyncHostToDevice(trans::GetRuntimePaddingShape(pk_node, 0), + LongToSize(tensor->data().nbytes()), tensor->data_type(), + tensor->data_c())) { + MS_LOG(EXCEPTION) << "SyncHostToDevice failed."; } } - tensor->set_dirty(false); } } } // namespace session diff --git a/mindspore/ccsrc/session/ascend_session.cc b/mindspore/ccsrc/session/ascend_session.cc index bae10ed943..f361cb26ca 100644 --- a/mindspore/ccsrc/session/ascend_session.cc +++ b/mindspore/ccsrc/session/ascend_session.cc @@ -18,6 +18,7 @@ #include #include #include +#include #include #include "operator/ops.h" #include "ir/tensor.h" @@ -45,6 +46,7 @@ #include "kernel/tbe/tbe_python_funcs.h" #include "utils/config_manager.h" #include "utils/base_ref_extends.h" +#include "debug/tensor_load.h" namespace mindspore { namespace session { @@ -137,16 +139,16 @@ std::vector GetRealArgs(const KernelGraphPtr graph, const VectorRef &ar } } if (graph_inputs.size() != valid_inputs.size()) { - MS_LOG(EXCEPTION) << "graph_inputs.size(): " << graph_inputs.size() + MS_LOG(EXCEPTION) << "Graph_inputs.size(): " << graph_inputs.size() << ", valid_inputs.size(): " << valid_inputs.size() << " not equal"; } if (real_args_size != graph_inputs.size()) { for (size_t j = 0; j < valid_inputs.size(); j++) { if (valid_inputs[j]) { - MS_LOG(INFO) << "index: " << j << ", nodes: " << graph_inputs[j]->DebugString(); + MS_LOG(INFO) << "Index: " << j << ", nodes: " << graph_inputs[j]->DebugString(); } } - MS_LOG(WARNING) << "real_args_size: " << real_args_size << ", graph_inputs.size(): " << graph_inputs.size() + MS_LOG(WARNING) << "Real_args_size: " << real_args_size << ", graph_inputs.size(): " << graph_inputs.size() << " not equal"; } return real_args; @@ -156,7 +158,7 @@ std::vector GetCNodes(const std::vector &anf_nodes) { std::vector cnodes = {}; size_t i = 0; for (const auto &anf : anf_nodes) { - MS_LOG(INFO) << "apply_list[" << i++ << "] = " << anf->DebugString(); + MS_LOG(INFO) << "Apply_list[" << i++ << "] = " << anf->DebugString(); MS_EXCEPTION_IF_NULL(anf); if (anf->isa()) { cnodes.push_back(anf->cast()); @@ -200,7 +202,8 @@ static std::vector> GetChildList(const std::vector ¶meters, const std::vector &args, - KernelGraph *child_graph) { + const KernelGraphPtr &graph, KernelGraphPtr child_graph, + const NotNull *> memo) { MS_EXCEPTION_IF_NULL(child_graph); MS_LOG(INFO) << "Start bind parameter of child graph:" << child_graph->graph_id(); if (args.empty()) { @@ -212,18 +215,25 @@ static void BindCallArgsWithParameter(const std::vector ¶meters, } child_graph->SetExecOrderByDefault(); for (size_t i = 0; i < parameters.size(); i++) { + MS_LOG(INFO) << "parameters[" << i << "]" << parameters[i]->DebugString() << ",args[" << i << "]" + << args[i]->DebugString(); if (args[i] == parameters[i]) { - child_graph->SetRealInput(parameters[i], args[i]); MS_LOG(INFO) << "Parameter and arg are same."; continue; } child_graph->SetRealInput(parameters[i], args[i]); + if (memo->find(child_graph) != memo->end() || !args[i]->isa()) { + MS_LOG(INFO) << "Add unreused arg,graph:" << graph->graph_id(); + child_graph->AddUnreuseArgs(args[i], graph); + } } } // if a call has kernel input, it's a child graph split from ME, so these kernel input should be set into real input of // graph.For example, call input = (prim,graph,kernel1,kernel2),then real_input = [kernel1,kernel2] -static void UpdateRealInput(NotNull graph) { +static void UpdateRealInput(NotNull graph, bool split_flag, + const NotNull *> memo) { + MS_EXCEPTION_IF_NULL(memo.get()); auto call_nodes = graph->FindNodeByPrimitive(prim::kPrimCall); for (auto &call_node : call_nodes) { MS_EXCEPTION_IF_NULL(call_node); @@ -233,8 +243,10 @@ static void UpdateRealInput(NotNull graph) { std::vector real_args = std::vector(call_node->inputs().begin() + 2, call_node->inputs().end()); std::vector child_inputs = child_graphs[0]->inputs(); - BindCallArgsWithParameter(child_inputs, real_args, child_graphs[0].get()); - call_node->set_inputs(std::vector(call_node->inputs().begin(), call_node->inputs().begin() + 2)); + BindCallArgsWithParameter(child_inputs, real_args, graph, child_graphs[0], memo); + if (split_flag) { + call_node->set_inputs(std::vector(call_node->inputs().begin(), call_node->inputs().begin() + 2)); + } } else if (child_graphs.size() == 2) { auto get_partial_args = [&](size_t input_index) -> std::vector { auto switch_node = call_node->input(1); @@ -243,15 +255,20 @@ static void UpdateRealInput(NotNull graph) { MS_EXCEPTION_IF_NULL(switch_cnode); auto partial = switch_cnode->input(input_index); MS_EXCEPTION_IF_NULL(partial); + if (IsValueNode(partial)) { + return {}; + } auto partial_cnode = partial->cast(); MS_EXCEPTION_IF_NULL(partial_cnode); auto ret = std::vector(partial_cnode->inputs().begin() + 2, partial_cnode->inputs().end()); - partial_cnode->set_inputs( - std::vector(partial_cnode->inputs().begin(), partial_cnode->inputs().begin() + 2)); + if (split_flag) { + partial_cnode->set_inputs( + std::vector(partial_cnode->inputs().begin(), partial_cnode->inputs().begin() + 2)); + } return ret; }; - BindCallArgsWithParameter(child_graphs[0]->inputs(), get_partial_args(2), child_graphs[0].get()); - BindCallArgsWithParameter(child_graphs[1]->inputs(), get_partial_args(3), child_graphs[1].get()); + BindCallArgsWithParameter(child_graphs[0]->inputs(), get_partial_args(2), graph, child_graphs[0], memo); + BindCallArgsWithParameter(child_graphs[1]->inputs(), get_partial_args(3), graph, child_graphs[1], memo); } } } @@ -274,7 +291,7 @@ static void RecurseToUpdateCallRealInput(NotNull graph, } // namespace GraphId AscendSession::CompileGraph(const AnfNodePtrList &lst, const AnfNodePtrList &outputs) { - MS_LOG(INFO) << "start"; + MS_LOG(INFO) << "Start"; // construct graph, if successfully, graph_sum_ + 1 auto graph = ConstructKernelGraph(lst, outputs); auto graph_id = graph->graph_id(); @@ -283,18 +300,23 @@ GraphId AscendSession::CompileGraph(const AnfNodePtrList &lst, const AnfNodePtrL } GraphId AscendSession::CompileGraph(NotNull func_graph) { - MS_LOG(INFO) << "start"; + MS_LOG(INFO) << "Start"; std::vector all_graphs; auto root_graph = ConstructKernelGraph(func_graph, &all_graphs); BackendOptimization(all_graphs); + // empty graph dont entry to backend + if (root_graph->execution_order().empty()) { + MS_LOG(INFO) << root_graph->ToString() << " is empty graph."; + root_graph->set_executable(false); + InitRuntimeResource(); + return root_graph->graph_id(); + } // split switch SplitGraphs(NOT_NULL(root_graph)); // insert goto labels and label_sets LinkChildGraphs(NOT_NULL(root_graph)); // resource initialize InitRuntimeResource(); - // assign label - AssignLabel(NOT_NULL(root_graph)); // recurse compile child root_graph std::set memo; RecurseCompileGraph(NOT_NULL(root_graph), NOT_NULL(&memo)); @@ -303,7 +325,7 @@ GraphId AscendSession::CompileGraph(NotNull func_graph) { // adjust kernel AdjustKernel(root_graph); // assign stream - AssignStream(root_graph); + AssignStream(NOT_NULL(root_graph)); // insert profiling point device::KernelAdjust::GetInstance().Profiling(NOT_NULL(root_graph.get())); // build kernel @@ -336,7 +358,7 @@ void AscendSession::SetFinalGraphSummaryFlag(const std::shared_ptr } void AscendSession::BuildGraph(GraphId graph_id) { - MS_LOG(INFO) << "start"; + MS_LOG(INFO) << "Start"; auto graph = GetGraph(graph_id); MS_EXCEPTION_IF_NULL(graph); // resource initialize @@ -367,6 +389,7 @@ void AscendSession::BuildGraph(GraphId graph_id) { MergeGraphExecOrder(); } else { auto single_graph = GetGraph(graph_id); + MS_EXCEPTION_IF_NULL(single_graph); CompileChildGraph(single_graph); // set the distinction label of single graph single_graph->set_stream_distinction_label(graph_id); @@ -375,7 +398,7 @@ void AscendSession::BuildGraph(GraphId graph_id) { // adjust execution order because merge child graph and other special operations AdjustKernel(graph); // Assign streams for control sink and hccl and so on - AssignStream(graph); + AssignStream(NOT_NULL(graph)); device::KernelAdjust::GetInstance().Profiling(NOT_NULL(graph.get())); // build kernel if node is cnode @@ -395,7 +418,7 @@ void AscendSession::BuildGraph(GraphId graph_id) { // sync the inital const tensor to device SyncInitialTenosrToDevice(); ExportChildGraphs(graph_id); - MS_LOG(INFO) << "end"; + MS_LOG(INFO) << "End"; } void AscendSession::CompileChildGraph(const KernelGraphPtr &child_graph) { @@ -437,7 +460,7 @@ void AscendSession::CompileChildGraph(const KernelGraphPtr &child_graph) { void AscendSession::RunGraph(const GraphId &graph_id, const std::vector &inputs, VectorRef *const outputs) { - MS_LOG(INFO) << "start"; + MS_LOG(INFO) << "Start"; auto kernel_graph = GetGraph(graph_id); MS_EXCEPTION_IF_NULL(kernel_graph); // if none of child graph and no anf output exists @@ -450,6 +473,12 @@ void AscendSession::RunGraph(const GraphId &graph_id, const std::vectorPreExecute(kernel_graph); + } +#endif { py::gil_scoped_release release; // run task on device @@ -459,8 +488,20 @@ void AscendSession::RunGraph(const GraphId &graph_id, const std::vectordebugger_enabled()) { + LoadTensor(kernel_graph); + } +#endif // dump used for debug Dump(kernel_graph); +#ifdef ENABLE_DEBUGGER + // debugger post-execution processing + if (debugger_) { + debugger_->PostExecute(); + } +#endif MS_LOG(INFO) << "Finish!"; } @@ -479,7 +520,7 @@ void AscendSession::RunOpExecTask(const std::shared_ptr &kernel_gra MS_EXCEPTION_IF_NULL(runtime_instance); bool ret_ok = runtime_instance->LaunchKernel(kernel_graph.get()); if (!ret_ok) { - MS_LOG(EXCEPTION) << "run task error!"; + MS_LOG(EXCEPTION) << "Run task error!"; } MS_LOG(INFO) << "Finish!"; } @@ -627,18 +668,12 @@ void AscendSession::RunOpAdjustKernel(const std::shared_ptr &kernel MS_LOG(INFO) << "Finish!"; } -void AscendSession::AssignStream(const std::shared_ptr &kernel_graph) const { +void AscendSession::AssignStream(NotNull kernel_graph) const { MS_LOG(INFO) << "Start!"; device::ascend::AscendStreamAssign::GetInstance().AssignStream(kernel_graph); MS_LOG(INFO) << "Finish!"; } -void AscendSession::AssignLabel(NotNull kernel_graph) const { - MS_LOG(INFO) << "Start!"; - device::ascend::AscendLabelAssign::GetInstance().AssignLabel(kernel_graph); - MS_LOG(INFO) << "Finish!"; -} - void AscendSession::BuildKernel(const std::shared_ptr &kernel_graph) const { MS_LOG(INFO) << "Start!"; struct timeval start_time, end_time; @@ -676,7 +711,7 @@ void AscendSession::RunOpMemoryAlloc(const std::vector &input MS_LOG(INFO) << "Finish!"; } -void AscendSession::RunOpMemoryClear(KernelGraph *kernel_graph) const { +void AscendSession::RunOpMemoryClear(const KernelGraph *kernel_graph) const { MS_EXCEPTION_IF_NULL(kernel_graph); auto runtime_instance = device::KernelRuntimeManager::Instance().GetKernelRuntime(kAscendDevice, device_id_); MS_EXCEPTION_IF_NULL(runtime_instance); @@ -757,9 +792,26 @@ void AscendSession::ExportChildGraphs(const GraphId graph_id) { #endif } +void AscendSession::LoadTensor(const std::shared_ptr &kernel_graph) const { + MS_LOG(INFO) << "Start!"; + MS_EXCEPTION_IF_NULL(kernel_graph); +#ifdef ENABLE_DEBUGGER + auto runtime_instance = device::KernelRuntimeManager::Instance().GetKernelRuntime(kAscendDevice, device_id_); + MS_EXCEPTION_IF_NULL(runtime_instance); + DebugServices *debug_services = debugger_->get_debug_services(); + TensorLoader *tensor_loader = debug_services->get_tensor_loader(); + tensor_loader->EmptyTensor(); + uint32_t iter_num = tensor_loader->GetIterNum(); + tensor_loader->set_iter_num(++iter_num); + (void)runtime_instance->LoadData(kernel_graph.get(), debugger_.get()); +#endif + MS_LOG(INFO) << "Finish!"; +} + GraphId AscendSession::SetFinalGraphInput(const std::vector &args) { MS_LOG(INFO) << "Start! Args size " << args.size(); auto final_graph = NewKernelGraph(); + MS_EXCEPTION_IF_NULL(final_graph); final_graph_id_ = final_graph->graph_id(); MS_LOG(INFO) << "Create a new final graph" << final_graph_id_ << " success"; // init private variables and bind them with final_graph_id @@ -790,7 +842,7 @@ GraphId AscendSession::SetFinalGraphInput(const std::vector &args) { final_graph_inputs->push_back(parameter_backend); } MS_EXCEPTION_IF_NULL(parameter_backend); - MS_LOG(INFO) << "parameter backend " << parameter_backend->DebugString() << " belong_graph_id " + MS_LOG(INFO) << "Parameter backend " << parameter_backend->DebugString() << " belong_graph_id " << AnfAlgo::GetGraphId(parameter_backend.get()); } MS_LOG(INFO) << "End final_graph_id " << final_graph_id_; @@ -806,7 +858,7 @@ void AscendSession::RecurseGetSummaryNodes(KernelGraph *graph, if (graph_order_iter == graph_execute_orders_.end()) { SessionBasic::GetSummaryNodes(graph); auto summary_nodes = graph->summary_nodes(); - (*summary).insert(summary_nodes.begin(), summary_nodes.end()); + summary->insert(summary_nodes.begin(), summary_nodes.end()); return; } // for every child graph, find summary nodes @@ -818,7 +870,7 @@ void AscendSession::RecurseGetSummaryNodes(KernelGraph *graph, } SessionBasic::GetSummaryNodes(child_graph.get()); auto child_graph_summary = child_graph->summary_nodes(); - (*summary).insert(child_graph_summary.begin(), child_graph_summary.end()); + summary->insert(child_graph_summary.begin(), child_graph_summary.end()); RecurseGetSummaryNodes(child_graph.get(), summary); } graph->set_summary_nodes(*summary); @@ -905,6 +957,7 @@ void AscendSession::SetFinalGraphOutput(const ValuePtr &value) { MS_EXCEPTION_IF_NULL(final_graph); final_graph->set_output(final_graph->NewCNode({NewValueNode(prim::kPrimMakeTuple), value_node})); final_graph->set_executable(false); + MS_EXCEPTION_IF_NULL(value); MS_LOG(INFO) << "Not anf output[" << value->ToString() << "]"; } @@ -937,15 +990,6 @@ void AscendSession::SetFinalGraphOutput(const BaseRef &output) { } } -KernelGraphPtr AscendSession::GetGraph(mindspore::GraphId graph_id) { - auto it = graphs_.find(graph_id); - if (it == graphs_.end()) { - MS_LOG(WARNING) << "Can't find graph " << graph_id; - return nullptr; - } - return it->second; -} - void AscendSession::InsertSwitchToGraph(GraphId condition_graph_id, GraphId true_graph_id) { MS_LOG(INFO) << "Start!"; MS_LOG(INFO) << "Condition graph id[" << condition_graph_id << "],true graph id[" << true_graph_id << "]"; @@ -953,7 +997,7 @@ void AscendSession::InsertSwitchToGraph(GraphId condition_graph_id, GraphId true MS_EXCEPTION_IF_NULL(condition_graph); tensor::TensorPtr tensor = std::make_shared(kNumberTypeInt32, std::vector{1}); int32_t *val = nullptr; - val = static_cast(tensor->data_c(true)); + val = static_cast(tensor->data_c()); MS_EXCEPTION_IF_NULL(val); *val = 0; auto value_node = std::make_shared(tensor); @@ -1011,6 +1055,7 @@ void AscendSession::CopyOutputOfIf(GraphId false_graph_id) { MS_LOG(INFO) << "The last graph of false branch is " << false_graph_id; // create fake output auto fake_output_graph = NewKernelGraph(); + MS_EXCEPTION_IF_NULL(fake_output_graph); graph_execute_order.push_back(fake_output_graph->graph_id()); graph_order_type.push_back(COMMON_GRAPH); fake_output_graph->set_output(CreateFakeOutput(fake_output_graph->graph_id(), final_graph->output())); @@ -1043,7 +1088,7 @@ void AscendSession::SwitchCompile(GraphId cond_graph_id, GraphId true_graph_id, auto cond_graph_index = ExecOrderOfChildGraph(final_graph_id_, cond_graph_id); auto &graph_order_type = GetGraphOrderType(final_graph_id_); if (cond_graph_index >= graph_order_type.size()) { - MS_LOG(EXCEPTION) << "cond_graph_index " << cond_graph_index << " out of range " << graph_order_types_.size(); + MS_LOG(EXCEPTION) << "Cond_graph_index " << cond_graph_index << " out of range " << graph_order_types_.size(); } graph_order_type[cond_graph_index] = CONDITION_GRAPH; // update distinction label of false graph,update before merge to sure the distinction @@ -1120,7 +1165,7 @@ void AscendSession::InsertAllAssigns() { MS_EXCEPTION_IF_NULL(to_graph); std::vector graph_inputs = to_graph->inputs(); if (input_idx >= graph_inputs.size()) { - MS_LOG(EXCEPTION) << "input_index " << input_idx << " out of range size " << graph_inputs.size(); + MS_LOG(EXCEPTION) << "Input_index " << input_idx << " out of range size " << graph_inputs.size(); } auto backend_parameter = graph_inputs[input_idx]; assigns.emplace_back(std::pair(front_anf, backend_parameter)); @@ -1144,7 +1189,7 @@ void AscendSession::InsertAllAssigns() { // insert active to graph void AscendSession::SetActive(GraphId from, GraphId to) { if (while_condition_graphs_.find(to) != while_condition_graphs_.end()) { - MS_LOG(WARNING) << " to " << to << " has been exits in map,from " << from << ",exist from " + MS_LOG(WARNING) << "To " << to << " has been exits in map,from " << from << ",exist from " << while_condition_graphs_[to]; return; } @@ -1180,7 +1225,7 @@ void AscendSession::SetChildGraphParameter(const AnfNodePtr &front_anf, GraphId MS_EXCEPTION_IF_NULL(to_graph); std::vector graph_inputs = to_graph->inputs(); if (input_idx >= graph_inputs.size()) { - MS_LOG(EXCEPTION) << "input_index " << input_idx << " out of range size " << graph_inputs.size(); + MS_LOG(EXCEPTION) << "Input_index " << input_idx << " out of range size " << graph_inputs.size(); } auto backend_parameter = graph_inputs[input_idx]; MS_EXCEPTION_IF_NULL(backend_parameter); @@ -1214,7 +1259,7 @@ void AscendSession::SetChildGraphParameter(const tensor::TensorPtr &front_tensor } void AscendSession::UpdateGraphOrder(GraphId to_graph_id) { - MS_LOG(INFO) << "to_graph_id " << to_graph_id; + MS_LOG(INFO) << "To_graph_id " << to_graph_id; auto &graph_order = GetGraphOrder(final_graph_id_); auto &graph_type = GetGraphOrderType(final_graph_id_); for (size_t i = 0; i < graph_order.size(); i++) { @@ -1232,6 +1277,8 @@ void AscendSession::UpdateGraphOrder(GraphId to_graph_id) { } size_t AscendSession::SetChildGraphInput(const KernelGraphPtr &graph, const AnfNodePtr &node, size_t input_index) { + MS_EXCEPTION_IF_NULL(graph); + MS_EXCEPTION_IF_NULL(node); auto output_num = AnfAlgo::GetOutputTensorNum(node); if (output_num > 1 && !AnfAlgo::CheckPrimitiveType(node, prim::kPrimTupleGetItem)) { return input_index + output_num; @@ -1246,6 +1293,7 @@ size_t AscendSession::SetChildGraphInput(const KernelGraphPtr &graph, const AnfN } size_t AscendSession::SetChildGraphInput(const KernelGraphPtr &graph, const ValuePtr &value, size_t input_index) { + MS_EXCEPTION_IF_NULL(graph); MS_EXCEPTION_IF_NULL(value); if (!value->isa()) { MS_LOG(EXCEPTION) << "Value Node should be a tensor, unexpected value: " << value->ToString(); @@ -1283,7 +1331,7 @@ void AscendSession::SetChildGraphInput(GraphId g, const VectorRef &args) { size_t input_index = 0; for (size_t i = 0; i < real_args.size(); i++) { if (input_index >= graph_inputs.size()) { - MS_LOG(EXCEPTION) << "input_index " << input_index << " out of range size " << graph_inputs.size(); + MS_LOG(EXCEPTION) << "Input_index " << input_index << " out of range size " << graph_inputs.size(); } auto &real_arg = real_args[i]; if (utils::isa(real_arg)) { @@ -1315,7 +1363,7 @@ GraphId AscendSession::GetGraphIdByNode(const AnfNodePtr &front_anf) const { } } MS_EXCEPTION_IF_NULL(front_anf); - MS_LOG(DEBUG) << "front_anf " << front_anf->DebugString() << " is not exist in any graph"; + MS_LOG(DEBUG) << "Front_anf " << front_anf->DebugString() << " is not exist in any graph"; return kInvalidGraphId; } @@ -1478,7 +1526,7 @@ void AscendSession::SyncInitialTenosrToDevice() { MS_EXCEPTION_IF_NULL(to_graph); std::vector graph_inputs = to_graph->inputs(); if (input_idx >= graph_inputs.size()) { - MS_LOG(EXCEPTION) << "input_index " << input_idx << " out of range size " << graph_inputs.size(); + MS_LOG(EXCEPTION) << "Input_index " << input_idx << " out of range size " << graph_inputs.size(); } auto backend_parameter = graph_inputs[input_idx]; // sync data from host to device @@ -1487,7 +1535,7 @@ void AscendSession::SyncInitialTenosrToDevice() { auto addr = AnfAlgo::GetOutputAddr(backend_parameter, 0); MS_EXCEPTION_IF_NULL(addr); if (!addr->SyncHostToDevice(trans::GetRuntimePaddingShape(backend_parameter, 0), tensor_size, - front_tensor->data_type(), front_tensor->data_c(false))) { + front_tensor->data_type(), front_tensor->data_c())) { MS_LOG(EXCEPTION) << "Tensor SyncHostToDevice fail!"; } } @@ -1512,6 +1560,7 @@ static void ConstructSplitedGraphOutput(const KernelGraphPtr &new_kernel_graph, new_kernel_graph->set_return(anf_node); } if (has_output_nodes.find(anf_node) == has_output_nodes.end()) { + MS_EXCEPTION_IF_NULL(anf_node); MS_LOG(INFO) << "Output[" << output_idx++ << "]:" << anf_node->DebugString(); make_tuple_inputs.push_back(anf_node); } @@ -1524,7 +1573,7 @@ static void ConstructSplitedGraphOutput(const KernelGraphPtr &new_kernel_graph, std::vector AscendSession::ConstructSplitedGraph(const KernelGraphPtr &new_kernel_graph, const std::vector &list) { MS_EXCEPTION_IF_NULL(new_kernel_graph); - MS_LOG(INFO) << "start contruct splited kernel graph:" << new_kernel_graph->graph_id(); + MS_LOG(INFO) << "Start contruct splited kernel graph:" << new_kernel_graph->graph_id(); MS_LOG(INFO) << "Construct input of kernel graph:" << new_kernel_graph->graph_id(); std::vector call_node_inputs; std::vector new_graph_inputs; @@ -1536,14 +1585,17 @@ std::vector AscendSession::ConstructSplitedGraph(const KernelGraphPt auto input = cnode->inputs()[input_idx]; MS_EXCEPTION_IF_NULL(input); AnfNodePtr new_parameter = nullptr; + // check whether input has been put into args of call, if mulptiple use of one parameter or cnode, only set one + // parameter in graph inputs and one arg in call node + auto call_input_it = std::find(call_node_inputs.begin(), call_node_inputs.end(), input); + if (call_input_it != call_node_inputs.end()) { + cnode->set_input(input_idx, new_graph_inputs[std::distance(call_node_inputs.begin(), call_input_it)]); + continue; + } // value node consider move to new graph if (input->isa()) { cnode->set_input(input_idx, input); continue; - } else if (input->isa()) { - // parameter reuse and should attention mulptiple use of one parameter - cnode->set_input(input_idx, input); - new_parameter = input; } else if (AnfAlgo::GetGraphId(input.get()) != new_kernel_graph->graph_id()) { // if is cnode and not in current child graph new_parameter = CreateNewParameterFromCNode(input, true, new_kernel_graph.get()); @@ -1552,12 +1604,8 @@ std::vector AscendSession::ConstructSplitedGraph(const KernelGraphPt // if is a cnode and in current graph continue; } - // if mulptiple use of one parameter or cnode, only set one parameter in graph inputs and one arg in call node - // args - if (std::find(call_node_inputs.begin(), call_node_inputs.end(), new_parameter) == call_node_inputs.end()) { - new_graph_inputs.push_back(new_parameter); - call_node_inputs.push_back(input); - } + new_graph_inputs.push_back(new_parameter); + call_node_inputs.push_back(input); } } // set graph inputs of new graph @@ -1568,7 +1616,7 @@ std::vector AscendSession::ConstructSplitedGraph(const KernelGraphPt MS_LOG(INFO) << "Construct output of kernel graph:" << new_kernel_graph->graph_id(); ConstructSplitedGraphOutput(new_kernel_graph, list); - MS_LOG(INFO) << "end"; + MS_LOG(INFO) << "End"; return call_node_inputs; } @@ -1585,7 +1633,7 @@ void AscendSession::SplitGraphs(NotNull root_graph) { // if root graph output is a call node ,the root graph is condition graph of 'if' sentence auto root_graph_output = AnfAlgo::VisitKernelWithReturnType(root_graph->output(), 0).first; if (AnfAlgo::CheckPrimitiveType(root_graph_output, prim::kPrimCall)) { - SplitGraph(root_graph, {prim::kPrimReturn}); + SplitGraph(root_graph, {prim::kPrimReturn}, NOT_NULL(&memo)); for (auto &child_graph : root_graph->child_graph_order()) { RecurseSplitGraph(NOT_NULL(child_graph), NOT_NULL(&memo)); } @@ -1626,8 +1674,10 @@ AnfNodePtr AscendSession::BindNewCallToNewGraph(NotNull graph, return new_call; } -void AscendSession::SplitGraph(NotNull graph, const std::set &cut_prims) { +void AscendSession::SplitGraph(NotNull graph, const std::set &cut_prims, + const NotNull *> memo) { MS_LOG(INFO) << "Start,graph_id:" << graph->graph_id(); + bool split_flag = false; auto apply_list = GetCNodes(TopoSort(graph->get_return())); // update the root graph child graph order AscendControlParser::UpdateChildGraphOrder(graph); @@ -1660,16 +1710,16 @@ void AscendSession::SplitGraph(NotNull graph, const std::setgraph_id() << "] end"; - // recurse to split child graph + UpdateRealInput(graph, split_flag, memo); + MS_LOG(INFO) << "Split graph[" << graph->graph_id() << "] end"; } void AscendSession::RecurseSplitGraph(NotNull graph, const NotNull *> memo) { memo->insert(graph.get()); - SplitGraph(graph, {prim::kPrimCall}); + SplitGraph(graph, {prim::kPrimCall}, memo); for (auto &child_graph : graph->child_graph_order()) { if (memo->find(child_graph) == memo->end()) { RecurseSplitGraph(NOT_NULL(child_graph), memo); @@ -1691,6 +1741,14 @@ void AscendSession::RecurseCompileGraph(NotNull graph, const Not continue; } RecurseCompileGraph(NOT_NULL(child_graph), memo); + // copy ref map to final graph + auto child_ref_map = child_graph->GetRefMap(); + for (auto &item : child_ref_map) { + if (graph->IsInRefOutputMap(item.first)) { + MS_LOG(EXCEPTION) << "The ref pair is already in final graph!"; + } + graph->AddRefCorrespondPairs(item.first, item.second); + } } } } // namespace session diff --git a/mindspore/ccsrc/session/ascend_session.h b/mindspore/ccsrc/session/ascend_session.h index 7857330115..531860c379 100755 --- a/mindspore/ccsrc/session/ascend_session.h +++ b/mindspore/ccsrc/session/ascend_session.h @@ -76,17 +76,17 @@ class AscendSession : public SessionBasic { void HardwareOptimize(const std::shared_ptr &kernel_graph) const; void AdjustKernel(const std::shared_ptr &kernel_graph) const; void RunOpAdjustKernel(const std::shared_ptr &kernel_graph) const; - void AssignStream(const std::shared_ptr &kernel_graph) const; - void AssignLabel(NotNull kernel_graph) const; + void AssignStream(NotNull kernel_graph) const; void BuildKernel(const std::shared_ptr &kernel_graph) const; void MemoryAlloc(KernelGraph *kernel_graph) const; void RunOpMemoryAlloc(const std::vector &input_tensors, KernelGraph *kernel_graph) const; - void RunOpMemoryClear(KernelGraph *kernel_graph) const; + void RunOpMemoryClear(const KernelGraph *kernel_graph) const; void GenerateTaskInfo(const std::shared_ptr &kernel_graph) const; void LoadTask(const std::shared_ptr &kernel_graph) const; void ExecTask(const std::shared_ptr &kernel_graph) const; void Dump(const std::shared_ptr &kernel_graph) const; void ExportChildGraphs(const GraphId graph_id); + void LoadTensor(const std::shared_ptr &kernel_graph) const; // below functions are used for run op void RunOpHardwareOptimize(const std::shared_ptr &kernel_graph) const; void RunOpExecTask(const std::shared_ptr &kernel_graph) const; @@ -99,7 +99,8 @@ class AscendSession : public SessionBasic { void SetFinalGraphOutput(const ValuePtr &value); void SetFinalGraphOutput(const VectorRef &vec_output); - void SplitGraph(NotNull graph, const std::set &cut_prims); + void SplitGraph(NotNull graph, const std::set &cut_prims, + const NotNull *> memo); // split graphs with recurse from root graph void SplitGraphs(NotNull root_graph); void BackendOptimization(const std::vector &all_graphs); @@ -127,8 +128,6 @@ class AscendSession : public SessionBasic { void InsertDependToGraph(GraphId graph_id, const AnfNodePtr &attch_node); // insert depend to graph, used to attch control nodes to graph void InsertControlDependToGraph(GraphId graph_id, const AnfNodePtr &first_node, const AnfNodePtr &second_node); - // Get graph by graph id ,if not exist return null ptr - KernelGraphPtr GetGraph(GraphId graph_id); // set child graph parameter if front arg is a anf void SetChildGraphParameter(const AnfNodePtr &front_anf, GraphId to_graph_id, size_t input_idx); // set child graph parameter if front arg is a tensor diff --git a/mindspore/ccsrc/session/cpu_session.cc b/mindspore/ccsrc/session/cpu_session.cc index e70e551022..1927df2f49 100644 --- a/mindspore/ccsrc/session/cpu_session.cc +++ b/mindspore/ccsrc/session/cpu_session.cc @@ -25,11 +25,15 @@ #include "predict/predict.h" #include "kernel/cpu/cpu_kernel_factory.h" #include "device/cpu/kernel_select_cpu.h" +#ifdef ENABLE_DEBUGGER +#include "debug/debugger/debugger.h" +#endif namespace mindspore { namespace session { ParameterPtr CPUSession::CreateNewParameterFromParameter(const AnfNodePtr &anf, bool valid_input, KernelGraph *graph) { MS_EXCEPTION_IF_NULL(anf); + MS_EXCEPTION_IF_NULL(graph); if (!anf->isa()) { MS_LOG(EXCEPTION) << "anf[" << anf->DebugString() << "] is not a parameter"; } @@ -78,7 +82,12 @@ void CPUSession::RunGraph(const GraphId &graph_id, const std::vectorsummary_nodes(); runtime_.IncreaseSummaryRefCount(summary_outputs); } - +#ifdef ENABLE_DEBUGGER + // debugger pre-execution processing + if (debugger_) { + debugger_->PreExecute(kernel_graph); + } +#endif bool ret = runtime_.Run(kernel_graph.get()); if (!ret) { MS_LOG(EXCEPTION) << "Run graph failed"; @@ -92,6 +101,12 @@ void CPUSession::RunGraph(const GraphId &graph_id, const std::vectorPostExecute(); + } +#endif MS_LOG(INFO) << "Run graph end"; } diff --git a/mindspore/ccsrc/session/gpu_session.cc b/mindspore/ccsrc/session/gpu_session.cc index e67a922567..7765e93758 100644 --- a/mindspore/ccsrc/session/gpu_session.cc +++ b/mindspore/ccsrc/session/gpu_session.cc @@ -23,6 +23,8 @@ #include "pre_activate/common/helper.h" #include "pre_activate/pass/communication_op_fusion.h" #include "pre_activate/pass/getitem_tuple.h" +#include "pre_activate/gpu/adam_weight_decay_fusion.h" +#include "pre_activate/gpu/adam_fusion.h" #include "device/kernel_runtime_manager.h" #include "predict/predict.h" #include "common/utils.h" @@ -53,6 +55,16 @@ void GPUSession::StartKernelRT() const { void GPUSession::Optimize(const std::shared_ptr &kernel_graph) { MS_EXCEPTION_IF_NULL(kernel_graph); + auto optimizer = std::make_shared(); + auto pm = std::make_shared(); + pm->AddPass(std::make_shared()); + pm->AddPass(std::make_shared()); + optimizer->AddPassManager(pm); + (void)optimizer->Optimize(kernel_graph); + kernel_graph->SetExecOrderByDefault(); +} + +void GPUSession::HardwareOptimize(const std::shared_ptr &kernel_graph) { auto optimizer = std::make_shared(); auto pm = std::make_shared(); pm->AddPass(std::make_shared()); @@ -112,10 +124,10 @@ void GPUSession::LoadInputData(const std::shared_ptr &kernel_graph, auto tensor_address = tensor->device_address(); bool need_sync = false; if (ms_context->enable_pynative_infer()) { - if (tensor_address.get() == nullptr || tensor_address != device_address) { + if (tensor_address == nullptr || tensor_address != device_address) { need_sync = true; } - } else if (tensor->is_dirty()) { + } else if (tensor->is_dirty() || tensor_address == nullptr) { need_sync = true; } else if (tensor_address != device_address) { if (tensor_address->DeviceType() == device_address->DeviceType()) { @@ -129,7 +141,7 @@ void GPUSession::LoadInputData(const std::shared_ptr &kernel_graph, MS_EXCEPTION_IF_NULL(device_address); if (!device_address->SyncHostToDevice(trans::GetRuntimePaddingShape(pk_node, 0), LongToSize(tensor->data().nbytes()), tensor->data_type(), - tensor->data_c(false))) { + tensor->data_c())) { MS_LOG(EXCEPTION) << "SyncHostToDevice failed."; } } @@ -151,14 +163,16 @@ GraphId GPUSession::CompileGraph(const AnfNodePtrList &lst, const AnfNodePtrList auto graph_id = graph_sum_; auto graph = ConstructKernelGraph(lst, outputs); MS_EXCEPTION_IF_NULL(graph); + // Optimize + Optimize(graph); // Select kernel build info SelectKernel(graph); // Convert kernel Graph to model predictmodel::StepConvertGraph(graph); // Start gpu kernel runtime StartKernelRT(); - // AllReduce Optimize - Optimize(graph); + // HardwareOptimize + HardwareOptimize(graph); // Assign CUDA streams AssignStream(graph); // Hide NoOp from execution graph diff --git a/mindspore/ccsrc/session/gpu_session.h b/mindspore/ccsrc/session/gpu_session.h index 0dfb815abe..4e46c2138d 100644 --- a/mindspore/ccsrc/session/gpu_session.h +++ b/mindspore/ccsrc/session/gpu_session.h @@ -51,6 +51,8 @@ class GPUSession : public SessionBasic { void Optimize(const std::shared_ptr &kernel_graph); + void HardwareOptimize(const std::shared_ptr &kernel_graph); + void AssignStream(const std::shared_ptr &kernel_graph); void BuildKernel(const std::shared_ptr &kernel_graph) const; diff --git a/mindspore/ccsrc/session/kernel_graph.cc b/mindspore/ccsrc/session/kernel_graph.cc index 7e9bb62aab..264e2c661b 100644 --- a/mindspore/ccsrc/session/kernel_graph.cc +++ b/mindspore/ccsrc/session/kernel_graph.cc @@ -33,6 +33,7 @@ constexpr auto kIsFeatureMapOutput = "IsFeatureMapOutput"; constexpr auto kIsFeatureMapInputList = "IsFeatureMapInputList"; void PushNoVisitedNode(const AnfNodePtr &node, std::queue *que, std::unordered_set *visited_nodes) { + MS_EXCEPTION_IF_NULL(node); MS_EXCEPTION_IF_NULL(que); MS_EXCEPTION_IF_NULL(visited_nodes); if (visited_nodes->find(node) == visited_nodes->end()) { @@ -43,7 +44,8 @@ void PushNoVisitedNode(const AnfNodePtr &node, std::queue *que, } std::vector GetCallRealOutputs(const AnfNodePtr &call_node) { - auto item_with_index = AnfAlgo::VisitKernelWithReturnType(call_node, 0); + auto item_with_index = + AnfAlgo::VisitKernelWithReturnType(call_node, 0, false, {prim::kPrimTupleGetItem, prim::kPrimMakeTuple}); AnfNodePtr node = item_with_index.first; MS_EXCEPTION_IF_NULL(node); if (AnfAlgo::CheckPrimitiveType(node, prim::kPrimMakeTuple)) { @@ -101,6 +103,23 @@ AnfNodePtr MakeValueNode(const AnfNodePtr &node) { AnfAlgo::SetSelectKernelBuildInfo(kernel_build_info_builder->Build(), new_value_node.get()); return new_value_node; } + +bool IsSameLabel(const CNodePtr &left, const CNodePtr &right) { + if (left == right) { + return true; + } + if (left == nullptr || right == nullptr) { + return false; + } + if (!IsPrimitiveCNode(left, GetCNodePrimitive(right))) { + return false; + } + if (AnfAlgo::HasNodeAttr(kAttrLabelIndex, left) && AnfAlgo::HasNodeAttr(kAttrLabelIndex, right)) { + return AnfAlgo::GetNodeAttr(left, kAttrLabelIndex) == + AnfAlgo::GetNodeAttr(right, kAttrLabelIndex); + } + return false; +} } // namespace std::vector KernelGraph::outputs() const { auto graph_output = output(); @@ -130,11 +149,10 @@ void KernelGraph::VisitNodeDescendants(const AnfNodePtr &node, std::queue active_nodes; for (const auto &output_edge : it->second) { auto next_node = output_edge.first; + MS_EXCEPTION_IF_NULL(next_node); if (node_input_num_.find(next_node) == node_input_num_.end()) { - MS_EXCEPTION_IF_NULL(next_node); MS_LOG(EXCEPTION) << "Can't find node[" << next_node->DebugString() << "]"; } - MS_EXCEPTION_IF_NULL(next_node); MS_LOG(DEBUG) << "Decrease input:" << next_node->DebugString() << ",node:" << node->DebugString() << ",num: " << node_input_num_[next_node] << ",decrease num:" << output_edge.second; if (node_input_num_[next_node] < output_edge.second) { @@ -146,7 +164,7 @@ void KernelGraph::VisitNodeDescendants(const AnfNodePtr &node, std::queuefind(next_node) == visited_nodes->end()) { (void)visited_nodes->insert(next_node); if (AnfAlgo::IsCommunicationOp(next_node)) { - MS_LOG(DEBUG) << "visit node:" << next_node->DebugString(); + MS_LOG(DEBUG) << "Visit node:" << next_node->DebugString(); visit_queue->push(next_node); } else { active_nodes.emplace_back(next_node); @@ -155,7 +173,8 @@ void KernelGraph::VisitNodeDescendants(const AnfNodePtr &node, std::queueDebugString(); + MS_EXCEPTION_IF_NULL(node); + MS_LOG(DEBUG) << "Visit node:" << node->DebugString(); visit_queue->push(node); } } @@ -217,6 +236,19 @@ void KernelGraph::SetExecOrderByDefault() { if (node == start_label_ || node == end_goto_) { continue; } + + if (IsSameLabel(node, end_goto_)) { + end_goto_ = node; + MS_LOG(INFO) << "Replace end_goto_ in kernel graph:" << graph_id(); + continue; + } + + if (IsSameLabel(node, start_label_)) { + start_label_ = node; + MS_LOG(INFO) << "Replace start_label_ in kernel graph:" << graph_id(); + continue; + } + re_order.push_back(node); } if (end_goto_ != nullptr) { @@ -327,6 +359,9 @@ CNodePtr KernelGraph::NewCNode(const CNodePtr &cnode) { FrontBackendlMapUpdate(cnode, new_cnode); } AnfAlgo::SetGraphId(graph_id_, cnode.get()); + if (IsInternalOutput(cnode)) { + ReplaceInternalOutput(cnode, new_cnode); + } return new_cnode; } @@ -352,18 +387,16 @@ ParameterPtr KernelGraph::NewParameter(const ParameterPtr ¶meter) { } else { kernel_info->SetFeatureMapFlag(true); } - // if output is a tuple tensor,now can use for loop to handle tuple tensor - output_tensor_num = AnfAlgo::GetOutputTensorNum(parameter); } new_parameter->set_kernel_info(kernel_info); // create kernel_build_info for new parameter auto kernel_build_info_builder = std::make_shared(); // create init data type, std::vector init_data_type = {}; - for (size_t i = 0; i < output_tensor_num; i++) { - TypeId infer_data_type = AnfAlgo::GetOutputInferDataType(new_parameter, i); - init_data_type.push_back(AnfAlgo::IsParameterWeight(new_parameter) ? kTypeUnknown : infer_data_type); - } + + TypeId infer_data_type = AnfAlgo::GetOutputInferDataType(new_parameter, 0); + init_data_type.push_back(AnfAlgo::IsParameterWeight(new_parameter) ? kTypeUnknown : infer_data_type); + // set the format of parameter to DEFAULT_FORMAT kernel_build_info_builder->SetOutputsFormat(std::vector(output_tensor_num, kOpFormat_DEFAULT)); // set parameter initaial device data type @@ -379,12 +412,12 @@ std::vector KernelGraph::SplitTupleValueNodeToNodeList(const ValueNo auto output_size = AnfAlgo::GetOutputTensorNum(value_node); std::vector convert_inputs; if (!node_value->isa()) { - MS_LOG(EXCEPTION) << "multiple output valuenode's value must be a value tuple but got " << node_value->ToString(); + MS_LOG(EXCEPTION) << "Multiple output valuenode's value must be a value tuple but got " << node_value->ToString(); } auto value_tuple = node_value->cast(); MS_EXCEPTION_IF_NULL(value_tuple); if (value_tuple->size() != output_size) { - MS_LOG(EXCEPTION) << "value tuple size" << value_tuple->size() + MS_LOG(EXCEPTION) << "Value tuple size" << value_tuple->size() << " is not mathced with the value node's output size" << output_size; } for (size_t index = 0; index < value_tuple->value().size(); ++index) { @@ -407,7 +440,7 @@ std::vector KernelGraph::SplitTupleValueNodeToNodeList(const ValueNo convert_inputs.emplace_back(new_value_node); } if (!RemoveValueNodeFromGraph(value_node)) { - MS_LOG(WARNING) << "failed to remove the value_node " << value_node->DebugString(); + MS_LOG(WARNING) << "Failed to remove the value_node " << value_node->DebugString(); } return convert_inputs; } @@ -428,10 +461,10 @@ void KernelGraph::FrontBackendlMapAdd(const AnfNodePtr &front_anf, const AnfNode MS_EXCEPTION_IF_NULL(front_anf); MS_EXCEPTION_IF_NULL(backend_anf); if (front_backend_anf_map_.find(front_anf) != front_backend_anf_map_.end()) { - MS_LOG(EXCEPTION) << "anf " << front_anf->DebugString() << " has been exist in the front_backend_anf_map_"; + MS_LOG(EXCEPTION) << "Anf " << front_anf->DebugString() << " has been exist in the front_backend_anf_map_"; } if (backend_front_anf_map_.find(backend_anf) != backend_front_anf_map_.end()) { - MS_LOG(EXCEPTION) << "kernel " << backend_anf->DebugString() << "has been exist in the backend_front_anf_map_"; + MS_LOG(EXCEPTION) << "Kernel " << backend_anf->DebugString() << "has been exist in the backend_front_anf_map_"; } front_backend_anf_map_[front_anf] = backend_anf; backend_front_anf_map_[backend_anf] = front_anf; @@ -441,15 +474,15 @@ void KernelGraph::FrontBackendlMapUpdate(const AnfNodePtr &old_backend_anf, cons MS_EXCEPTION_IF_NULL(old_backend_anf); MS_EXCEPTION_IF_NULL(new_backend_anf); if (old_backend_anf == new_backend_anf) { - MS_LOG(DEBUG) << "old same with new:" << old_backend_anf->DebugString(); + MS_LOG(DEBUG) << "Old same with new:" << old_backend_anf->DebugString(); return; } if (backend_front_anf_map_.find(old_backend_anf) == backend_front_anf_map_.end()) { - MS_LOG(DEBUG) << "old_backend_anf " << old_backend_anf->DebugString() << " is not exist in the map"; + MS_LOG(DEBUG) << "Old_backend_anf " << old_backend_anf->DebugString() << " is not exist in the map"; return; } if (front_backend_anf_map_.find(backend_front_anf_map_[old_backend_anf]) == front_backend_anf_map_.end()) { - MS_LOG(EXCEPTION) << "anf is not exist in the map ,old " << old_backend_anf->DebugString(); + MS_LOG(EXCEPTION) << "Anf is not exist in the map ,old " << old_backend_anf->DebugString(); } front_backend_anf_map_[backend_front_anf_map_[old_backend_anf]] = new_backend_anf; backend_front_anf_map_[new_backend_anf] = backend_front_anf_map_[old_backend_anf]; @@ -482,6 +515,8 @@ void KernelGraph::TensorValueNodeMapAdd(const tensor::TensorPtr &tensor, const V } void KernelGraph::AddDependEdge(const AnfNodePtr &node, const AnfNodePtr &input, size_t depend_edge_num) { + MS_EXCEPTION_IF_NULL(node); + MS_EXCEPTION_IF_NULL(input); MS_LOG(DEBUG) << "Input:" << input->DebugString() << ", node:" << node->DebugString() << ",num:" << depend_edge_num; auto output_depend_edge = std::pair(node, depend_edge_num); // add output depend edge of input @@ -520,6 +555,47 @@ std::vector KernelGraph::GetOutputNodes(const AnfNodePtr &node) { return output_nodes; } +// Find control_depend real input nodes. +void GetAllFatherRealNode(const AnfNodePtr &anf_node, std::vector *result, std::set *visited) { + MS_EXCEPTION_IF_NULL(anf_node); + MS_EXCEPTION_IF_NULL(result); + MS_EXCEPTION_IF_NULL(visited); + if (visited->find(anf_node) != visited->end()) { + MS_LOG(WARNING) << "Node:" << anf_node->fullname_with_scope() << " has alreday been visited"; + return; + } + visited->insert(anf_node); + if (AnfAlgo::IsRealKernel(anf_node)) { + result->emplace_back(anf_node); + return; + } + if (!anf_node->isa()) { + return; + } + auto cnode = anf_node->cast(); + MS_EXCEPTION_IF_NULL(cnode); + if (cnode->inputs().empty()) { + MS_LOG(EXCEPTION) << "Illegal null input of cnode(%s)" << anf_node->DebugString(); + } + auto input0 = cnode->input(0); + if (IsPrimitive(input0, prim::kPrimMakeTuple)) { + for (size_t i = 1; i < cnode->inputs().size(); ++i) { + GetAllFatherRealNode(cnode->input(i), result, visited); + } + } else if (IsPrimitive(input0, prim::kPrimTupleGetItem)) { + if (cnode->inputs().size() != kTupleGetItemInputSize) { + MS_LOG(EXCEPTION) << "The node tuple_get_item must have 2 inputs!"; + } + GetAllFatherRealNode(cnode->input(kRealInputNodeIndexInTupleGetItem), result, visited); + } else if (IsPrimitive(input0, prim::kPrimDepend)) { + if (cnode->inputs().size() != kDependInputSize) { + MS_LOG(EXCEPTION) << "Depend node must have 2 inputs!"; + } + GetAllFatherRealNode(cnode->input(kRealInputIndexInDepend), result, visited); + GetAllFatherRealNode(cnode->input(kDependAttachNodeIndex), result, visited); + } +} + // update the depend relations of control depend void KernelGraph::UpdateControlDependRelations(const std::vector &depends) { for (const auto &node : depends) { @@ -538,18 +614,36 @@ void KernelGraph::UpdateControlDependRelations(const std::vector &de MS_EXCEPTION_IF_NULL(depend_node); std::vector prior_nodes = {prior_node}; std::vector depend_nodes = {depend_node}; - MS_LOG(INFO) << "Prior node[" << prior_node->DebugString() << "], depend node[" << depend_node->DebugString(); - if (prior_node->isa()) { + int depend_mode = 0; + if (AnfAlgo::HasNodeAttr(kControlDependMode, cnode)) { + depend_mode = AnfAlgo::GetNodeAttr(cnode, kControlDependMode); + } + MS_LOG(INFO) << "Prior node[" << prior_node->DebugString() << "], depend node[" << depend_node->DebugString() + << "], depend_mode :" << depend_mode << "."; + if (prior_node->isa() && depend_mode == 1) { prior_nodes = GetOutputNodes(prior_node); } if (depend_node->isa()) { - depend_nodes = GetOutputNodes(depend_node); + depend_nodes = depend_mode == 1 ? GetOutputNodes(depend_node) : std::vector{}; + } + + std::vector real_prior_nodes; + std::set prior_visited; + for (const auto &tmp : prior_nodes) { + GetAllFatherRealNode(tmp, &real_prior_nodes, &prior_visited); } - for (auto &first_node : prior_nodes) { + + std::vector real_depend_nodes; + std::set depend_visited; + for (const auto &tmp : depend_nodes) { + GetAllFatherRealNode(tmp, &real_depend_nodes, &depend_visited); + } + + for (auto &first_node : real_prior_nodes) { if (AnfAlgo::CheckPrimitiveType(first_node, prim::kPrimControlDepend)) { continue; } - for (auto &second_node : depend_nodes) { + for (auto &second_node : real_depend_nodes) { if (AnfAlgo::CheckPrimitiveType(second_node, prim::kPrimControlDepend)) { continue; } @@ -565,6 +659,8 @@ void KernelGraph::UpdateControlDependRelations(const std::vector &de bool KernelGraph::HandleControlDependNode(const AnfNodePtr &node, std::queue *que, std::unordered_set *visited_nodes) { MS_EXCEPTION_IF_NULL(node); + MS_EXCEPTION_IF_NULL(que); + MS_EXCEPTION_IF_NULL(visited_nodes); if (!node->isa()) { return false; } @@ -590,6 +686,7 @@ bool KernelGraph::HandleControlDependNode(const AnfNodePtr &node, std::queue *seed_nodes) { + MS_EXCEPTION_IF_NULL(seed_nodes); node_output_edges_.clear(); node_input_num_.clear(); node_input_edges_.clear(); @@ -631,14 +728,14 @@ bool KernelGraph::IsInRefOutputMap(const AnfWithOutIndex &pair) const { return r AnfWithOutIndex KernelGraph::GetRefCorrespondOutput(const AnfWithOutIndex &out_pair) const { if (!IsInRefOutputMap(out_pair)) { - MS_LOG(EXCEPTION) << "out_pair is not in RefOutputMap"; + MS_LOG(EXCEPTION) << "Out_pair is not in RefOutputMap"; } return ref_out_in_map_.at(out_pair); } void KernelGraph::AddRefCorrespondPairs(const AnfWithOutIndex &final_pair, const AnfWithOutIndex &origin_pair) { if (IsInRefOutputMap(final_pair)) { - MS_LOG(EXCEPTION) << "out_pair is already in RefOutputMap"; + MS_LOG(EXCEPTION) << "Out_pair is already in RefOutputMap"; } (void)ref_out_in_map_.insert(std::make_pair(final_pair, origin_pair)); } @@ -682,10 +779,9 @@ void KernelGraph::ReplaceNode(NotNull old_anf_node, NotNullsecond; - (void)node_output_edges_.erase(old_anf_node); } + // if change the ir of graph, regenerate execution order of graph + SetExecOrderByDefault(); // update graph inputs in child graph auto it_real_inputs = std::find_if(real_inputs_.begin(), real_inputs_.end(), [&old_anf_node](const std::pair> &n) -> bool { @@ -701,7 +797,7 @@ void KernelGraph::ReplaceNode(NotNull old_anf_node, NotNullDebugString() << " already exist in real inputs, will be rewrited."; + MS_LOG(WARNING) << new_anf_node->DebugString() << " Already exist in real inputs, will be rewrited."; iter->second = old_args; } else { real_inputs_.emplace_back(new_anf_node, old_args); @@ -744,7 +840,7 @@ std::vector KernelGraph::FindNodeByPrimitive(const PrimitivePtr &primi void KernelGraph::SetRealInput(const AnfNodePtr ¶meter, const AnfNodePtr &arg) { MS_EXCEPTION_IF_NULL(parameter); MS_EXCEPTION_IF_NULL(arg); - MS_LOG(INFO) << "parameter: " << parameter->DebugString() << ", real input : " << arg->DebugString(); + MS_LOG(INFO) << "Parameter: " << parameter->DebugString() << ", real input : " << arg->DebugString(); MS_EXCEPTION_IF_NULL(parameter); MS_EXCEPTION_IF_NULL(arg); auto iter = std::find_if( @@ -758,6 +854,10 @@ void KernelGraph::SetRealInput(const AnfNodePtr ¶meter, const AnfNodePtr &ar } } +void KernelGraph::AddUnreuseArgs(const AnfNodePtr &arg, const std::shared_ptr &from_graph) { + unreuse_args_[arg] = from_graph; +} + void KernelGraph::UpdateCallRealInput() { MS_LOG(INFO) << "Update graph id: " << graph_id_; std::vector>> real_inputs_map; @@ -768,10 +868,19 @@ void KernelGraph::UpdateCallRealInput() { std::vector new_real_inputs; for (auto &real_input : real_inputs) { // if real input is a call node ,find the child graph output act as the new real input - auto item_with_index = AnfAlgo::VisitKernelWithReturnType(real_input, 0); - MS_EXCEPTION_IF_NULL(item_with_index.first); - auto tmp_real_input = GetCallRealOutputs(item_with_index.first); + auto tmp_real_input = GetCallRealOutputs(real_input); std::copy(tmp_real_input.begin(), tmp_real_input.end(), std::back_inserter(new_real_inputs)); + // replace the call in unreuse_args_ + auto unreuse_arg_it = unreuse_args_.find(real_input); + if (unreuse_arg_it != unreuse_args_.end()) { + auto old_graph = unreuse_arg_it->second; + for (auto new_real_input : new_real_inputs) { + // if call reference graph output is parameter, it will be allowed to reuse + if (!new_real_input->isa()) { + unreuse_args_[new_real_input] = old_graph; + } + } + } } real_inputs_map.emplace_back(parameter, new_real_inputs); } @@ -779,7 +888,7 @@ void KernelGraph::UpdateCallRealInput() { } void KernelGraph::PrintGraphExecuteOrder() const { - MS_LOG(INFO) << "graph:" << graph_id_ << "execution order"; + MS_LOG(INFO) << "Graph:" << graph_id_ << "execution order"; for (size_t i = 0; i < execution_order_.size(); i++) { CNodePtr cur_cnode_ptr = execution_order_[i]; MS_EXCEPTION_IF_NULL(cur_cnode_ptr); @@ -801,13 +910,83 @@ void KernelGraph::PrintGraphExecuteOrder() const { } } - MS_LOG(INFO) << "index[" << i << "], node name[" << cur_cnode_ptr->fullname_with_scope() << "], logic id[" + MS_LOG(INFO) << "Index[" << i << "], node name[" << cur_cnode_ptr->fullname_with_scope() << "], logic id[" << AnfAlgo::GetStreamDistinctionLabel(cur_cnode_ptr.get()) << "], stream id[" << AnfAlgo::GetStreamId(cur_cnode_ptr) << "], node info[" << cur_cnode_ptr->DebugString() << "]" << event_str << label_str; } } +void KernelGraph::AddInternalOutput(const AnfNodePtr &front_node, const AnfNodePtr &node) { + if (front_node == nullptr || node == nullptr) { + MS_LOG(INFO) << "Front node or node is nullptr"; + return; + } + MS_LOG(INFO) << "Add internal node " << node->DebugString() << " with front node " << front_node->DebugString(); + front_to_internal_outputs_map_[front_node] = node; + internal_outputs_to_front_map_[node] = front_node; +} + +void KernelGraph::ReplaceInternalOutput(const AnfNodePtr &node, const AnfNodePtr &new_node) { + if (new_node == nullptr || node == nullptr) { + MS_LOG(INFO) << "New node or node is nullptr"; + return; + } + if (node == new_node) { + MS_LOG(INFO) << "New node and node is the same"; + return; + } + auto iter = internal_outputs_to_front_map_.find(node); + if (iter == internal_outputs_to_front_map_.end()) { + MS_LOG(INFO) << "Node is not internal output"; + return; + } + MS_LOG(INFO) << "Replace internal node " << node->DebugString() << " To " << new_node->DebugString(); + internal_outputs_to_front_map_[new_node] = iter->second; + front_to_internal_outputs_map_[iter->second] = new_node; + internal_outputs_to_front_map_.erase(iter); +} + +AnfNodePtr KernelGraph::GetInternalOutputByFrontNode(const AnfNodePtr &front_node) const { + auto iter = front_to_internal_outputs_map_.find(front_node); + if (iter != front_to_internal_outputs_map_.end()) { + return iter->second; + } + return nullptr; +} + +bool KernelGraph::IsInternalOutput(const AnfNodePtr &node) const { + if (internal_outputs_to_front_map_.find(node) != internal_outputs_to_front_map_.end()) { + return true; + } + return false; +} + +AnfNodePtr KernelGraph::GetFrontNodeByInternalOutput(const AnfNodePtr &node) const { + auto iter = internal_outputs_to_front_map_.find(node); + if (iter != internal_outputs_to_front_map_.end()) { + return iter->second; + } + return nullptr; +} + +void KernelGraph::AddFinalOutputKernel(const AnfNodePtr &node) { + if (node == nullptr) { + return; + } + (void)final_output_kernels_.insert(node); +} + +bool KernelGraph::IsFinalOutputKernel(const AnfNodePtr &node) const { + if (node == nullptr) { + return false; + } + if (final_output_kernels_.find(node) != final_output_kernels_.end()) { + return true; + } + return false; +} + std::string KernelGraph::ToString() const { return std::string("kernel_graph_").append(std::to_string(graph_id_)); } KernelGraph::~KernelGraph() { device::KernelRuntimeManager::Instance().ClearGraphResource(graph_id_); } diff --git a/mindspore/ccsrc/session/kernel_graph.h b/mindspore/ccsrc/session/kernel_graph.h index 3009ab0ce9..6861d43de0 100644 --- a/mindspore/ccsrc/session/kernel_graph.h +++ b/mindspore/ccsrc/session/kernel_graph.h @@ -75,7 +75,7 @@ class KernelGraph : public FuncGraph { // add value node tensor relation map void TensorValueNodeMapAdd(const tensor::TensorPtr &tensor, const ValueNodePtr &value_node); // get all value nodes of graph - std::unordered_set graph_value_nodes() { return graph_value_nodes_; } + const std::unordered_set graph_value_nodes() const { return graph_value_nodes_; } // add value node to graph void AddValueNodeToGraph(const ValueNodePtr &value_node); // ref output is in map @@ -130,6 +130,9 @@ class KernelGraph : public FuncGraph { // get real inputs const std::vector>> &real_inputs() const { return real_inputs_; } void SetRealInput(const AnfNodePtr ¶meter, const AnfNodePtr &arg); + // mark unreused args + void AddUnreuseArgs(const AnfNodePtr &arg, const std::shared_ptr &from_graph); + const std::map> &unreuse_args() const { return unreuse_args_; } // used to dump ir std::string ToString() const override; // update the real input if the node is a call @@ -144,6 +147,13 @@ class KernelGraph : public FuncGraph { void PrintGraphExecuteOrder() const; const std::map> &summary_nodes() const { return summary_nodes_; } void set_summary_nodes(const std::map> &nodes) { summary_nodes_ = nodes; } + void AddInternalOutput(const AnfNodePtr &front_node, const AnfNodePtr &node); + void ReplaceInternalOutput(const AnfNodePtr &node, const AnfNodePtr &new_node); + AnfNodePtr GetInternalOutputByFrontNode(const AnfNodePtr &front_node) const; + bool IsInternalOutput(const AnfNodePtr &node) const; + AnfNodePtr GetFrontNodeByInternalOutput(const AnfNodePtr &node) const; + void AddFinalOutputKernel(const AnfNodePtr &node); + bool IsFinalOutputKernel(const AnfNodePtr &node) const; private: // remove value node form graph @@ -198,10 +208,14 @@ class KernelGraph : public FuncGraph { std::shared_ptr parent_graph_; // record real parameters,inputs_ is the formal parameters std::vector>> real_inputs_; + std::map> unreuse_args_; CNodePtr start_label_; CNodePtr end_goto_; bool null_output_; + std::unordered_map front_to_internal_outputs_map_; + std::unordered_map internal_outputs_to_front_map_; + std::set final_output_kernels_; }; } // namespace session using KernelGraphPtr = std::shared_ptr; diff --git a/mindspore/ccsrc/session/session.cc b/mindspore/ccsrc/session/session.cc index 90e02b37ff..ae70fc77aa 100644 --- a/mindspore/ccsrc/session/session.cc +++ b/mindspore/ccsrc/session/session.cc @@ -33,9 +33,14 @@ namespace py = pybind11; namespace mindspore::inference { std::shared_ptr LoadModel(const char *model_buf, size_t size, const std::string &device) { - inference::Session::RegAllOp(); - auto anf_graph = lite::AnfConverter::RunAnfConverter(model_buf, size); - return anf_graph; + try { + inference::Session::RegAllOp(); + auto anf_graph = lite::AnfConverter::RunAnfConverter(model_buf, size); + return anf_graph; + } catch (std::exception &e) { + MS_LOG(ERROR) << "Inference LoadModel failed"; + return nullptr; + } } void ExitInference() { @@ -51,12 +56,17 @@ void ExitInference() { } std::shared_ptr MSSession::CreateSession(const std::string &device, uint32_t device_id) { - auto session = std::make_shared(); - auto ret = session->Init(device, device_id); - if (ret != 0) { + try { + auto session = std::make_shared(); + auto ret = session->Init(device, device_id); + if (ret != 0) { + return nullptr; + } + return session; + } catch (std::exception &e) { + MS_LOG(ERROR) << "Inference CreatSession failed"; return nullptr; } - return session; } void Session::RegAllOp() { @@ -113,47 +123,71 @@ void Session::RegAllOp() { uint32_t Session::CompileGraph(std::shared_ptr funcGraphPtr) { MS_ASSERT(session_impl_ != nullptr); - auto graph_id = session_impl_->CompileGraph(NOT_NULL(funcGraphPtr)); - py::gil_scoped_release gil_release; - return graph_id; + try { + auto graph_id = session_impl_->CompileGraph(NOT_NULL(funcGraphPtr)); + py::gil_scoped_release gil_release; + return graph_id; + } catch (std::exception &e) { + MS_LOG(ERROR) << "Inference CompileGraph failed"; + return static_cast(-1); + } } MultiTensor Session::RunGraph(uint32_t graph_id, const std::vector> &inputs) { - std::vector inTensors; - inTensors.resize(inputs.size()); - bool has_error = false; - std::transform(inputs.begin(), inputs.end(), inTensors.begin(), - [&has_error](const std::shared_ptr &tensor_ptr) -> tensor::TensorPtr { - if (tensor_ptr == nullptr) { - MS_LOG(WARNING) << "input MSTensor is nullptr, return nullptr"; - has_error = true; - return nullptr; - } - auto tensor = static_cast(tensor_ptr.get()); - if (tensor == nullptr) { - MS_LOG(ERROR) << "Can not cast input MSTensor to tensor"; - has_error = true; - return nullptr; - } - return tensor->tensor(); - }); - if (has_error) { - MS_LOG(ERROR) << "Init Tensor failed, returning empty result"; - std::vector> multiTensor; - return multiTensor; - } - VectorRef outputs; - session_impl_->RunGraph(graph_id, inTensors, &outputs); + try { + std::vector inTensors; + inTensors.resize(inputs.size()); + bool has_error = false; + std::transform(inputs.begin(), inputs.end(), inTensors.begin(), + [&has_error](const std::shared_ptr &tensor_ptr) -> tensor::TensorPtr { + if (tensor_ptr == nullptr) { + MS_LOG(WARNING) << "input MSTensor is nullptr, return nullptr"; + has_error = true; + return nullptr; + } + auto tensor = static_cast(tensor_ptr.get()); + if (tensor == nullptr) { + MS_LOG(ERROR) << "Can not cast input MSTensor to tensor"; + has_error = true; + return nullptr; + } + return tensor->tensor(); + }); + if (has_error) { + MS_LOG(ERROR) << "Init Tensor failed, returning empty result"; + std::vector> multiTensor; + return multiTensor; + } + VectorRef outputs; + session_impl_->RunGraph(graph_id, inTensors, &outputs); - return TransformVectorRefToMultiTensor(outputs); + return TransformVectorRefToMultiTensor(outputs); + } catch (std::exception &e) { + MS_LOG(ERROR) << "Inference Rungraph failed"; + return MultiTensor(); + } } - +namespace { +string AjustTargetName(const std::string &device) { + if (device == kAscendDevice) { + return std::string(kAscendDevice) + "Inference"; + } else { + MS_LOG(ERROR) << "Only support device Ascend right now"; + return ""; + } +} +} // namespace int Session::Init(const std::string &device, uint32_t device_id) { RegAllOp(); auto ms_context = MsContext::GetInstance(); ms_context->set_execution_mode(kGraphMode); - ms_context->set_device_target(kAscendDevice); - session_impl_ = session::SessionFactory::Get().Create(device); + ms_context->set_device_id(device_id); + auto ajust_device = AjustTargetName(device); + if (ajust_device == "") { + return -1; + } + ms_context->set_device_target(device); + session_impl_ = session::SessionFactory::Get().Create(ajust_device); if (session_impl_ == nullptr) { MS_LOG(ERROR) << "Session create failed!, please make sure target device:" << device << " is available."; return -1; diff --git a/mindspore/ccsrc/session/session_basic.cc b/mindspore/ccsrc/session/session_basic.cc index ff6fa8ff94..91e430182c 100644 --- a/mindspore/ccsrc/session/session_basic.cc +++ b/mindspore/ccsrc/session/session_basic.cc @@ -60,7 +60,7 @@ PyObject *GetParamDefaultInputTensor(const AnfNodePtr &node) { BaseRef CreateOneTensor(const AnfNodePtr &node, size_t output_index, const KernelGraph &graph, const std::vector &input_tensors) { MS_EXCEPTION_IF_NULL(node); - MS_LOG(INFO) << "create tensor for output[" << node->DebugString() << "] index[" << output_index << "]"; + MS_LOG(INFO) << "Create tensor for output[" << node->DebugString() << "] index[" << output_index << "]"; // if node is a value node, no need sync addr from device to host if (!AnfAlgo::OutputAddrExist(node, output_index)) { if (node->isa()) { @@ -71,34 +71,40 @@ BaseRef CreateOneTensor(const AnfNodePtr &node, size_t output_index, const Kerne if (node->isa()) { for (size_t input_idx = 0; input_idx < graph.inputs().size(); input_idx++) { if (input_idx >= input_tensors.size()) { - MS_LOG(EXCEPTION) << "input idx:" << input_idx << "out of range:" << input_tensors.size(); + MS_LOG(EXCEPTION) << "Input idx:" << input_idx << "out of range:" << input_tensors.size(); } if (graph.inputs()[input_idx] == node) { return input_tensors[input_idx]; } } - MS_LOG(EXCEPTION) << "parameter : " << node->DebugString() << "has no output addr"; + MS_LOG(EXCEPTION) << "Parameter : " << node->DebugString() << "has no output addr"; } } // if proccess reach here,it remarks item_with_index is a real node(Parameter,or executable CNode) - auto address = AnfAlgo::GetOutputAddr(node, output_index); + auto address = AnfAlgo::GetMutableOutputAddr(node, output_index); MS_EXCEPTION_IF_NULL(address); auto shape = AnfAlgo::GetOutputInferShape(node, output_index); TypeId type_id = kNumberTypeFloat32; type_id = AnfAlgo::GetOutputInferDataType(node, output_index); std::vector temp_shape; + if (graph.IsInternalOutput(node)) { + temp_shape.emplace_back(1); + tensor::TensorPtr tensor = std::make_shared(type_id, temp_shape); + tensor->set_device_address(address); + tensor->set_dirty(false); + return tensor; + } (void)std::copy(shape.begin(), shape.end(), std::back_inserter(temp_shape)); tensor::TensorPtr tensor = std::make_shared(type_id, temp_shape); // if in paynative mode,data only copyed to host when user want to print data auto ms_context = MsContext::GetInstance(); MS_EXCEPTION_IF_NULL(ms_context); if (ms_context->execution_mode() == kPynativeMode || ms_context->device_target() == kGPUDevice) { - tensor->set_device_address(AnfAlgo::GetMutableOutputAddr(node, output_index)); + tensor->set_device_address(address); tensor->set_dirty(false); } else if (!address->SyncDeviceToHost(trans::GetRuntimePaddingShape(node, output_index), - LongToSize(tensor->data().nbytes()), tensor->data_type(), - tensor->data_c(true))) { - MS_LOG(INFO) << "output sync device to host error!!!"; + LongToSize(tensor->data().nbytes()), tensor->data_type(), tensor->data_c())) { + MS_LOG(INFO) << "Output sync device to host error!!!"; tensor->set_dirty(false); } return tensor; @@ -107,10 +113,10 @@ BaseRef CreateOneTensor(const AnfNodePtr &node, size_t output_index, const Kerne BaseRef CreatTensorForOutput(const AnfNodePtr &anf, const KernelGraph &graph, const std::vector &input_tensors) { MS_EXCEPTION_IF_NULL(anf); - MS_LOG(INFO) << "create tensor for output[" << anf->DebugString() << "]"; + MS_LOG(INFO) << "Create tensor for output[" << anf->DebugString() << "]"; auto item_with_index = AnfAlgo::VisitKernelWithReturnType(anf, 0); MS_EXCEPTION_IF_NULL(item_with_index.first); - MS_LOG(INFO) << "create tensor for output after visit:" << item_with_index.first->DebugString(); + MS_LOG(INFO) << "Create tensor for output after visit:" << item_with_index.first->DebugString(); // special handle for maketuple if (AnfAlgo::CheckPrimitiveType(item_with_index.first, prim::kPrimMakeTuple)) { auto cnode = item_with_index.first->cast(); @@ -134,7 +140,7 @@ BaseRef CreatTupleForOutput(const AnfNodePtr &anf, const KernelGraph &graph, const std::vector &input_tensors) { MS_EXCEPTION_IF_NULL(anf); if (!AnfAlgo::IsRealKernel(anf)) { - MS_LOG(EXCEPTION) << "anf[" << anf->DebugString() << "] should be a executable kernel"; + MS_LOG(EXCEPTION) << "Anf[" << anf->DebugString() << "] should be a executable kernel"; } if (anf->isa()) { return CreateOneTensor(anf, 0, graph, input_tensors); @@ -165,49 +171,8 @@ ValueNodePtr CreateNewValueNode(const AnfNodePtr &anf, KernelGraph *graph) { return new_value_node; } -std::vector CreateParameterFromTuple(const AnfNodePtr &node, bool valid_input, KernelGraph *graph) { - MS_EXCEPTION_IF_NULL(node); - MS_EXCEPTION_IF_NULL(graph); - std::vector parameters; - std::vector pre_graph_out = {node}; - // If a cnode is a call, it's input0 is a cnode too, so it doesn't have primitive - if (!AnfAlgo::IsRealKernel(node)) { - pre_graph_out = AnfAlgo::GetAllOutput(node, {prim::kPrimTupleGetItem}); - } - auto valid_inputs = graph->MutableValidInputs(); - MS_EXCEPTION_IF_NULL(valid_inputs); - auto graph_inputs = graph->MutableInputs(); - MS_EXCEPTION_IF_NULL(graph_inputs); - auto create_parameter = [&](const AbstractBasePtr &abstract) -> void { - auto parameter = graph->NewParameter(); - MS_EXCEPTION_IF_NULL(parameter); - parameter->set_abstract(abstract); - auto new_parameter = graph->NewParameter(parameter); - parameters.push_back(new_parameter); - valid_inputs->push_back(valid_input); - graph_inputs->push_back(new_parameter); - }; - for (const auto &out_node : pre_graph_out) { - MS_EXCEPTION_IF_NULL(out_node); - auto abstract = out_node->abstract(); - MS_EXCEPTION_IF_NULL(abstract); - // create multiple parameters if is a tuple output real kernel - if (abstract->isa() && !AnfAlgo::CheckPrimitiveType(out_node, prim::kPrimTupleGetItem)) { - auto tuple_abstract = abstract->cast(); - MS_EXCEPTION_IF_NULL(tuple_abstract); - MS_LOG(INFO) << "tuple_size [" << tuple_abstract->size() << "]"; - for (size_t output_idx = 0; output_idx < tuple_abstract->size(); output_idx++) { - create_parameter((*tuple_abstract)[output_idx]); - } - continue; - } - // create single parameter if is a abstract real kernel - create_parameter(out_node->abstract()); - } - return parameters; -} - size_t LoadCtrlInputTensor(const std::shared_ptr &graph, std::vector *inputs) { + MS_EXCEPTION_IF_NULL(graph); MS_LOG(INFO) << "Load kInputCtrlTensors"; auto inputs_params = graph->input_ctrl_tensors(); if (inputs_params == nullptr) { @@ -218,7 +183,7 @@ size_t LoadCtrlInputTensor(const std::shared_ptr &graph, std::vecto } auto tensor = (*inputs_params)[0]; MS_EXCEPTION_IF_NULL(tensor); - auto *val = static_cast(tensor->data_c(true)); + auto *val = static_cast(tensor->data_c()); MS_EXCEPTION_IF_NULL(val); *val = 0; tensor->set_dirty(true); @@ -275,10 +240,10 @@ ParameterPtr ConstructRunOpParameter(const std::shared_ptr &graph, } void DumpGraphOutput(const Any &any, size_t recurse_level = 0) { - MS_LOG(INFO) << "graph outputs:"; + MS_LOG(INFO) << "Graph outputs:"; const size_t max_deep = 10; if (recurse_level > max_deep) { - MS_LOG(INFO) << "recurse too deep"; + MS_LOG(INFO) << "Recurse too deep"; return; } std::string tab_str; @@ -315,11 +280,108 @@ bool ExistSummaryNode(const KernelGraph *graph) { } // namespace GraphId SessionBasic::graph_sum_ = 0; + +KernelGraphPtr SessionBasic::GetGraph(mindspore::GraphId graph_id) { + auto it = graphs_.find(graph_id); + if (it == graphs_.end()) { + MS_LOG(WARNING) << "Can't find graph " << graph_id; + return nullptr; + } + return it->second; +} + +void SessionBasic::InitInternalOutputParameter(const AnfNodePtr &out_node, const AnfNodePtr ¶meter) { + auto graph_id = GetGraphIdByNode(out_node); + if (graph_id == kInvalidGraphId) { + return; + } + auto node_graph = GetGraph(graph_id); + if (node_graph == nullptr) { + return; + } + MS_LOG(INFO) << "Init parameter with pre graph output node: " << out_node->DebugString(); + auto ref_node = node_graph->GetInternalOutputByFrontNode(out_node); + if (ref_node == nullptr) { + MS_LOG(INFO) << "No corresponding internal output for output node"; + return; + } + auto real_kernel = AnfAlgo::VisitKernel(ref_node, 0); + auto ref_real_node = real_kernel.first; + auto ref_real_node_index = real_kernel.second; + if (ref_real_node->isa() && node_graph->IsInternalOutput(ref_real_node) && + node_graph->IsFinalOutputKernel(ref_real_node)) { + auto kernel_info = ref_real_node->kernel_info(); + if (kernel_info == nullptr || kernel_info->select_kernel_build_info() == nullptr) { + MS_LOG(INFO) << "No kernel info"; + return; + } + auto address = AnfAlgo::GetMutableOutputAddr(ref_real_node, ref_real_node_index); + if (address == nullptr) { + MS_LOG(INFO) << "No kernel address"; + return; + } + auto format = AnfAlgo::GetOutputFormat(ref_real_node, ref_real_node_index); + auto type = AnfAlgo::GetOutputDeviceDataType(ref_real_node, ref_real_node_index); + parameter->set_kernel_info(std::make_shared()); + auto d_kernel_info = parameter->kernel_info(); + MS_EXCEPTION_IF_NULL(d_kernel_info); + kernel::KernelBuildInfo::KernelBuildInfoBuilder builder; + builder.SetOutputsDeviceType({type}); + builder.SetOutputsFormat({format}); + d_kernel_info->set_select_kernel_build_info(builder.Build()); + AnfAlgo::SetOutputAddr(address, 0, parameter.get()); + } +} + +std::vector SessionBasic::CreateParameterFromTuple(const AnfNodePtr &node, bool valid_input, + KernelGraph *graph) { + MS_EXCEPTION_IF_NULL(node); + MS_EXCEPTION_IF_NULL(graph); + std::vector parameters; + std::vector pre_graph_out = {node}; + // If a cnode is a call, it's input0 is a cnode too, so it doesn't have primitive + if (!AnfAlgo::IsRealKernel(node)) { + pre_graph_out = AnfAlgo::GetAllOutput(node, {prim::kPrimTupleGetItem}); + } + auto valid_inputs = graph->MutableValidInputs(); + MS_EXCEPTION_IF_NULL(valid_inputs); + auto graph_inputs = graph->MutableInputs(); + MS_EXCEPTION_IF_NULL(graph_inputs); + auto create_parameter = [&](const AbstractBasePtr &abstract) -> void { + auto parameter = graph->NewParameter(); + MS_EXCEPTION_IF_NULL(parameter); + parameter->set_abstract(abstract); + auto new_parameter = graph->NewParameter(parameter); + parameters.push_back(new_parameter); + valid_inputs->push_back(valid_input); + graph_inputs->push_back(new_parameter); + }; + for (const auto &out_node : pre_graph_out) { + MS_EXCEPTION_IF_NULL(out_node); + auto abstract = out_node->abstract(); + MS_EXCEPTION_IF_NULL(abstract); + // create multiple parameters if is a tuple output real kernel + if (abstract->isa() && !AnfAlgo::CheckPrimitiveType(out_node, prim::kPrimTupleGetItem)) { + auto tuple_abstract = abstract->cast(); + MS_EXCEPTION_IF_NULL(tuple_abstract); + MS_LOG(INFO) << "Tuple_size [" << tuple_abstract->size() << "]"; + for (size_t output_idx = 0; output_idx < tuple_abstract->size(); output_idx++) { + create_parameter((*tuple_abstract)[output_idx]); + } + continue; + } + // create single parameter if is a abstract real kernel + create_parameter(out_node->abstract()); + InitInternalOutputParameter(out_node, parameters[parameters.size() - 1]); + } + return parameters; +} + ParameterPtr SessionBasic::CreateNewParameterFromParameter(const AnfNodePtr &anf, bool valid_input, KernelGraph *graph) { MS_EXCEPTION_IF_NULL(anf); if (!anf->isa()) { - MS_LOG(EXCEPTION) << "anf[" << anf->DebugString() << "] is not a parameter"; + MS_LOG(EXCEPTION) << "Anf[" << anf->DebugString() << "] is not a parameter"; } MS_EXCEPTION_IF_NULL(graph); auto m_tensor = GetParamDefaultInputTensor(anf); @@ -387,9 +449,15 @@ CNodePtr SessionBasic::CreateNewCNode(const CNodePtr &cnode, bool valid_input, K auto new_fg = BasicClone(fg); cnode_inputs.push_back(std::make_shared(new_fg)); } + auto origin_inputs = cnode->inputs(); + bool optimize_depend = false; + if (IsPrimitiveCNode(cnode, prim::kPrimDepend) && origin_inputs.size() == 3 && + origin_inputs[kRealInputIndexInDepend]->isa()) { + optimize_depend = true; + } // if has multiple depends,only select first depend as parameter - for (size_t input_idx = 1; input_idx < cnode->inputs().size(); input_idx++) { - auto anf = cnode->inputs()[input_idx]; + for (size_t input_idx = 1; input_idx < origin_inputs.size(); input_idx++) { + auto anf = origin_inputs[input_idx]; MS_EXCEPTION_IF_NULL(anf); // anf has been created before if (graph->GetBackendAnfByFrontAnf(anf) != nullptr) { @@ -414,6 +482,9 @@ CNodePtr SessionBasic::CreateNewCNode(const CNodePtr &cnode, bool valid_input, K (*other_graph_cnode)[anf] = new_parameter; } continue; + } else if (optimize_depend && input_idx == kDependAttachNodeIndex) { + cnode_inputs.push_back(origin_inputs[kRealInputIndexInDepend]); + continue; } else if (anf->isa()) { *from_other_graph = true; // the input node is a cnode from other graph @@ -430,13 +501,49 @@ CNodePtr SessionBasic::CreateNewCNode(const CNodePtr &cnode, bool valid_input, K return new_cnode; } +static std::vector CreateSwitchOrPartialNode(const CNodePtr &cnode, KernelGraph *graph) { + MS_EXCEPTION_IF_NULL(cnode); + MS_EXCEPTION_IF_NULL(graph); + // create primitive of cnode:call(partial or switch) + std::vector cnode_inputs = { + graph->NewValueNode(NewValueNode(std::make_shared(prim::kPrimCall->name())))}; + auto attr_input = cnode->input(kAnfPrimitiveIndex); + MS_EXCEPTION_IF_NULL(attr_input); + auto cnode_input = graph->GetBackendAnfByFrontAnf(attr_input); + if (cnode_input == nullptr) { + MS_LOG(EXCEPTION) << "CNode input[0] is CNode:" << attr_input->DebugString() + << ", but input[0] has not been created."; + } + // if the node is partial, insert the inputs of partial to the call + if (AnfAlgo::CheckPrimitiveType(cnode_input, prim::kPrimPartial)) { + auto partial_node = attr_input->cast(); + MS_EXCEPTION_IF_NULL(partial_node); + auto partial_inputs = partial_node->inputs(); + std::transform(partial_inputs.begin() + kFirstDataInputIndex, partial_inputs.end(), + std::back_inserter(cnode_inputs), [&graph](const AnfNodePtr &node) { + MS_EXCEPTION_IF_NULL(graph->GetBackendAnfByFrontAnf(node)); + return graph->GetBackendAnfByFrontAnf(node); + }); + return cnode_inputs; + } else if (AnfAlgo::CheckPrimitiveType(cnode_input, prim::kPrimSwitch)) { + cnode_inputs.emplace_back(cnode_input); + return cnode_inputs; + } + MS_LOG(EXCEPTION) << "CNode input[0] must be partial or switch."; +} + CNodePtr SessionBasic::CreateNewCNode(const CNodePtr &cnode, KernelGraph *graph) { MS_EXCEPTION_IF_NULL(cnode); MS_EXCEPTION_IF_NULL(graph); std::vector cnode_inputs; auto attr_input = cnode->input(kAnfPrimitiveIndex); MS_EXCEPTION_IF_NULL(attr_input); - if (IsValueNode(attr_input)) { + if (AnfAlgo::IsGraphKernel(cnode)) { + auto fg = AnfAlgo::GetCNodeFuncGraphPtr(cnode); + MS_EXCEPTION_IF_NULL(fg); + auto new_fg = BasicClone(fg); + cnode_inputs.push_back(std::make_shared(new_fg)); + } else if (IsValueNode(attr_input)) { // create primitive of cnode:call cnode_inputs = {graph->NewValueNode(NewValueNode(std::make_shared(prim::kPrimCall->name())))}; // create a ValueNode as input of cnode:call @@ -449,18 +556,7 @@ CNodePtr SessionBasic::CreateNewCNode(const CNodePtr &cnode, KernelGraph *graph) } } } else if (attr_input->isa()) { - // create primitive of cnode:call(switch) - cnode_inputs = {graph->NewValueNode(NewValueNode(std::make_shared(prim::kPrimCall->name())))}; - if (graph->GetBackendAnfByFrontAnf(attr_input) != nullptr) { - auto cnode_input = graph->GetBackendAnfByFrontAnf(attr_input); - if (!AnfAlgo::CheckPrimitiveType(cnode_input, prim::kPrimSwitch)) { - MS_LOG(EXCEPTION) << "CNode input[0] must be switch."; - } - cnode_inputs.emplace_back(cnode_input); - } else { - MS_LOG(EXCEPTION) << "CNode input[0] is CNode:" << attr_input->DebugString() - << ", but input[0] has not been created."; - } + cnode_inputs = CreateSwitchOrPartialNode(cnode, graph); } else { // get primitive of old node auto prim = AnfAlgo::GetCNodePrimitive(cnode); @@ -519,7 +615,7 @@ ParameterPtr SessionBasic::CreateNewParameter(const AnfNodePtr &anf, KernelGraph MS_EXCEPTION_IF_NULL(anf); MS_EXCEPTION_IF_NULL(graph); if (!anf->isa()) { - MS_LOG(EXCEPTION) << "anf[" << anf->DebugString() << "] is not a parameter"; + MS_LOG(EXCEPTION) << "Anf[" << anf->DebugString() << "] is not a parameter"; } auto m_tensor = GetParamDefaultInputTensor(anf); @@ -589,6 +685,22 @@ KernelGraphPtr SessionBasic::ConstructKernelGraph(const AnfNodePtrList &lst, con return graph; } +void SessionBasic::CreateCNodeKernelGraph(const AnfNodePtr node, KernelGraphPtr graph) { + MS_EXCEPTION_IF_NULL(node); + MS_EXCEPTION_IF_NULL(graph); + auto cnode = node->cast(); + MS_EXCEPTION_IF_NULL(cnode); + // create a new cnode object + auto new_cnode = CreateNewCNode(cnode, graph.get()); + MS_EXCEPTION_IF_NULL(new_cnode); + new_cnode->set_abstract(cnode->abstract()); + new_cnode->set_fullname_with_scope(cnode->fullname_with_scope()); + new_cnode->set_scope(cnode->scope()); + graph->FrontBackendlMapAdd(node, new_cnode); + if (AnfAlgo::CheckPrimitiveType(new_cnode, prim::kPrimReturn)) { + graph->set_return(new_cnode); + } +} std::shared_ptr SessionBasic::ConstructKernelGraph(const FuncGraphPtr &func_graph, std::vector *all_out_graph) { MS_EXCEPTION_IF_NULL(func_graph); @@ -626,18 +738,7 @@ std::shared_ptr SessionBasic::ConstructKernelGraph(const FuncGraphP } continue; } else { - auto cnode = node->cast(); - MS_EXCEPTION_IF_NULL(cnode); - // create a new cnode object - auto new_cnode = CreateNewCNode(cnode, graph.get()); - MS_EXCEPTION_IF_NULL(new_cnode); - new_cnode->set_abstract(cnode->abstract()); - new_cnode->set_fullname_with_scope(cnode->fullname_with_scope()); - new_cnode->set_scope(cnode->scope()); - graph->FrontBackendlMapAdd(node, new_cnode); - if (AnfAlgo::CheckPrimitiveType(new_cnode, prim::kPrimReturn)) { - graph->set_return(new_cnode); - } + CreateCNodeKernelGraph(node, graph); } } // if a graph jump back unconditionally, return op of this graph will never be executed, so output is null. @@ -666,7 +767,7 @@ void SessionBasic::AddParameterToGraphInputs(const std::vector ¶ MS_LOG(INFO) << "Can't find parameter:" << parameter->DebugString(); continue; } - MS_LOG(INFO) << "graph[" << graph->graph_id() << "],parameter:" << parameter->DebugString(); + MS_LOG(INFO) << "Graph[" << graph->graph_id() << "],parameter:" << parameter->DebugString(); graph_inputs->push_back(backend_parameter); } } @@ -682,7 +783,7 @@ void SessionBasic::LoadInputData(const std::shared_ptr &kernel_grap } auto input_nodes = kernel_graph->inputs(); if ((inputs.size() + input_ctrl_size) - 1 != input_nodes.size()) { - MS_LOG(EXCEPTION) << "tensor input:" << inputs.size() << " is not equal graph inputs:" << input_nodes.size() + MS_LOG(EXCEPTION) << "Tensor input:" << inputs.size() << " is not equal graph inputs:" << input_nodes.size() << ", input_ctrl_size:" << input_ctrl_size; } auto ms_context = MsContext::GetInstance(); @@ -715,7 +816,7 @@ void SessionBasic::LoadInputData(const std::shared_ptr &kernel_grap MS_EXCEPTION_IF_NULL(device_address); if (!device_address->SyncHostToDevice(trans::GetRuntimePaddingShape(pk_node, 0), LongToSize(tensor->data().nbytes()), tensor->data_type(), - tensor->data_c(false))) { + tensor->data_c())) { MS_LOG(EXCEPTION) << "SyncHostToDevice failed."; } } @@ -736,7 +837,7 @@ void SessionBasic::UpdateOutputs(const std::shared_ptr &kernel_grap auto anf_outputs = kernel_graph->outputs(); for (auto &item : anf_outputs) { MS_EXCEPTION_IF_NULL(item); - MS_LOG(INFO) << "update output[" << item->DebugString() << "]"; + MS_LOG(INFO) << "Update output[" << item->DebugString() << "]"; if (AnfAlgo::IsTupleOutput(item) && AnfAlgo::IsRealKernel(item)) { outputs->emplace_back(CreatTupleForOutput(item, *kernel_graph, input_tensors)); continue; @@ -767,7 +868,7 @@ void SessionBasic::GetSummaryNodes(KernelGraph *graph) { auto cnode = n->cast(); MS_EXCEPTION_IF_NULL(cnode); if (cnode->inputs().size() <= kSummaryGetItem) { - MS_LOG(EXCEPTION) << "the node Summary should have 2 inputs at least!"; + MS_LOG(EXCEPTION) << "The node Summary should have 2 inputs at least!"; } auto node = cnode->input(kSummaryGetItem); MS_EXCEPTION_IF_NULL(node); @@ -810,7 +911,7 @@ void SessionBasic::Summary(KernelGraph *graph) { continue; } if (!address->SyncDeviceToHost(trans::GetRuntimePaddingShape(node, index), LongToSize(tensor->data().nbytes()), - tensor->data_type(), tensor->data_c(true))) { + tensor->data_type(), tensor->data_c())) { MS_LOG(ERROR) << "Failed to sync output from device to host."; } tensor->set_dirty(false); @@ -825,11 +926,34 @@ CNodePtr SessionBasic::ConstructOutput(const AnfNodePtrList &outputs, const std: std::vector output_args; for (const auto &output : outputs) { MS_EXCEPTION_IF_NULL(output); - MS_LOG(INFO) << "output:" << output->DebugString(); + MS_LOG(INFO) << "Output:" << output->DebugString(); } auto FindEqu = [graph, outputs](const AnfNodePtr &out) -> AnfNodePtr { auto backend_anf = graph->GetBackendAnfByFrontAnf(out); if (backend_anf != nullptr) { + auto front_real_kernel = AnfAlgo::VisitKernel(out, 0); + auto backend_real_kernel = AnfAlgo::VisitKernel(backend_anf, 0); + MS_EXCEPTION_IF_NULL(out); + auto out_func_graph = out->func_graph(); + MS_EXCEPTION_IF_NULL(out_func_graph); + auto out_func_graph_manager = out_func_graph->manager(); + if (out_func_graph_manager == nullptr) { + return backend_anf; + } + auto node_users = out_func_graph_manager->node_users(); + auto users = node_users[out]; + bool internal_output = true; + std::string kernel_target = GetCNodeTarget(front_real_kernel.first); + for (auto user : users) { + if (!AnfAlgo::IsRealKernel(user.first) || kernel_target != GetCNodeTarget(user.first)) { + internal_output = false; + break; + } + } + if (internal_output) { + MS_LOG(INFO) << "Internal output1: " << out->DebugString() << "To " << backend_real_kernel.first->DebugString(); + graph->AddInternalOutput(out, backend_real_kernel.first); + } return backend_anf; } MS_LOG(EXCEPTION) << "Can't find the node in the equiv map!"; @@ -844,13 +968,13 @@ void SessionBasic::CreateOutputNode(const CNodePtr &cnode, const std::shared_ptr MS_LOG(INFO) << "Start!"; std::vector make_tuple_inputs; make_tuple_inputs.push_back(NewValueNode(prim::kPrimMakeTuple)); + MS_EXCEPTION_IF_NULL(graph); if (AnfRuntimeAlgorithm::GetOutputTensorNum(cnode) > 1) { for (size_t output_index = 0; output_index < AnfRuntimeAlgorithm::GetOutputTensorNum(cnode); output_index++) { auto idx = NewValueNode(SizeToInt(output_index)); MS_EXCEPTION_IF_NULL(idx); auto imm = std::make_shared(output_index); idx->set_abstract(std::make_shared(imm)); - MS_EXCEPTION_IF_NULL(graph); auto getitem = graph->NewCNode({NewValueNode(prim::kPrimTupleGetItem), cnode, idx}); std::vector types = {AnfAlgo::GetOutputInferDataType(cnode, output_index)}; std::vector> shapes = {AnfAlgo::GetOutputInferShape(cnode, output_index)}; diff --git a/mindspore/ccsrc/session/session_basic.h b/mindspore/ccsrc/session/session_basic.h index 27171b7589..cf85dd0225 100755 --- a/mindspore/ccsrc/session/session_basic.h +++ b/mindspore/ccsrc/session/session_basic.h @@ -32,6 +32,9 @@ #include "utils/contract.h" #include "pynative/pynative_execute.h" #include "device/kernel_info.h" +#ifdef ENABLE_DEBUGGER +#include "debug/debugger/debugger.h" +#endif namespace mindspore { using GraphId = uint32_t; @@ -48,7 +51,11 @@ using OpRunInfoPtr = std::shared_ptr; class SessionBasic { public: - SessionBasic() : context_(nullptr), summary_callback_(nullptr), device_id_(0) {} + SessionBasic() : context_(nullptr), summary_callback_(nullptr), device_id_(0) { +#ifdef ENABLE_DEBUGGER + debugger_ = nullptr; +#endif + } virtual void Init(uint32_t device_id) { device_id_ = device_id; } @@ -70,6 +77,8 @@ class SessionBasic { virtual void RegisterSummaryCallBackFunc(const CallBackFunc &callback); + void CreateCNodeKernelGraph(const AnfNodePtr node, KernelGraphPtr graph); + std::shared_ptr ConstructKernelGraph(const AnfNodePtrList &lst, const AnfNodePtrList &outputs); std::shared_ptr ConstructKernelGraph(const FuncGraphPtr &func_graph, std::vector *all_out_graph); @@ -92,7 +101,17 @@ class SessionBasic { virtual void SetActive(GraphId, GraphId) {} virtual void GetSummaryNodes(KernelGraph *graph); +#ifdef ENABLE_DEBUGGER + // set debugger + void SetDebugger() { + debugger_ = Debugger::GetInstance(); + debugger_->Init(device_id_); + } +#endif + protected: + // Get graph by graph id ,if not exist return null ptr + KernelGraphPtr GetGraph(GraphId graph_id); virtual void LoadInputData(const std::shared_ptr &kernel_graph, const std::vector &inputs_const) const; void UpdateOutputs(const std::shared_ptr &kernel_graph, VectorRef *const outputs, @@ -110,11 +129,13 @@ class SessionBasic { BaseRef TransformBaseRefListToTuple(const BaseRef &base_ref); // create a new kernel graph and update the graph sum KernelGraphPtr NewKernelGraph(); + std::vector CreateParameterFromTuple(const AnfNodePtr &node, bool valid_input, KernelGraph *graph); virtual ParameterPtr CreateNewParameterFromParameter(const AnfNodePtr &anf, bool valid_input, KernelGraph *graph); ValueNodePtr CreateValueNodeKernelGraph(const AnfNodePtr &anf, KernelGraph *graph); ParameterPtr CreateNewParameter(const AnfNodePtr &anf, KernelGraph *graph); AnfNodePtr CreateNewParameterFromCNode(const AnfNodePtr &anf, bool valid_input, KernelGraph *graph); void AddParameterToGraphInputs(const std::vector ¶meters, KernelGraph *graph); + void InitInternalOutputParameter(const AnfNodePtr &out_node, const AnfNodePtr ¶meter); std::unordered_map> graphs_; std::unordered_map> run_op_graphs_; @@ -123,6 +144,9 @@ class SessionBasic { CallBackFunc summary_callback_; static GraphId graph_sum_; uint32_t device_id_; +#ifdef ENABLE_DEBUGGER + std::shared_ptr debugger_; +#endif }; using SessionPtr = std::shared_ptr; diff --git a/mindspore/ccsrc/transform/convert.cc b/mindspore/ccsrc/transform/convert.cc index a5726b078a..f88e31fcd2 100644 --- a/mindspore/ccsrc/transform/convert.cc +++ b/mindspore/ccsrc/transform/convert.cc @@ -55,9 +55,7 @@ const char kNameSimpleMeanGrad[] = "SimpleMeanGrad"; const char kNameAllReduce[] = "AllReduce"; const char kNameBroadcast[] = "Broadcast"; const char kNameAllgather[] = "AllGather"; -const char kNameHostAllgather[] = "HostAllGather"; const char kNameReduceScatter[] = "ReduceScatter"; -const char kNameHostReduceScatter[] = "HostReduceScatter"; const char kNameReduceSum[] = "ReduceSum"; const char kNameIsFinite[] = "isFinite"; const char kNameReciprocal[] = "Reciprocal"; @@ -136,6 +134,7 @@ const char kNameAssignSub[] = "AssignSub"; const char kNameNPUAllocFloatStatus[] = "NPUAllocFloatStatus"; const char kNameNPUClearFloatStatus[] = "NPUClearFloatStatus"; const char kNameReshape[] = "Reshape"; +const char kNameTransShape[] = "TransShape"; const char kNameRealDiv[] = "RealDiv"; const char kNameTile[] = "Tile"; const char kNameCos[] = "Cos"; @@ -244,6 +243,7 @@ std::unordered_map &DfGraphConvertor::get_adpt_ma {string(kNameBatchNorm), ADPT_DESC(BatchNorm)}, {string(kNameBatchNormGrad), ADPT_DESC(BatchNormGrad)}, {string(kNameReshape), ADPT_DESC(Reshape)}, + {string(kNameTransShape), ADPT_DESC(TransShape)}, {string(kNameFlattenGrad), ADPT_DESC(Reshape)}, {prim::kPrimFlatten->name(), ADPT_DESC(Flatten)}, {string(kNameAddN), ADPT_DESC(AddN)}, @@ -371,6 +371,7 @@ std::unordered_map &DfGraphConvertor::get_adpt_ma {prim::kPrimImageSummary->name(), ADPT_DESC(Summary)}, {prim::kPrimTensorSummary->name(), ADPT_DESC(Summary)}, {prim::kPrimHistogramSummary->name(), ADPT_DESC(Summary)}, + {prim::kPrimDebug->name(), ADPT_DESC(Summary)}, {prim::kPrimTensorAdd->name(), std::make_shared(std::make_shared>(ExtraAttr({{"mode", MakeValue(1)}})), std::make_shared>(ExtraAttr({{"mode", MakeValue(1)}})))}, @@ -1647,7 +1648,7 @@ bool DfGraphConvertor::GetControlDependList(const CNodePtr &node, dst_ops_list->insert(dst_ops_list->end(), converted_list.begin(), converted_list.end()); } if (src_ops_list->empty() || dst_ops_list->empty()) { - MS_LOG(WARNING) << "Control depend node's src or dest node is not a apply node, ignore it"; + MS_LOG(DEBUG) << "Control depend node's src or dest node is not a CNode, ignore it"; error_ = SUCCESS; } return true; @@ -1691,6 +1692,8 @@ void DfGraphConvertor::ConvertControlDependNode(const CNodePtr node) { }); } else if (src_ops_list->size() == 1 && dst_ops_list->size() == 1) { control_edges.push_back({(*src_ops_list)[0], (*dst_ops_list)[0]}); + } else if (src_ops_list->empty() || dst_ops_list->empty()) { + MS_LOG(DEBUG) << "Depend list of src or dst is empty, ignore it"; } else { MS_LOG(ERROR) << "Convert control depend node to operator failed, depend src:" << src_ops_list->size() << " -> dst:" << dst_ops_list->size(); diff --git a/mindspore/ccsrc/transform/op_declare.cc b/mindspore/ccsrc/transform/op_declare.cc index 7e5e69beb6..9d86b30b17 100644 --- a/mindspore/ccsrc/transform/op_declare.cc +++ b/mindspore/ccsrc/transform/op_declare.cc @@ -442,6 +442,12 @@ INPUT_MAP(Reshape) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(shape)}}; ATTR_MAP(Reshape) = EMPTY_ATTR_MAP; OUTPUT_MAP(Reshape) = {{0, OUTPUT_DESC(y)}}; +// TransShape +INPUT_MAP(TransShape) = {{1, INPUT_DESC(x)}}; +INPUT_ATTR_MAP(TransShape) = {{2, ATTR_DESC(outShape, AnyTraits(), AnyTraits>())}}; +ATTR_MAP(TransShape) = EMPTY_ATTR_MAP; +OUTPUT_MAP(TransShape) = {{0, OUTPUT_DESC(y)}}; + // BiasAdd INPUT_MAP(BiasAdd) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(bias)}}; ATTR_MAP(BiasAdd) = {{"data_format", ATTR_DESC(data_format, AnyTraits())}}; diff --git a/mindspore/ccsrc/transform/op_declare.h b/mindspore/ccsrc/transform/op_declare.h index f64dc7b671..baa819f71f 100755 --- a/mindspore/ccsrc/transform/op_declare.h +++ b/mindspore/ccsrc/transform/op_declare.h @@ -112,6 +112,9 @@ DECLARE_OP_USE_INPUT_ATTR(DepthwiseConv2DBackpropInputD) DECLARE_OP_USE_OUTPUT(DepthwiseConv2DBackpropInputD) DECLARE_OP_ADAPTER(Reshape) DECLARE_OP_USE_OUTPUT(Reshape) +DECLARE_OP_ADAPTER(TransShape) +DECLARE_OP_USE_INPUT_ATTR(TransShape) +DECLARE_OP_USE_OUTPUT(TransShape) DECLARE_OP_ADAPTER(Iou) DECLARE_OP_USE_OUTPUT(Iou) DECLARE_OP_ADAPTER(ResizeNearestNeighborV2D) diff --git a/mindspore/ccsrc/transform/util.cc b/mindspore/ccsrc/transform/util.cc index 3f856fe564..b848ec117b 100644 --- a/mindspore/ccsrc/transform/util.cc +++ b/mindspore/ccsrc/transform/util.cc @@ -342,7 +342,7 @@ MeTensorPtr TransformUtil::GenerateMeTensor(const GeTensorPtr &ge_tensor, const MeTensor me_tensor(me_type, me_dims); // Get the writable data pointer of the tensor and cast it to its data type - auto me_data_ptr = reinterpret_cast(me_tensor.data_c(true)); + auto me_data_ptr = reinterpret_cast(me_tensor.data_c()); size_t me_data_size = static_cast(me_tensor.data().nbytes()); MS_EXCEPTION_IF_NULL(me_data_ptr); MS_EXCEPTION_IF_NULL(ge_tensor); diff --git a/mindspore/ccsrc/utils/context/ms_context.cc b/mindspore/ccsrc/utils/context/ms_context.cc index 35e053dd53..0e3542d1cb 100644 --- a/mindspore/ccsrc/utils/context/ms_context.cc +++ b/mindspore/ccsrc/utils/context/ms_context.cc @@ -69,7 +69,11 @@ MsContext::MsContext(const std::string &policy, const std::string &target) { enable_task_sink_ = true; ir_fusion_flag_ = true; enable_hccl_ = false; +#ifdef ENABLE_DEBUGGER + enable_mem_reuse_ = false; +#else enable_mem_reuse_ = true; +#endif enable_gpu_summary_ = true; precompile_only_ = false; auto_mixed_precision_flag_ = false; @@ -85,6 +89,7 @@ MsContext::MsContext(const std::string &policy, const std::string &target) { max_device_memory_ = kDefaultMaxDeviceMemory; print_file_path_ = ""; enable_graph_kernel_ = false; + enable_sparse_flag_ = false; } std::shared_ptr MsContext::GetInstance() { @@ -300,19 +305,6 @@ void MsContext::GetGeOptions(std::map *ge_options) con (*ge_options)["ge.exec.profilingOptions"] = profiling_options_; } - // only not supported in ge - auto tbe_plugin_path = common::GetEnv("ME_TBE_PLUGIN_PATH"); - if (!tbe_plugin_path.empty()) { - char real_path[PATH_MAX] = {0}; - if (nullptr == realpath(tbe_plugin_path.c_str(), real_path)) { - MS_LOG(ERROR) << "Ms tbe plugin Path error, " << tbe_plugin_path; - } else { - tbe_plugin_path = real_path; - (*ge_options)["ge.TBE_plugin_path"] = tbe_plugin_path; - } - } else { - MS_LOG(ERROR) << "Set TBE plugin path failed!"; - } (*ge_options)["rank_table_file"] = ""; auto env_ddk_version = common::GetEnv("DDK_VERSION"); if (!env_ddk_version.empty()) { @@ -356,18 +348,6 @@ void MsContext::GetGeOptions(std::map *ge_options) con MS_LOG(INFO) << "Use AICPU, make sure aicpu lib is set in OPTION_EXEC_EXTERN_PLUGIN_PATH."; } - // all libs are set in same env variable "OPTION_EXEC_EXTERN_PLUGIN_PATH", such as FE, HCCL, AICPU, etc - auto load_path = common::GetEnv("OPTION_EXEC_EXTERN_PLUGIN_PATH"); - if (!load_path.empty()) { - char real_path[PATH_MAX] = {0}; - if (realpath(load_path.c_str(), real_path)) { - load_path = real_path; - (*ge_options)["ge.soLoadPath"] = load_path; - } - } else { - MS_LOG(ERROR) << "Set lib load path failed!"; - } - auto proto_lib_path = common::GetEnv("OPTION_PROTO_LIB_PATH"); if (!proto_lib_path.empty()) { char real_path[PATH_MAX] = {0}; @@ -376,7 +356,7 @@ void MsContext::GetGeOptions(std::map *ge_options) con (*ge_options)["ge.opsProtoLibPath"] = proto_lib_path; } } else { - MS_LOG(ERROR) << "Set proto lib path failed!"; + MS_LOG(WARNING) << "Set proto lib path failed!"; } // Enable auto mixed precision according to the context options diff --git a/mindspore/ccsrc/utils/context/ms_context.h b/mindspore/ccsrc/utils/context/ms_context.h index 9afe1fa5aa..3bca16f8ee 100644 --- a/mindspore/ccsrc/utils/context/ms_context.h +++ b/mindspore/ccsrc/utils/context/ms_context.h @@ -161,6 +161,9 @@ class MsContext { void set_enable_graph_kernel(bool enable_graph_kernel) { enable_graph_kernel_ = enable_graph_kernel; } bool enable_graph_kernel() const { return enable_graph_kernel_; } + bool enable_sparse_flag() const { return enable_sparse_flag_; } + void set_enable_sparse_flag(bool enable_sparse_flag) { enable_sparse_flag_ = enable_sparse_flag; } + private: MsContext(const std::string &backend_policy, const std::string &target); void GetGeOptions(std::map *ge_options) const; @@ -204,6 +207,7 @@ class MsContext { float max_device_memory_; std::string print_file_path_; bool enable_graph_kernel_; + bool enable_sparse_flag_; }; } // namespace mindspore diff --git a/mindspore/ccsrc/utils/convert_utils.cc b/mindspore/ccsrc/utils/convert_utils.cc index 6e28e38ed1..8cb071b769 100644 --- a/mindspore/ccsrc/utils/convert_utils.cc +++ b/mindspore/ccsrc/utils/convert_utils.cc @@ -105,7 +105,7 @@ py::object ValuePtrToPyData(const ValuePtr &value) { i++; } ret = rets; - } else if (value->isa()) { + } else if (value->isa()) { ret = py::ellipsis(); } else if (value->isa()) { auto slice = value->cast(); @@ -579,11 +579,12 @@ tensor::TensorPtr ScalarToTensor(const ScalarPtr &scalar) { } tensor::TensorPtr tensor = nullptr; if (scalar->isa()) { - tensor = std::make_shared(py::float_(GetValue(scalar)), kFloat32); + tensor = std::make_shared(static_cast(GetValue(scalar)), kFloat32); } else if (scalar->isa()) { - tensor = std::make_shared(py::int_(GetValue(scalar)), kInt32); + tensor = std::make_shared(static_cast(GetValue(scalar)), kInt32); } else if (scalar->isa()) { - tensor = std::make_shared(py::array(py::bool_(GetValue(scalar))), kBool); + const int64_t bool_value = GetValue(scalar) ? 1 : 0; + tensor = std::make_shared(bool_value, kBool); } else { auto type = scalar->type(); auto type_str = (type == nullptr) ? "nullptr" : type->ToString(); diff --git a/mindspore/ccsrc/utils/convert_utils_base.h b/mindspore/ccsrc/utils/convert_utils_base.h index 76d8930324..3638a43e6a 100644 --- a/mindspore/ccsrc/utils/convert_utils_base.h +++ b/mindspore/ccsrc/utils/convert_utils_base.h @@ -102,6 +102,16 @@ inline void IntMulWithOverflowCheck(int a, int b, int *c) { *c = out; } +inline size_t SizetMulWithOverflowCheck(size_t a, size_t b) { + size_t out = a * b; + if (a != 0) { + if ((out / a) != b) { + MS_LOG(EXCEPTION) << "Mul: a(" << a << ") * b(" << b << ") result is overflow"; + } + } + return out; +} + inline uint8_t *AddressOffset(void *address, size_t offset) { MS_EXCEPTION_IF_NULL(address); return static_cast(address) + offset; diff --git a/mindspore/ccsrc/utils/error_code.h b/mindspore/ccsrc/utils/error_code.h deleted file mode 100644 index f6d70666c4..0000000000 --- a/mindspore/ccsrc/utils/error_code.h +++ /dev/null @@ -1,114 +0,0 @@ -/** - * Copyright 2019 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef MINDSPORE_CCSRC_UTILS_LOG_DE_ERROR_CODE_H_ -#define MINDSPORE_CCSRC_UTILS_LOG_DE_ERROR_CODE_H_ - -#include -#include - -namespace DataEngineBase { -// system ID -const int SYSID_MD = 20; - -// Runtime location -enum LogRuntime { - RT_HOST = 0b01, - RT_DEVICE = 0b10, -}; - -// sub model -enum SubModuleId { - COMMON_MODULE = 0, - DATAGET_MODULE, - MINDRECORD_MODULE, -}; - -// error code type -enum ErrorCodeType { - ERROR_CODE = 0b01, - EXCEPTION_CODE = 0b10, -}; - -// error level -enum ErrorLevel { - COMMON_LEVEL = 0b000, - SUGGESTION_LEVEL = 0b001, - MINOR_LEVEL = 0b010, - MAJOR_LEVEL = 0b011, - CRITICAL_LEVEL = 0b100, -}; - -// code compose(4 byte), runtime: 2 bit, type: 2 bit, level: 3 bit, sysid: 8 bit, modid: 5 bit, value: 12 bit -#define DE_ERRORNO(runtime, type, level, sysid, submodid, name, value, desc) \ - constexpr DataEngineBase::Status name = ((0xFF & ((uint8_t)runtime)) << 30) | ((0xFF & ((uint8_t)type)) << 28) | \ - ((0xFF & ((uint8_t)level)) << 25) | ((0xFF & ((uint8_t)sysid)) << 17) | \ - ((0xFF & ((uint8_t)submodid)) << 12) | (0x0FFF & ((uint16_t)value)); \ - const DataEngineBase::ErrorNoRegisterar g_##name##_errorno(name, desc); - -// each module defines error codes using the following macros -#define DE_ERRORNO_COMMON(name, value, desc) \ - DE_ERRORNO(DataEngineBase::RT_HOST, DataEngineBase::ERROR_CODE, DataEngineBase::COMMON_LEVEL, \ - DataEngineBase::SYSID_MD, DataEngineBase::COMMON_MODULE, name, value, desc) - -#define DE_ERRORNO_DATASET(name, value, desc) \ - DE_ERRORNO(DataEngineBase::RT_HOST, DataEngineBase::ERROR_CODE, DataEngineBase::COMMON_LEVEL, \ - DataEngineBase::SYSID_MD, DataEngineBase::DATAGET_MODULE, name, value, desc) - -#define DE_ERRORNO_MINDRECORD(name, value, desc) \ - DE_ERRORNO(DataEngineBase::RT_HOST, DataEngineBase::ERROR_CODE, DataEngineBase::COMMON_LEVEL, \ - DataEngineBase::SYSID_MD, DataEngineBase::MINDRECORD_MODULE, name, value, desc) - -// get error code description -#define DE_GET_ERRORNO_STR(value) DataEngineBase::StatusFactory::Instance()->GetErrDesc(value) - -class StatusFactory { - public: - static StatusFactory *Instance() { - static StatusFactory instance; - return &instance; - } - - void RegisterErrorNo(uint32_t err, const std::string &desc) { - if (err_desc_.find(err) != err_desc_.end()) return; - err_desc_[err] = desc; - } - - std::string GetErrDesc(uint32_t err) { - auto iter_find = err_desc_.find(err); - if (iter_find == err_desc_.end()) return ""; - return iter_find->second; - } - - protected: - StatusFactory() = default; - - ~StatusFactory() = default; - - private: - std::map err_desc_; -}; - -class ErrorNoRegisterar { - public: - ErrorNoRegisterar(uint32_t err, const std::string &desc) { StatusFactory::Instance()->RegisterErrorNo(err, desc); } - - ~ErrorNoRegisterar() = default; -}; - -using Status = uint32_t; -} // namespace DataEngineBase -#endif // MINDSPORE_CCSRC_UTILS_LOG_DE_ERROR_CODE_H_ diff --git a/mindspore/ccsrc/utils/graph_utils.cc b/mindspore/ccsrc/utils/graph_utils.cc index e956498a3f..03ac14573d 100644 --- a/mindspore/ccsrc/utils/graph_utils.cc +++ b/mindspore/ccsrc/utils/graph_utils.cc @@ -23,10 +23,10 @@ #include #include #include -#include #include #include #include +#include #include #include "common/utils.h" @@ -37,9 +37,11 @@ namespace mindspore { std::vector TopoSort(const AnfNodePtr &root, const SuccFunc &succ, const IncludeFunc &include) { size_t seen = NewSeenGeneration(); - std::list todo(1, root); + std::deque todo(1024); std::unordered_map rank; std::vector res; + todo.clear(); + todo.push_back(root); while (!todo.empty()) { AnfNodePtr node = todo.back(); diff --git a/mindspore/ccsrc/utils/load_onnx/anf_model_parser.cc b/mindspore/ccsrc/utils/load_onnx/anf_model_parser.cc index e44eb23001..c3dfa5194f 100644 --- a/mindspore/ccsrc/utils/load_onnx/anf_model_parser.cc +++ b/mindspore/ccsrc/utils/load_onnx/anf_model_parser.cc @@ -22,12 +22,14 @@ #include #include "google/protobuf/io/zero_copy_stream_impl.h" #include "ir/tensor.h" +#include "ir/tensor_py.h" #include "ir/param_value_py.h" #include "operator/ops.h" #include "pipeline/static_analysis/abstract_value.h" #include "proto/onnx.pb.h" #include "utils/log_adapter.h" +using mindspore::tensor::TensorPy; using std::string; namespace mindspore { @@ -117,11 +119,11 @@ bool MSANFModelParser::BuildParameterForFuncGraph(const ParameterPtr &node, cons if (default_para_map_.find(value_proto.name()) != default_para_map_.end()) { const onnx::TensorProto initialize_proto = default_para_map_[value_proto.name()]; std::string initial_data = initialize_proto.raw_data(); - auto *tensor_data_buf = reinterpret_cast(tensor_info->data_c(true)); + auto *tensor_data_buf = reinterpret_cast(tensor_info->data_c()); MS_EXCEPTION_IF_NULL(tensor_data_buf); memcpy_s(tensor_data_buf, tensor_info->data().nbytes(), initial_data.data(), initial_data.size()); - py::array array_data = tensor_info->data(); + py::array array_data = TensorPy::AsNumpy(*tensor_info); ParamValuePyPtr para_value_ptr = std::make_shared(); MS_EXCEPTION_IF_NULL(para_value_ptr); para_value_ptr->set_value(array_data); @@ -249,7 +251,7 @@ bool MSANFModelParser::ObtainValueNodeInTensorForm(const std::string &value_node } tensor::TensorPtr tensor_info = std::make_shared(kDefaultValueSwitchMap[attr_tensor_type], shape); const std::string &tensor_buf = attr_tensor.raw_data(); - auto *tensor_data_buf = reinterpret_cast(tensor_info->data_c(true)); + auto *tensor_data_buf = reinterpret_cast(tensor_info->data_c()); memcpy_s(tensor_data_buf, tensor_info->data().nbytes(), tensor_buf.data(), tensor_buf.size()); auto new_value_node = NewValueNode(MakeValue(tensor_info)); MS_EXCEPTION_IF_NULL(new_value_node); diff --git a/mindspore/ccsrc/utils/log_adapter.cc b/mindspore/ccsrc/utils/log_adapter.cc index d16fbead9b..3588754dae 100644 --- a/mindspore/ccsrc/utils/log_adapter.cc +++ b/mindspore/ccsrc/utils/log_adapter.cc @@ -463,7 +463,7 @@ void InitSubModulesLogLevel() { // set submodule's log level auto submodule = GetEnv("MS_SUBMODULE_LOG_v"); - MS_LOG(INFO) << "MS_SUBMODULE_LOG_v=`" << submodule << "`"; + MS_LOG(DEBUG) << "MS_SUBMODULE_LOG_v=`" << submodule << "`"; LogConfigParser parser(submodule); auto configs = parser.Parse(); for (const auto &cfg : configs) { @@ -489,22 +489,14 @@ void InitSubModulesLogLevel() { } // namespace mindspore extern "C" { -// shared lib init hook #if defined(_WIN32) || defined(_WIN64) -__attribute__((constructor)) void mindspore_log_init(void) { +__attribute__((constructor)) void common_log_init(void) { #else -void mindspore_log_init(void) { +void common_log_init(void) { #endif #ifdef USE_GLOG // do not use glog predefined log prefix FLAGS_log_prefix = false; - static bool is_glog_initialzed = false; - if (!is_glog_initialzed) { -#if !defined(_WIN32) && !defined(_WIN64) - google::InitGoogleLogging("mindspore"); -#endif - is_glog_initialzed = true; - } // set default log level to WARNING if (mindspore::GetEnv("GLOG_v").empty()) { FLAGS_v = mindspore::WARNING; @@ -525,4 +517,22 @@ void mindspore_log_init(void) { #endif mindspore::InitSubModulesLogLevel(); } + +// shared lib init hook +#if defined(_WIN32) || defined(_WIN64) +__attribute__((constructor)) void mindspore_log_init(void) { +#else +void mindspore_log_init(void) { +#endif +#ifdef USE_GLOG + static bool is_glog_initialzed = false; + if (!is_glog_initialzed) { +#if !defined(_WIN32) && !defined(_WIN64) + google::InitGoogleLogging("mindspore"); +#endif + is_glog_initialzed = true; + } +#endif + common_log_init(); +} } diff --git a/mindspore/ccsrc/utils/tensorprint_utils.cc b/mindspore/ccsrc/utils/tensorprint_utils.cc index 0d464e88a8..ee53345f31 100644 --- a/mindspore/ccsrc/utils/tensorprint_utils.cc +++ b/mindspore/ccsrc/utils/tensorprint_utils.cc @@ -87,7 +87,7 @@ bool PrintTensorToString(const char *str_data_ptr, mindspore::tensor::Tensor *co const size_t &memory_size) { MS_EXCEPTION_IF_NULL(str_data_ptr); MS_EXCEPTION_IF_NULL(print_tensor); - auto *tensor_data_ptr = static_cast(print_tensor->data_c(true)); + auto *tensor_data_ptr = static_cast(print_tensor->data_c()); MS_EXCEPTION_IF_NULL(tensor_data_ptr); auto cp_ret = memcpy_s(tensor_data_ptr, static_cast(print_tensor->data().nbytes()), str_data_ptr, memory_size); @@ -214,10 +214,10 @@ bool ConvertDataItem2Tensor(const std::vector &items) { bool SaveDataItem2File(const std::vector &items, const std::string &print_file_path, prntpb::Print print, std::fstream *output) { - bool ret_end_sequence = false; + bool ret_end_thread = false; for (auto &item : items) { if (item.dataType_ == tdt::TDT_END_OF_SEQUENCE) { - ret_end_sequence = true; + ret_end_thread = true; break; } prntpb::Print_Value *value = print.add_value(); @@ -225,14 +225,16 @@ bool SaveDataItem2File(const std::vector &items, const std::strin MS_EXCEPTION_IF_NULL(str_data_ptr); if (item.tensorShape_ == kShapeScalar || item.tensorShape_ == kShapeNone) { if (!judgeLengthValid(str_data_ptr->size(), item.tensorType_)) { - MS_LOG(EXCEPTION) << "Print op receive data length is invalid."; + MS_LOG(ERROR) << "Print op receive data length is invalid."; + ret_end_thread = true; } } std::vector tensor_shape; size_t totaldims = 1; if (!ParseTensorShape(item.tensorShape_, &tensor_shape, &totaldims)) { - MS_LOG(EXCEPTION) << "Tensor print can not parse tensor shape, receive info" << item.tensorShape_; + MS_LOG(ERROR) << "Tensor print can not parse tensor shape, receive info" << item.tensorShape_; + ret_end_thread = true; } if (item.tensorType_ == "string") { @@ -252,11 +254,12 @@ bool SaveDataItem2File(const std::vector &items, const std::strin } if (!print.SerializeToOstream(output)) { - MS_LOG(EXCEPTION) << "Save print file:" << print_file_path << " fail."; + MS_LOG(ERROR) << "Save print file:" << print_file_path << " fail."; + ret_end_thread = true; } print.Clear(); } - return ret_end_sequence; + return ret_end_thread; } void TensorPrint::operator()() { diff --git a/mindspore/ccsrc/utils/utils.h b/mindspore/ccsrc/utils/utils.h index 3c118ec444..477ac350a8 100644 --- a/mindspore/ccsrc/utils/utils.h +++ b/mindspore/ccsrc/utils/utils.h @@ -44,6 +44,7 @@ constexpr auto kBNGrad3OpName = "BNGrad3"; constexpr auto kClearZeroOpName = "ClearZero"; constexpr auto kAtomicAddrCleanOpName = "AtomicAddrClean"; constexpr auto kGetNextOpName = "GetNext"; +constexpr auto kEndOfSequence = "EndOfSequence"; constexpr auto kAllReduceOpName = "AllReduce"; constexpr auto kAllGatherOpName = "AllGather"; constexpr auto kHostAllGatherOpName = "HostAllGather"; @@ -161,6 +162,8 @@ constexpr auto kNMSWithMaskOpName = "NMSWithMask"; constexpr auto kSoftmaxGradExtOpName = "SoftmaxGradExt"; constexpr auto kStridedReadOpName = "StridedRead"; constexpr auto kStridedWriteOpName = "StridedWrite"; +constexpr auto kFusedAdamWeightDecayName = "FusedAdamWeightDecay"; +constexpr auto kFusedAdamName = "FusedAdam"; // attr key name constexpr auto kAttrInputNames = "input_names"; @@ -247,9 +250,11 @@ constexpr auto kTupleGetItemInputSize = 3; // index define of control depend constexpr auto kControlDependPriorIndex = 1; constexpr auto kControlDependBehindIndex = 2; +constexpr auto kControlDependMode = "depend_mode"; // index define of depend constexpr auto kRealInputIndexInDepend = 1; constexpr auto kDependAttachNodeIndex = 2; +constexpr auto kDependInputSize = 3; // format constexpr auto kOpFormat_DEFAULT = "DefaultFormat"; constexpr auto kOpFormat_NC1KHKWHWC0 = "NC1KHKWHWC0"; @@ -287,7 +292,7 @@ const std::set kFloatDataTypeSet = {kNumberTypeFloat16, kNumberTypeFloat static inline void ChangeFileMode(const std::string &file_name, mode_t mode) { try { if (chmod(file_name.c_str(), mode) != 0) { - MS_LOG(WARNING) << "Change file `" << file_name << "` to mode " << std::oct << mode << " fail."; + MS_LOG(DEBUG) << "Change file `" << file_name << "` to mode " << std::oct << mode << " fail."; } } catch (std::exception &e) { MS_LOG(DEBUG) << "File `" << file_name << "` change mode failed! May be not exist."; diff --git a/mindspore/ccsrc/vm/backend.cc b/mindspore/ccsrc/vm/backend.cc index 3fde263c9d..47bc69bbbb 100644 --- a/mindspore/ccsrc/vm/backend.cc +++ b/mindspore/ccsrc/vm/backend.cc @@ -353,7 +353,10 @@ void MsBackend::CreateOtherSession(const std::string &target) { if (other_sess_ == nullptr) { MS_LOG(EXCEPTION) << "Session create failed!, please make sure target device:" << target << " is available."; } - other_sess_->Init(0); + auto context_ptr = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(context_ptr); + uint32_t device_id = context_ptr->device_id(); + other_sess_->Init(device_id); other_sess_->RegisterSummaryCallBackFunc(callbacks::SummarySaveCallback); other_device_ = target; } @@ -362,5 +365,9 @@ GraphId MsBackend::CompileGraph(NotNull fg) { return target_sess_- VectorRef MsBackend::RunGraph(GraphId graph_id, const VectorRef &args) { return MsRunGraph(graph_id, args); } +#ifdef ENABLE_DEBUGGER +void MsBackend::SetDebugger() { target_sess_->SetDebugger(); } +#endif + } // namespace compile } // namespace mindspore diff --git a/mindspore/ccsrc/vm/backend.h b/mindspore/ccsrc/vm/backend.h index 0e0b02c055..3a93cf930f 100644 --- a/mindspore/ccsrc/vm/backend.h +++ b/mindspore/ccsrc/vm/backend.h @@ -69,6 +69,8 @@ class Backend { bool is_switch_call() const { return is_switch_call_; } void set_simu_flag(bool simu) { simu_flag_ = simu; } + virtual void SetDebugger() {} + protected: std::string name_; LinkFuncType convert_fn_; @@ -109,6 +111,10 @@ class MsBackend : public Backend { VectorRef RunGraph(GraphId graph_id, const VectorRef &args); void CreateOtherSession(const std::string &target); +#ifdef ENABLE_DEBUGGER + void SetDebugger() override; +#endif + private: session::SessionPtr target_sess_; session::SessionPtr other_sess_; diff --git a/mindspore/ccsrc/vm/segment_runner.cc b/mindspore/ccsrc/vm/segment_runner.cc index 9b2ee51b3f..db27506134 100644 --- a/mindspore/ccsrc/vm/segment_runner.cc +++ b/mindspore/ccsrc/vm/segment_runner.cc @@ -28,6 +28,7 @@ #include #include "utils/log_adapter.h" +#include "utils/utils.h" #include "ir/manager.h" #include "ir/func_graph_cloner.h" #include "operator/ops.h" @@ -85,7 +86,6 @@ std::tuple TransformSegmentToAnfGr if (lst.empty()) { MS_LOG(EXCEPTION) << "Input anf node list is empty"; } - auto ref = [&eqv, &inputs, &fg](const AnfNodePtr &a) -> AnfNodePtr { if (a->isa() && !IsValueNode(a)) { eqv[a] = a; @@ -95,17 +95,14 @@ std::tuple TransformSegmentToAnfGr eqv[a]->set_abstract(a->abstract()); eqv[a]->set_kernel_info(a->kernel_info_ptr()); } - return eqv[a]; }; - // Merge CNodes into a AnfGraph that represents a linear instruction segment for (auto n : lst) { if (!n->isa()) { MS_LOG(EXCEPTION) << "Inst is not CNode"; } auto &inps = n->cast()->inputs(); - if (inps.empty()) { MS_LOG(EXCEPTION) << "Input is empty"; } @@ -114,21 +111,22 @@ std::tuple TransformSegmentToAnfGr inps[0]->cast()->value()->cast()->has_attr(FUNC_GRAPH_ATTR_GRAPH_KERNEL))) { MS_LOG(EXCEPTION) << "Input[0] Must be a Primitive valuenode"; } - auto fn = inps[0]; - std::vector args{fn}; - (void)std::transform(std::begin(inps) + 1, std::end(inps), std::back_inserter(args), ref); - + if (IsPrimitive(fn, prim::kPrimDepend) && inps.size() == 3 && inps[kRealInputIndexInDepend]->isa() && + eqv.find(inps[kDependAttachNodeIndex]) == eqv.end()) { + args.emplace_back(inps[kRealInputIndexInDepend]); + args.emplace_back(inps[kRealInputIndexInDepend]); + } else { + (void)std::transform(std::begin(inps) + 1, std::end(inps), std::back_inserter(args), ref); + } eqv[n] = fg->NewCNode(args); eqv[n]->set_abstract(n->abstract()); eqv[n]->set_kernel_info(n->kernel_info_ptr()); } - std::vector eqv_keys; (void)std::transform(std::begin(eqv), std::end(eqv), std::back_inserter(eqv_keys), [](const std::pair &elem) -> AnfNodePtr { return elem.first; }); - auto outputs = GetOutput(lst, lst[0]->func_graph()->manager()->node_users(), eqv_keys); AnfNodePtr fg_output; if (outputs.size() > 1) { diff --git a/mindspore/ccsrc/vm/transform.cc b/mindspore/ccsrc/vm/transform.cc index c1fba78be8..80d2fc9df9 100644 --- a/mindspore/ccsrc/vm/transform.cc +++ b/mindspore/ccsrc/vm/transform.cc @@ -52,45 +52,6 @@ const std::vector &GetMsNonlinearOps() { } namespace { -std::string GetCNodeTarget(const AnfNodePtr &node) { - auto context_ptr = MsContext::GetInstance(); - MS_EXCEPTION_IF_NULL(context_ptr); - std::string default_target = context_ptr->device_target(); - if (!node->isa()) { - return default_target; - } - auto cnode = node->cast(); - MS_EXCEPTION_IF_NULL(cnode); - auto attr_input = cnode->input(kAnfPrimitiveIndex); - if (attr_input == nullptr) { - return default_target; - } - auto value_node = attr_input->cast(); - if (value_node == nullptr) { - return default_target; - } - auto value = value_node->value(); - if (value == nullptr) { - return default_target; - } - if (!value->isa()) { - return default_target; - } - auto primitive = value->cast(); - auto att_target = primitive->GetAttr("primitive_target"); - if (att_target != nullptr) { - if (!att_target->isa()) { - MS_LOG(EXCEPTION) << "Only support string CPU|GPU|Ascend for primitive_target"; - } - auto target = GetValue(att_target); - if (kTargetSet.find(target) == kTargetSet.end()) { - MS_LOG(EXCEPTION) << "Only support string CPU|GPU|Ascend for primitive_target"; - } - return target; - } - return default_target; -} - bool ContainMultiTarget(const std::vector &nodes) { auto context_ptr = MsContext::GetInstance(); MS_EXCEPTION_IF_NULL(context_ptr); @@ -136,29 +97,12 @@ void CalcNodeRefCount(const FuncGraphPtr &graph, std::map *n } } -bool IsGetItemNode(const AnfNodePtr &node) { - MS_EXCEPTION_IF_NULL(node); - if (node->isa()) { - auto cnode = node->cast(); - auto &inputs = cnode->inputs(); - if (inputs.empty()) { - MS_LOG(EXCEPTION) << "Inputs of apply node is empty"; - } - if (!IsValueNode(inputs[0])) { - return true; - } - PrimitivePtr node_prim = GetValueNode(inputs[0]); - return node_prim->name() == prim::kPrimTupleGetItem->name(); - } - return false; -} - -std::vector ReorderGetItemNode(const std::vector &nodes) { +std::vector OptimizeGetItemOrder(const std::vector &nodes) { std::vector result; std::map> insert_positions; std::map node_positions; for (auto &node : nodes) { - if (IsGetItemNode(node)) { + if (node->isa() && IsPrimitiveCNode(node, prim::kPrimTupleGetItem)) { auto cnode = node->cast(); MS_EXCEPTION_IF_NULL(cnode); auto &inputs = cnode->inputs(); @@ -241,7 +185,7 @@ std::vector SplitSort(const FuncGraphPtr &graph, const std::string & } } std::reverse(result.begin(), result.end()); - return ReorderGetItemNode(result); + return result; } } // namespace @@ -309,19 +253,12 @@ bool CompileGraph::IsCut(const AnfNodePtr &node) { return false; } -VectorRef CompileGraph::SplitNodes(const FuncGraphPtr &graph) { +VectorRef CompileGraph::SplitNodesWithTarget(const std::vector &input_nodes, const FuncGraphPtr &graph) { MS_EXCEPTION_IF_NULL(graph); + auto nodes = OptimizeGetItemOrder(input_nodes); VectorRef splits; VectorRef split; - auto nodes = TopoSort(graph->get_return()); - if (ContainMultiTarget(nodes)) { - auto context_ptr = MsContext::GetInstance(); - MS_EXCEPTION_IF_NULL(context_ptr); - std::string default_target = context_ptr->device_target(); - nodes = SplitSort(graph, default_target); - } std::string last_target; - MS_LOG(DEBUG) << "Split all nodes size:" << nodes.size(); for (auto &node : nodes) { MS_EXCEPTION_IF_NULL(node); if (IsCut(node)) { @@ -343,6 +280,36 @@ VectorRef CompileGraph::SplitNodes(const FuncGraphPtr &graph) { return splits; } +VectorRef CompileGraph::SplitNodes(const FuncGraphPtr &graph) { + MS_EXCEPTION_IF_NULL(graph); + auto nodes = TopoSort(graph->get_return()); + MS_LOG(DEBUG) << "Split all nodes size:" << nodes.size(); + + if (ContainMultiTarget(nodes)) { + auto context_ptr = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(context_ptr); + std::string default_target = context_ptr->device_target(); + nodes = SplitSort(graph, default_target); + return SplitNodesWithTarget(nodes, graph); + } + + VectorRef splits; + VectorRef split; + for (auto &node : nodes) { + MS_EXCEPTION_IF_NULL(node); + if (IsCut(node)) { + if (split.size() != 0) { + splits.push_back(split); + } + splits.push_back(node); + split.clear(); + } else if (node->isa()) { + split.push_back(node); + } + } + return splits; +} + // Push the value node on the stack. void CompileGraph::Push(const AnfNodePtr &node) { MS_EXCEPTION_IF_NULL(node); diff --git a/mindspore/ccsrc/vm/transform.h b/mindspore/ccsrc/vm/transform.h index f2d54198d6..a02478fc1b 100644 --- a/mindspore/ccsrc/vm/transform.h +++ b/mindspore/ccsrc/vm/transform.h @@ -78,6 +78,7 @@ class CompileGraph { } private: + VectorRef SplitNodesWithTarget(const std::vector &input_nodes, const FuncGraphPtr &graph); void PushParameters(const FuncGraphPtr &func_graph); bool SplitGraph(const FuncGraphPtr &func_graph); int LinConvert(const FuncGraphPtr &func_graph, const AnfNodePtrList &node_list, const std::string &target = ""); diff --git a/mindspore/common/__init__.py b/mindspore/common/__init__.py index ead8aee556..c896805d75 100644 --- a/mindspore/common/__init__.py +++ b/mindspore/common/__init__.py @@ -17,10 +17,10 @@ from . import dtype from .api import ms_function from .dtype import * from .parameter import Parameter, ParameterTuple -from .tensor import MetaTensor, Tensor +from .tensor import MetaTensor, Tensor, IndexedSlices __all__ = [ - "MetaTensor", "Tensor", # tensor + "MetaTensor", "Tensor", "IndexedSlices", # tensor 'ms_function', # api 'Parameter', 'ParameterTuple', # parameter "dtype" diff --git a/mindspore/common/api.py b/mindspore/common/api.py index 4fad3e455b..050baf9f79 100644 --- a/mindspore/common/api.py +++ b/mindspore/common/api.py @@ -158,7 +158,9 @@ class _MindSporeFunction: # replace key with obj info and object ext info when fn is a method if self.obj is not None: self.obj.__parse_method__ = method_name - generate_name = self.obj.__module__ + "." + str(self.obj.create_time) + generate_name = self.obj.__module__ + "." + if self.obj.__class__.__name__ != "ClipByNorm": + generate_name = generate_name + str(self.obj.create_time) if self.identify_obj is not None: generate_name = generate_name + str(id(self.identify_obj)) diff --git a/mindspore/common/dtype.py b/mindspore/common/dtype.py index 46b111d2f6..85bb1c52d6 100644 --- a/mindspore/common/dtype.py +++ b/mindspore/common/dtype.py @@ -96,7 +96,9 @@ type_refkey = typing.RefKeyType() tensor_type = typing.TensorType anything_type = typing.TypeAnything slice_type = typing.Slice -ellipsis_type = typing.Ellipsis +ellipsis_type = typing.TypeEllipsis +list_type = typing.List +tuple_type = typing.Tuple number_type = (int8, int16, diff --git a/mindspore/common/initializer.py b/mindspore/common/initializer.py index 83586272ee..48a142a23f 100644 --- a/mindspore/common/initializer.py +++ b/mindspore/common/initializer.py @@ -343,10 +343,6 @@ def initializer(init, shape=None, dtype=mstype.float32): shape = tuple(shape) elif isinstance(shape, numbers.Number): shape = (shape,) - try: - np.ndarray(shape) - except ValueError: - raise ValueError("Error shape={}".format(shape)) if isinstance(init, Initializer): init.shape = init.shape if init.shape is not None else shape diff --git a/mindspore/common/parameter.py b/mindspore/common/parameter.py index 773f6a99a6..571cc9cb40 100644 --- a/mindspore/common/parameter.py +++ b/mindspore/common/parameter.py @@ -52,13 +52,16 @@ class Parameter: layerwise_parallel (bool): A kind of model parallel mode. When layerwise_parallel is true in paralle mode, broadcast and gradients communication would not be applied on parameters. Default: False. sparse_grad (str): Set if the parameter's gradient is sparse. Default: empty. + has_indexed_slices (bool): Set if the parameter's gradient is indexed_slices. Default: false. """ - def __init__(self, default_input, name, requires_grad=True, layerwise_parallel=False, sparse_grad=""): + def __init__(self, default_input, name, requires_grad=True, layerwise_parallel=False, + sparse_grad="", has_indexed_slices_grad=False): self.set_parameter_data(default_input) self.name = name self.requires_grad = requires_grad self.layerwise_parallel = layerwise_parallel self.sparse_grad = sparse_grad + self.has_indexed_slices_grad = has_indexed_slices_grad self._is_init = False self._sliced = False self.clone_info = _CloneInfo() @@ -186,6 +189,17 @@ class Parameter: raise TypeError("`sparse_grad` parameter must be str type") self._sparse_grad = value + @property + def has_indexed_slices_grad(self): + """Return whether the parameter's gradient is indexed_slices.""" + return self._has_indexed_slices_grad + + @has_indexed_slices_grad.setter + def has_indexed_slices_grad(self, value=False): + if not isinstance(value, bool): + raise TypeError("`has_indexed_slices_grad` parameter must be bool type") + self._has_indexed_slices_grad = value + @property def data(self): return self.default_input diff --git a/mindspore/common/python_pass_register.py b/mindspore/common/python_pass_register.py new file mode 100644 index 0000000000..36eb37adc7 --- /dev/null +++ b/mindspore/common/python_pass_register.py @@ -0,0 +1,80 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Python pass register""" +from inspect import isfunction +from mindspore._c_expression import PyPassManager_ +from mindspore._c_expression import phase + +class PyPassManager(PyPassManager_): + r""" + Used to registe and unregiste python passes which can be used to alter graphs. + + Args: + pipeline_phase (phase): Specify the stage in which the pass will run in the pipeline. Default: phase.opt. + run_only_once (bool): Specify whether or not to run pass only once. Default: False. + multigraph (bool): Whether or not the pattern exists across graphs. Default: True. + + Raises: + TypeError: If argument has invalid type. + """ + def __init__(self, pipeline_phase=phase.opt, run_only_once=False, multi_graph=True): + if not isinstance(pipeline_phase, phase): + raise TypeError(f"Expecting phase, got : ({type(pipeline_phase)}){pipeline_phase}") + if not isinstance(run_only_once, bool): + raise TypeError(f"Expecting bool, got : ({type(run_only_once)}){run_only_once}") + if not isinstance(multi_graph, bool): + raise TypeError(f"Expecting bool, got : ({type(multi_graph)}){multi_graph}") + PyPassManager_.__init__(self) + self.phase_ = pipeline_phase + self.run_only_once_ = run_only_once + self.multi_graph_ = multi_graph + + def registe(self, py_pass): + if not isfunction(py_pass): + raise TypeError(f"Expecting function pass, got : ({type(py_pass)}){py_pass}") + pattern, target = py_pass() + pass_name = py_pass.__name__ + if not isfunction(pattern): + raise TypeError(f"Expecting function pattern, got : ({type(pattern)}){pattern}") + if not isfunction(target): + raise TypeError(f"Expecting function target, got : ({type(target)}){target}") + super().registe(pass_name, pattern, target, self.phase_, self.run_only_once_, self.multi_graph_) + + def unregiste(self, py_pass, pipeline_phase=phase.opt): + if not isinstance(pipeline_phase, phase): + raise TypeError(f"Expecting phase, got : ({type(pipeline_phase)}){pipeline_phase}") + if isinstance(py_pass, str): + super().unregiste(py_pass, pipeline_phase) + return + if isfunction(py_pass): + super().unregiste(py_pass.__name__, pipeline_phase) + return + raise TypeError(f"Expecting py_pass to be string or function, got ({type(py_pass)}){py_pass}") + + def __call__(self, py_pass): + self.registe(py_pass) + return py_pass + +def registe_pass(pipeline_phase=phase.opt, run_only_once=False, multi_graph=True): + """ + Examples: + >>> @registe_pass() + >>> def toy_pass(): + >>> def pattern(): + >>> pass + >>> def target(): + >>> pass + """ + return PyPassManager(pipeline_phase, run_only_once, multi_graph) diff --git a/mindspore/common/tensor.py b/mindspore/common/tensor.py index 0a631b954f..043ab4f6cf 100644 --- a/mindspore/common/tensor.py +++ b/mindspore/common/tensor.py @@ -21,7 +21,11 @@ from .._checkparam import check_type, check_typename from . import dtype as mstype from ._register_for_tensor import tensor_operator_registry -__all__ = ['Tensor', 'MetaTensor'] +__all__ = ['Tensor', 'MetaTensor', 'IndexedSlices'] +np_types = (np.int8, np.int16, np.int32, np.int64, + np.uint8, np.uint16, np.uint32, np.uint64, np.float16, + np.float32, np.float64, np.bool_) + class Tensor(Tensor_): @@ -54,6 +58,10 @@ class Tensor(Tensor_): """ def __init__(self, input_data, dtype=None): + # If input data is numpy number, convert it to np array + if isinstance(input_data, np_types): + input_data = np.array(input_data) + # If input_data is tuple/list/numpy.ndarray, it's support in check_type method. check_type('tensor input_data', input_data, (Tensor_, float, int)) if dtype is not None: @@ -61,9 +69,9 @@ class Tensor(Tensor_): if isinstance(input_data, np.ndarray) and (not input_data.flags['FORC']): input_data = np.ascontiguousarray(input_data) if dtype is None: - super(Tensor, self).__init__(input_data) + Tensor_.__init__(self, input_data) else: - super(Tensor, self).__init__(input_data, dtype) + Tensor_.__init__(self, input_data, dtype) self._virtual_flag = False self._init_flag = False @@ -101,17 +109,18 @@ class Tensor(Tensor_): out = tensor_operator_registry.get('__neg__')(self) return out + def __pos__(self): + return self + def __iadd__(self, other): - out = self.__add__(other) - return out + return self.__add__(other) def __radd__(self, other): out = tensor_operator_registry.get('__add__')(self, other) return out def __imul__(self, other): - out = self.__mul__(other) - return out + return self.__mul__(other) def __rmul__(self, other): out = tensor_operator_registry.get('__mul__')(self, other) @@ -130,8 +139,7 @@ class Tensor(Tensor_): return out def __isub__(self, other): - out = self.__sub__(other) - return out + return self.__sub__(other) def __rsub__(self, other): out = tensor_operator_registry.get('__sub__')(other, self) @@ -168,6 +176,18 @@ class Tensor(Tensor_): return 1 return out[0] + def __mod__(self, other): + return tensor_operator_registry.get('__mod__')(self, other) + + def __imod__(self, other): + return self.__mod__(other) + + def __floordiv__(self, other): + return tensor_operator_registry.get('__floordiv__')(self, other) + + def __ifloordiv__(self, other): + return self.__floordiv__(other) + def __str__(self): if self.dtype == mstype.type_none: return "Unknown Tensor type!" @@ -197,3 +217,8 @@ class Tensor(Tensor_): raise TypeError("init_flag must be bool.") self.set_init_flag(value) self._init_flag = value + + +class IndexedSlices: + def __init__(self, indices, values, dense_shape): + raise NotImplementedError diff --git a/mindspore/context.py b/mindspore/context.py index ad601f8fab..b5be6c3213 100644 --- a/mindspore/context.py +++ b/mindspore/context.py @@ -25,7 +25,6 @@ from mindspore._c_expression import MSContext from mindspore._checkparam import args_type_check from mindspore.parallel._auto_parallel_context import _set_auto_parallel_context, _get_auto_parallel_context, \ _reset_auto_parallel_context -from mindspore.parallel.mpi._mpi_config import _set_mpi_config, _get_mpi_config __all__ = ['GRAPH_MODE', 'PYNATIVE_MODE', 'set_context', 'get_context', 'set_auto_parallel_context', 'get_auto_parallel_context', 'reset_auto_parallel_context'] @@ -356,6 +355,14 @@ class _Context: def check_bprop(self, check_bprop_flag): self._context_handle.set_check_bprop_flag(check_bprop_flag) + @property + def enable_sparse(self): + return self._context_handle.get_enable_sparse_flag() + + @enable_sparse.setter + def enable_sparse(self, enable_sparse_flag): + self._context_handle.set_enable_sparse_flag(enable_sparse_flag) + @property def max_device_memory(self): return self._context_handle.get_max_device_memory() @@ -511,7 +518,8 @@ def reset_auto_parallel_context(): save_graphs_path=str, save_ms_model=bool, save_ms_model_path=str, enable_dump=bool, save_dump_path=str, enable_reduce_precision=bool, variable_memory_max_size=str, enable_profiling=bool, profiling_options=str, enable_auto_mixed_precision=bool, - check_bprop=bool, max_device_memory=str, print_file_path=str) + enable_graph_kernel=bool, check_bprop=bool, max_device_memory=str, print_file_path=str, + enable_sparse=bool) def set_context(**kwargs): """ Sets context for running environment. @@ -539,6 +547,8 @@ def set_context(**kwargs): save_ms_model_path (str): Path to save converted lite model. Default: "." save_graphs_path (str): Path to save graphs. Default: "." enable_auto_mixed_precision (bool): Whether to enable auto mixed precision. Default: True. + enable_graph_kernel (bool): Whether to enable composition of basic primitives. These primitives would be + compiled into a fused kernel automatically. Default: False. reserve_class_name_in_scope (bool) : Whether to save the network class name in the scope. Default: True. enable_reduce_precision (bool): Whether to enable precision reduction. Default: True. enable_dump (bool): Whether to enable dump. Default: False. @@ -564,6 +574,9 @@ def set_context(**kwargs): check_bprop (bool): Whether to check bprop. Default: False. max_device_memory (str): Sets the maximum memory available for device, currently only supported on GPU. The format is "xxGB". Default: "1024GB". + print_file_path (str): The path of print data to save. If this parameter is set, print data is saved to + a file by default, and turn off printing to the screen. + enable_sparse (bool): Whether to enable sparse feature. Default: False. Raises: ValueError: If input key is not an attribute in context. @@ -584,6 +597,7 @@ def set_context(**kwargs): >>> save_graphs_path="/mindspore") >>> context.set_context(enable_profiling=True, profiling_options="training_trace") >>> context.set_context(max_device_memory="3.5GB") + >>> context.set_context(print_file_path="print.pb") """ for key, value in kwargs.items(): if not hasattr(_context(), key): @@ -608,40 +622,3 @@ def get_context(attr_key): raise ValueError( "Get context keyword %s is not recognized!" % attr_key) return getattr(_context(), attr_key) - -@args_type_check(enable_mpi=bool) -def set_mpi_config(**kwargs): - """ - Sets mpi config for running environment. - - mpi config should be configured before running your program. If there is no configuration, - mpi moudle will be disabled by default. - - Note: - Attribute name is required for setting attributes. - - Args: - enable_mpi (bool): Whether to enable mpi. Default: False. - - Raises: - ValueError: If input key is not an attribute in mpi config. - - Examples: - >>> mpiconfig.set_mpi_config(enable_mpi=True) - """ - _set_mpi_config(**kwargs) - -def get_mpi_config(attr_key): - """ - Gets mpi config attribute value according to the input key. - - Args: - attr_key (str): The key of the attribute. - - Returns: - Object, The value of given attribute key. - - Raises: - ValueError: If input key is not an attribute in context. - """ - return _get_mpi_config(attr_key) diff --git a/mindspore/dataset/core/configuration.py b/mindspore/dataset/core/configuration.py index d3175cd181..5376c668c4 100644 --- a/mindspore/dataset/core/configuration.py +++ b/mindspore/dataset/core/configuration.py @@ -185,7 +185,8 @@ class ConfigurationManager: >>> # "numParallelWorkers": 4, >>> # "workerConnectorSize": 16, >>> # "opConnectorSize": 16, - >>> # "seed": 5489 + >>> # "seed": 5489, + >>> # "monitorSamplingInterval": 30 >>> # } """ self.config.load(file) diff --git a/mindspore/dataset/engine/datasets.py b/mindspore/dataset/engine/datasets.py index ca6f7ca33e..ae0dc6789e 100644 --- a/mindspore/dataset/engine/datasets.py +++ b/mindspore/dataset/engine/datasets.py @@ -1040,7 +1040,7 @@ class Dataset: Args: columns (list[str], optional): List of columns to be used to specify the order of columns - (defaults=None, means all columns). + (default=None, means all columns). Returns: Iterator, list of ndarray. @@ -2575,6 +2575,22 @@ class MnistDataset(MappableDataset): - False - not allowed + Citation of Mnist dataset. + + .. code-block:: + + @article{lecun2010mnist, + title = {MNIST handwritten digit database}, + author = {LeCun, Yann and Cortes, Corinna and Burges, CJ}, + journal = {ATT Labs [Online]}, + volume = {2}, + year = {2010}, + howpublished = {http://yann.lecun.com/exdb/mnist}, + description = {The MNIST database of handwritten digits has a training set of 60,000 examples, + and a test set of 10,000 examples. It is a subset of a larger set available from + NIST. The digits have been size-normalized and centered in a fixed-size image.} + } + Args: dataset_dir (str): Path to the root directory that contains the dataset. num_samples (int, optional): The number of images to be included in the dataset @@ -3053,7 +3069,7 @@ class GeneratorDataset(MappableDataset): sampler (Sampler/Iterable, optional): Object used to choose samples from the dataset. Random accessible input is required (default=None, expected order behavior shown in the table). num_shards (int, optional): Number of shards that the dataset should be divided into (default=None). - This argument should be specified only when 'num_samples' is "None". Random accessible input is required. + When this argument is specified, 'num_samples' will not effect. Random accessible input is required. shard_id (int, optional): The shard ID within num_shards (default=None). This argument should be specified only when num_shards is also specified. Random accessible input is required. @@ -3141,6 +3157,9 @@ class GeneratorDataset(MappableDataset): self.column_names.append(col["name"]) self.column_types.append(DataType(col["type"])) + if source is not None and hasattr(source, "__len__"): + self._dataset_size = len(source) + def get_args(self): args = super().get_args() args["source"] = self.source @@ -3161,6 +3180,7 @@ class GeneratorDataset(MappableDataset): return self._dataset_size if self._dataset_size is None: return None + return min(rows_from_sampler, self._dataset_size) # manually set dataset_size as a temporary solution. @@ -3382,7 +3402,7 @@ class ManifestDataset(MappableDataset): class_indexing (dict, optional): A str-to-int mapping from label name to index (default=None, the folder names will be sorted alphabetically and each class will be given a unique index starting from 0). - decode (bool, optional): decode the images after reading (defaults=False). + decode (bool, optional): decode the images after reading (default=False). num_shards (int, optional): Number of shards that the dataset should be divided into (default=None). shard_id (int, optional): The shard ID within num_shards (default=None). This @@ -3536,6 +3556,20 @@ class Cifar10Dataset(MappableDataset): - False - not allowed + Citation of Cifar10 dataset. + + .. code-block:: + + @techreport{Krizhevsky09, + author = {Alex Krizhevsky}, + title = {Learning multiple layers of features from tiny images}, + institution = {}, + year = {2009}, + howpublished = {http://www.cs.toronto.edu/~kriz/cifar.html}, + description = {The CIFAR-10 dataset consists of 60000 32x32 colour images in 10 classes, + with 6000 images per class. There are 50000 training images and 10000 test images.} + } + Args: dataset_dir (str): Path to the root directory that contains the dataset. num_samples (int, optional): The number of images to be included in the dataset. @@ -3658,6 +3692,22 @@ class Cifar100Dataset(MappableDataset): - False - not allowed + Citation of Cifar100 dataset. + + .. code-block:: + + @techreport{Krizhevsky09, + author = {Alex Krizhevsky}, + title = {Learning multiple layers of features from tiny images}, + institution = {}, + year = {2009}, + howpublished = {http://www.cs.toronto.edu/~kriz/cifar.html}, + description = {This dataset is just like the CIFAR-10, except it has 100 classes containing 600 images + each. There are 500 training images and 100 testing images per class. The 100 classes in + the CIFAR-100 are grouped into 20 superclasses. Each image comes with a "fine" label (the + class to which it belongs) and a "coarse" label (the superclass to which it belongs).} + } + Args: dataset_dir (str): Path to the root directory that contains the dataset. num_samples (int, optional): The number of images to be included in the dataset. @@ -3973,8 +4023,6 @@ class Schema: else: raise RuntimeError("Unknown field %s" % k) - if self.dataset_type is None: - raise RuntimeError("DatasetType field is missing.") if self.columns is None: raise RuntimeError("Columns are missing.") if self.num_rows is not None: @@ -3990,8 +4038,8 @@ class VOCDataset(MappableDataset): A source dataset for reading and parsing VOC dataset. The generated dataset has two columns : - task='Detection' : ['image', 'annotation']. - task='Segmentation' : ['image', 'target'] + task='Detection' : ['image', 'annotation']; + task='Segmentation' : ['image', 'target']. The shape of both column 'image' and 'target' is [image_size] if decode flag is False, or [H, W, C] otherwise. The type of both tensor 'image' and 'target' is uint8. @@ -4026,11 +4074,32 @@ class VOCDataset(MappableDataset): - False - not allowed + Citation of VOC dataset. + + .. code-block:: + + @article{Everingham10, + author = {Everingham, M. and Van~Gool, L. and Williams, C. K. I. and Winn, J. and Zisserman, A.}, + title = {The Pascal Visual Object Classes (VOC) Challenge}, + journal = {International Journal of Computer Vision}, + volume = {88}, + year = {2010}, + number = {2}, + month = {jun}, + pages = {303--338}, + biburl = {http://host.robots.ox.ac.uk/pascal/VOC/pubs/everingham10.html#bibtex}, + howpublished = {http://host.robots.ox.ac.uk/pascal/VOC/voc{year}/index.html}, + description = {The PASCAL Visual Object Classes (VOC) challenge is a benchmark in visual + object category recognition and detection, providing the vision and machine + learning communities with a standard dataset of images and annotation, and + standard evaluation procedures.} + } + Args: dataset_dir (str): Path to the root directory that contains the dataset. task (str): Set the task type of reading voc data, now only support "Segmentation" or "Detection" - (default="Segmentation") - mode(str): Set the data list txt file to be readed (default="train") + (default="Segmentation"). + mode (str): Set the data list txt file to be readed (default="train"). class_indexing (dict, optional): A str-to-int mapping from label name to index (default=None, the folder names will be sorted alphabetically and each class will be given a unique index starting from 0). @@ -4049,9 +4118,9 @@ class VOCDataset(MappableDataset): argument should be specified only when num_shards is also specified. Raises: - RuntimeError: If xml of Annotations is a invalid format - RuntimeError: If xml of Annotations loss attribution of "object" - RuntimeError: If xml of Annotations loss attribution of "bndbox" + RuntimeError: If xml of Annotations is a invalid format. + RuntimeError: If xml of Annotations loss attribution of "object". + RuntimeError: If xml of Annotations loss attribution of "bndbox". RuntimeError: If sampler and shuffle are specified at the same time. RuntimeError: If sampler and sharding are specified at the same time. RuntimeError: If num_shards is specified but shard_id is None. @@ -4165,10 +4234,10 @@ class CocoDataset(MappableDataset): """ A source dataset for reading and parsing COCO dataset. - CocoDataset support four kinds of task: - 2017 Train/Val/Test Detection, Keypoints, Stuff, Panoptic. + CocoDataset support four kinds of task: 2017 Train/Val/Test Detection, Keypoints, Stuff, Panoptic. The generated dataset has multi-columns : + - task='Detection', column: [['image', dtype=uint8], ['bbox', dtype=float32], ['category_id', dtype=uint32], ['iscrowd', dtype=uint32]]. - task='Stuff', column: [['image', dtype=uint8], ['segmentation',dtype=float32], ['iscrowd',dtype=uint32]]. @@ -4206,11 +4275,35 @@ class CocoDataset(MappableDataset): - False - not allowed + Citation of Coco dataset. + + .. code-block:: + + @article{DBLP:journals/corr/LinMBHPRDZ14, + author = {Tsung{-}Yi Lin and Michael Maire and Serge J. Belongie and + Lubomir D. Bourdev and Ross B. Girshick and James Hays and + Pietro Perona and Deva Ramanan and Piotr Doll{\'{a}}r and C. Lawrence Zitnick}, + title = {Microsoft {COCO:} Common Objects in Context}, + journal = {CoRR}, + volume = {abs/1405.0312}, + year = {2014}, + url = {http://arxiv.org/abs/1405.0312}, + archivePrefix = {arXiv}, + eprint = {1405.0312}, + timestamp = {Mon, 13 Aug 2018 16:48:13 +0200}, + biburl = {https://dblp.org/rec/journals/corr/LinMBHPRDZ14.bib}, + bibsource = {dblp computer science bibliography, https://dblp.org}, + description = {COCO is a large-scale object detection, segmentation, and captioning dataset. + It contains 91 common object categories with 82 of them having more than 5,000 + labeled instances. In contrast to the popular ImageNet dataset, COCO has fewer + categories but more instances per category.} + } + Args: dataset_dir (str): Path to the root directory that contains the dataset. annotation_file (str): Path to the annotation json. task (str): Set the task type of reading coco data, now support 'Detection'/'Stuff'/'Panoptic'/'Keypoint' - (default='Detection') + (default='Detection'). num_samples (int, optional): The number of images to be included in the dataset (default=None, all images). num_parallel_workers (int, optional): Number of workers to read the data @@ -4328,6 +4421,35 @@ class CelebADataset(MappableDataset): The generated dataset has two columns ['image', 'attr']. The type of the image tensor is uint8. The attr tensor is uint32 and one hot type. + Citation of CelebA dataset. + + .. code-block:: + + @article{DBLP:journals/corr/LiuLWT14, + author = {Ziwei Liu and Ping Luo and Xiaogang Wang and Xiaoou Tang}, + title = {Deep Learning Face Attributes in the Wild}, + journal = {CoRR}, + volume = {abs/1411.7766}, + year = {2014}, + url = {http://arxiv.org/abs/1411.7766}, + archivePrefix = {arXiv}, + eprint = {1411.7766}, + timestamp = {Tue, 10 Dec 2019 15:37:26 +0100}, + biburl = {https://dblp.org/rec/journals/corr/LiuLWT14.bib}, + bibsource = {dblp computer science bibliography, https://dblp.org}, + howpublished = {http://mmlab.ie.cuhk.edu.hk/projects/CelebA.html}, + description = {CelebFaces Attributes Dataset (CelebA) is a large-scale face attributes dataset + with more than 200K celebrity images, each with 40 attribute annotations. + The images in this dataset cover large pose variations and background clutter. + CelebA has large diversities, large quantities, and rich annotations, including + * 10,177 number of identities, + * 202,599 number of face images, and + * 5 landmark locations, 40 binary attributes annotations per image. + The dataset can be employed as the training and test sets for the following computer + vision tasks: face attribute recognition, face detection, landmark (or facial part) + localization, and face editing & synthesis.} + } + Args: dataset_dir (str): Path to the root directory that contains the dataset. num_parallel_workers (int, optional): Number of workers to read the data (default=value set in the config). @@ -4420,6 +4542,24 @@ class CLUEDataset(SourceDataset): models, corpus and leaderboard. Here we bring in classification task of CLUE, which are AFQMC, TNEWS, IFLYTEK, CMNLI, WSC and CSL. + Citation of CLUE dataset. + + .. code-block:: + + @article{CLUEbenchmark, + title = {CLUE: A Chinese Language Understanding Evaluation Benchmark}, + author = {Liang Xu, Xuanwei Zhang, Lu Li, Hai Hu, Chenjie Cao, Weitang Liu, Junyi Li, Yudong Li, + Kai Sun, Yechen Xu, Yiming Cui, Cong Yu, Qianqian Dong, Yin Tian, Dian Yu, Bo Shi, Jun Zeng, + Rongzhao Wang, Weijian Xie, Yanting Li, Yina Patterson, Zuoyu Tian, Yiwen Zhang, He Zhou, + Shaoweihua Liu, Qipeng Zhao, Cong Yue, Xinrui Zhang, Zhengliang Yang, Zhenzhong Lan}, + journal = {arXiv preprint arXiv:2004.05986}, + year = {2020}, + howpublished = {https://github.com/CLUEbenchmark/CLUE}, + description = {CLUE, a Chinese Language Understanding Evaluation benchmark. It contains eight different + tasks, including single-sentence classification, sentence pair classification, and machine + reading comprehension.} + } + Args: dataset_files (str or list[str]): String or list of files to be read or glob strings to search for a pattern of files. The list will be sorted in a lexicographical order. @@ -4446,7 +4586,6 @@ class CLUEDataset(SourceDataset): >>> import mindspore.dataset as ds >>> dataset_files = ["/path/to/1", "/path/to/2"] # contains 1 or multiple text files >>> dataset = ds.CLUEDataset(dataset_files=dataset_files, task='AFQMC', usage='train') - """ @check_cluedataset @@ -4741,6 +4880,11 @@ class _NumpySlicesDataset: else: self.data = (np.array(data),) + # check whether the data length in each column is equal + data_len = [len(data_item) for data_item in self.data] + if data_len[1:] != data_len[:-1]: + raise ValueError("Data length in each column is not equal.") + # Init column_name if column_list is not None: self.column_list = column_list @@ -4760,7 +4904,7 @@ class _NumpySlicesDataset: def process_dict(self, input_data): """ - Convert the dict like data into tuple format, when input is a tuple of dict then compose it into a dict first. + Convert the dict like data into tuple format, when input is a tuple of dicts then compose it into a dict first. """ # Convert pandas like dict(has "values" column) into General dict data_keys = list(input_data.keys()) @@ -4829,7 +4973,7 @@ class NumpySlicesDataset(GeneratorDataset): sampler (Sampler/Iterable, optional): Object used to choose samples from the dataset. Random accessible input is required (default=None, expected order behavior shown in the table). num_shards (int, optional): Number of shards that the dataset should be divided into (default=None). - This argument should be specified only when 'num_samples' is "None". Random accessible input is required. + When this argument is specified, 'num_samples' will not effect. Random accessible input is required. shard_id (int, optional): The shard ID within num_shards (default=None). This argument should be specified only when num_shards is also specified. Random accessible input is required. diff --git a/mindspore/dataset/engine/iterators.py b/mindspore/dataset/engine/iterators.py index 4946fb3252..1d2d28c1c0 100644 --- a/mindspore/dataset/engine/iterators.py +++ b/mindspore/dataset/engine/iterators.py @@ -172,13 +172,13 @@ class Iterator: # Convert python node into C node and add to C layer execution tree in postorder traversal. def __convert_node_postorder(self, node): op_type = self.__get_dataset_type(node) - c_node = self.depipeline.AddNodeToTree(op_type, node.get_args()) + c_nodes = self.depipeline.AddNodeToTree(op_type, node.get_args()) for py_child in node.children: c_child = self.__convert_node_postorder(py_child) - self.depipeline.AddChildToParentNode(c_child, c_node) + self.depipeline.AddChildToParentNode(c_child, c_nodes["bottom"]) - return c_node + return c_nodes["top"] def __batch_node(self, dataset, level): """Recursively get batch node in the dataset tree.""" diff --git a/mindspore/dataset/engine/serializer_deserializer.py b/mindspore/dataset/engine/serializer_deserializer.py index 833f660f16..9d3339e26d 100644 --- a/mindspore/dataset/engine/serializer_deserializer.py +++ b/mindspore/dataset/engine/serializer_deserializer.py @@ -22,7 +22,7 @@ import sys from mindspore import log as logger from . import datasets as de from ..transforms.vision.utils import Inter, Border - +from ..core.configuration import config def serialize(dataset, json_filepath=None): """ @@ -164,6 +164,8 @@ def traverse(node): node_repr[k] = v.to_json() elif k in set(['schema', 'dataset_files', 'dataset_dir', 'schema_file_path']): expand_path(node_repr, k, v) + elif k == "num_parallel_workers" and v is None: + node_repr[k] = config.get_num_parallel_workers() else: node_repr[k] = v diff --git a/mindspore/dataset/engine/validators.py b/mindspore/dataset/engine/validators.py index 005f7072aa..744a9b94be 100644 --- a/mindspore/dataset/engine/validators.py +++ b/mindspore/dataset/engine/validators.py @@ -153,6 +153,7 @@ def check_sampler_shuffle_shard_options(param_dict): raise RuntimeError("sampler and sharding cannot be specified at the same time.") if num_shards is not None: + check_positive_int32(num_shards, "num_shards") if shard_id is None: raise RuntimeError("num_shards is specified and currently requires shard_id as well.") if shard_id < 0 or shard_id >= num_shards: @@ -529,6 +530,7 @@ def check_generatordataset(method): # These two parameters appear together. raise ValueError("num_shards and shard_id need to be passed in together") if num_shards is not None: + check_positive_int32(num_shards, "num_shards") if shard_id >= num_shards: raise ValueError("shard_id should be less than num_shards") @@ -604,8 +606,15 @@ def check_bucket_batch_by_length(method): nreq_param_list = ['column_names', 'bucket_boundaries', 'bucket_batch_sizes'] check_param_type(nreq_param_list, param_dict, list) + nbool_param_list = ['pad_to_bucket_boundary', 'drop_remainder'] + check_param_type(nbool_param_list, param_dict, bool) + # check column_names: must be list of string. column_names = param_dict.get("column_names") + + if not column_names: + raise ValueError("column_names cannot be empty") + all_string = all(isinstance(item, str) for item in column_names) if not all_string: raise TypeError("column_names should be a list of str.") @@ -641,9 +650,9 @@ def check_bucket_batch_by_length(method): if not all_int: raise TypeError("bucket_batch_sizes should be a list of int.") - all_non_negative = all(item >= 0 for item in bucket_batch_sizes) + all_non_negative = all(item > 0 for item in bucket_batch_sizes) if not all_non_negative: - raise ValueError("bucket_batch_sizes cannot contain any negative numbers.") + raise ValueError("bucket_batch_sizes should be a list of positive numbers.") if param_dict.get('pad_info') is not None: check_type(param_dict["pad_info"], "pad_info", dict) @@ -1065,12 +1074,12 @@ def check_split(method): if all_int: all_positive = all(item > 0 for item in sizes) if not all_positive: - raise ValueError("sizes is a list of int, but there should be no negative numbers.") + raise ValueError("sizes is a list of int, but there should be no negative or zero numbers.") if all_float: all_valid_percentages = all(0 < item <= 1 for item in sizes) if not all_valid_percentages: - raise ValueError("sizes is a list of float, but there should be no numbers outside the range [0, 1].") + raise ValueError("sizes is a list of float, but there should be no numbers outside the range (0, 1].") epsilon = 0.00001 if not abs(sum(sizes) - 1) < epsilon: @@ -1110,10 +1119,10 @@ def check_gnn_list_or_ndarray(param, param_name): for m in param: if not isinstance(m, int): raise TypeError( - "Each membor in {0} should be of type int. Got {1}.".format(param_name, type(m))) + "Each member in {0} should be of type int. Got {1}.".format(param_name, type(m))) elif isinstance(param, np.ndarray): if not param.dtype == np.int32: - raise TypeError("Each membor in {0} should be of type int32. Got {1}.".format( + raise TypeError("Each member in {0} should be of type int32. Got {1}.".format( param_name, param.dtype)) else: raise TypeError("Wrong input type for {0}, should be list or numpy.ndarray, got {1}".format( @@ -1196,15 +1205,15 @@ def check_gnn_get_sampled_neighbors(method): # check neighbor_nums; required argument neighbor_nums = param_dict.get("neighbor_nums") check_gnn_list_or_ndarray(neighbor_nums, 'neighbor_nums') - if len(neighbor_nums) > 6: - raise ValueError("Wrong number of input members for {0}, should be less than or equal to 6, got {1}".format( + if not neighbor_nums or len(neighbor_nums) > 6: + raise ValueError("Wrong number of input members for {0}, should be between 1 and 6, got {1}".format( 'neighbor_nums', len(neighbor_nums))) # check neighbor_types; required argument neighbor_types = param_dict.get("neighbor_types") check_gnn_list_or_ndarray(neighbor_types, 'neighbor_types') - if len(neighbor_nums) > 6: - raise ValueError("Wrong number of input members for {0}, should be less than or equal to 6, got {1}".format( + if not neighbor_types or len(neighbor_types) > 6: + raise ValueError("Wrong number of input members for {0}, should be between 1 and 6, got {1}".format( 'neighbor_types', len(neighbor_types))) if len(neighbor_nums) != len(neighbor_types): @@ -1256,7 +1265,7 @@ def check_gnn_random_walk(method): return new_method -def check_aligned_list(param, param_name, membor_type): +def check_aligned_list(param, param_name, member_type): """Check whether the structure of each member of the list is the same.""" if not isinstance(param, list): @@ -1264,27 +1273,27 @@ def check_aligned_list(param, param_name, membor_type): if not param: raise TypeError( "Parameter {0} or its members are empty".format(param_name)) - membor_have_list = None + member_have_list = None list_len = None - for membor in param: - if isinstance(membor, list): - check_aligned_list(membor, param_name, membor_type) - if membor_have_list not in (None, True): + for member in param: + if isinstance(member, list): + check_aligned_list(member, param_name, member_type) + if member_have_list not in (None, True): raise TypeError("The type of each member of the parameter {0} is inconsistent".format( param_name)) - if list_len is not None and len(membor) != list_len: + if list_len is not None and len(member) != list_len: raise TypeError("The size of each member of parameter {0} is inconsistent".format( param_name)) - membor_have_list = True - list_len = len(membor) + member_have_list = True + list_len = len(member) else: - if not isinstance(membor, membor_type): - raise TypeError("Each membor in {0} should be of type int. Got {1}.".format( - param_name, type(membor))) - if membor_have_list not in (None, False): + if not isinstance(member, member_type): + raise TypeError("Each member in {0} should be of type int. Got {1}.".format( + param_name, type(member))) + if member_have_list not in (None, False): raise TypeError("The type of each member of the parameter {0} is inconsistent".format( param_name)) - membor_have_list = False + member_have_list = False def check_gnn_get_node_feature(method): @@ -1300,7 +1309,7 @@ def check_gnn_get_node_feature(method): check_aligned_list(node_list, 'node_list', int) elif isinstance(node_list, np.ndarray): if not node_list.dtype == np.int32: - raise TypeError("Each membor in {0} should be of type int32. Got {1}.".format( + raise TypeError("Each member in {0} should be of type int32. Got {1}.".format( node_list, node_list.dtype)) else: raise TypeError("Wrong input type for {0}, should be list or numpy.ndarray, got {1}".format( diff --git a/mindspore/dataset/text/__init__.py b/mindspore/dataset/text/__init__.py index 7c43a2888c..04eb90a0b6 100644 --- a/mindspore/dataset/text/__init__.py +++ b/mindspore/dataset/text/__init__.py @@ -24,7 +24,7 @@ from .utils import to_str, to_bytes, JiebaMode, Vocab, NormalizeForm __all__ = [ "Lookup", "JiebaTokenizer", "UnicodeCharTokenizer", "Ngram", - "to_str", "to_bytes", "JiebaMode", "Vocab", "WordpieceTokenizer", "TruncateSequencePair", "ToNumber", + "to_str", "to_bytes", "Vocab", "WordpieceTokenizer", "TruncateSequencePair", "ToNumber", "PythonTokenizer" ] @@ -33,4 +33,4 @@ if platform.system().lower() != 'windows': RegexReplace, RegexTokenizer, BasicTokenizer, BertTokenizer, PythonTokenizer __all__.append(["UnicodeScriptTokenizer", "WhitespaceTokenizer", "CaseFold", "NormalizeUTF8", - "RegexReplace", "RegexTokenizer", "BasicTokenizer", "BertTokenizer", "NormalizeForm"]) + "RegexReplace", "RegexTokenizer", "BasicTokenizer", "BertTokenizer"]) diff --git a/mindspore/dataset/text/transforms.py b/mindspore/dataset/text/transforms.py index fe970e06cc..8b0d47df25 100644 --- a/mindspore/dataset/text/transforms.py +++ b/mindspore/dataset/text/transforms.py @@ -119,10 +119,12 @@ class JiebaTokenizer(cde.JiebaTokenizerOp): the dictionary can be obtained on the official website of cppjieba. mp_path (str): the dictionary file is used by MPSegment algorithm, the dictionary can be obtained on the official website of cppjieba. - mode (JiebaMode, optional): "MP" model will tokenize with MPSegment algorithm, - "HMM" mode will tokenize with Hiddel Markov Model Segment algorithm, - "MIX" model will tokenize with a mix of MPSegment and HMMSegment algorithm - (default="MIX"). + mode (JiebaMode, optional): Valid values can be any of [JiebaMode.MP, JiebaMode.HMM, + JiebaMode.MIX](default=JiebaMode.MIX). + + - JiebaMode.MP, tokenize with MPSegment algorithm. + - JiebaMode.HMM, tokenize with Hiddel Markov Model Segment algorithm. + - JiebaMode.MIX, tokenize with a mix of MPSegment and HMMSegment algorithm. """ @check_jieba_init @@ -190,7 +192,8 @@ class JiebaTokenizer(cde.JiebaTokenizerOp): if not os.path.exists(file_path): raise ValueError( "user dict file {} is not exist".format(file_path)) - file_dict = open(file_path) + real_file_path = os.path.realpath(file_path) + file_dict = open(real_file_path) data_re = re.compile('^(.+?)( [0-9]+)?$', re.U) words_list = [] for item in file_dict: @@ -200,8 +203,9 @@ class JiebaTokenizer(cde.JiebaTokenizerOp): words = data_re.match(data).groups() if len(words) != 2: raise ValueError( - "user dict file {} format error".format(file_path)) + "user dict file {} format error".format(real_file_path)) words_list.append(words) + file_dict.close() return words_list def __decode(self, data): @@ -285,10 +289,16 @@ if platform.system().lower() != 'windows': Apply normalize operation on utf-8 string tensor. Args: - normalize_form (NormalizeForm, optional): Valid values are "NONE", "NFC", "NFKC", "NFD", "NFKD". - If set "NONE", will do nothing for input string tensor. - If set to any of "NFC", "NFKC", "NFD", "NFKD", will apply normalize operation(default="NFKC"). - See http://unicode.org/reports/tr15/ for details. + normalize_form (NormalizeForm, optional): Valid values can be any of [NormalizeForm.NONE, + NormalizeForm.NFC, NormalizeForm.NFKC, NormalizeForm.NFD, + NormalizeForm.NFKD](default=NormalizeForm.NFKC). + And you can see http://unicode.org/reports/tr15/ for details. + + - NormalizeForm.NONE, do nothing for input string tensor. + - NormalizeForm.NFC, normalize with Normalization Form C. + - NormalizeForm.NFKC, normalize with Normalization Form KC. + - NormalizeForm.NFD, normalize with Normalization Form D. + - NormalizeForm.NFKD, normalize with Normalization Form KD. """ def __init__(self, normalize_form=NormalizeForm.NFKC): diff --git a/mindspore/dataset/text/utils.py b/mindspore/dataset/text/utils.py index 766de76e01..7347a4b854 100644 --- a/mindspore/dataset/text/utils.py +++ b/mindspore/dataset/text/utils.py @@ -24,6 +24,9 @@ import mindspore._c_dataengine as cde from .validators import check_from_file, check_from_list, check_from_dict, check_from_dataset +__all__ = [ + "Vocab", "to_str", "to_bytes" +] class Vocab(cde.Vocab): """ diff --git a/mindspore/dataset/transforms/c_transforms.py b/mindspore/dataset/transforms/c_transforms.py index ffe711b106..48e986202c 100644 --- a/mindspore/dataset/transforms/c_transforms.py +++ b/mindspore/dataset/transforms/c_transforms.py @@ -75,28 +75,30 @@ class Slice(cde.SliceOp): Slice operation to extract a tensor out using the given n slices. The functionality of Slice is similar to NumPy indexing feature. - (Currently only rank 1 Tensors are supported) + (Currently only rank-1 tensors are supported). Args: - *slices(Variable length argument list): Maximum `n` number of arguments to slice a tensor of rank `n`. + *slices(Variable length argument list, supported types are, int, list(int), slice, None or Ellipses): + Maximum `n` number of arguments to slice a tensor of rank `n`. One object in slices can be one of: - 1. int: slice this index only. Negative index is supported. - 2. slice object: slice the generated indices from the slice object. Similar to `start:stop:step`. - 3. None: slice the whole dimension. Similar to `:` in python indexing. - 4. Ellipses ...: slice all dimensions between the two slices. + 1. :py:obj:`int`: Slice this index only. Negative index is supported. + 2. :py:obj:`list(int)`: Slice these indices ion the list only. Negative indices are supdeported. + 3. :py:obj:`slice`: Slice the generated indices from the slice object. Similar to `start:stop:step`. + 4. :py:obj:`None`: Slice the whole dimension. Similar to `:` in python indexing. + 5. :py:obj:`Ellipses`: Slice all dimensions between the two slices. Similar to `...` in python indexing. Examples: - >>> # Data before - >>> # | col | - >>> # +---------+ - >>> # | [1,2,3] | - >>> # +---------| - >>> data = data.map(operations=Slice(slice(1,3))) # slice indices 1 and 2 only - >>> # Data after - >>> # | col | - >>> # +------------+ - >>> # | [1,2] | - >>> # +------------| + >>> # Data before + >>> # | col | + >>> # +---------+ + >>> # | [1,2,3] | + >>> # +---------| + >>> data = data.map(operations=Slice(slice(1,3))) # slice indices 1 and 2 only + >>> # Data after + >>> # | col | + >>> # +---------+ + >>> # | [2,3] | + >>> # +---------| """ @check_slice_op @@ -167,7 +169,7 @@ class PadEnd(cde.PadEndOp): Pad input tensor according to `pad_shape`, need to have same rank. Args: - pad_shape (list of `int`): list on integers representing the shape needed. Dimensions that set to `None` will + pad_shape (list(int)): list on integers representing the shape needed. Dimensions that set to `None` will not be padded (i.e., original dim will be used). Shorter dimensions will truncate the values. pad_value (python types (str, bytes, int, float, or bool), optional): value used to pad. Default to 0 or empty string in case of Tensors of strings. diff --git a/mindspore/dataset/transforms/vision/c_transforms.py b/mindspore/dataset/transforms/vision/c_transforms.py index c2497f9629..43ac037541 100644 --- a/mindspore/dataset/transforms/vision/c_transforms.py +++ b/mindspore/dataset/transforms/vision/c_transforms.py @@ -151,7 +151,7 @@ class RandomCrop(cde.RandomCropOp): class RandomCropWithBBox(cde.RandomCropWithBBoxOp): """ - Crop the input image at a random location and adjust bounding boxes for crop area + Crop the input image at a random location and adjust bounding boxes accordingly. Args: size (int or sequence): The output size of the cropped image. @@ -202,7 +202,7 @@ class RandomHorizontalFlip(cde.RandomHorizontalFlipOp): Flip the input image horizontally, randomly with a given probability. Args: - prob (float): Probability of the image being flipped (default=0.5). + prob (float, optional): Probability of the image being flipped (default=0.5). """ @check_prob @@ -213,11 +213,10 @@ class RandomHorizontalFlip(cde.RandomHorizontalFlipOp): class RandomHorizontalFlipWithBBox(cde.RandomHorizontalFlipWithBBoxOp): """ - Flip the input image horizontally, randomly with a given probability. - Maintains data integrity by also flipping bounding boxes in an object detection pipeline. + Flip the input image horizontally, randomly with a given probability and adjust bounding boxes accordingly. Args: - prob (float): Probability of the image being flipped (default=0.5). + prob (float, optional): Probability of the image being flipped (default=0.5). """ @check_prob @@ -231,7 +230,7 @@ class RandomVerticalFlip(cde.RandomVerticalFlipOp): Flip the input image vertically, randomly with a given probability. Args: - prob (float): Probability of the image being flipped (default=0.5). + prob (float, optional): Probability of the image being flipped (default=0.5). """ @check_prob @@ -242,7 +241,7 @@ class RandomVerticalFlip(cde.RandomVerticalFlipOp): class RandomVerticalFlipWithBBox(cde.RandomVerticalFlipWithBBoxOp): """ - Flip the input image vertically, randomly with a given probability and adjust bounding boxes as well + Flip the input image vertically, randomly with a given probability and adjust bounding boxes accordingly. Args: prob (float, optional): Probability of the image being flipped (default=0.5). @@ -256,8 +255,7 @@ class RandomVerticalFlipWithBBox(cde.RandomVerticalFlipWithBBoxOp): class BoundingBoxAugment(cde.BoundingBoxAugmentOp): """ - Apply a given image transform on a random selection of bounding box regions - of a given image. + Apply a given image transform on a random selection of bounding box regions of a given image. Args: transform: C++ transformation function to be applied on random selection @@ -265,6 +263,7 @@ class BoundingBoxAugment(cde.BoundingBoxAugmentOp): ratio (float, optional): Ratio of bounding boxes to apply augmentation on. Range: [0,1] (default=0.3). """ + @check_bounding_box_augment_cpp def __init__(self, transform, ratio=0.3): self.ratio = ratio @@ -302,9 +301,39 @@ class Resize(cde.ResizeOp): super().__init__(*size, interpoltn) +class ResizeWithBBox(cde.ResizeWithBBoxOp): + """ + Resize the input image to the given size and adjust bounding boxes accordingly. + + Args: + size (int or sequence): The output size of the resized image. + If size is an int, smaller edge of the image will be resized to this value with + the same image aspect ratio. + If size is a sequence of length 2, it should be (height, width). + interpolation (Inter mode, optional): Image interpolation mode (default=Inter.LINEAR). + It can be any of [Inter.LINEAR, Inter.NEAREST, Inter.BICUBIC]. + + - Inter.LINEAR, means interpolation method is bilinear interpolation. + + - Inter.NEAREST, means interpolation method is nearest-neighbor interpolation. + + - Inter.BICUBIC, means interpolation method is bicubic interpolation. + """ + + @check_resize_interpolation + def __init__(self, size, interpolation=Inter.LINEAR): + self.size = size + self.interpolation = interpolation + interpoltn = DE_C_INTER_MODE[interpolation] + if isinstance(size, int): + super().__init__(size, interpolation=interpoltn) + else: + super().__init__(*size, interpoltn) + + class RandomResizedCropWithBBox(cde.RandomCropAndResizeWithBBoxOp): """ - Crop the input image to a random size and aspect ratio and adjust the Bounding Boxes accordingly + Crop the input image to a random size and aspect ratio and adjust bounding boxes accordingly. Args: size (int or sequence): The size of the output image. @@ -326,6 +355,7 @@ class RandomResizedCropWithBBox(cde.RandomCropAndResizeWithBBoxOp): max_attempts (int, optional): The maximum number of attempts to propose a valid crop_area (default=10). If exceeded, fall back to use center_crop instead. """ + @check_random_resize_crop def __init__(self, size, scale=(0.08, 1.0), ratio=(3. / 4., 4. / 3.), interpolation=Inter.BILINEAR, max_attempts=10): @@ -499,6 +529,27 @@ class RandomResize(cde.RandomResizeOp): super().__init__(*size) +class RandomResizeWithBBox(cde.RandomResizeWithBBoxOp): + """ + Tensor operation to resize the input image using a randomly selected interpolation mode and adjust + bounding boxes accordingly. + + Args: + size (int or sequence): The output size of the resized image. + If size is an int, smaller edge of the image will be resized to this value with + the same image aspect ratio. + If size is a sequence of length 2, it should be (height, width). + """ + + @check_resize + def __init__(self, size): + self.size = size + if isinstance(size, int): + super().__init__(size) + else: + super().__init__(*size) + + class HWC2CHW(cde.ChannelSwapOp): """ Transpose the input image; shape (H, W, C) to shape (C, H, W). diff --git a/mindspore/mindrecord/filewriter.py b/mindspore/mindrecord/filewriter.py index 62bcc2df79..f8b5448c23 100644 --- a/mindspore/mindrecord/filewriter.py +++ b/mindspore/mindrecord/filewriter.py @@ -240,7 +240,9 @@ class FileWriter: def set_header_size(self, header_size): """ - Set the size of header. + Set the size of header which contains shard information, schema information, \ + page meta information, etc. The larger the header, the more training data \ + a single mindrecord file can store. Args: header_size (int): Size of header, between 16KB and 128MB. @@ -256,7 +258,9 @@ class FileWriter: def set_page_size(self, page_size): """ - Set the size of Page. + Set the size of page which mainly refers to the block to store training data, \ + and the training data will be split into raw page and blob page in mindrecord. \ + The larger the page, the more training data a single page can store. Args: page_size (int): Size of page, between 32KB and 256MB. diff --git a/mindspore/mindrecord/tools/imagenet_to_mr.py b/mindspore/mindrecord/tools/imagenet_to_mr.py index e941e76477..0aa870384e 100644 --- a/mindspore/mindrecord/tools/imagenet_to_mr.py +++ b/mindspore/mindrecord/tools/imagenet_to_mr.py @@ -114,7 +114,7 @@ class ImageNetToMR: if not image_bytes: logger.warning("The image file: {} is invalid.".format(file_name)) continue - data["data"] = image_bytes + data["image"] = image_bytes yield data def transform(self): @@ -126,8 +126,8 @@ class ImageNetToMR: """ t0_total = time.time() - imagenet_schema_json = {"label": {"type": "int64"}, - "data": {"type": "bytes"}, + imagenet_schema_json = {"label": {"type": "int32"}, + "image": {"type": "bytes"}, "file_name": {"type": "string"}} logger.info("transformed MindRecord schema is: {}".format(imagenet_schema_json)) diff --git a/mindspore/mindrecord/tools/tfrecord_to_mr.py b/mindspore/mindrecord/tools/tfrecord_to_mr.py index e8c52001fd..c351f4307f 100644 --- a/mindspore/mindrecord/tools/tfrecord_to_mr.py +++ b/mindspore/mindrecord/tools/tfrecord_to_mr.py @@ -113,7 +113,7 @@ class TFRecordToMR: feature_dict = {"context": {"xxxx": tf.io.FixedLenFeature([], tf.string), \ "yyyy": tf.io.VarLenFeature(tf.int64)}, \ "sequence": {"zzzz": tf.io.FixedLenSequenceFeature([], tf.float32)}} - bytes_fields (list): the bytes fields which are in feature_dict. + bytes_fields (list, optional): the bytes fields which are in feature_dict and can be images bytes. Raises: ValueError: If parameter is invalid. @@ -147,7 +147,7 @@ class TFRecordToMR: self.feature_dict = feature_dict bytes_fields_list = [] - if bytes_fields: + if bytes_fields is not None: if not isinstance(bytes_fields, list): raise ValueError("Parameter bytes_fields: {} must be list(str).".format(bytes_fields)) for item in bytes_fields: @@ -161,6 +161,9 @@ class TFRecordToMR: if not isinstance(self.feature_dict[item].shape, list): raise ValueError("Parameter feature_dict[{}].shape should be a list.".format(item)) + if self.feature_dict[item].dtype != tf.string: + raise ValueError("Parameter bytes_field: {} should be tf.string in feature_dict.".format(item)) + casted_bytes_field = _cast_name(item) bytes_fields_list.append(casted_bytes_field) @@ -172,7 +175,7 @@ class TFRecordToMR: for key, val in self.feature_dict.items(): if not val.shape: self.scalar_set.add(_cast_name(key)) - if key in self.bytes_fields_list: + if _cast_name(key) in self.bytes_fields_list: mindrecord_schema[_cast_name(key)] = {"type": "bytes"} else: mindrecord_schema[_cast_name(key)] = {"type": _cast_type(val.dtype)} @@ -182,8 +185,8 @@ class TFRecordToMR: if val.shape[0] < 1: raise ValueError("Parameter feature_dict[{}].shape[0] should > 0".format(key)) if val.dtype == tf.string: - raise ValueError("Parameter feautre_dict[{}].dtype is tf.string which shape[0] \ - is not None. It is not supported.".format(key)) + raise ValueError("Parameter feautre_dict[{}].dtype is tf.string which shape[0] " \ + "is not None. It is not supported.".format(key)) self.list_set.add(_cast_name(key)) mindrecord_schema[_cast_name(key)] = {"type": _cast_type(val.dtype), "shape": [val.shape[0]]} self.mindrecord_schema = mindrecord_schema @@ -219,12 +222,12 @@ class TFRecordToMR: index_id = index_id + 1 for key, val in features.items(): cast_key = _cast_name(key) - if key in self.scalar_set: + if cast_key in self.scalar_set: self._get_data_when_scalar_field(ms_dict, cast_key, key, val) else: if not isinstance(val.numpy(), np.ndarray) and not isinstance(val.numpy(), list): - raise ValueError("he response key: {}, value: {} from TFRecord should be a ndarray or list." - .format(key, val)) + raise ValueError("The response key: {}, value: {} from TFRecord should be a ndarray or " \ + "list.".format(key, val)) # list set ms_dict[cast_key] = \ np.asarray(val, _cast_string_type_to_np_type(self.mindrecord_schema[cast_key]["type"])) diff --git a/mindspore/nn/cell.py b/mindspore/nn/cell.py index c046c2e1bf..cffe00a920 100755 --- a/mindspore/nn/cell.py +++ b/mindspore/nn/cell.py @@ -81,6 +81,7 @@ class Cell: self.enable_hook = False self._bprop_debug = False self._is_run = False + self.cell_type = None @property def is_run(self): @@ -140,6 +141,14 @@ class Cell: for cell_name, cell in cells_name: cell._param_prefix = cell_name + def update_cell_type(self, cell_type): + """ + Update current cell type mainly identify if quantization aware training network. + + After invoked, can set the cell type to 'cell_type'. + """ + self.cell_type = cell_type + @cell_init_args.setter def cell_init_args(self, value): if not isinstance(value, str): @@ -186,7 +195,7 @@ class Cell: raise AttributeError("'{}' object has no attribute '{}'.".format(type(self).__name__, name)) def __del__(self): - _pynative_exec.clear("resource") + _pynative_exec.clear(str(id(self))) if hasattr(self, "_create_time"): _executor.del_net_res(str(self._create_time)) @@ -822,7 +831,7 @@ class GraphKernel(Cell): """ Base class for GraphKernel. - A `GraphKernel` a composite of basic primitives and can be compiled into a fused kernel automaticly when + A `GraphKernel` a composite of basic primitives and can be compiled into a fused kernel automatically when context.set_context(enable_graph_kernel=True). Examples: diff --git a/mindspore/nn/layer/basic.py b/mindspore/nn/layer/basic.py index b1d5af48c9..ecff453fab 100644 --- a/mindspore/nn/layer/basic.py +++ b/mindspore/nn/layer/basic.py @@ -173,7 +173,8 @@ class Dense(Cell): bias_init (Union[Tensor, str, Initializer, numbers.Number]): The trainable bias_init parameter. The dtype is same as input x. The values of str refer to the function `initializer`. Default: 'zeros'. has_bias (bool): Specifies whether the layer uses a bias vector. Default: True. - activation (str): Regularizer function applied to the output of the layer, eg. 'relu'. Default: None. + activation (str): activate function applied to the output of the fully connected layer, eg. 'relu'. + Default: None. Raises: ValueError: If weight_init or bias_init shape is incorrect. @@ -284,7 +285,6 @@ class ClipByNorm(Cell): self.reduce_sum = P.ReduceSum(keep_dims=True) self.select_ = P.Select() self.greater_ = P.Greater() - self.axis = () self.cast = P.Cast() self.zero = Tensor(np.array([0.0]).astype(np.float32)) self.sqrt = P.Sqrt() @@ -299,7 +299,7 @@ class ClipByNorm(Cell): def construct(self, x, clip_norm): """add ms_function decorator for pynative mode""" mul_x = F.square(x) - l2sum = self.cast(self.reduce_sum(mul_x, self.axis), mstype.float32) + l2sum = self.cast(self.reduce_sum(mul_x), mstype.float32) cond = self.greater_(l2sum, self.zero) ones_ = self.fill(self.dtype(cond), self.shape(cond), 1.0) diff --git a/mindspore/nn/layer/conv.py b/mindspore/nn/layer/conv.py index b2a0de9cbe..52ec9f2d63 100644 --- a/mindspore/nn/layer/conv.py +++ b/mindspore/nn/layer/conv.py @@ -17,11 +17,12 @@ from mindspore import log as logger from mindspore.ops import operations as P from mindspore.common.parameter import Parameter from mindspore.common.initializer import initializer +from mindspore._checkparam import ParamValidator as validator, Rel from mindspore._checkparam import check_bool, twice, check_int_positive, check_int_non_negative from mindspore._extends import cell_attr_register from ..cell import Cell -__all__ = ['Conv2d', 'Conv2dTranspose'] +__all__ = ['Conv2d', 'Conv2dTranspose', 'DepthwiseConv2d'] class _Conv(Cell): """ @@ -397,3 +398,150 @@ class Conv2dTranspose(_Conv): self.weight, self.bias) return s + + +class DepthwiseConv2d(Cell): + r""" + 2D depthwise convolution layer. + + Applies a 2D depthwise convolution over an input tensor which is typically of shape: + math:`(N, C_{in}, H_{in}, W_{in})`, where :math:`N` is batch size and :math:`C_{in}` is channel number. + For each batch of shape:math:`(C_{in}, H_{in}, W_{in})`, the formula is defined as: + + .. math:: + + out_j = \sum_{i=0}^{C_{in} - 1} ccor(W_{ij}, X_i) + b_j, + + where :math:`ccor` is cross correlation operator, :math:`C_{in}` is the input channel number, :math:`j` ranges + from :math:`0` to :math:`C_{out} - 1`, :math:`W_{ij}` corresponds to :math:`i`-th channel of the :math:`j`-th + filter and :math:`out_{j}` corresponds to the :math:`j`-th channel of the output. :math:`W_{ij}` is a slice + of kernel and it has shape :math:`(\text{ks_h}, \text{ks_w})`, where :math:`\text{ks_h}` and + :math:`\text{ks_w}` are height and width of the convolution kernel. The full kernel has shape + :math:`(C_{out}, C_{in} // \text{group}, \text{ks_h}, \text{ks_w})`, where group is the group number + to split the input in the channel dimension. + + If the 'pad_mode' is set to be "valid", the output height and width will be + :math:`\left \lfloor{1 + \frac{H_{in} + 2 \times \text{padding} - \text{ks_h} - + (\text{ks_h} - 1) \times (\text{dilation} - 1) }{\text{stride}}} \right \rfloor` and + :math:`\left \lfloor{1 + \frac{W_{in} + 2 \times \text{padding} - \text{ks_w} - + (\text{ks_w} - 1) \times (\text{dilation} - 1) }{\text{stride}}} \right \rfloor` respectively. + + The first introduction can be found in paper `Gradient Based Learning Applied to Document Recognition + `_. + + Args: + in_channels (int): The number of input channel :math:`C_{in}`. + out_channels (int): The number of output channel :math:`C_{out}`. + kernel_size (Union[int, tuple[int]]): The data type is int or tuple with 2 integers. Specifies the height + and width of the 2D convolution window. Single int means the value if for both height and width of + the kernel. A tuple of 2 ints means the first value is for the height and the other is for the + width of the kernel. + stride (Union[int, tuple[int]]): The distance of kernel moving, an int number that represents + the height and width of movement are both strides, or a tuple of two int numbers that + represent height and width of movement respectively. Default: 1. + pad_mode (str): Specifies padding mode. The optional values are + "same", "valid", "pad". Default: "same". + + - same: Adopts the way of completion. Output height and width will be the same as the input. + Total number of padding will be calculated for horizontal and vertical + direction and evenly distributed to top and bottom, left and right if possible. Otherwise, the + last extra padding will be done from the bottom and the right side. If this mode is set, `padding` + must be 0. + + - valid: Adopts the way of discarding. The possibly largest height and width of output will be return + without padding. Extra pixels will be discarded. If this mode is set, `padding` + must be 0. + + - pad: Implicit paddings on both sides of the input. The number of `padding` will be padded to the input + Tensor borders. `padding` should be greater than or equal to 0. + + padding (int): Implicit paddings on both sides of the input. Default: 0. + dilation (Union[int, tuple[int]]): The data type is int or tuple with 2 integers. Specifies the dilation rate + to use for dilated convolution. If set to be :math:`k > 1`, there will + be :math:`k - 1` pixels skipped for each sampling location. Its value should + be greater or equal to 1 and bounded by the height and width of the + input. Default: 1. + group (int): Split filter into groups, `in_ channels` and `out_channels` should be + divisible by the number of groups. Default: 1. + has_bias (bool): Specifies whether the layer uses a bias vector. Default: False. + weight_init (Union[Tensor, str, Initializer, numbers.Number]): Initializer for the convolution kernel. + It can be a Tensor, a string, an Initializer or a numbers.Number. When a string is specified, + values from 'TruncatedNormal', 'Normal', 'Uniform', 'HeUniform' and 'XavierUniform' distributions as well + as constant 'One' and 'Zero' distributions are possible. Alias 'xavier_uniform', 'he_uniform', 'ones' + and 'zeros' are acceptable. Uppercase and lowercase are both acceptable. Refer to the values of + Initializer for more details. Default: 'normal'. + bias_init (Union[Tensor, str, Initializer, numbers.Number]): Initializer for the bias vector. Possible + Initializer and string are the same as 'weight_init'. Refer to the values of + Initializer for more details. Default: 'zeros'. + + Inputs: + - **input** (Tensor) - Tensor of shape :math:`(N, C_{in}, H_{in}, W_{in})`. + + Outputs: + Tensor of shape :math:`(N, C_{out}, H_{out}, W_{out})`. + + Examples: + >>> net = nn.DepthwiseConv2d(120, 240, 4, has_bias=False, weight_init='normal') + >>> input = Tensor(np.ones([1, 120, 1024, 640]), mindspore.float32) + >>> net(input).shape + (1, 240, 1024, 640) + """ + def __init__(self, + in_channels, + out_channels, + kernel_size, + stride=1, + pad_mode='same', + padding=0, + dilation=1, + group=1, + has_bias=False, + weight_init='normal', + bias_init='zeros'): + super(DepthwiseConv2d, self).__init__() + self.kernel_size = twice(kernel_size) + self.stride = twice(stride) + self.dilation = twice(dilation) + self.in_channels = check_int_positive(in_channels) + self.out_channels = check_int_positive(out_channels) + validator.check_integer('group', group, in_channels, Rel.EQ) + validator.check_integer('group', group, out_channels, Rel.EQ) + validator.check_integer('group', group, 1, Rel.GE) + self.pad_mode = pad_mode + self.padding = padding + self.dilation = dilation + self.group = group + self.has_bias = has_bias + self.conv = P.DepthwiseConv2dNative(channel_multiplier=1, + kernel_size=self.kernel_size, + pad_mode=self.pad_mode, + pad=self.padding, + stride=self.stride, + dilation=self.dilation) + self.bias_add = P.BiasAdd() + weight_shape = [1, in_channels, *self.kernel_size] + self.weight = Parameter(initializer(weight_init, weight_shape), name='weight') + if check_bool(has_bias): + self.bias = Parameter(initializer(bias_init, [out_channels]), name='bias') + else: + if bias_init != 'zeros': + logger.warning("value of `has_bias` is False, value of `bias_init` will be ignore.") + self.bias = None + + def construct(self, x): + out = self.conv(x, self.weight) + if self.has_bias: + out = self.bias_add(out, self.bias) + return out + + def extend_repr(self): + s = 'input_channels={}, output_channels={}, kernel_size={}, stride={}, ' \ + 'pad_mode={}, padding={}, dilation={}, group={},' \ + 'has_bias={}, weight_init={}, bias_init={}'.format( + self.in_channels, self.out_channels, self.kernel_size, self.stride, + self.pad_mode, self.padding, self.dilation, self.group, + self.has_bias, self.weight_init, self.bias_init) + + if self.has_bias: + s += ', bias={}'.format(self.bias) + return s diff --git a/mindspore/nn/layer/normalization.py b/mindspore/nn/layer/normalization.py index 4c7ea9d4d6..d6c920b620 100644 --- a/mindspore/nn/layer/normalization.py +++ b/mindspore/nn/layer/normalization.py @@ -84,14 +84,13 @@ class _BatchNorm(Cell): self.dtype = P.DType() self.reshape = P.Reshape() self.is_ascend = context.get_context("device_target") == "Ascend" - self.is_graph_mode = context.get_context("mode") == context.GRAPH_MODE self.momentum = 1.0 - momentum if context.get_context("enable_ge"): self.is_ge_backend = True else: self.is_ge_backend = False - if self.is_graph_mode and (self.is_ge_backend or self.is_ascend): + if self.is_ge_backend or self.is_ascend: self.bn_train = P.BatchNorm(is_training=True, epsilon=self.eps) else: @@ -153,7 +152,7 @@ class _BatchNorm(Cell): if self.is_ge_backend and self.is_global: axes, re_shape = _shape_infer(F.shape(x), self.num_features) y = self._global_sync(x, axes, re_shape) - elif self.is_graph_mode and (self.is_ge_backend or self.is_ascend): + elif self.is_ge_backend or self.is_ascend: if self.is_global: axes, re_shape = _shape_infer(F.shape(x), self.num_features) y = self._global_sync(x, axes, re_shape) diff --git a/mindspore/nn/layer/quant.py b/mindspore/nn/layer/quant.py index 14731c6262..f0c82937c5 100644 --- a/mindspore/nn/layer/quant.py +++ b/mindspore/nn/layer/quant.py @@ -16,6 +16,7 @@ from functools import partial import numpy as np + import mindspore.common.dtype as mstype from mindspore.ops import operations as P from mindspore.ops import functional as F @@ -23,10 +24,9 @@ from mindspore.common.parameter import Parameter from mindspore.common.initializer import initializer from mindspore.common.tensor import Tensor from mindspore._checkparam import check_int_positive, check_bool, twice -from mindspore._checkparam import Validator as validator, Rel -from mindspore.nn.cell import Cell -from mindspore.nn.layer.activation import get_activation +from mindspore._checkparam import Rel import mindspore.context as context + from .normalization import BatchNorm2d from .activation import get_activation from ..cell import Cell @@ -82,7 +82,7 @@ class Conv2dBnAct(Cell): bias_init (Union[Tensor, str, Initializer, numbers.Number]): Initializer for the bias vector. Possible Initializer and string are the same as 'weight_init'. Refer to the values of Initializer for more details. Default: 'zeros'. - batchnorm (bool): Specifies to used batchnorm or not. Default: None. + has_bn (bool): Specifies to used batchnorm or not. Default: False. activation (string): Specifies activation type. The optional values are as following: 'softmax', 'logsoftmax', 'relu', 'relu6', 'tanh', 'gelu', 'sigmoid', 'prelu', 'leakyrelu', 'hswish', 'hsigmoid'. Default: None. @@ -94,7 +94,7 @@ class Conv2dBnAct(Cell): Tensor of shape :math:`(N, C_{out}, H_{out}, W_{out})`. Examples: - >>> net = Conv2dBnAct(120, 240, 4, batchnorm=True, activation='ReLU') + >>> net = Conv2dBnAct(120, 240, 4, has_bn=True, activation='ReLU') >>> input = Tensor(np.ones([1, 120, 1024, 640]), mindspore.float32) >>> net(input).shape (1, 240, 1024, 640) @@ -112,28 +112,39 @@ class Conv2dBnAct(Cell): has_bias=False, weight_init='normal', bias_init='zeros', - batchnorm=None, + has_bn=False, activation=None): super(Conv2dBnAct, self).__init__() - self.conv = conv.Conv2d( - in_channels, - out_channels, - kernel_size, - stride, - pad_mode, - padding, - dilation, - group, - has_bias, - weight_init, - bias_init) - self.has_bn = batchnorm is not None + + if context.get_context('device_target') == "Ascend" and group > 1: + self.conv = conv.DepthwiseConv2d(in_channels, + out_channels, + kernel_size=kernel_size, + stride=stride, + pad_mode=pad_mode, + padding=padding, + dilation=dilation, + group=group, + has_bias=has_bias, + weight_init=weight_init, + bias_init=bias_init) + else: + self.conv = conv.Conv2d(in_channels, + out_channels, + kernel_size=kernel_size, + stride=stride, + pad_mode=pad_mode, + padding=padding, + dilation=dilation, + group=group, + has_bias=has_bias, + weight_init=weight_init, + bias_init=bias_init) + + self.has_bn = validator.check_bool("has_bn", has_bn) self.has_act = activation is not None - self.batchnorm = batchnorm - if batchnorm is True: + if has_bn: self.batchnorm = BatchNorm2d(out_channels) - elif batchnorm is not None: - validator.check_isinstance('batchnorm', batchnorm, (BatchNorm2d,)) self.activation = get_activation(activation) def construct(self, x): @@ -160,7 +171,7 @@ class DenseBnAct(Cell): same as input x. The values of str refer to the function `initializer`. Default: 'zeros'. has_bias (bool): Specifies whether the layer uses a bias vector. Default: True. activation (str): Regularizer function applied to the output of the layer, eg. 'relu'. Default: None. - batchnorm (bool): Specifies to used batchnorm or not. Default: None. + has_bn (bool): Specifies to used batchnorm or not. Default: False. activation (string): Specifies activation type. The optional values are as following: 'softmax', 'logsoftmax', 'relu', 'relu6', 'tanh', 'gelu', 'sigmoid', 'prelu', 'leakyrelu', 'hswish', 'hsigmoid'. Default: None. @@ -183,7 +194,7 @@ class DenseBnAct(Cell): weight_init='normal', bias_init='zeros', has_bias=True, - batchnorm=None, + has_bn=False, activation=None): super(DenseBnAct, self).__init__() self.dense = basic.Dense( @@ -192,12 +203,10 @@ class DenseBnAct(Cell): weight_init, bias_init, has_bias) - self.has_bn = batchnorm is not None + self.has_bn = validator.check_bool("has_bn", has_bn) self.has_act = activation is not None - if batchnorm is True: + if has_bn: self.batchnorm = BatchNorm2d(out_channels) - elif batchnorm is not None: - validator.check_isinstance('batchnorm', batchnorm, (BatchNorm2d,)) self.activation = get_activation(activation) def construct(self, x): @@ -312,6 +321,10 @@ class FakeQuantWithMinMax(Cell): quant_delay=0): """init FakeQuantWithMinMax layer""" super(FakeQuantWithMinMax, self).__init__() + validator.check_type("min_init", min_init, [int, float]) + validator.check_type("max_init", max_init, [int, float]) + validator.check("min_init", min_init, "max_init", max_init, rel=Rel.LT) + validator.check_integer('quant_delay', quant_delay, 0, Rel.GE) self.min_init = min_init self.max_init = max_init self.num_bits = num_bits @@ -1180,15 +1193,16 @@ class QuantBlock(Cell): self.dequant = dequant_op self.dequant_scale = dequant_scale self.bias = bias - self.has_bias = bias is None + self.has_bias = bias is not None self.activation = activation - self.has_act = activation is None + self.has_act = activation is not None + self.bias_add = P.BiasAdd() def construct(self, x): x = self.quant(x) x = self.core_op(x, self.weight) if self.has_bias: - output = self.bias_add(output, self.bias) + x = self.bias_add(x, self.bias) if self.has_act: x = self.activation(x) x = self.dequant(x, self.dequant_scale) diff --git a/mindspore/nn/loss/loss.py b/mindspore/nn/loss/loss.py index 4639229c41..3f97fbf83c 100644 --- a/mindspore/nn/loss/loss.py +++ b/mindspore/nn/loss/loss.py @@ -369,7 +369,7 @@ class CosineEmbeddingLoss(_Loss): >>> x2 = Tensor(np.array([[0.4, 1.2], [-0.4, -0.9]]), mindspore.float32) >>> y = Tensor(np.array([1,-1]), mindspore.int32) >>> cosine_embedding_loss = P.CosineEmbeddingLoss() - >>> cosine_embedding_loss(x1, x2, target) + >>> cosine_embedding_loss(x1, x2, y) [0.0003426671] """ def __init__(self, margin=0.0, reduction="mean"): diff --git a/mindspore/nn/optim/adam.py b/mindspore/nn/optim/adam.py index 5a40d30d5a..b73c284aab 100755 --- a/mindspore/nn/optim/adam.py +++ b/mindspore/nn/optim/adam.py @@ -29,8 +29,9 @@ from .optimizer import Optimizer _adam_opt = C.MultitypeFuncGraph("adam_opt") -@_adam_opt.register("Tensor", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor", "Bool") -def _update_run_op(beta1, beta2, eps, lr, weight_decay_tensor, param, m, v, gradient, decay_flag): +@_adam_opt.register("Tensor", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor", + "Tensor", "Bool", "Bool") +def _update_run_op(beta1, beta2, eps, lr, weight_decay_tensor, param, m, v, gradient, decay_flag, optim_filter): """ Update parameters. @@ -44,38 +45,44 @@ def _update_run_op(beta1, beta2, eps, lr, weight_decay_tensor, param, m, v, grad m (Tensor): m value of parameters. v (Tensor): v value of parameters. gradient (Tensor): Gradient of parameters. + decay_flag (bool): Applies weight decay or not. + optim_filter (bool): Applies parameter update or not. Returns: Tensor, the new value of v after updating. """ - op_mul = P.Mul() - op_square = P.Square() - op_sqrt = P.Sqrt() - op_cast = P.Cast() - op_reshape = P.Reshape() - op_shape = P.Shape() + if optim_filter: + op_mul = P.Mul() + op_square = P.Square() + op_sqrt = P.Sqrt() + op_cast = P.Cast() + op_reshape = P.Reshape() + op_shape = P.Shape() - param_fp32 = op_cast(param, mstype.float32) - m_fp32 = op_cast(m, mstype.float32) - v_fp32 = op_cast(v, mstype.float32) - gradient_fp32 = op_cast(gradient, mstype.float32) + param_fp32 = op_cast(param, mstype.float32) + m_fp32 = op_cast(m, mstype.float32) + v_fp32 = op_cast(v, mstype.float32) + gradient_fp32 = op_cast(gradient, mstype.float32) - next_m = op_mul(beta1, m_fp32) + op_mul(op_cast(F.tuple_to_array((1.0,)), mstype.float32) - beta1, gradient_fp32) + next_m = op_mul(beta1, m_fp32) + op_mul(op_cast(F.tuple_to_array((1.0,)), mstype.float32) + - beta1, gradient_fp32) - next_v = op_mul(beta2, v_fp32) + op_mul(op_cast(F.tuple_to_array((1.0,)), mstype.float32) - - beta2, op_square(gradient_fp32)) + next_v = op_mul(beta2, v_fp32) + op_mul(op_cast(F.tuple_to_array((1.0,)), mstype.float32) + - beta2, op_square(gradient_fp32)) - update = next_m / (eps + op_sqrt(next_v)) - if decay_flag: - update = op_mul(weight_decay_tensor, param_fp32) + update - update_with_lr = op_mul(lr, update) - next_param = param_fp32 - op_reshape(update_with_lr, op_shape(param_fp32)) + update = next_m / (eps + op_sqrt(next_v)) + if decay_flag: + update = op_mul(weight_decay_tensor, param_fp32) + update - next_v = F.depend(next_v, F.assign(param, op_cast(next_param, mstype.float16))) - next_v = F.depend(next_v, F.assign(m, op_cast(next_m, mstype.float16))) - next_v = F.depend(next_v, F.assign(v, op_cast(next_v, mstype.float16))) - return next_v + update_with_lr = op_mul(lr, update) + next_param = param_fp32 - op_reshape(update_with_lr, op_shape(param_fp32)) + + next_param = F.depend(next_param, F.assign(param, op_cast(next_param, F.dtype(param)))) + next_param = F.depend(next_param, F.assign(m, op_cast(next_m, F.dtype(m)))) + next_param = F.depend(next_param, F.assign(v, op_cast(next_v, F.dtype(v)))) + return next_param + return gradient def _check_param_value(beta1, beta2, eps, weight_decay, prim_name): @@ -157,7 +164,7 @@ class Adam(Optimizer): The sparse strategy is applied while the SparseGatherV2 operator being used for forward network and the `sparse_grad` of `Parameter` being set. The sparse feature is under continuous development. The sparse - behavior is currently performed on the CPU, weight decay is not supported. + behavior is currently performed on the CPU. Args: params (Union[list[Parameter], list[dict]]): When the `params` is a list of `Parameter` which will be updated, @@ -174,8 +181,7 @@ class Adam(Optimizer): - order_params: Optional. If "order_params" in the keys, the value should be the order of parameters and the order will be followed in optimizer. There are no other keys in the `dict` and the parameters which - in the value of 'order_params' but not in any group will use default learning rate and default weight - decay. + in the value of 'order_params' should be in one of group parameters. learning_rate (Union[int, float, Tensor, Iterable]): A value for the learning rate. When the learning_rate is Iterable or a Tensor and the dims of the Tensor is 1, @@ -213,16 +219,14 @@ class Adam(Optimizer): >>> >>> #2) Use parameter groups and set different values >>> conv_params = list(filter(lambda x: 'conv' in x.name, net.trainable_params())) - >>> bias_params = list(filter(lambda x: 'bias' in x.name, net.trainable_params())) + >>> no_conv_params = list(filter(lambda x: 'conv' not in x.name, net.trainable_params())) >>> group_params = [{'params': conv_params, 'weight_decay': 0.01}, - >>> {'params': bias_params, 'lr': 0.01}, + >>> {'params': no_conv_params, 'lr': 0.01}, >>> {'order_params': net.trainable_params()}] >>> opt = nn.Adam(group_params, learning_rate=0.1, weight_decay=0.0) >>> # The conv_params's parameters will use a learning rate of default value 0.1 and a weight decay of 0.01. - >>> # The bias_params's parameters will use a learning rate of 0.01 and a weight decay of default value 0.0. + >>> # The no_conv_params's parameters will use a learning rate of 0.01 and a weight decay of default value 0.0. >>> # The final parameters order in which the optimizer will be followed is the value of 'order_params'. - >>> # The parameters which in the value of 'order_params' but not in any group will use a learning rate - >>> # of default value 0.1 and a weight decay of default value 0.0. >>> >>> loss = nn.SoftmaxCrossEntropyWithLogits() >>> model = Model(net, loss_fn=loss, optimizer=optim) @@ -300,7 +304,7 @@ class AdamWeightDecay(Optimizer): - **gradients** (tuple[Tensor]) - The gradients of `params`, the shape is the same as `params`. Outputs: - tuple[Parameter], the updated velocity value, the shape is the same as `params`. + tuple[bool], all elements are True. Examples: >>> net = Net() @@ -328,11 +332,13 @@ class AdamWeightDecay(Optimizer): def construct(self, gradients): lr = self.get_lr() - updated_velocity = self.hyper_map(F.partial(_adam_opt, self.beta1, self.beta2, self.eps, lr, - self.weight_decay_tensor), - self.params, self.moments1, self.moments2, gradients, self.decay_flag) - - return updated_velocity + optim_result = self.hyper_map(F.partial(_adam_opt, self.beta1, self.beta2, self.eps, lr, + self.weight_decay_tensor), + self.params, self.moments1, self.moments2, gradients, + self.decay_flag, self.optim_filter) + if self.use_parallel: + optim_result = self.broadcast_params(optim_result) + return optim_result class AdamWeightDecayDynamicLR(Optimizer): @@ -363,7 +369,7 @@ class AdamWeightDecayDynamicLR(Optimizer): - **gradients** (tuple[Tensor]) - The gradients of `params`, the shape is the same as `params`. Outputs: - tuple[Parameter], the updated velocity value, the shape is the same as `params`. + tuple[bool], all elements are True. Examples: >>> net = Net() @@ -424,12 +430,14 @@ class AdamWeightDecayDynamicLR(Optimizer): warmup_lr = self.start_learning_rate * warmup_percent is_warmup = self.cast(self.greater(self.warmup_steps, self.global_step), mstype.float32) lr = (self.one - is_warmup) * lr + is_warmup * warmup_lr - updated_velocity = self.hyper_map(F.partial(_adam_opt, self.beta1, self.beta2, self.eps, lr, - self.weight_decay_tensor), - self.params, self.moments1, self.moments2, gradients, self.decay_flag) - + optim_result = self.hyper_map(F.partial(_adam_opt, self.beta1, self.beta2, self.eps, lr, + self.weight_decay_tensor), + self.params, self.moments1, self.moments2, gradients, + self.decay_flag, self.optim_filter) + if self.use_parallel: + optim_result = self.broadcast_params(optim_result) added_global_step = self.global_step + self.one F.control_depend(lr, added_global_step) self.global_step = added_global_step - return updated_velocity + return optim_result diff --git a/mindspore/nn/optim/ftrl.py b/mindspore/nn/optim/ftrl.py index a40d6737cb..b2954430b4 100644 --- a/mindspore/nn/optim/ftrl.py +++ b/mindspore/nn/optim/ftrl.py @@ -16,6 +16,7 @@ from mindspore.ops import functional as F, composite as C, operations as P from mindspore.common import Tensor import mindspore.common.dtype as mstype +from mindspore.ops.operations import _inner_ops as inner from mindspore._checkparam import Validator as validator from mindspore._checkparam import Rel from .optimizer import Optimizer, _apply_decay, _grad_scale @@ -73,7 +74,7 @@ class FTRL(Optimizer): Note: The sparse strategy is applied while the SparseGatherV2 operator being used for forward network and the `sparse_grad` of `Parameter` being set. The sparse feature is under continuous development. The sparse - behavior is currently performed on the CPU, weight decay is not supported. + behavior is currently performed on the CPU. Args: params (list[Parameter]): A list of parameter, which will be updated. The element in `params` @@ -116,7 +117,7 @@ class FTRL(Optimizer): self.decay_tf = tuple((lambda: True)() for x in self.parameters) self.hyper_map = C.HyperMap() self.opt = P.ApplyFtrl(use_locking=use_locking) - self.sparse_opt = P.SparseApplyFtrl(learning_rate, l1, l2, lr_power, use_locking=use_locking) + self.sparse_opt = inner.SparseApplyFtrlNoReturn(learning_rate, l1, l2, lr_power, use_locking=use_locking) def construct(self, grads): params = self.parameters @@ -124,7 +125,7 @@ class FTRL(Optimizer): linear = self.linear lr = self.learning_rate if self.weight_decay > 0.0: - grads = self.hyper_map(F.partial(_apply_decay, self.weight_decay), self.decay_tf, params, grads) + grads = self.map_(F.partial(_apply_decay, self.weight_decay), self.decay_tf, params, grads) grads = self.scale_grad(grads) success = self.map_(F.partial(_ftrl_opt, self.opt, self.sparse_opt, lr, self.l1, self.l2, self.lr_power), diff --git a/mindspore/nn/optim/lamb.py b/mindspore/nn/optim/lamb.py index 832b35d66f..93c7edbce8 100755 --- a/mindspore/nn/optim/lamb.py +++ b/mindspore/nn/optim/lamb.py @@ -32,11 +32,10 @@ num_one = Tensor(np.ones([1]), mstype.float32) _lamb_opt = C.MultitypeFuncGraph("lamb_opt") - -@_lamb_opt.register("Tensor", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor", - "Tensor", "Tensor", "Tensor", "Tensor", "Bool") +@_lamb_opt.register("Tensor", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor", + "Tensor", "Bool", "Bool") def _update_run_op(beta1, beta2, eps, lr, weight_decay_tensor, global_step, param, m, v, - gradient, decay_flag): + gradient, decay_flag, optim_filter): """ Update parameters. @@ -52,66 +51,66 @@ def _update_run_op(beta1, beta2, eps, lr, weight_decay_tensor, global_step, para v (Tensor): v value of parameters. gradient (Tensor): Gradient of parameters. decay_flag (bool): Specifies whether param update with weight decay. + optim_filter(bool): Applies parameter update or not. Returns: Tensor, the new value of v after updating. """ - op_mul = P.Mul() - op_sqrt = P.Sqrt() - op_rsqrt = P.Rsqrt() - op_square = P.Square() - op_cast = P.Cast() - op_reshape = P.Reshape() - op_shape = P.Shape() - op_pow = P.Pow() - op_norm = layer.Norm() - op_select = P.Select() - op_greater = P.Greater() - op_fill = P.Fill() - op_dtype = P.DType() - - param_fp32 = op_cast(param, mstype.float32) - m_fp32 = op_cast(m, mstype.float32) - v_fp32 = op_cast(v, mstype.float32) - gradient_fp32 = op_cast(gradient, mstype.float32) - - next_m = op_mul(beta1, m_fp32) + op_mul(op_cast(num_one, - mstype.float32) - beta1, gradient_fp32) - - next_v = op_mul(beta2, v_fp32) + op_mul(op_cast(num_one, - mstype.float32) - beta2, op_square(gradient_fp32)) - - next_mm = next_m / (op_cast(num_one, mstype.float32) - - op_pow(beta1, op_cast(global_step + num_one, mstype.float32))) - next_vv = next_v / (op_cast(num_one, mstype.float32) - - op_pow(beta2, op_cast(global_step + num_one, mstype.float32))) - w_norm = op_norm(param_fp32) - g_norm = op_norm(gradient_fp32) - - g_norm_hat = op_norm(op_mul(next_mm, op_rsqrt( - next_vv + eps)) + weight_decay_tensor * param_fp32) - zeros = F.zeros_like(w_norm) - ones = op_fill(op_dtype(w_norm), op_shape(w_norm), 1.0) - trust_ratio = op_select( - op_greater(w_norm, zeros), - op_select(op_greater(g_norm, zeros), w_norm / g_norm_hat, ones), - ones) - tens = op_fill(op_dtype(trust_ratio), op_shape(trust_ratio), 10.0) - trust_ratio = C.clip_by_value(trust_ratio, zeros, tens) - update = next_mm / (op_sqrt(next_vv) + eps) - - if decay_flag: - update = update + op_mul(weight_decay_tensor, param_fp32) - - update_with_lr = op_mul(op_mul(trust_ratio, lr), update) - - next_param = param_fp32 - op_reshape(update_with_lr, op_shape(param_fp32)) - - next_v = F.depend(next_v, F.assign(param, next_param)) - next_v = F.depend(next_v, F.assign(m, next_m)) - next_v = F.depend(next_v, F.assign(v, next_v)) - - return next_v + if optim_filter: + op_mul = P.Mul() + op_sqrt = P.Sqrt() + op_rsqrt = P.Rsqrt() + op_square = P.Square() + op_cast = P.Cast() + op_reshape = P.Reshape() + op_shape = P.Shape() + op_pow = P.Pow() + op_norm = layer.Norm() + op_select = P.Select() + op_greater = P.Greater() + op_fill = P.Fill() + op_dtype = P.DType() + + param_fp32 = op_cast(param, mstype.float32) + m_fp32 = op_cast(m, mstype.float32) + v_fp32 = op_cast(v, mstype.float32) + gradient_fp32 = op_cast(gradient, mstype.float32) + + next_m = op_mul(beta1, m_fp32) + op_mul(op_cast(num_one, mstype.float32) - beta1, gradient_fp32) + + next_v = op_mul(beta2, v_fp32) + op_mul(op_cast(num_one, mstype.float32) - beta2, op_square(gradient_fp32)) + + next_mm = next_m / (op_cast(num_one, mstype.float32) + - op_pow(beta1, op_cast(global_step + num_one, mstype.float32))) + next_vv = next_v / (op_cast(num_one, mstype.float32) - + op_pow(beta2, op_cast(global_step + num_one, mstype.float32))) + w_norm = op_norm(param_fp32) + g_norm = op_norm(gradient_fp32) + + g_norm_hat = op_norm(op_mul(next_mm, op_rsqrt(next_vv + eps)) + weight_decay_tensor * param_fp32) + zeros = F.zeros_like(w_norm) + ones = op_fill(op_dtype(w_norm), op_shape(w_norm), 1.0) + trust_ratio = op_select( + op_greater(w_norm, zeros), + op_select(op_greater(g_norm, zeros), w_norm / g_norm_hat, ones), + ones) + tens = op_fill(op_dtype(trust_ratio), op_shape(trust_ratio), 10.0) + trust_ratio = C.clip_by_value(trust_ratio, zeros, tens) + update = next_mm / (op_sqrt(next_vv) + eps) + + if decay_flag: + update = update + op_mul(weight_decay_tensor, param_fp32) + + update_with_lr = op_mul(op_mul(trust_ratio, lr), update) + + next_param = param_fp32 - op_reshape(update_with_lr, op_shape(param_fp32)) + + next_param = F.depend(next_param, F.assign(param, next_param)) + next_param = F.depend(next_param, F.assign(m, next_m)) + next_param = F.depend(next_param, F.assign(v, next_v)) + + return next_param + return gradient lamb_opt_graph_kernel = C.MultitypeFuncGraph("lamb_opt_graph_kernel") @@ -238,7 +237,7 @@ class Lamb(Optimizer): - **gradients** (tuple[Tensor]) - The gradients of `params`, the shape is the same as `params`. Outputs: - tuple[Parameter], the updated velocity value, the shape is the same as `params`. + tuple[bool], all elements are True. Examples: >>> net = Net() @@ -311,18 +310,21 @@ class Lamb(Optimizer): self.warmup_steps, self.global_step), mstype.float32) lr = (self.one - is_warmup) * lr + is_warmup * warmup_lr if self.enable_graph_kernel: - updated_velocity = self.hyper_map(F.partial(lamb_opt_graph_kernel, - self.beta1, self.beta2, self.eps, lr, - self.weight_decay_tensor, self.global_step), - self.params, self.moments1, self.moments2, gradients, self.decay_flag) + optim_result = self.hyper_map(F.partial(lamb_opt_graph_kernel, + self.beta1, self.beta2, self.eps, lr, + self.weight_decay_tensor, self.global_step), + self.params, self.moments1, self.moments2, gradients, self.decay_flag) else: - updated_velocity = self.hyper_map(F.partial(_lamb_opt, - self.beta1, self.beta2, self.eps, lr, - self.weight_decay_tensor, self.global_step), - self.params, self.moments1, self.moments2, gradients, self.decay_flag) + optim_result = self.hyper_map(F.partial(_lamb_opt, + self.beta1, self.beta2, self.eps, lr, + self.weight_decay_tensor, self.global_step), + self.params, self.moments1, self.moments2, gradients, + self.decay_flag, self.optim_filter) + if self.use_parallel: + optim_result = self.broadcast_params(optim_result) added_global_step = self.global_step + self.one F.control_depend(lr, added_global_step) self.global_step = added_global_step - return updated_velocity + return optim_result diff --git a/mindspore/nn/optim/lars.py b/mindspore/nn/optim/lars.py index b55d1c5574..7b05b372eb 100755 --- a/mindspore/nn/optim/lars.py +++ b/mindspore/nn/optim/lars.py @@ -13,22 +13,18 @@ # limitations under the License. # ============================================================================ """lars optimizer""" -from typing import Iterable -from mindspore.common import dtype as mstype -from mindspore.common import Tensor -from mindspore.common.initializer import initializer -from mindspore.common.parameter import Parameter from mindspore.ops import operations as P from mindspore.ops import composite as C from mindspore.ops import functional as F from mindspore._checkparam import Validator as validator +from mindspore.common import Tensor, Parameter, dtype as mstype from .optimizer import _grad_scale, Optimizer _lars_opt = C.MultitypeFuncGraph("lars_opt") -@_lars_opt.register("Function", "Number", "Tensor", "Tensor", "Tensor", "Bool", "Bool") -def _tensor_run_opt(lars, weight_decay, learning_rate, gradient, weight, decay_flag, lars_flag): +@_lars_opt.register("Function", "Tensor", "Number", "Tensor", "Tensor", "Bool", "Bool") +def _tensor_run_opt(lars, learning_rate, weight_decay, gradient, weight, decay_flag, lars_flag): """Apply lars optimizer to the weight parameter.""" if lars_flag: op_reduce_sum = P.SquareSumAll() @@ -42,10 +38,12 @@ def _tensor_run_opt(lars, weight_decay, learning_rate, gradient, weight, decay_f return gradient -def _check_param_value(optimizer, epsilon, hyperpara, use_clip, prim_name): +def _check_param_value(optimizer, epsilon, coefficient, use_clip, prim_name): validator.check_value_type("optimizer", optimizer, Optimizer, prim_name) + if "Adam" in optimizer.cls_name or "Lamb" in optimizer.cls_name: + raise TypeError("LARS can not be used with ", optimizer.cls_name) validator.check_value_type("epsilon", epsilon, [float], prim_name) - validator.check_value_type("hyperpara", hyperpara, [float], prim_name) + validator.check_value_type("coefficient", coefficient, [float], prim_name) validator.check_value_type("use_clip", use_clip, [bool], prim_name) class LARS(Optimizer): @@ -58,14 +56,10 @@ class LARS(Optimizer): Args: optimizer (Optimizer): MindSpore optimizer for which to wrap and modify gradients. epsilon (float): Term added to the denominator to improve numerical stability. Default: 1e-05. - hyperpara (float): Trust coefficient for calculating the local learning rate. Default: 0.001. - weight_decay (float): Weight decay (L2 penalty). It should be equal to or greater than 0. Default: 0.0. + coefficient (float): Trust coefficient for calculating the local learning rate. Default: 0.001. use_clip (bool): Whether to use clip operation for calculating the local learning rate. Default: False. - decay_filter (Function): A function to determine whether apply weight decay on parameters. Default: - lambda x: 'LayerNorm' not in x.name and 'bias' not in x.name. lars_filter (Function): A function to determine whether apply lars algorithm. Default: lambda x: 'LayerNorm' not in x.name and 'bias' not in x.name. - loss_scale (float): A floating point value for the loss scale. It should be greater than 0. Default: 1.0. Inputs: - **gradients** (tuple[Tensor]) - The gradients of `params` in optimizer, the shape is @@ -78,51 +72,54 @@ class LARS(Optimizer): >>> net = Net() >>> loss = nn.SoftmaxCrossEntropyWithLogits() >>> opt = nn.Momentum(net.trainable_params(), 0.1, 0.9) - >>> opt_lars = nn.LARS(opt, epsilon=1e-08, hyperpara=0.02) + >>> opt_lars = nn.LARS(opt, epsilon=1e-08, coefficient=0.02) >>> model = Model(net, loss_fn=loss, optimizer=opt_lars, metrics=None) """ - def __init__(self, optimizer, epsilon=1e-05, hyperpara=0.001, weight_decay=0.0, use_clip=False, - decay_filter=lambda x: 'LayerNorm' not in x.name and 'bias' not in x.name, - lars_filter=lambda x: 'LayerNorm' not in x.name and 'bias' not in x.name, loss_scale=1.0): - super(LARS, self).__init__(0.0, [Parameter(Tensor(0.0), name="trivial")], weight_decay, loss_scale) - if optimizer.is_group: - raise RuntimeError(f"The {self.cls_name} optimizer cannot support group setting.") - _check_param_value(optimizer, epsilon, hyperpara, use_clip, self.cls_name) + def __init__(self, optimizer, epsilon=1e-05, coefficient=0.001, use_clip=False, + lars_filter=lambda x: 'LayerNorm' not in x.name and 'bias' not in x.name): + super(LARS, self).__init__(0.0, [Parameter(Tensor(0.0), name="fake_param")]) + _check_param_value(optimizer, epsilon, coefficient, use_clip, self.cls_name) self.opt = optimizer - self.parameters = optimizer.parameters - self.learning_rate = optimizer.learning_rate - self.lars = P.LARSUpdate(epsilon, hyperpara, use_clip) - self.reciprocal_scale = 1.0 / loss_scale - self.weight_decay = weight_decay + self.lars = P.LARSUpdate(epsilon, coefficient, use_clip) self.cast = P.Cast() - self.decay_flag = tuple(decay_filter(x) for x in self.parameters) + self.parameters = optimizer.parameters + if use_clip is True: + self.learning_rate = optimizer.learning_rate + self.dynamic_lr = optimizer.dynamic_lr + self.gather = optimizer.gather + self.assignadd = optimizer.assignadd + self.global_step = optimizer.global_step + else: + self.learning_rate = Parameter(Tensor(0.0, dtype=mstype.float32), name="fake_lr") + self.reciprocal_scale = optimizer.reciprocal_scale + optimizer.reciprocal_scale = 1.0 + self.is_group = optimizer.is_group + if self.is_group: + self.weight_decay = tuple(map(lambda x: x / optimizer.loss_scale, optimizer.weight_decay)) + else: + self.weight_decay = optimizer.weight_decay / optimizer.loss_scale + optimizer.exec_weight_decay = False + optimizer.weight_decay = 0.0 + self.decay_flags = optimizer.decay_flags self.lars_flag = tuple(lars_filter(x) for x in self.parameters) self.hyper_map = C.HyperMap() - self.dynamic_lr = False - self.gather = None - self.global_step = None - self.axis = None - if isinstance(self.learning_rate.default_input, Iterable) or \ - (isinstance(self.learning_rate.default_input, Tensor) and self.learning_rate.default_input.dim() == 1): - self.dynamic_lr = True - self.assignadd = P.AssignAdd() - self.gather = P.GatherV2() - self.global_step = Parameter(initializer(0, [1], mstype.int32), name="lars_global_step") - self.axis = 0 def construct(self, gradients): params = self.parameters if self.dynamic_lr: - lr = self.gather(self.learning_rate, self.global_step, self.axis) + lr = self.gather(self.learning_rate, self.global_step, 0) F.control_depend(lr, self.assignadd(self.global_step, 1)) else: lr = self.learning_rate if self.reciprocal_scale != 1.0: gradients = self.hyper_map(F.partial(_grad_scale, self.reciprocal_scale), gradients) - - grad_t = self.hyper_map(F.partial(_lars_opt, self.lars, self.weight_decay, lr), - gradients, params, self.decay_flag, self.lars_flag) + if self.is_group: + grad_t = self.hyper_map(F.partial(_lars_opt, self.lars, lr), self.weight_decay, + gradients, params, self.decay_flags, self.lars_flag) + else: + grad_t = self.hyper_map(F.partial(_lars_opt, self.lars, lr, self.weight_decay), + gradients, params, self.decay_flags, self.lars_flag) success = self.opt(grad_t) return success diff --git a/mindspore/nn/optim/lazyadam.py b/mindspore/nn/optim/lazyadam.py index 48d33bf798..4b97d2eb20 100644 --- a/mindspore/nn/optim/lazyadam.py +++ b/mindspore/nn/optim/lazyadam.py @@ -94,8 +94,7 @@ class LazyAdam(Optimizer): The sparse strategy is applied while the SparseGatherV2 operator being used for forward network and the `sparse_grad` of `Parameter` being set. The sparse behavior, to be notice, is not equivalent to the original Adam algorithm, as only the current indices parames will be updated. The sparse feature is under - continuous development. The sparse behavior is currently performed on the CPU, weight decay is - not supported. + continuous development. The sparse behavior is currently performed on the CPU. Args: params (Union[list[Parameter], list[dict]]): When the `params` is a list of `Parameter` which will be updated, @@ -110,6 +109,10 @@ class LazyAdam(Optimizer): - weight_decay: Optional. If "weight_decay" in the keys, the value of corresponding weight decay will be used. If not, the `weight_decay` in the API will be used. + - order_params: Optional. If "order_params" in the keys, the value should be the order of parameters and + the order will be followed in optimizer. There are no other keys in the `dict` and the parameters which + in the value of 'order_params' should be in one of group parameters. + learning_rate (Union[float, Tensor, Iterable]): A value for the learning rate. When the learning_rate is Iterable or a Tensor and the dims of the Tensor is 1, use dynamic learning rate, then the i-th step will @@ -147,12 +150,13 @@ class LazyAdam(Optimizer): >>> #2) Use parameter groups and set different values >>> conv_params = list(filter(lambda x: 'conv' in x.name, net.trainable_params())) >>> no_conv_params = list(filter(lambda x: 'conv' not in x.name, net.trainable_params())) - >>> group_params = [{'params': conv_params, 'weight_decay': 0.01, 'lr': 0.01}, - >>> {'params': no_conv_params}] + >>> group_params = [{'params': conv_params, 'weight_decay': 0.01}, + >>> {'params': no_conv_params, 'lr': 0.01}, + >>> {'order_params': net.trainable_params()}] >>> opt = nn.LazyAdam(group_params, learning_rate=0.1, weight_decay=0.0) - >>> # the conv_params's parameters will use a learning rate of 0.01 and a weight decay of 0.01 - >>> # the no_cov_params's parameters don't set learning and weight decay. So they will use a - >>> # learning rate of 0.1 and a weight decay of 0.0. + >>> # The conv_params's parameters will use a learning rate of default value 0.1 and a weight decay of 0.01. + >>> # The no_conv_params's parameters will use a learning rate of 0.01 and a weight decay of default value 0.0. + >>> # The final parameters order in which the optimizer will be followed is the value of 'order_params'. >>> >>> loss = nn.SoftmaxCrossEntropyWithLogits() >>> model = Model(net, loss_fn=loss, optimizer=optim) diff --git a/mindspore/nn/optim/momentum.py b/mindspore/nn/optim/momentum.py index ebdc5d86bf..1e8ce85570 100755 --- a/mindspore/nn/optim/momentum.py +++ b/mindspore/nn/optim/momentum.py @@ -64,8 +64,7 @@ class Momentum(Optimizer): - order_params: Optional. If "order_params" in the keys, the value should be the order of parameters and the order will be followed in optimizer. There are no other keys in the `dict` and the parameters which - in the value of 'order_params' but not in any group will use default learning rate and default weight - decay. + in the value of 'order_params' should be in one of group parameters. learning_rate (Union[int, float, Tensor, Iterable]): A value for the learning rate. When the learning_rate is Iterable or a Tensor and the dims of the Tensor is 1, @@ -97,16 +96,14 @@ class Momentum(Optimizer): >>> >>> #2) Use parameter groups and set different values >>> conv_params = list(filter(lambda x: 'conv' in x.name, net.trainable_params())) - >>> bias_params = list(filter(lambda x: 'bias' in x.name, net.trainable_params())) + >>> no_conv_params = list(filter(lambda x: 'conv' not in x.name, net.trainable_params())) >>> group_params = [{'params': conv_params, 'weight_decay': 0.01}, - >>> {'params': bias_params, 'lr': 0.01}, + >>> {'params': no_conv_params, 'lr': 0.01}, >>> {'order_params': net.trainable_params()}] >>> opt = nn.Momentum(group_params, learning_rate=0.1, momentum=0.9, weight_decay=0.0) >>> # The conv_params's parameters will use a learning rate of default value 0.1 and a weight decay of 0.01. - >>> # The bias_params's parameters will use a learning rate of 0.01 and a weight decay of default value 0.0. + >>> # The no_conv_params's parameters will use a learning rate of 0.01 and a weight decay of default value 0.0. >>> # The final parameters order in which the optimizer will be followed is the value of 'order_params'. - >>> # The parameters which in the value of 'order_params' but not in any group will use a learning rate - >>> # of default value 0.1 and a weight decay of default value 0.0. >>> >>> loss = nn.SoftmaxCrossEntropyWithLogits() >>> model = Model(net, loss_fn=loss, optimizer=optim, metrics=None) diff --git a/mindspore/nn/optim/optimizer.py b/mindspore/nn/optim/optimizer.py index 45eb604bf5..4d6471f424 100755 --- a/mindspore/nn/optim/optimizer.py +++ b/mindspore/nn/optim/optimizer.py @@ -22,11 +22,14 @@ from mindspore.ops import functional as F, composite as C, operations as P from mindspore.nn.cell import Cell from mindspore.common.parameter import Parameter, ParameterTuple from mindspore.common.initializer import initializer +from mindspore.common.tensor import Tensor import mindspore.common.dtype as mstype from mindspore._checkparam import Validator as validator from mindspore._checkparam import Rel -from mindspore.common.tensor import Tensor from mindspore import log as logger +from mindspore.parallel._utils import _get_global_rank, _get_device_num, _get_parallel_mode +from mindspore.parallel._auto_parallel_context import auto_parallel_context +from mindspore.train.parallel_utils import ParallelMode __all__ = ['Optimizer'] @@ -74,8 +77,7 @@ class Optimizer(Cell): - order_params: Optional. If "order_params" in the keys, the value should be the order of parameters and the order will be followed in optimizer. There are no other keys in the `dict` and the parameters which - in the value of 'order_params' but not in any group will use default learning rate and default weight - decay. + in the value of 'order_params' should be in one of group parameters. weight_decay (float): A floating point value for the weight decay. It should be equal to or greater than 0. If the type of `weight_decay` input is int, it will be converted to float. Default: 0.0. @@ -155,6 +157,27 @@ class Optimizer(Cell): self.param_length = len(self.parameters) self.map_ = C.Map() + use_parallel = auto_parallel_context().get_enable_parallel_optimizer() + self.use_parallel = use_parallel + if use_parallel: + if self.cls_name not in ["Lamb", "AdamWeightDecayDynamicLR", "AdamWeightDecay"]: + raise RuntimeError("Optimizer segmentation does not support optimizer {}".format(self.cls_name)) + if _get_parallel_mode() not in [ParallelMode.HYBRID_PARALLEL, ParallelMode.DATA_PARALLEL, + ParallelMode.AUTO_PARALLEL]: + raise RuntimeError("Optimizer segmentation does not support parallel mode {}".format + (_get_parallel_mode())) + self.dev_num = _get_device_num() + if self.dev_num > self.param_length: + raise RuntimeError("Optimizer segmentation can not be applied when the number of parameters {} is" + " less than the number of devices {}".format(self.param_length, self.dev_num)) + self.param_rank = self._get_parameter_group_id() + self.optim_filter = tuple(map(lambda x: x == _get_global_rank(), self.param_rank)) + self.param_names = [] + for param in self.parameters: + self.param_names.append(param.name) + else: + self.optim_filter = (True,) * self.param_length + def decay_weight(self, gradients): """ Weight decay. @@ -171,12 +194,12 @@ class Optimizer(Cell): params = self.parameters if self.is_group: if self.exec_weight_decay: - gradients = self.hyper_map(F.partial(_apply_decay), self.weight_decay, self.decay_flags, - params, gradients) + gradients = self.map_(F.partial(_apply_decay), self.weight_decay, self.decay_flags, + params, gradients) else: if self.weight_decay > 0: - gradients = self.hyper_map(F.partial(_apply_decay, self.weight_decay), self.decay_flags, - params, gradients) + gradients = self.map_(F.partial(_apply_decay, self.weight_decay), self.decay_flags, + params, gradients) return gradients @@ -219,8 +242,32 @@ class Optimizer(Cell): raise TypeError("Learning rate should be float, Tensor or Iterable.") return lr + def _check_group_params(self, parameters): + """Check group params.""" + parse_keys = ['params', 'lr', 'weight_decay', 'order_params'] + for group_param in parameters: + invalid_key = list(filter(lambda x: x not in parse_keys, group_param.keys())) + if invalid_key: + raise KeyError(f'The key "{invalid_key}" cannot be recognized in group params.') + + if 'order_params' in group_param.keys(): + if len(group_param.keys()) > 1: + raise ValueError("The order params dict in group parameters should " + "only include the 'order_params' key.") + if not isinstance(group_param['order_params'], Iterable): + raise TypeError("The value of 'order_params' should be an Iterable type.") + continue + + if not group_param['params']: + raise ValueError("Optimizer got an empty group parameter list.") + + for param in group_param['params']: + if not isinstance(param, Parameter): + raise TypeError("The group param should be an iterator of Parameter type.") + def _parse_group_params(self, parameters, learning_rate): """Parse group params.""" + self._check_group_params(parameters) if self.dynamic_lr: dynamic_lr_length = learning_rate.size() else: @@ -250,9 +297,6 @@ class Optimizer(Cell): if dynamic_lr_length not in (lr_length, 0): raise ValueError("The dynamic learning rate in group should be the same size.") - if not group_param['params']: - raise ValueError("Optimizer got an empty group parameter list.") - dynamic_lr_length = lr_length self.dynamic_lr_length = dynamic_lr_length @@ -306,24 +350,28 @@ class Optimizer(Cell): self.group_weight_decay.append(weight_decay_) if self.is_group_params_ordered: - self._order_and_adjust_group_params(ordered_parameters, learning_rate, weight_decay) + self._order_and_adjust_group_params(ordered_parameters) - def _order_and_adjust_group_params(self, ordered_parameters, learning_rate, weight_decay): + def _order_and_adjust_group_params(self, ordered_parameters): """ - Order group parameter, learning rate and weight decay in group params. And assign the parameters - which in the value of 'order_params' but not in any group to default value. + Order group parameter, learning rate and weight decay in group params. """ - params_length = len(ordered_parameters) - ordered_learning_rate = [Parameter(learning_rate, name="lr_" + param.name) for param in ordered_parameters] - ordered_weight_decay = [weight_decay * self.loss_scale] * params_length + params_length = len(self.group_params) + if len(ordered_parameters) != len(self.group_params): + raise ValueError(f"The value of 'order_params' should be same with all group parameters.") + + ordered_params = [None] * params_length + ordered_learning_rate = [None] * params_length + ordered_weight_decay = [None] * params_length params_name = [param.name for param in ordered_parameters] for param, lr, wd in zip(self.group_params, self.group_lr, self.group_weight_decay): index = params_name.index(param.name) + ordered_params[index] = param ordered_learning_rate[index] = lr ordered_weight_decay[index] = wd - self.group_params = list(ordered_parameters) + self.group_params = ordered_params self.group_lr = ordered_learning_rate self.group_weight_decay = ordered_weight_decay @@ -383,15 +431,70 @@ class Optimizer(Cell): lr = self.learning_rate return lr + def _get_parameter_group_id(self): + """ + Get the parameter partition group id, which is less than the number of devices. + + Returns: + tuple, the group id tuple of parameters. + """ + rank_list = () + count = 0 + for _ in range(self.param_length): + rank_list = rank_list + (count,) + count = count + 1 + if count == self.dev_num: + count = 0 + return rank_list + + def broadcast_params(self, optim_result): + """ + Apply Broadcast operations in the sequential order of parameter groups. + + Returns: + bool, the status flag. + """ + param_group = [] + key_group = [] + for _ in range(self.dev_num): + param_group.append(F.make_tuple()) + key_group.append(F.make_tuple()) + for i in range(self.param_length): + param_group[self.param_rank[i]] = param_group[self.param_rank[i]] + (optim_result[i],) + key = P.MakeRefKey(self.param_names[i])() + key_group[self.param_rank[i]] = key_group[self.param_rank[i]] + (key,) + new_param_group = [] + for root in range(self.dev_num): + ops = P.Broadcast(root) + next_params = ops(param_group[root]) + new_param_group.append(next_params) + for i in range(F.tuple_len(next_params)): + F.assign(key_group[root][i], next_params[i]) + status = True + for i in range(self.dev_num - 1): + status = F.control_depend(new_param_group[i][0], new_param_group[i+1]) + + return status + def construct(self, *hyper_params): raise NotImplementedError op_add = P.AddN() +op_gather = P.GatherV2() _apply_decay = C.MultitypeFuncGraph("apply_decay") +@_apply_decay.register("Number", "Bool", "Tensor", "Tuple") +def _tensor_apply_decay_with_sparse(weight_decay, if_apply, weight, gradient): + """Get grad with weight_decay.""" + if if_apply: + weight = op_gather(weight, gradient[0], 0) + return gradient[0], op_add((weight * weight_decay, gradient[1])), gradient[2] + return gradient + + @_apply_decay.register("Number", "Bool", "Tensor", "Tensor") def _tensor_apply_decay(weight_decay, if_apply, weight, gradient): """Get grad with weight_decay.""" diff --git a/mindspore/nn/optim/proximal_ada_grad.py b/mindspore/nn/optim/proximal_ada_grad.py index 380720404a..75f3994e2a 100644 --- a/mindspore/nn/optim/proximal_ada_grad.py +++ b/mindspore/nn/optim/proximal_ada_grad.py @@ -16,6 +16,7 @@ from mindspore.ops import functional as F, composite as C, operations as P from mindspore.common import Tensor import mindspore.common.dtype as mstype +from mindspore.ops.operations import _inner_ops as inner from mindspore._checkparam import Validator as validator from mindspore._checkparam import Rel from .optimizer import Optimizer @@ -60,7 +61,7 @@ class ProximalAdagrad(Optimizer): Note: The sparse strategy is applied while the SparseGatherV2 operator being used for forward network and the `sparse_grad` of `Parameter` being set as True. The sparse feature is under continuous development. The sparse - behavior is currently performed on the CPU, weight decay is not supported. + behavior is currently performed on the CPU. Args: params (list[Parameter]): A list of parameter, which will be updated. The element in `params` @@ -99,7 +100,7 @@ class ProximalAdagrad(Optimizer): self.weight_decay = weight_decay self.hyper_map = C.HyperMap() self.opt = P.ApplyProximalAdagrad(use_locking=use_locking) - self.sparse_opt = P.SparseApplyProximalAdagrad(use_locking=use_locking) + self.sparse_opt = inner.SparseApplyProximalAdagradNoReturn(use_locking=use_locking) def construct(self, grads): params = self.parameters diff --git a/mindspore/nn/optim/rmsprop.py b/mindspore/nn/optim/rmsprop.py index 05c42fb444..8e8885aff7 100644 --- a/mindspore/nn/optim/rmsprop.py +++ b/mindspore/nn/optim/rmsprop.py @@ -107,8 +107,7 @@ class RMSProp(Optimizer): - order_params: Optional. If "order_params" in the keys, the value should be the order of parameters and the order will be followed in optimizer. There are no other keys in the `dict` and the parameters which - in the value of 'order_params' but not in any group will use default learning rate and default weight - decay. + in the value of 'order_params' should be in one of group parameters. learning_rate (Union[float, Tensor, Iterable]): A value for the learning rate. When the learning_rate is Iterable or a Tensor and the dims of the Tensor is 1, @@ -140,16 +139,14 @@ class RMSProp(Optimizer): >>> >>> #2) Use parameter groups and set different values >>> conv_params = list(filter(lambda x: 'conv' in x.name, net.trainable_params())) - >>> bias_params = list(filter(lambda x: 'bias' in x.name, net.trainable_params())) + >>> no_conv_params = list(filter(lambda x: 'conv' not in x.name, net.trainable_params())) >>> group_params = [{'params': conv_params, 'weight_decay': 0.01}, - >>> {'params': bias_params, 'lr': 0.01}, + >>> {'params': no_conv_params, 'lr': 0.01}, >>> {'order_params': net.trainable_params()}] >>> opt = nn.RMSProp(group_params, learning_rate=0.1, weight_decay=0.0) >>> # The conv_params's parameters will use a learning rate of default value 0.1 and a weight decay of 0.01. - >>> # The bias_params's parameters will use a learning rate of 0.01 and a weight decay of default value 0.0. + >>> # The no_conv_params's parameters will use a learning rate of 0.01 and a weight decay of default value 0.0. >>> # The final parameters order in which the optimizer will be followed is the value of 'order_params'. - >>> # The parameters which in the value of 'order_params' but not in any group will use a learning rate - >>> # of default value 0.1 and a weight decay of default value 0.0. >>> >>> loss = nn.SoftmaxCrossEntropyWithLogits() >>> model = Model(net, loss_fn=loss, optimizer=optim) diff --git a/mindspore/nn/optim/sgd.py b/mindspore/nn/optim/sgd.py index d2680a38e5..382f095627 100755 --- a/mindspore/nn/optim/sgd.py +++ b/mindspore/nn/optim/sgd.py @@ -64,8 +64,7 @@ class SGD(Optimizer): - order_params: Optional. If "order_params" in the keys, the value should be the order of parameters and the order will be followed in optimizer. There are no other keys in the `dict` and the parameters which - in the value of 'order_params' but not in any group will use default learning rate and default weight - decay. + in the value of 'order_params' should be in one of group parameters. learning_rate (Union[float, Tensor, Iterable]): A value for the learning rate. When the learning_rate is Iterable or a Tensor and the dims of the Tensor is 1, @@ -98,16 +97,14 @@ class SGD(Optimizer): >>> >>> #2) Use parameter groups and set different values >>> conv_params = list(filter(lambda x: 'conv' in x.name, net.trainable_params())) - >>> bias_params = list(filter(lambda x: 'bias' in x.name, net.trainable_params())) + >>> no_conv_params = list(filter(lambda x: 'conv' not in x.name, net.trainable_params())) >>> group_params = [{'params': conv_params, 'weight_decay': 0.01}, - >>> {'params': bias_params, 'lr': 0.01}, + >>> {'params': no_conv_params, 'lr': 0.01}, >>> {'order_params': net.trainable_params()}] >>> opt = nn.SGD(group_params, learning_rate=0.1, weight_decay=0.0) >>> # The conv_params's parameters will use a learning rate of default value 0.1 and a weight decay of 0.01. - >>> # The bias_params's parameters will use a learning rate of 0.01 and a weight decay of default value 0.0. + >>> # The no_conv_params's parameters will use a learning rate of 0.01 and a weight decay of default value 0.0. >>> # The final parameters order in which the optimizer will be followed is the value of 'order_params'. - >>> # The parameters which in the value of 'order_params' but not in any group will use a learning rate - >>> # of default value 0.1 and a weight decay of default value 0.0. >>> >>> loss = nn.SoftmaxCrossEntropyWithLogits() >>> model = Model(net, loss_fn=loss, optimizer=optim) diff --git a/mindspore/nn/wrap/cell_wrapper.py b/mindspore/nn/wrap/cell_wrapper.py index f0d920f51f..9e3d00cc95 100644 --- a/mindspore/nn/wrap/cell_wrapper.py +++ b/mindspore/nn/wrap/cell_wrapper.py @@ -220,7 +220,9 @@ class DataWrapper(Cell): def __init__(self, network, dataset_types, dataset_shapes, queue_name): super(DataWrapper, self).__init__(auto_prefix=False, flags=network.get_flags()) - + # Also copy the flag in `network` construct + flags = getattr(network.__class__.construct, "_mindspore_flags", {}) + self.add_flags(**flags) self.get_next = P.GetNext(dataset_types, dataset_shapes, len(dataset_types), queue_name) self.network = network diff --git a/mindspore/nn/wrap/grad_reducer.py b/mindspore/nn/wrap/grad_reducer.py index c66bfbe646..9354b42e55 100644 --- a/mindspore/nn/wrap/grad_reducer.py +++ b/mindspore/nn/wrap/grad_reducer.py @@ -16,18 +16,22 @@ from mindspore.nn.cell import Cell from mindspore.communication.management import GlobalComm, get_group_size from mindspore.ops import functional as F, composite as C, operations as P -from mindspore.ops.operations.comm_ops import AllReduce, ReduceOp +from mindspore.ops.operations.comm_ops import AllReduce, ReduceOp, AllGather import mindspore.common.dtype as mstype reduce_opt = C.MultitypeFuncGraph("reduce_opt") _all_reduce = AllReduce() +_all_gather = None -def _init_optimizer_allreduce(): +def _init_optimizer_communication(): global _all_reduce + global _all_gather + _all_reduce = AllReduce(ReduceOp.SUM, GlobalComm.WORLD_COMM_GROUP) _all_reduce.add_prim_attr('fusion', 1) + _all_gather = AllGather(GlobalComm.WORLD_COMM_GROUP) @reduce_opt.register("Function", "Number", "Bool", "Tensor") @@ -72,8 +76,8 @@ def _tensors_allreduce_mean_with_sparse(mul, degree, allreduce_filter, grad): degree = F.scalar_cast(degree, F.dtype(grad[1])) dout = _all_gather(grad[1]) cast_op = P.Cast() - dout = mul(dout, cast_op(F.scalar_to_array(1.0/degree), F.dtype(dout))) - grad = (indices, dout, dout[2]) + dout = mul(dout, cast_op(F.scalar_to_array(1.0 / degree), F.dtype(dout))) + grad = (indices, dout, grad[2]) return grad @@ -110,7 +114,7 @@ def _tensors_allreduce_with_sparse(allreduce_filter, grad): if allreduce_filter: indices = _all_gather(grad[0]) dout = _all_gather(grad[1]) - grad = (indices, dout, dout[2]) + grad = (indices, dout, grad[2]) return grad @@ -131,6 +135,20 @@ def _tensors_get_datatype(grad): return F.dtype(grad) +@_get_datatype.register("Tuple") +def _tensors_get_datatype_with_sparse(grad): + """ + Acquire gradient datatype. + + Args: + grad (Tuple): The gradient tensor before operation. + + Returns: + mstype, the datatype of gradient. + """ + return F.dtype(grad[1]) + + _cast_datatype = C.MultitypeFuncGraph("_cast_datatype") @@ -149,6 +167,22 @@ def _tensors_cast_datatype(datatype, grad): return F.cast(grad, datatype) +@_cast_datatype.register("TypeType", "Tuple") +def _tensors_cast_datatype_with_sparse(datatype, grad): + """ + Cast gradient to datatype. + + Args: + datatype (mstype): the destination datatype of gradient. + grad (Tuple): The gradient tensor before operation. + + Returns: + Tuple, the gradient tuple after operation. + """ + dout = F.cast(grad[1], datatype) + return (grad[0], dout, grad[2]) + + class DistributedGradReducer(Cell): """ A distributed optimizer. @@ -224,7 +258,7 @@ class DistributedGradReducer(Cell): def __init__(self, parameters, mean=True, degree=None): super(DistributedGradReducer, self).__init__(auto_prefix=False) - self.hyper_map = C.HyperMap() + self.map_ = C.Map() self.mul = P.Mul() if degree is None: self.degree = get_group_size() @@ -234,19 +268,27 @@ class DistributedGradReducer(Cell): self.degree = degree self.mean = mean self.allreduce_filter = tuple(x.layerwise_parallel is False for x in parameters) - _init_optimizer_allreduce() + _init_optimizer_communication() def construct(self, grads): - # In some circumstances, the data precision of grads could be mixed with float16 and float32. Thus, the - # result of AllReduce is unreliable. To solve the problem, grads should be cast to float32 before AllReduce, - # and cast back after the operation. - datatypes = self.hyper_map(F.partial(_get_datatype), grads) - grads = self.hyper_map(F.partial(_cast_datatype, mstype.float32), grads) + """ + In some circumstances, the data precision of grads could be mixed with float16 and float32. Thus, the + result of AllReduce is unreliable. To solve the problem, grads should be cast to float32 before AllReduce, + and cast back after the operation. + + Args: + grads (Union[Tensor, tuple[Tensor]]): The gradient tensor or tuple before operation. + + Returns: + new_grads (Union[Tensor, tuple[Tensor]]), the gradient tensor or tuple after operation. + """ + datatypes = self.map_(F.partial(_get_datatype), grads) + grads = self.map_(F.partial(_cast_datatype, mstype.float32), grads) if self.mean: - new_grad = self.hyper_map(F.partial(reduce_opt, self.mul, self.degree), self.allreduce_filter, grads) + new_grad = self.map_(F.partial(reduce_opt, self.mul, self.degree), self.allreduce_filter, grads) else: - new_grad = self.hyper_map(F.partial(reduce_opt), self.allreduce_filter, grads) + new_grad = self.map_(F.partial(reduce_opt), self.allreduce_filter, grads) - new_grad = self.hyper_map(F.partial(_cast_datatype), datatypes, new_grad) + new_grad = self.map_(F.partial(_cast_datatype), datatypes, new_grad) return new_grad diff --git a/mindspore/nn/wrap/loss_scale.py b/mindspore/nn/wrap/loss_scale.py index 2bae6bbc5c..a9aa4d781b 100644 --- a/mindspore/nn/wrap/loss_scale.py +++ b/mindspore/nn/wrap/loss_scale.py @@ -234,8 +234,8 @@ class TrainOneStepWithLossScaleCell(Cell): if scale_update_cell: self.loss_scale = Parameter(Tensor(scale_update_cell.get_loss_scale(), dtype=mstype.float32), name="loss_scale") - self.add_flags(has_effect=True) + @C.add_flags(has_effect=True) def construct(self, data, label, sens=None): weights = self.weights loss = self.network(data, label) diff --git a/mindspore/ops/_grad/grad_array_ops.py b/mindspore/ops/_grad/grad_array_ops.py index 1155fb7c03..d1494bc051 100644 --- a/mindspore/ops/_grad/grad_array_ops.py +++ b/mindspore/ops/_grad/grad_array_ops.py @@ -715,3 +715,13 @@ def get_bprop_reverse_sequence(self): dx = reverse_sequence_grad(dout, seq_lengths) return dx, zeros_like(seq_lengths) return bprop + + +@bprop_getters.register(P.TransShape) +def get_bprop_trans_shape(self): + """Generate bprop for TransShape""" + op = P.TransShape() + def bprop(x, shape, out, dout): + dx = op(dout, shape_op(x)) + return (dx, zeros_like(shape)) + return bprop diff --git a/mindspore/ops/_grad/grad_comm_ops.py b/mindspore/ops/_grad/grad_comm_ops.py index 7477d50895..34df18beba 100644 --- a/mindspore/ops/_grad/grad_comm_ops.py +++ b/mindspore/ops/_grad/grad_comm_ops.py @@ -18,9 +18,9 @@ import mindspore.common.dtype as mstype from mindspore.ops import functional as F from .. import operations as P from ..composite.multitype_ops.zeros_like_impl import zeros_like -from ..operations.comm_ops import (AllGather, HostAllGather, AllReduce, _AlltoAll, Broadcast, +from ..operations.comm_ops import (AllGather, _HostAllGather, AllReduce, _AlltoAll, Broadcast, _GetTensorSlice, _MirrorOperator, ReduceOp, - ReduceScatter, HostReduceScatter, _VirtualDiv) + ReduceScatter, _HostReduceScatter, _VirtualDiv) from .grad_base import bprop_getters @@ -93,10 +93,10 @@ def get_bprop_all_gather(self): return bprop -@bprop_getters.register(HostAllGather) +@bprop_getters.register(_HostAllGather) def get_bprop_host_all_gather(self): - """Generate bprop for HostAllGather""" - host_all_gather_grad = HostReduceScatter(ReduceOp.SUM, self.group) + """Generate bprop for _HostAllGather""" + host_all_gather_grad = _HostReduceScatter(ReduceOp.SUM, self.group) if self.instance_name: instance_name = "grad" + self.instance_name host_all_gather_grad.set_prim_instance_name(instance_name) @@ -126,10 +126,10 @@ def get_bprop_reduce_scatter(self): return bprop -@bprop_getters.register(HostReduceScatter) +@bprop_getters.register(_HostReduceScatter) def get_bprop_host_reduce_scatter(self): - """Generate bprop for HostReduceScatter""" - host_reduce_scatter_grad = HostAllGather(self.group) + """Generate bprop for _HostReduceScatter""" + host_reduce_scatter_grad = _HostAllGather(self.group) if self.instance_name: instance_name = "grad" + self.instance_name host_reduce_scatter_grad.set_prim_instance_name(instance_name) diff --git a/mindspore/ops/_grad/grad_debug_ops.py b/mindspore/ops/_grad/grad_debug_ops.py index 1cb756219a..6e31556b14 100644 --- a/mindspore/ops/_grad/grad_debug_ops.py +++ b/mindspore/ops/_grad/grad_debug_ops.py @@ -66,3 +66,12 @@ def get_bprop_insert_gradient_of(self): def bprop(x, out, dout): return (f(dout),) return bprop + + +@bprop_getters.register(P.Debug) +def get_bprop_debug(self): + """Generate bprop for Debug""" + + def bprop(x, out, dout): + return dout + return bprop diff --git a/mindspore/ops/_grad/grad_math_ops.py b/mindspore/ops/_grad/grad_math_ops.py index 1e4f932442..975e918817 100755 --- a/mindspore/ops/_grad/grad_math_ops.py +++ b/mindspore/ops/_grad/grad_math_ops.py @@ -306,6 +306,18 @@ def get_bprop_floormod(self): return bprop +@bprop_getters.register(P.Mod) +def get_bprop_mod(self): + """Grad definition for `Mod` operation.""" + + def bprop(x, y, out, dout): + bc_x = dout + bc_y = -dout * (x // y) + return binop_grad_common(x, y, bc_x, bc_y) + + return bprop + + @bprop_getters.register(P.Square) def get_bprop_square(self): """Grad definition for `Square` operation.""" diff --git a/mindspore/ops/_grad/grad_nn_ops.py b/mindspore/ops/_grad/grad_nn_ops.py index 508883f221..3a86a05943 100755 --- a/mindspore/ops/_grad/grad_nn_ops.py +++ b/mindspore/ops/_grad/grad_nn_ops.py @@ -146,6 +146,40 @@ def get_bprop_max_pool_with_argmax(self): return bprop +@bprop_getters.register(G.MaxPoolGrad) +def get_bprop_max_pool_grad_grad(self): + """Grad definition for `MaxPoolGrad` operation.""" + maxpool_grad_grad = G.MaxPoolGradGrad( + ksize=self.ksize, + strides=self.strides, + padding=self.padding) + + def bprop(x1, x2, grad, out, dout): + dx1 = zeros_like(x1) + dx2 = zeros_like(x2) + dgrad = maxpool_grad_grad(x1, x2, dout) + return (dx1, dx2, dgrad) + + return bprop + + +@bprop_getters.register(G.MaxPoolGradGrad) +def get_bprop_max_pool_grad_grad_grad(self): + """Grad definition for `MaxPoolGradGrad` operation.""" + maxpool_grad = G.MaxPoolGrad( + ksize=self.ksize, + strides=self.strides, + padding=self.padding) + + def bprop(x1, x2, grad, out, dout): + dx1 = zeros_like(x1) + dx2 = zeros_like(x2) + dgrad = maxpool_grad(x1, x2, dout) + return (dx1, dx2, dgrad) + + return bprop + + @bprop_getters.register(P.MaxPool) def get_bprop_max_pool_grad(self): """Grad definition for `MaxPool` operation.""" @@ -336,6 +370,21 @@ def get_bprop_softplus(self): return bprop +@bprop_getters.register(P.Softsign) +def get_bprop_softsign(self): + """Grad definition for `Softsign` operation.""" + mul = P.Mul() + absolute = P.Abs() + div = P.Div() + square = P.Square() + + def bprop(x, out, dout): + dx = mul(dout, div(1, square(1 + absolute(x)))) + return (dx,) + + return bprop + + @bprop_getters.register(P.Tanh) def get_bprop_tanh(self): """Grad definition for `Tanh` operation.""" @@ -726,8 +775,20 @@ def get_bprop_basic_lstm_cell(self): def bprop(x, h, c, w, b, out, dout): _, _, it, jt, ft, ot, tanhct = out dct, dht, _, _, _, _, _ = dout - dgate, dct_1 = basic_lstm_cell_cstate_grad(c, dht, dct, it, ft, jt, ot, tanhct) + dgate, dct_1 = basic_lstm_cell_cstate_grad(c, dht, dct, it, jt, ft, ot, tanhct) dxt, dht = basic_lstm_cell_input_grad(dgate, w) dw, db = basic_lstm_cell_weight_grad(F.depend(x, dxt), h, dgate) return dxt, dht, dct_1, dw, db return bprop + + +@bprop_getters.register(P.LRN) +def get_bprop_lrn(self): + """Grad definition for `LRN` operation.""" + grad = G.LRNGrad(self.depth_radius, self.bias, self.alpha, self.beta) + + def bprop(x, out, dout): + dx = grad(dout, x, out) + return (dx,) + + return bprop diff --git a/mindspore/ops/_op_impl/_custom_op/batchnorm_fold2.py b/mindspore/ops/_op_impl/_custom_op/batchnorm_fold2.py index 7e98517057..9daab5a75f 100644 --- a/mindspore/ops/_op_impl/_custom_op/batchnorm_fold2.py +++ b/mindspore/ops/_op_impl/_custom_op/batchnorm_fold2.py @@ -30,7 +30,6 @@ batchnorm_fold2_op_info = TBERegOp("BatchNormFold2_D") \ .compute_cost(10) \ .kernel_name("batchnorm_fold2") \ .partial_flag(True) \ - .op_pattern("formatAgnostic") \ .input(0, "x", None, "required", None) \ .input(1, "beta", None, "required", None) \ .input(2, "gamma", None, "required", None) \ diff --git a/mindspore/ops/_op_impl/_custom_op/batchnorm_fold2_grad.py b/mindspore/ops/_op_impl/_custom_op/batchnorm_fold2_grad.py index 824da62d19..9994a88f30 100644 --- a/mindspore/ops/_op_impl/_custom_op/batchnorm_fold2_grad.py +++ b/mindspore/ops/_op_impl/_custom_op/batchnorm_fold2_grad.py @@ -30,7 +30,6 @@ batchnorm_fold2_grad_op_info = TBERegOp("BatchNormFold2GradD") \ .compute_cost(10) \ .kernel_name("batchnorm_fold2_grad") \ .partial_flag(True) \ - .op_pattern("formatAgnostic") \ .input(0, "dout", None, "required", None) \ .input(1, "dout_reduce", None, "required", None) \ .input(2, "dout_x_reduce", None, "required", None) \ diff --git a/mindspore/ops/_op_impl/_custom_op/batchnorm_fold2_grad_reduce.py b/mindspore/ops/_op_impl/_custom_op/batchnorm_fold2_grad_reduce.py index 7806c6834e..92b91ff712 100644 --- a/mindspore/ops/_op_impl/_custom_op/batchnorm_fold2_grad_reduce.py +++ b/mindspore/ops/_op_impl/_custom_op/batchnorm_fold2_grad_reduce.py @@ -31,7 +31,6 @@ batchnorm_fold2_grad_reduce_op_info = TBERegOp("BatchNormFold2GradReduce") \ .compute_cost(10) \ .kernel_name("batchnorm_fold2_grad_reduce") \ .partial_flag(True) \ - .op_pattern("formatAgnostic") \ .input(0, "dout", None, "required", None) \ .input(1, "x", None, "required", None) \ .output(0, "dout_reduce", True, "required", "all") \ diff --git a/mindspore/ops/_op_impl/_custom_op/correction_mul.py b/mindspore/ops/_op_impl/_custom_op/correction_mul.py index ce92d2bbc5..49cd35cc11 100644 --- a/mindspore/ops/_op_impl/_custom_op/correction_mul.py +++ b/mindspore/ops/_op_impl/_custom_op/correction_mul.py @@ -30,7 +30,6 @@ correction_mul_op_info = TBERegOp("CorrectionMul") \ .compute_cost(10) \ .kernel_name("correction_mul") \ .partial_flag(True) \ - .op_pattern("formatAgnostic") \ .attr("channel_axis", "optional", "int", "all") \ .input(0, "x", None, "required", None) \ .input(1, "batch_std", None, "required", None) \ diff --git a/mindspore/ops/_op_impl/_custom_op/correction_mul_grad.py b/mindspore/ops/_op_impl/_custom_op/correction_mul_grad.py index da3a634454..6c11ce6855 100644 --- a/mindspore/ops/_op_impl/_custom_op/correction_mul_grad.py +++ b/mindspore/ops/_op_impl/_custom_op/correction_mul_grad.py @@ -30,7 +30,6 @@ correction_mul_grad_op_info = TBERegOp("CorrectionMulGrad") \ .compute_cost(10) \ .kernel_name("correction_mul_grad") \ .partial_flag(True) \ - .op_pattern("formatAgnostic") \ .attr("channel_axis", "optional", "int", "all") \ .input(0, "dout", None, "required", None) \ .input(1, "x", None, "required", None) \ @@ -128,7 +127,6 @@ correction_mul_grad_reduce_op_info = TBERegOp("CorrectionMulGradReduce") \ .compute_cost(10) \ .kernel_name("correction_mul_grad_reduce") \ .partial_flag(True) \ - .op_pattern("formatAgnostic") \ .attr("channel_axis", "optional", "int", "all") \ .input(0, "dout", None, "required", None) \ .output(0, "d_batch_std", True, "required", "all") \ diff --git a/mindspore/ops/_op_impl/_custom_op/fake_quant_perchannel.py b/mindspore/ops/_op_impl/_custom_op/fake_quant_perchannel.py index f6c133c808..dae2d7058d 100644 --- a/mindspore/ops/_op_impl/_custom_op/fake_quant_perchannel.py +++ b/mindspore/ops/_op_impl/_custom_op/fake_quant_perchannel.py @@ -99,11 +99,15 @@ def fake_quant_perchannel(x, min_val, max_val, y, min_dtype = min_val.get("dtype") max_shape = max_val.get("ori_shape") max_dtype = max_val.get("dtype") - + # for Dense weight quant, 2d[co,ci] -> 4d[1,co,ci,1], channel_axis_ need change to 1. + if channel_axis == 0 and x_shape_[0] != min_shape[0] and x_shape_[1] == min_shape[0]: + channel_axis_ = 1 + else: + channel_axis_ = channel_axis util.check_kernel_name(kernel_name) util.check_shape_rule(x_shape) - util.check_shape_rule(min_shape, 1, 1, x_shape_[channel_axis]) - util.check_shape_rule(max_shape, 1, 1, x_shape_[channel_axis]) + util.check_shape_rule(min_shape, 1, 1, x_shape_[channel_axis_]) + util.check_shape_rule(max_shape, 1, 1, x_shape_[channel_axis_]) util.check_tensor_shape_size(x_shape) util.check_tensor_shape_size(min_shape) util.check_tensor_shape_size(max_shape) @@ -126,8 +130,8 @@ def fake_quant_perchannel(x, min_val, max_val, y, quant_min = quant_min + 1 shape_c = [1] * len(x_shape) - shape_c[channel_axis] = min_val.get("ori_shape")[0] - if x_format == "NC1HWC0" and channel_axis == 1: + shape_c[channel_axis_] = min_val.get("ori_shape")[0] + if x_format == "NC1HWC0" and channel_axis_ == 1: shape_c = min_val.get("shape") input_data = tvm.placeholder(x_shape, name="x", dtype=x_dtype) min_data = tvm.placeholder(shape_c, name="min_val", dtype=x_dtype) diff --git a/mindspore/ops/_op_impl/_custom_op/fake_quant_perchannel_grad.py b/mindspore/ops/_op_impl/_custom_op/fake_quant_perchannel_grad.py index 4e9053fcb1..795aab52a3 100644 --- a/mindspore/ops/_op_impl/_custom_op/fake_quant_perchannel_grad.py +++ b/mindspore/ops/_op_impl/_custom_op/fake_quant_perchannel_grad.py @@ -124,11 +124,15 @@ def fake_quant_perchannel_grad(dout, x, min_val, max_val, dx, min_dtype = min_val.get("dtype") max_shape = max_val.get("ori_shape") max_dtype = max_val.get("dtype") - + # for Dense weight quant, 2d[co,ci] -> 4d[1,co,ci,1], channel_axis_ need change to 1. + if channel_axis == 0 and x_shape_[0] != min_shape[0] and x_shape_[1] == min_shape[0]: + channel_axis_ = 1 + else: + channel_axis_ = channel_axis util.check_kernel_name(kernel_name) util.check_shape_rule(x_shape) - util.check_shape_rule(min_shape, 1, 1, x_shape_[channel_axis]) - util.check_shape_rule(max_shape, 1, 1, x_shape_[channel_axis]) + util.check_shape_rule(min_shape, 1, 1, x_shape_[channel_axis_]) + util.check_shape_rule(max_shape, 1, 1, x_shape_[channel_axis_]) util.check_tensor_shape_size(x_shape) util.check_tensor_shape_size(min_shape) util.check_tensor_shape_size(max_shape) @@ -151,8 +155,8 @@ def fake_quant_perchannel_grad(dout, x, min_val, max_val, dx, quant_min = quant_min + 1 shape_c = [1] * len(x_shape) - shape_c[channel_axis] = min_val.get("ori_shape")[0] - if x_format == "NC1HWC0" and channel_axis == 1: + shape_c[channel_axis_] = min_val.get("ori_shape")[0] + if x_format == "NC1HWC0" and channel_axis_ == 1: shape_c = min_val.get("shape") dout_data = tvm.placeholder(x_shape, name="dout", dtype=x_dtype) input_data = tvm.placeholder(x_shape, name="x", dtype=x_dtype) diff --git a/mindspore/ops/_op_impl/_custom_op/minmax_update_perchannel.py b/mindspore/ops/_op_impl/_custom_op/minmax_update_perchannel.py index 1ff63464c3..f29fc53325 100644 --- a/mindspore/ops/_op_impl/_custom_op/minmax_update_perchannel.py +++ b/mindspore/ops/_op_impl/_custom_op/minmax_update_perchannel.py @@ -88,11 +88,15 @@ def minmax_update_perchannel(x, min_val, max_val, min_up, max_up, min_dtype = min_val.get("dtype") max_shape = max_val.get("ori_shape") max_dtype = max_val.get("dtype") - + # for Dense weight quant, 2d[co,ci] -> 4d[1,co,ci,1], channel_axis_ need change to 1. + if channel_axis == 0 and x_shape[0] != min_shape[0] and x_shape[1] == min_shape[0]: + channel_axis_ = 1 + else: + channel_axis_ = channel_axis util.check_kernel_name(kernel_name) util.check_shape_rule(x_shape) - util.check_shape_rule(min_shape, 1, 1, x_shape[channel_axis]) - util.check_shape_rule(max_shape, 1, 1, x_shape[channel_axis]) + util.check_shape_rule(min_shape, 1, 1, x_shape[channel_axis_]) + util.check_shape_rule(max_shape, 1, 1, x_shape[channel_axis_]) util.check_tensor_shape_size(x_shape) util.check_tensor_shape_size(min_shape) util.check_tensor_shape_size(max_shape) @@ -105,7 +109,7 @@ def minmax_update_perchannel(x, min_val, max_val, min_up, max_up, util.check_dtype_rule(min_dtype, check_list) util.check_dtype_rule(max_dtype, check_list) - if channel_axis == 0: + if channel_axis_ == 0: shape_c = min_val.get("ori_shape") else: shape_c = [min_val.get("shape")[1], min_val.get("shape")[-1]] @@ -113,7 +117,7 @@ def minmax_update_perchannel(x, min_val, max_val, min_up, max_up, min_data = tvm.placeholder(shape_c, name="min_val", dtype=x_dtype) max_data = tvm.placeholder(shape_c, name="max_val", dtype=x_dtype) res_list = minmax_update_perchannel_compute(input_data, min_data, max_data, - ema, ema_decay, channel_axis) + ema, ema_decay, channel_axis_) with tvm.target.cce(): sch = generic.auto_schedule(res_list) diff --git a/mindspore/ops/_op_impl/aicpu/__init__.py b/mindspore/ops/_op_impl/aicpu/__init__.py index bbf1013e0a..15a30cfae2 100644 --- a/mindspore/ops/_op_impl/aicpu/__init__.py +++ b/mindspore/ops/_op_impl/aicpu/__init__.py @@ -42,3 +42,4 @@ from .uniform_real import _uniform_real_aicpu from .laplace import _laplace_aicpu from .strided_slice import _strided_slice_aicpu from .strided_slice_grad import _strided_slice_grad_aicpu +from .end_of_sequence import _end_of_sequence_aicpu diff --git a/mindspore/ops/_op_impl/aicpu/end_of_sequence.py b/mindspore/ops/_op_impl/aicpu/end_of_sequence.py new file mode 100644 index 0000000000..da70cc12f5 --- /dev/null +++ b/mindspore/ops/_op_impl/aicpu/end_of_sequence.py @@ -0,0 +1,30 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""EndOfSequence op""" +from mindspore.ops.op_info_register import op_info_register, AiCPURegOp, DataType + +end_of_sequence_op_info = AiCPURegOp("EndOfSequence") \ + .fusion_type("OPAQUE") \ + .input(0, "x", "required") \ + .output(0, "y", "required") \ + .dtype_format(DataType.U8_Default, DataType.U8_Default) \ + .get_op_info() + + +@op_info_register(end_of_sequence_op_info) +def _end_of_sequence_aicpu(): + """EndOfSequence AiCPU register""" + return diff --git a/mindspore/ops/_op_impl/akg/__init__.py b/mindspore/ops/_op_impl/akg/__init__.py index f38b99f5e4..fd86dbf999 100644 --- a/mindspore/ops/_op_impl/akg/__init__.py +++ b/mindspore/ops/_op_impl/akg/__init__.py @@ -47,6 +47,7 @@ from .gather_v2 import _gather_v2_akg from .less import _less_akg from .log import _log_akg from .matmul import _matmul_akg +from .batchmatmul import _batchmatmul_akg from .max_pool_grad_with_argmax import _max_pool_grad_with_argmax_akg from .max_pool_with_argmax import _max_pool_with_argmax_akg from .max import _max_akg diff --git a/mindspore/ops/_op_impl/akg/batchmatmul.py b/mindspore/ops/_op_impl/akg/batchmatmul.py new file mode 100644 index 0000000000..f5da71aa25 --- /dev/null +++ b/mindspore/ops/_op_impl/akg/batchmatmul.py @@ -0,0 +1,73 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""BatchMatMul op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "BatchMatMul", + "imply_type": "AutoDiff", + "fusion_type": "OPAQUE", + "attr": [ + { + "name": "transpose_a", + "param_type": "optional", + "type": "bool" + }, + { + "name": "transpose_b", + "param_type": "optional", + "type": "bool" + } + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16" + ], + "format": [ + "FRACTAL_NZ" + ], + "name": "x1" + }, + { + "index": 1, + "dtype": [ + "float16" + ], + "format": [ + "FRACTAL_NZ" + ], + "name": "x2" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16" + ], + "format": [ + "FRACTAL_NZ" + ], + "name": "output" + } + ] +}""") +def _batchmatmul_akg(): + """BatchMatMul AKG register""" + return diff --git a/mindspore/ops/_op_impl/tbe/__init__.py b/mindspore/ops/_op_impl/tbe/__init__.py index 631ec1bf44..35785a085c 100644 --- a/mindspore/ops/_op_impl/tbe/__init__.py +++ b/mindspore/ops/_op_impl/tbe/__init__.py @@ -34,6 +34,12 @@ from .apply_ada_max import _apply_ada_max_tbe from .apply_adadelta import _apply_adadelta_tbe from .apply_adagrad import _apply_adagrad_tbe from .apply_adagrad_v2 import _apply_adagrad_v2_tbe +from .apply_add_sign import _apply_add_sign_tbe +from .apply_power_sign import _apply_power_sign_tbe +from .apply_gradient_descent import _apply_gradient_descent_tbe +from .apply_proximal_gradient_descent import _apply_proximal_gradient_descent_tbe +from .sparse_apply_ftrl_v2 import _sparse_apply_ftrl_v2_tbe +from .sparse_apply_adagrad_v2 import _sparse_apply_adagrad_v2_tbe from .approximate_equal import _approximate_equal_tbe from .adam_apply_one import _adam_apply_one_tbe from .assign import _assign_tbe @@ -116,6 +122,7 @@ from .round import _round_tbe from .tanh import _tanh_tbe from .tanh_grad import _tanh_grad_tbe from .softmax import _softmax_tbe +from .softsign import _softsign_tbe from .softplus import _softplus_tbe from .softplus_grad import _softplus_grad_tbe from .softmax_grad_ext import _softmax_grad_ext_tbe @@ -267,3 +274,13 @@ from .lin_space import _lin_space_tbe from .matrix_diag import _matrix_diag_tbe from .matrix_diag_part import _matrix_diag_part_tbe from .matrix_set_diag import _matrix_set_diag_tbe +from .lrn import _lrn_tbe +from .lrn_grad import _lrn_grad_tbe +from .scatter_max import _scatter_max_tbe +from .scatter_min import _scatter_min_tbe +from .scatter_sub import _scatter_sub_tbe +from .scatter_mul import _scatter_mul_tbe +from .scatter_div import _scatter_div_tbe +from .mod import _mod_tbe +from .max_pool_grad_grad import _max_pool_grad_grad_tbe +from .max_pool_grad_grad_with_argmax import _max_pool_grad_grad_with_argmax_tbe diff --git a/mindspore/ops/_op_impl/tbe/accumulate_n_v2.py b/mindspore/ops/_op_impl/tbe/accumulate_n_v2.py index fdd72a9494..b16233c37e 100644 --- a/mindspore/ops/_op_impl/tbe/accumulate_n_v2.py +++ b/mindspore/ops/_op_impl/tbe/accumulate_n_v2.py @@ -27,11 +27,11 @@ accumulate_n_v2_op_info = TBERegOp("AccumulateNV2") \ .input(0, "x", False, "dynamic", "all") \ .output(0, "y", False, "required", "all") \ .op_pattern("broadcast") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.I32_Default, DataType.I32_Default) \ - .dtype_format(DataType.I8_Default, DataType.I8_Default) \ - .dtype_format(DataType.U8_Default, DataType.U8_Default) \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ + .dtype_format(DataType.I32_None, DataType.I32_None) \ + .dtype_format(DataType.I8_None, DataType.I8_None) \ + .dtype_format(DataType.U8_None, DataType.U8_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/acos.py b/mindspore/ops/_op_impl/tbe/acos.py index 98516f4496..974276ca20 100644 --- a/mindspore/ops/_op_impl/tbe/acos.py +++ b/mindspore/ops/_op_impl/tbe/acos.py @@ -26,10 +26,8 @@ acos_op_info = TBERegOp("ACos") \ .op_pattern("formatAgnostic") \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD) \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/acosh.py b/mindspore/ops/_op_impl/tbe/acosh.py index 0bf8755bc0..bbc35f2533 100644 --- a/mindspore/ops/_op_impl/tbe/acosh.py +++ b/mindspore/ops/_op_impl/tbe/acosh.py @@ -26,10 +26,8 @@ acosh_op_info = TBERegOp("Acosh") \ .op_pattern("formatAgnostic") \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD) \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/add.py b/mindspore/ops/_op_impl/tbe/add.py index d3db3de0ad..01eb6e881e 100644 --- a/mindspore/ops/_op_impl/tbe/add.py +++ b/mindspore/ops/_op_impl/tbe/add.py @@ -23,16 +23,11 @@ add_op_info = TBERegOp("Add") \ .compute_cost(10) \ .kernel_name("add") \ .partial_flag(True) \ + .op_pattern("dynamicFormat") \ .input(0, "x1", False, "required", "all") \ .input(1, "x2", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .op_pattern("dynamicFormat") \ - .dtype_format(DataType.I32_Default, DataType.I32_Default, DataType.I32_Default) \ - .dtype_format(DataType.I32_5HD, DataType.I32_5HD, DataType.I32_5HD) \ - .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD) \ + .dtype_format(DataType.None_None, DataType.None_None, DataType.None_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/apply_add_sign.py b/mindspore/ops/_op_impl/tbe/apply_add_sign.py new file mode 100644 index 0000000000..4a9c3c4be2 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/apply_add_sign.py @@ -0,0 +1,65 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""ApplyAddSignD op""" +from mindspore.ops.op_info_register import op_info_register, TBERegOp, DataType + +apply_add_sign_d_op_info = TBERegOp("ApplyAddSign") \ + .fusion_type("OPAQUE") \ + .async_flag(False) \ + .binfile_name("apply_add_sign_d.so") \ + .compute_cost(10) \ + .kernel_name("apply_add_sign_d") \ + .partial_flag(True) \ + .input(0, "var", False, "required", "all") \ + .input(1, "m", False, "required", "all") \ + .input(2, "lr", False, "required", "all") \ + .input(3, "alpha", False, "required", "all") \ + .input(4, "sign_decay", False, "required", "all") \ + .input(5, "beta", False, "required", "all") \ + .input(6, "grad", False, "required", "all") \ + .output(0, "var", False, "required", "all") \ + .output(1, "m", False, "required", "all") \ + .dtype_format(DataType.F16_5HD, DataType.F16_5HD, DataType.F16_Default, DataType.F16_Default, + DataType.F16_Default, DataType.F16_Default, DataType.F16_5HD, DataType.F16_5HD, + DataType.F16_5HD) \ + .dtype_format(DataType.F16_C1HWNCoC0, DataType.F16_C1HWNCoC0, DataType.F16_Default, DataType.F16_Default, + DataType.F16_Default, DataType.F16_Default, DataType.F16_C1HWNCoC0, DataType.F16_C1HWNCoC0, + DataType.F16_C1HWNCoC0) \ + .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.F16_Default, DataType.F16_Default, + DataType.F16_Default, DataType.F16_Default, DataType.F16_Default, DataType.F16_Default, + DataType.F16_Default) \ + .dtype_format(DataType.F16_FracZ, DataType.F16_FracZ, DataType.F16_Default, DataType.F16_Default, + DataType.F16_Default, DataType.F16_Default, DataType.F16_FracZ, DataType.F16_FracZ, + DataType.F16_FracZ) \ + .dtype_format(DataType.F32_5HD, DataType.F32_5HD, DataType.F32_Default, DataType.F32_Default, + DataType.F32_Default, DataType.F32_Default, DataType.F32_5HD, DataType.F32_5HD, + DataType.F32_5HD) \ + .dtype_format(DataType.F32_C1HWNCoC0, DataType.F32_C1HWNCoC0, DataType.F32_Default, DataType.F32_Default, + DataType.F32_Default, DataType.F32_Default, DataType.F32_C1HWNCoC0, DataType.F32_C1HWNCoC0, + DataType.F32_C1HWNCoC0) \ + .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, + DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, + DataType.F32_Default) \ + .dtype_format(DataType.F32_FracZ, DataType.F32_FracZ, DataType.F32_Default, DataType.F32_Default, + DataType.F32_Default, DataType.F32_Default, DataType.F32_FracZ, DataType.F32_FracZ, + DataType.F32_FracZ) \ + .get_op_info() + + +@op_info_register(apply_add_sign_d_op_info) +def _apply_add_sign_tbe(): + """ApplyAddSignD TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/apply_gradient_descent.py b/mindspore/ops/_op_impl/tbe/apply_gradient_descent.py new file mode 100644 index 0000000000..c3276f8595 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/apply_gradient_descent.py @@ -0,0 +1,44 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""ApplyGradientDescent op""" +from mindspore.ops.op_info_register import op_info_register, TBERegOp, DataType + +apply_gradient_descent_op_info = TBERegOp("ApplyGradientDescent") \ + .fusion_type("OPAQUE") \ + .async_flag(False) \ + .binfile_name("apply_gradient_descent.so") \ + .compute_cost(10) \ + .kernel_name("apply_gradient_descent") \ + .partial_flag(True) \ + .input(0, "var", False, "required", "all") \ + .input(1, "alpha", False, "required", "all") \ + .input(2, "delta", False, "required", "all") \ + .output(0, "var", False, "required", "all") \ + .dtype_format(DataType.F16_5HD, DataType.F16_Default, DataType.F16_5HD, DataType.F16_5HD) \ + .dtype_format(DataType.F16_FracZ, DataType.F16_Default, DataType.F16_FracZ, DataType.F16_FracZ) \ + .dtype_format(DataType.F16_C1HWNCoC0, DataType.F16_Default, DataType.F16_C1HWNCoC0, DataType.F16_C1HWNCoC0) \ + .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.F16_Default, DataType.F16_Default) \ + .dtype_format(DataType.F32_5HD, DataType.F32_Default, DataType.F32_5HD, DataType.F32_5HD) \ + .dtype_format(DataType.F32_FracZ, DataType.F32_Default, DataType.F32_FracZ, DataType.F32_FracZ) \ + .dtype_format(DataType.F32_C1HWNCoC0, DataType.F32_Default, DataType.F32_C1HWNCoC0, DataType.F32_C1HWNCoC0) \ + .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, DataType.F32_Default) \ + .get_op_info() + + +@op_info_register(apply_gradient_descent_op_info) +def _apply_gradient_descent_tbe(): + """ApplyGradientDescent TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/apply_power_sign.py b/mindspore/ops/_op_impl/tbe/apply_power_sign.py new file mode 100644 index 0000000000..136391e91e --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/apply_power_sign.py @@ -0,0 +1,65 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""ApplyPowerSignD op""" +from mindspore.ops.op_info_register import op_info_register, TBERegOp, DataType + +apply_power_sign_d_op_info = TBERegOp("ApplyPowerSign") \ + .fusion_type("OPAQUE") \ + .async_flag(False) \ + .binfile_name("apply_power_sign_d.so") \ + .compute_cost(10) \ + .kernel_name("apply_power_sign_d") \ + .partial_flag(True) \ + .input(0, "var", False, "required", "all") \ + .input(1, "m", False, "required", "all") \ + .input(2, "lr", False, "required", "all") \ + .input(3, "logbase", False, "required", "all") \ + .input(4, "sign_decay", False, "required", "all") \ + .input(5, "beta", False, "required", "all") \ + .input(6, "grad", False, "required", "all") \ + .output(0, "var", False, "required", "all") \ + .output(1, "m", False, "required", "all") \ + .dtype_format(DataType.F16_5HD, DataType.F16_5HD, DataType.F16_Default, DataType.F16_Default, + DataType.F16_Default, DataType.F16_Default, DataType.F16_5HD, DataType.F16_5HD, + DataType.F16_5HD) \ + .dtype_format(DataType.F16_C1HWNCoC0, DataType.F16_C1HWNCoC0, DataType.F16_Default, DataType.F16_Default, + DataType.F16_Default, DataType.F16_Default, DataType.F16_C1HWNCoC0, DataType.F16_C1HWNCoC0, + DataType.F16_C1HWNCoC0) \ + .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.F16_Default, DataType.F16_Default, + DataType.F16_Default, DataType.F16_Default, DataType.F16_Default, DataType.F16_Default, + DataType.F16_Default) \ + .dtype_format(DataType.F16_FracZ, DataType.F16_FracZ, DataType.F16_Default, DataType.F16_Default, + DataType.F16_Default, DataType.F16_Default, DataType.F16_FracZ, DataType.F16_FracZ, + DataType.F16_FracZ) \ + .dtype_format(DataType.F32_5HD, DataType.F32_5HD, DataType.F32_Default, DataType.F32_Default, + DataType.F32_Default, DataType.F32_Default, DataType.F32_5HD, DataType.F32_5HD, + DataType.F32_5HD) \ + .dtype_format(DataType.F32_C1HWNCoC0, DataType.F32_C1HWNCoC0, DataType.F32_Default, DataType.F32_Default, + DataType.F32_Default, DataType.F32_Default, DataType.F32_C1HWNCoC0, DataType.F32_C1HWNCoC0, + DataType.F32_C1HWNCoC0) \ + .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, + DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, + DataType.F32_Default) \ + .dtype_format(DataType.F32_FracZ, DataType.F32_FracZ, DataType.F32_Default, DataType.F32_Default, + DataType.F32_Default, DataType.F32_Default, DataType.F32_FracZ, DataType.F32_FracZ, + DataType.F32_FracZ) \ + .get_op_info() + + +@op_info_register(apply_power_sign_d_op_info) +def _apply_power_sign_tbe(): + """ApplyPowerSignD TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/apply_proximal_gradient_descent.py b/mindspore/ops/_op_impl/tbe/apply_proximal_gradient_descent.py new file mode 100644 index 0000000000..aca521d84b --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/apply_proximal_gradient_descent.py @@ -0,0 +1,54 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""ApplyProximalGradientDescent op""" +from mindspore.ops.op_info_register import op_info_register, TBERegOp, DataType + +apply_proximal_gradient_descent_op_info = TBERegOp("ApplyProximalGradientDescent") \ + .fusion_type("OPAQUE") \ + .async_flag(False) \ + .binfile_name("apply_proximal_gradient_descent.so") \ + .compute_cost(10) \ + .kernel_name("apply_proximal_gradient_descent") \ + .partial_flag(True) \ + .input(0, "var", False, "required", "all") \ + .input(1, "alpha", False, "required", "all") \ + .input(2, "l1", False, "required", "all") \ + .input(3, "l2", False, "required", "all") \ + .input(4, "delta", False, "required", "all") \ + .output(0, "var", False, "required", "all") \ + .dtype_format(DataType.F16_5HD, DataType.F16_Default, DataType.F16_Default, DataType.F16_Default, + DataType.F16_5HD, DataType.F16_5HD) \ + .dtype_format(DataType.F16_FracZ, DataType.F16_Default, DataType.F16_Default, DataType.F16_Default, + DataType.F16_FracZ, DataType.F16_FracZ) \ + .dtype_format(DataType.F16_C1HWNCoC0, DataType.F16_Default, DataType.F16_Default, DataType.F16_Default, + DataType.F16_C1HWNCoC0, DataType.F16_C1HWNCoC0) \ + .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.F16_Default, DataType.F16_Default, + DataType.F16_Default, DataType.F16_Default) \ + .dtype_format(DataType.F32_5HD, DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, + DataType.F32_5HD, DataType.F32_5HD) \ + .dtype_format(DataType.F32_FracZ, DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, + DataType.F32_FracZ, DataType.F32_FracZ) \ + .dtype_format(DataType.F32_C1HWNCoC0, DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, + DataType.F32_C1HWNCoC0, DataType.F32_C1HWNCoC0) \ + .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, + DataType.F32_Default, DataType.F32_Default) \ + .get_op_info() + + +@op_info_register(apply_proximal_gradient_descent_op_info) +def _apply_proximal_gradient_descent_tbe(): + """ApplyProximalGradientDescent TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/approximate_equal.py b/mindspore/ops/_op_impl/tbe/approximate_equal.py index 62b8a0c16d..195918a19d 100644 --- a/mindspore/ops/_op_impl/tbe/approximate_equal.py +++ b/mindspore/ops/_op_impl/tbe/approximate_equal.py @@ -28,10 +28,8 @@ approximate_equal_op_info = TBERegOp("ApproximateEqual") \ .input(0, "x1", False, "required", "all") \ .input(1, "x2", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD, DataType.BOOL_5HD) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD, DataType.BOOL_5HD) \ + .dtype_format(DataType.F16_None, DataType.F16_None, DataType.BOOL_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None, DataType.BOOL_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/asin.py b/mindspore/ops/_op_impl/tbe/asin.py index b4e3dbcca3..7a6bdfa087 100644 --- a/mindspore/ops/_op_impl/tbe/asin.py +++ b/mindspore/ops/_op_impl/tbe/asin.py @@ -26,8 +26,8 @@ asin_op_info = TBERegOp("Asin") \ .op_pattern("formatAgnostic") \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD) \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/asinh.py b/mindspore/ops/_op_impl/tbe/asinh.py index 1720f48638..5702b865ad 100644 --- a/mindspore/ops/_op_impl/tbe/asinh.py +++ b/mindspore/ops/_op_impl/tbe/asinh.py @@ -26,8 +26,8 @@ asinh_op_info = TBERegOp("Asinh") \ .op_pattern("formatAgnostic") \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD) \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/atan.py b/mindspore/ops/_op_impl/tbe/atan.py index 293839eaf0..6dcbce0d70 100644 --- a/mindspore/ops/_op_impl/tbe/atan.py +++ b/mindspore/ops/_op_impl/tbe/atan.py @@ -26,10 +26,8 @@ atan_op_info = TBERegOp("Atan") \ .op_pattern("formatAgnostic") \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD) \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/atan2.py b/mindspore/ops/_op_impl/tbe/atan2.py index 26ffdcb59a..f6f11af10c 100644 --- a/mindspore/ops/_op_impl/tbe/atan2.py +++ b/mindspore/ops/_op_impl/tbe/atan2.py @@ -27,10 +27,8 @@ atan2_op_info = TBERegOp("Atan2") \ .input(0, "x1", False, "required", "all") \ .input(1, "x2", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD) \ + .dtype_format(DataType.F16_None, DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/atanh.py b/mindspore/ops/_op_impl/tbe/atanh.py index f60b01967c..d5de93a7bd 100644 --- a/mindspore/ops/_op_impl/tbe/atanh.py +++ b/mindspore/ops/_op_impl/tbe/atanh.py @@ -26,10 +26,8 @@ atanh_op_info = TBERegOp("Atanh") \ .op_pattern("formatAgnostic") \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD) \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/basic_lstm_cell_c_state_grad.py b/mindspore/ops/_op_impl/tbe/basic_lstm_cell_c_state_grad.py index 440b1ce2c7..1e42c1d6fe 100644 --- a/mindspore/ops/_op_impl/tbe/basic_lstm_cell_c_state_grad.py +++ b/mindspore/ops/_op_impl/tbe/basic_lstm_cell_c_state_grad.py @@ -29,8 +29,8 @@ basic_lstm_cell_c_state_grad_op_info = TBERegOp("BasicLSTMCellCStateGrad") \ .input(1, "dht", False, "required", "all") \ .input(2, "dct", False, "required", "all") \ .input(3, "it", False, "required", "all") \ - .input(4, "ft", False, "required", "all") \ - .input(5, "jt", False, "required", "all") \ + .input(4, "jt", False, "required", "all") \ + .input(5, "ft", False, "required", "all") \ .input(6, "ot", False, "required", "all") \ .input(7, "tanhct", False, "required", "all") \ .output(0, "dgate", False, "required", "all") \ diff --git a/mindspore/ops/_op_impl/tbe/batch_matmul.py b/mindspore/ops/_op_impl/tbe/batch_matmul.py index 02f2dd5880..fd1935d369 100644 --- a/mindspore/ops/_op_impl/tbe/batch_matmul.py +++ b/mindspore/ops/_op_impl/tbe/batch_matmul.py @@ -30,10 +30,9 @@ batch_matmul_op_info = TBERegOp("BatchMatMul") \ .input(2, "bias", False, "optional", "all") \ .output(0, "y", False, "required", "all") \ .op_pattern("dynamicFormat") \ - .dtype_format(DataType.I32_Default, DataType.I32_Default, DataType.I32_Default, DataType.I32_Default) \ - .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_FracNZ, DataType.F16_FracNZ, DataType.F16_Default, DataType.F16_FracNZ) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, DataType.F32_Default) \ + .dtype_format(DataType.I32_None, DataType.I32_None, DataType.I32_None, DataType.I32_None) \ + .dtype_format(DataType.F16_None, DataType.F16_None, DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None, DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/batch_to_space_nd.py b/mindspore/ops/_op_impl/tbe/batch_to_space_nd.py index ad5060e7c1..f942a3836d 100644 --- a/mindspore/ops/_op_impl/tbe/batch_to_space_nd.py +++ b/mindspore/ops/_op_impl/tbe/batch_to_space_nd.py @@ -25,8 +25,8 @@ batch_to_space_nd_op_info = TBERegOp("BatchToSpaceND") \ .partial_flag(True) \ .attr("block_shape", "required", "listInt", "all") \ .attr("crops", "required", "listListInt", "all") \ - .input(0, "x", False, "required", "all") \ - .output(0, "y", False, "required", "all") \ + .input(0, "x", False, "required", "all", reshape_type="NH") \ + .output(0, "y", False, "required", "all", reshape_type="NH") \ .dtype_format(DataType.F16_5HD, DataType.F16_5HD) \ .dtype_format(DataType.F32_5HD, DataType.F32_5HD) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/bessel_i0e.py b/mindspore/ops/_op_impl/tbe/bessel_i0e.py index ad0030d93a..d1d05d43cb 100644 --- a/mindspore/ops/_op_impl/tbe/bessel_i0e.py +++ b/mindspore/ops/_op_impl/tbe/bessel_i0e.py @@ -26,8 +26,8 @@ bessel_i0e_op_info = TBERegOp("BesselI0e") \ .op_pattern("formatAgnostic") \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD) \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/bessel_i1e.py b/mindspore/ops/_op_impl/tbe/bessel_i1e.py index 39abb5dad4..a9bc270d0f 100644 --- a/mindspore/ops/_op_impl/tbe/bessel_i1e.py +++ b/mindspore/ops/_op_impl/tbe/bessel_i1e.py @@ -26,8 +26,8 @@ bessel_i1e_op_info = TBERegOp("BesselI1e") \ .op_pattern("formatAgnostic") \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD) \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/bias_add.py b/mindspore/ops/_op_impl/tbe/bias_add.py index 5ab1916299..8fe792d2ff 100644 --- a/mindspore/ops/_op_impl/tbe/bias_add.py +++ b/mindspore/ops/_op_impl/tbe/bias_add.py @@ -28,9 +28,9 @@ bias_add_grad_op_info = TBERegOp("BiasAdd") \ .input(1, "bias", False, "required", "all") \ .output(0, "y", False, "required", "all") \ .op_pattern("dynamicFormat") \ - .dtype_format(DataType.I32_Default, DataType.I32_Default, DataType.I32_Default) \ - .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default) \ + .dtype_format(DataType.I32_None, DataType.I32_None, DataType.I32_None) \ + .dtype_format(DataType.F16_None, DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/binary_cross_entropy.py b/mindspore/ops/_op_impl/tbe/binary_cross_entropy.py index bbb4dcab0b..1f73d3bfe7 100644 --- a/mindspore/ops/_op_impl/tbe/binary_cross_entropy.py +++ b/mindspore/ops/_op_impl/tbe/binary_cross_entropy.py @@ -28,10 +28,8 @@ binary_cross_entropy_op_info = TBERegOp("BinaryCrossEntropy") \ .input(1, "y", False, "required", "all") \ .input(2, "weight", False, "optional", "all") \ .output(0, "output", False, "required", "all") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD, DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD) \ + .op_pattern("dynamicFormat") \ + .dtype_format(DataType.None_None, DataType.None_None, DataType.None_None, DataType.None_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/bitwise_and.py b/mindspore/ops/_op_impl/tbe/bitwise_and.py index 30a79e5243..332cf32ee1 100644 --- a/mindspore/ops/_op_impl/tbe/bitwise_and.py +++ b/mindspore/ops/_op_impl/tbe/bitwise_and.py @@ -26,8 +26,9 @@ bitwise_and_op_info = TBERegOp("BitwiseAnd") \ .input(0, "x1", False, "required", "all") \ .input(1, "x2", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.I16_Default, DataType.I16_Default, DataType.I16_Default) \ - .dtype_format(DataType.U16_Default, DataType.U16_Default, DataType.U16_Default) \ + .op_pattern("broadcast") \ + .dtype_format(DataType.I16_None, DataType.I16_None, DataType.I16_None) \ + .dtype_format(DataType.U16_None, DataType.U16_None, DataType.U16_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/bitwise_or.py b/mindspore/ops/_op_impl/tbe/bitwise_or.py index a80bc908b0..6af6a03eb1 100644 --- a/mindspore/ops/_op_impl/tbe/bitwise_or.py +++ b/mindspore/ops/_op_impl/tbe/bitwise_or.py @@ -26,8 +26,9 @@ bitwise_or_op_info = TBERegOp("BitwiseOr") \ .input(0, "x1", False, "required", "all") \ .input(1, "x2", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.I16_Default, DataType.I16_Default, DataType.I16_Default) \ - .dtype_format(DataType.U16_Default, DataType.U16_Default, DataType.U16_Default) \ + .op_pattern("broadcast") \ + .dtype_format(DataType.I16_None, DataType.I16_None, DataType.I16_None) \ + .dtype_format(DataType.U16_None, DataType.U16_None, DataType.U16_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/bitwise_xor.py b/mindspore/ops/_op_impl/tbe/bitwise_xor.py index c606877ff6..8073d13e62 100644 --- a/mindspore/ops/_op_impl/tbe/bitwise_xor.py +++ b/mindspore/ops/_op_impl/tbe/bitwise_xor.py @@ -26,8 +26,9 @@ bitwise_xor_op_info = TBERegOp("BitwiseXor") \ .input(0, "x1", False, "required", "all") \ .input(1, "x2", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.I16_Default, DataType.I16_Default, DataType.I16_Default) \ - .dtype_format(DataType.U16_Default, DataType.U16_Default, DataType.U16_Default) \ + .op_pattern("broadcast") \ + .dtype_format(DataType.I16_None, DataType.I16_None, DataType.I16_None) \ + .dtype_format(DataType.U16_None, DataType.U16_None, DataType.U16_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/bn_training_reduce.py b/mindspore/ops/_op_impl/tbe/bn_training_reduce.py index f33cba2110..0982202d2b 100644 --- a/mindspore/ops/_op_impl/tbe/bn_training_reduce.py +++ b/mindspore/ops/_op_impl/tbe/bn_training_reduce.py @@ -27,8 +27,8 @@ bn_training_reduce_op_info = TBERegOp("BNTrainingReduce") \ .output(0, "sum", False, "required", "all") \ .output(1, "square_sum", False, "required", "all") \ .op_pattern("dynamicFormat") \ - .dtype_format(DataType.F16_5HD, DataType.F32_5HD, DataType.F32_5HD) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD) \ + .dtype_format(DataType.F16_None, DataType.F32_None, DataType.F32_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/bn_training_reduce_grad.py b/mindspore/ops/_op_impl/tbe/bn_training_reduce_grad.py index 89736a0097..9e39a0325d 100644 --- a/mindspore/ops/_op_impl/tbe/bn_training_reduce_grad.py +++ b/mindspore/ops/_op_impl/tbe/bn_training_reduce_grad.py @@ -33,10 +33,10 @@ bn_training_reduce_grad_op_info = TBERegOp("BNTrainingReduceGrad") \ .input(6, "batch_variance", False, "required", "all") \ .output(0, "y", False, "required", "all", reshape_type="NC") \ .op_pattern("dynamicFormat") \ - .dtype_format(DataType.F16_5HD, DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD, - DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD, - DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD) \ + .dtype_format(DataType.F16_None, DataType.F32_None, DataType.F32_None, DataType.F32_None, + DataType.F32_None, DataType.F32_None, DataType.F32_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None, DataType.F32_None, DataType.F32_None, + DataType.F32_None, DataType.F32_None, DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/bn_training_update_grad.py b/mindspore/ops/_op_impl/tbe/bn_training_update_grad.py index 1aa822a3c1..3739e9cfaf 100644 --- a/mindspore/ops/_op_impl/tbe/bn_training_update_grad.py +++ b/mindspore/ops/_op_impl/tbe/bn_training_update_grad.py @@ -31,10 +31,10 @@ bn_training_update_grad_op_info = TBERegOp("BNTrainingUpdateGrad") \ .output(0, "diff_scale", False, "required", "all") \ .output(1, "diff_offset", False, "required", "all") \ .op_pattern("dynamicFormat") \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD, DataType.F32_5HD, DataType.F32_5HD, - DataType.F32_5HD, DataType.F32_5HD) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD, - DataType.F32_5HD, DataType.F32_5HD) \ + .dtype_format(DataType.F16_None, DataType.F16_None, DataType.F32_None, DataType.F32_None, + DataType.F32_None, DataType.F32_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None, DataType.F32_None, DataType.F32_None, + DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/bn_training_update_v2.py b/mindspore/ops/_op_impl/tbe/bn_training_update_v2.py index a54d91a483..1a0f65c6e5 100644 --- a/mindspore/ops/_op_impl/tbe/bn_training_update_v2.py +++ b/mindspore/ops/_op_impl/tbe/bn_training_update_v2.py @@ -33,12 +33,12 @@ bn_training_update_v2_op_info = TBERegOp("BNTrainingUpdateV2") \ .output(1, "batch_mean", False, "required", "all") \ .output(2, "batch_variance", False, "required", "all") \ .op_pattern("dynamicFormat") \ - .dtype_format(DataType.F16_5HD, DataType.F32_5HD, DataType.F32_5HD, - DataType.F32_5HD, DataType.F32_5HD, DataType.F16_5HD, - DataType.F32_5HD, DataType.F32_5HD) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD, - DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD, - DataType.F32_5HD, DataType.F32_5HD) \ + .dtype_format(DataType.F16_None, DataType.F32_None, DataType.F32_None, + DataType.F32_None, DataType.F32_None, DataType.F16_None, + DataType.F32_None, DataType.F32_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None, DataType.F32_None, + DataType.F32_None, DataType.F32_None, DataType.F32_None, + DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/ceil.py b/mindspore/ops/_op_impl/tbe/ceil.py index d9a127603f..769d1738da 100644 --- a/mindspore/ops/_op_impl/tbe/ceil.py +++ b/mindspore/ops/_op_impl/tbe/ceil.py @@ -25,8 +25,9 @@ ceil_op_info = TBERegOp("Ceil") \ .partial_flag(True) \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ + .op_pattern("formatAgnostic") \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/check_valid.py b/mindspore/ops/_op_impl/tbe/check_valid.py index 9c489b64c5..c5e218b049 100644 --- a/mindspore/ops/_op_impl/tbe/check_valid.py +++ b/mindspore/ops/_op_impl/tbe/check_valid.py @@ -26,8 +26,9 @@ check_valid_op_info = TBERegOp("CheckValid") \ .input(0, "bbox_tensor", False, "required", "all") \ .input(1, "img_tas", False, "required", "all") \ .output(0, "valid_tensor", False, "required", "all") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.I8_Default) \ - .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.BOOL_Default) \ + .op_pattern("broadcast") \ + .dtype_format(DataType.F16_None, DataType.F16_None, DataType.I8_None) \ + .dtype_format(DataType.F16_None, DataType.F16_None, DataType.BOOL_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/clip_by_value.py b/mindspore/ops/_op_impl/tbe/clip_by_value.py index 4ddc6c3c0f..3b51c14160 100644 --- a/mindspore/ops/_op_impl/tbe/clip_by_value.py +++ b/mindspore/ops/_op_impl/tbe/clip_by_value.py @@ -28,12 +28,10 @@ clip_by_value_op_info = TBERegOp("ClipByValue") \ .input(1, "clip_value_min", False, "required", "all") \ .input(2, "clip_value_max", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.I32_Default, DataType.I32_Default, DataType.I32_Default, DataType.I32_Default) \ - .dtype_format(DataType.I32_5HD, DataType.I32_5HD, DataType.I32_5HD, DataType.I32_5HD) \ - .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD, DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD) \ + .op_pattern("broadcast") \ + .dtype_format(DataType.I32_None, DataType.I32_None, DataType.I32_None, DataType.I32_None) \ + .dtype_format(DataType.F16_None, DataType.F16_None, DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None, DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/concat.py b/mindspore/ops/_op_impl/tbe/concat.py index 0bf636016f..af8a6e24d8 100644 --- a/mindspore/ops/_op_impl/tbe/concat.py +++ b/mindspore/ops/_op_impl/tbe/concat.py @@ -27,28 +27,7 @@ concat_op_info = TBERegOp("Concat") \ .input(0, "input_values", False, "dynamic", "all") \ .output(0, "output_data", False, "required", "all") \ .op_pattern("dynamicFormat") \ - .dtype_format(DataType.BOOL_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.BOOL_5HD, DataType.BOOL_5HD) \ - .dtype_format(DataType.I8_Default, DataType.I8_Default) \ - .dtype_format(DataType.I8_5HD, DataType.I8_5HD) \ - .dtype_format(DataType.U8_Default, DataType.U8_Default) \ - .dtype_format(DataType.U8_5HD, DataType.U8_5HD) \ - .dtype_format(DataType.I16_Default, DataType.I16_Default) \ - .dtype_format(DataType.I16_5HD, DataType.I16_5HD) \ - .dtype_format(DataType.U16_Default, DataType.U16_Default) \ - .dtype_format(DataType.U16_5HD, DataType.U16_5HD) \ - .dtype_format(DataType.I32_Default, DataType.I32_Default) \ - .dtype_format(DataType.I32_5HD, DataType.I32_5HD) \ - .dtype_format(DataType.U32_Default, DataType.U32_Default) \ - .dtype_format(DataType.U32_5HD, DataType.U32_5HD) \ - .dtype_format(DataType.I64_Default, DataType.I64_Default) \ - .dtype_format(DataType.I64_5HD, DataType.I64_5HD) \ - .dtype_format(DataType.U64_Default, DataType.U64_Default) \ - .dtype_format(DataType.U64_5HD, DataType.U64_5HD) \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD) \ + .dtype_format(DataType.None_None, DataType.None_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/confusion_transpose_d.py b/mindspore/ops/_op_impl/tbe/confusion_transpose_d.py index e52ae01520..f84e9d4292 100644 --- a/mindspore/ops/_op_impl/tbe/confusion_transpose_d.py +++ b/mindspore/ops/_op_impl/tbe/confusion_transpose_d.py @@ -28,26 +28,8 @@ confusion_transpose_d_op_info = TBERegOp("ConfusionTransposeD") \ .attr("transpose_first", "required", "bool", "all") \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.I8_FracNZ, DataType.I8_FracNZ) \ - .dtype_format(DataType.I8_Default, DataType.I8_Default) \ - .dtype_format(DataType.U8_FracNZ, DataType.U8_FracNZ) \ - .dtype_format(DataType.U8_Default, DataType.U8_Default) \ - .dtype_format(DataType.I16_FracNZ, DataType.I16_FracNZ) \ - .dtype_format(DataType.I16_Default, DataType.I16_Default) \ - .dtype_format(DataType.U16_FracNZ, DataType.U16_FracNZ) \ - .dtype_format(DataType.U16_Default, DataType.U16_Default) \ - .dtype_format(DataType.I32_FracNZ, DataType.I32_FracNZ) \ - .dtype_format(DataType.I32_Default, DataType.I32_Default) \ - .dtype_format(DataType.U32_FracNZ, DataType.U32_FracNZ) \ - .dtype_format(DataType.U32_Default, DataType.U32_Default) \ - .dtype_format(DataType.I64_FracNZ, DataType.I64_FracNZ) \ - .dtype_format(DataType.I64_Default, DataType.I64_Default) \ - .dtype_format(DataType.U64_FracNZ, DataType.U64_FracNZ) \ - .dtype_format(DataType.U64_Default, DataType.U64_Default) \ - .dtype_format(DataType.F16_FracNZ, DataType.F16_FracNZ) \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F32_FracNZ, DataType.F32_FracNZ) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ + .op_pattern("dynamicFormat") \ + .dtype_format(DataType.None_None, DataType.None_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/conv2d.py b/mindspore/ops/_op_impl/tbe/conv2d.py index a2879d521a..5531b573c3 100644 --- a/mindspore/ops/_op_impl/tbe/conv2d.py +++ b/mindspore/ops/_op_impl/tbe/conv2d.py @@ -33,7 +33,9 @@ conv2d_op_info = TBERegOp("Conv2D") \ .input(2, "bias", False, "optional", "all") \ .input(3, "offset_w", False, "optional", "all") \ .output(0, "y", True, "required", "all") \ + .op_pattern("dynamicFormat") \ .dtype_format(DataType.F16_None, DataType.F16_None, DataType.F16_None, DataType.I8_None, DataType.F16_None) \ + .dtype_format(DataType.I8_None, DataType.I8_None, DataType.I32_None, DataType.I8_None, DataType.I32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/conv2d_backprop_filter.py b/mindspore/ops/_op_impl/tbe/conv2d_backprop_filter.py index 04b55bb2a3..c309a4f2ab 100644 --- a/mindspore/ops/_op_impl/tbe/conv2d_backprop_filter.py +++ b/mindspore/ops/_op_impl/tbe/conv2d_backprop_filter.py @@ -27,6 +27,8 @@ conv2d_backprop_filter_op_info = TBERegOp("Conv2DBackpropFilter") \ .attr("stride", "required", "listInt", "all") \ .attr("pad_list", "required", "listInt", "all") \ .attr("dilation", "required", "listInt", "all") \ + .attr("groups", "optional", "int", "all") \ + .attr("data_format", "optional", "str", "all") \ .input(0, "out_backprop", False, "required", "all") \ .input(1, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ diff --git a/mindspore/ops/_op_impl/tbe/cos.py b/mindspore/ops/_op_impl/tbe/cos.py index 3acb0c2a7e..4d8d8047aa 100644 --- a/mindspore/ops/_op_impl/tbe/cos.py +++ b/mindspore/ops/_op_impl/tbe/cos.py @@ -26,10 +26,8 @@ cos_op_info = TBERegOp("Cos") \ .op_pattern("formatAgnostic") \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD) \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/cosh.py b/mindspore/ops/_op_impl/tbe/cosh.py index 75d48293e9..b6710669d9 100644 --- a/mindspore/ops/_op_impl/tbe/cosh.py +++ b/mindspore/ops/_op_impl/tbe/cosh.py @@ -26,8 +26,8 @@ cosh_op_info = TBERegOp("Cosh") \ .op_pattern("formatAgnostic") \ .input(0, "x", False, "required", "all") \ .output(0, "y", True, "required", "all") \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD) \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/div.py b/mindspore/ops/_op_impl/tbe/div.py index 2a83745399..de36df902f 100644 --- a/mindspore/ops/_op_impl/tbe/div.py +++ b/mindspore/ops/_op_impl/tbe/div.py @@ -26,16 +26,12 @@ div_op_info = TBERegOp("Div") \ .input(0, "x1", False, "required", "all") \ .input(1, "x2", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.I8_Default, DataType.I8_Default, DataType.I8_Default) \ - .dtype_format(DataType.I8_5HD, DataType.I8_5HD, DataType.I8_5HD) \ - .dtype_format(DataType.U8_Default, DataType.U8_Default, DataType.U8_Default) \ - .dtype_format(DataType.U8_5HD, DataType.U8_5HD, DataType.U8_5HD) \ - .dtype_format(DataType.I32_Default, DataType.I32_Default, DataType.I32_Default) \ - .dtype_format(DataType.I32_5HD, DataType.I32_5HD, DataType.I32_5HD) \ - .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD) \ + .op_pattern("broadcast") \ + .dtype_format(DataType.I8_None, DataType.I8_None, DataType.I8_None) \ + .dtype_format(DataType.U8_None, DataType.U8_None, DataType.U8_None) \ + .dtype_format(DataType.I32_None, DataType.I32_None, DataType.I32_None) \ + .dtype_format(DataType.F16_None, DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/div_no_nan.py b/mindspore/ops/_op_impl/tbe/div_no_nan.py index 893b38042e..431c1303b0 100644 --- a/mindspore/ops/_op_impl/tbe/div_no_nan.py +++ b/mindspore/ops/_op_impl/tbe/div_no_nan.py @@ -26,16 +26,12 @@ div_no_nan_op_info = TBERegOp("DivNoNan") \ .input(0, "x1", False, "required", "all") \ .input(1, "x2", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.I8_Default, DataType.I8_Default, DataType.I8_Default) \ - .dtype_format(DataType.I8_5HD, DataType.I8_5HD, DataType.I8_5HD) \ - .dtype_format(DataType.U8_Default, DataType.U8_Default, DataType.U8_Default) \ - .dtype_format(DataType.U8_5HD, DataType.U8_5HD, DataType.U8_5HD) \ - .dtype_format(DataType.I32_Default, DataType.I32_Default, DataType.I32_Default) \ - .dtype_format(DataType.I32_5HD, DataType.I32_5HD, DataType.I32_5HD) \ - .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD) \ + .op_pattern("broadcast") \ + .dtype_format(DataType.I8_None, DataType.I8_None, DataType.I8_None) \ + .dtype_format(DataType.U8_None, DataType.U8_None, DataType.U8_None) \ + .dtype_format(DataType.I32_None, DataType.I32_None, DataType.I32_None) \ + .dtype_format(DataType.F16_None, DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/dropout_do_mask.py b/mindspore/ops/_op_impl/tbe/dropout_do_mask.py index a24e02f964..0b5a5e87f4 100644 --- a/mindspore/ops/_op_impl/tbe/dropout_do_mask.py +++ b/mindspore/ops/_op_impl/tbe/dropout_do_mask.py @@ -28,8 +28,7 @@ drop_out_do_mask_op_info = TBERegOp("DropoutDoMask") \ .input(2, "keep_prob", False, "required", "all") \ .output(0, "y", False, "required", "all") \ .op_pattern("dynamicFormat") \ - .dtype_format(DataType.F16_Default, DataType.U8_Default, DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F32_Default, DataType.U8_Default, DataType.F32_Default, DataType.F32_Default) \ + .dtype_format(DataType.None_None, DataType.None_None, DataType.None_None, DataType.None_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/elu.py b/mindspore/ops/_op_impl/tbe/elu.py index e61e2851af..75e9d3e889 100644 --- a/mindspore/ops/_op_impl/tbe/elu.py +++ b/mindspore/ops/_op_impl/tbe/elu.py @@ -27,8 +27,8 @@ elu_op_info = TBERegOp("Elu") \ .attr("alpha", "optional", "float", "all", "1.0") \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/equal.py b/mindspore/ops/_op_impl/tbe/equal.py index 594fb51cb5..aabb51edad 100644 --- a/mindspore/ops/_op_impl/tbe/equal.py +++ b/mindspore/ops/_op_impl/tbe/equal.py @@ -26,16 +26,12 @@ equal_op_info = TBERegOp("Equal") \ .input(0, "x1", False, "required", "all") \ .input(1, "x2", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.I8_Default, DataType.I8_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.I8_5HD, DataType.I8_5HD, DataType.BOOL_5HD) \ - .dtype_format(DataType.U8_Default, DataType.U8_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.U8_5HD, DataType.U8_5HD, DataType.BOOL_5HD) \ - .dtype_format(DataType.I32_Default, DataType.I32_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.I32_5HD, DataType.I32_5HD, DataType.BOOL_5HD) \ - .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD, DataType.BOOL_5HD) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD, DataType.BOOL_5HD) \ + .op_pattern("broadcast") \ + .dtype_format(DataType.I8_None, DataType.I8_None, DataType.BOOL_None) \ + .dtype_format(DataType.U8_None, DataType.U8_None, DataType.BOOL_None) \ + .dtype_format(DataType.I32_None, DataType.I32_None, DataType.BOOL_None) \ + .dtype_format(DataType.F16_None, DataType.F16_None, DataType.BOOL_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None, DataType.BOOL_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/erf.py b/mindspore/ops/_op_impl/tbe/erf.py index 4c4893d505..ff34a4db14 100644 --- a/mindspore/ops/_op_impl/tbe/erf.py +++ b/mindspore/ops/_op_impl/tbe/erf.py @@ -26,8 +26,8 @@ erf_op_info = TBERegOp("Erf") \ .op_pattern("formatAgnostic") \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/erfc.py b/mindspore/ops/_op_impl/tbe/erfc.py index 7b0eccf52e..93d9dec070 100644 --- a/mindspore/ops/_op_impl/tbe/erfc.py +++ b/mindspore/ops/_op_impl/tbe/erfc.py @@ -26,8 +26,8 @@ erfc_op_info = TBERegOp("Erfc") \ .op_pattern("formatAgnostic") \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/exp.py b/mindspore/ops/_op_impl/tbe/exp.py index 545845a3b0..9448399866 100644 --- a/mindspore/ops/_op_impl/tbe/exp.py +++ b/mindspore/ops/_op_impl/tbe/exp.py @@ -25,10 +25,9 @@ exp_op_info = TBERegOp("Exp") \ .partial_flag(True) \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD) \ + .op_pattern("formatAgnostic") \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/expm1.py b/mindspore/ops/_op_impl/tbe/expm1.py index a126aca36f..f8f5479b6f 100644 --- a/mindspore/ops/_op_impl/tbe/expm1.py +++ b/mindspore/ops/_op_impl/tbe/expm1.py @@ -26,8 +26,8 @@ expm1_op_info = TBERegOp("Expm1") \ .op_pattern("formatAgnostic") \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/floor.py b/mindspore/ops/_op_impl/tbe/floor.py index e02bce4aa3..dbdd7927ef 100644 --- a/mindspore/ops/_op_impl/tbe/floor.py +++ b/mindspore/ops/_op_impl/tbe/floor.py @@ -25,8 +25,9 @@ floor_op_info = TBERegOp("Floor") \ .partial_flag(True) \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ + .op_pattern("formatAgnostic") \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/floor_div.py b/mindspore/ops/_op_impl/tbe/floor_div.py index 74fd594901..c700e8d15a 100644 --- a/mindspore/ops/_op_impl/tbe/floor_div.py +++ b/mindspore/ops/_op_impl/tbe/floor_div.py @@ -26,11 +26,12 @@ floordiv_op_info = TBERegOp("FloorDiv") \ .input(0, "x1", False, "required", "all") \ .input(1, "x2", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.I8_Default, DataType.I8_Default, DataType.I8_Default) \ - .dtype_format(DataType.U8_Default, DataType.U8_Default, DataType.U8_Default) \ - .dtype_format(DataType.I32_Default, DataType.I32_Default, DataType.I32_Default) \ - .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default) \ + .op_pattern("broadcast") \ + .dtype_format(DataType.I8_None, DataType.I8_None, DataType.I8_None) \ + .dtype_format(DataType.U8_None, DataType.U8_None, DataType.U8_None) \ + .dtype_format(DataType.I32_None, DataType.I32_None, DataType.I32_None) \ + .dtype_format(DataType.F16_None, DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/floor_mod.py b/mindspore/ops/_op_impl/tbe/floor_mod.py index 031f160e0a..72cd6499d4 100644 --- a/mindspore/ops/_op_impl/tbe/floor_mod.py +++ b/mindspore/ops/_op_impl/tbe/floor_mod.py @@ -26,9 +26,10 @@ floor_mod_op_info = TBERegOp("FloorMod") \ .input(0, "x1", False, "required", "all") \ .input(1, "x2", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.I32_Default, DataType.I32_Default, DataType.I32_Default) \ + .op_pattern("broadcast") \ + .dtype_format(DataType.F16_None, DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None, DataType.F32_None) \ + .dtype_format(DataType.I32_None, DataType.I32_None, DataType.I32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/fused_mul_add.py b/mindspore/ops/_op_impl/tbe/fused_mul_add.py index fa104fb561..3a7efee792 100644 --- a/mindspore/ops/_op_impl/tbe/fused_mul_add.py +++ b/mindspore/ops/_op_impl/tbe/fused_mul_add.py @@ -28,21 +28,7 @@ fused_mul_add_op_info = TBERegOp("FusedMulAdd") \ .input(2, "x3", False, "required", "all") \ .output(0, "y", False, "required", "all") \ .op_pattern("dynamicFormat") \ - .dtype_format(DataType.I32_Default, DataType.I32_Default, DataType.I32_Default, DataType.I32_Default) \ - .dtype_format(DataType.I32_5HD, DataType.I32_5HD, DataType.I32_5HD, DataType.I32_5HD) \ - .dtype_format(DataType.I32_FracZ, DataType.I32_FracZ, DataType.I32_FracZ, DataType.I32_FracZ) \ - .dtype_format(DataType.I32_FracNZ, DataType.I32_FracNZ, DataType.I32_FracNZ, DataType.I32_FracNZ) \ - .dtype_format(DataType.I32_C1HWNCoC0, DataType.I32_C1HWNCoC0, DataType.I32_C1HWNCoC0, DataType.I32_C1HWNCoC0) \ - .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD, DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F16_FracZ, DataType.F16_FracZ, DataType.F16_FracZ, DataType.F16_FracZ) \ - .dtype_format(DataType.F16_FracNZ, DataType.F16_FracNZ, DataType.F16_FracNZ, DataType.F16_FracNZ) \ - .dtype_format(DataType.F16_C1HWNCoC0, DataType.F16_C1HWNCoC0, DataType.F16_C1HWNCoC0, DataType.F16_C1HWNCoC0) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD) \ - .dtype_format(DataType.F32_FracZ, DataType.F32_FracZ, DataType.F32_FracZ, DataType.F32_FracZ) \ - .dtype_format(DataType.F32_FracNZ, DataType.F32_FracNZ, DataType.F32_FracNZ, DataType.F32_FracNZ) \ - .dtype_format(DataType.F32_C1HWNCoC0, DataType.F32_C1HWNCoC0, DataType.F32_C1HWNCoC0, DataType.F32_C1HWNCoC0) \ + .dtype_format(DataType.None_None, DataType.None_None, DataType.None_None, DataType.None_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/gelu.py b/mindspore/ops/_op_impl/tbe/gelu.py index 9d4b2ed7f3..a539cb8ef3 100644 --- a/mindspore/ops/_op_impl/tbe/gelu.py +++ b/mindspore/ops/_op_impl/tbe/gelu.py @@ -25,14 +25,9 @@ gelu_op_info = TBERegOp("Gelu") \ .partial_flag(True) \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F16_FracZ, DataType.F16_FracZ) \ - .dtype_format(DataType.F16_FracNZ, DataType.F16_FracNZ) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD) \ - .dtype_format(DataType.F32_FracZ, DataType.F32_FracZ) \ - .dtype_format(DataType.F32_FracNZ, DataType.F32_FracNZ) \ + .op_pattern("formatAgnostic") \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/greater.py b/mindspore/ops/_op_impl/tbe/greater.py index 90c680ab04..acbc7ef729 100644 --- a/mindspore/ops/_op_impl/tbe/greater.py +++ b/mindspore/ops/_op_impl/tbe/greater.py @@ -26,16 +26,12 @@ greater_op_info = TBERegOp("Greater") \ .input(0, "x1", False, "required", "all") \ .input(1, "x2", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.I8_Default, DataType.I8_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.I8_5HD, DataType.I8_5HD, DataType.BOOL_5HD) \ - .dtype_format(DataType.U8_Default, DataType.U8_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.U8_5HD, DataType.U8_5HD, DataType.BOOL_5HD) \ - .dtype_format(DataType.I32_Default, DataType.I32_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.I32_5HD, DataType.I32_5HD, DataType.BOOL_5HD) \ - .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD, DataType.BOOL_5HD) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD, DataType.BOOL_5HD) \ + .op_pattern("broadcast") \ + .dtype_format(DataType.I8_None, DataType.I8_None, DataType.BOOL_None) \ + .dtype_format(DataType.U8_None, DataType.U8_None, DataType.BOOL_None) \ + .dtype_format(DataType.I32_None, DataType.I32_None, DataType.BOOL_None) \ + .dtype_format(DataType.F16_None, DataType.F16_None, DataType.BOOL_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None, DataType.BOOL_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/greater_equal.py b/mindspore/ops/_op_impl/tbe/greater_equal.py index 5609f15f18..86aba300c1 100644 --- a/mindspore/ops/_op_impl/tbe/greater_equal.py +++ b/mindspore/ops/_op_impl/tbe/greater_equal.py @@ -26,16 +26,12 @@ greater_equal_op_info = TBERegOp("GreaterEqual") \ .input(0, "x1", False, "required", "all") \ .input(1, "x2", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.I8_Default, DataType.I8_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.I8_5HD, DataType.I8_5HD, DataType.BOOL_5HD) \ - .dtype_format(DataType.U8_Default, DataType.U8_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.U8_5HD, DataType.U8_5HD, DataType.BOOL_5HD) \ - .dtype_format(DataType.I32_Default, DataType.I32_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.I32_5HD, DataType.I32_5HD, DataType.BOOL_5HD) \ - .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD, DataType.BOOL_5HD) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD, DataType.BOOL_5HD) \ + .op_pattern("broadcast") \ + .dtype_format(DataType.I8_None, DataType.I8_None, DataType.BOOL_None) \ + .dtype_format(DataType.U8_None, DataType.U8_None, DataType.BOOL_None) \ + .dtype_format(DataType.I32_None, DataType.I32_None, DataType.BOOL_None) \ + .dtype_format(DataType.F16_None, DataType.F16_None, DataType.BOOL_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None, DataType.BOOL_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/inv.py b/mindspore/ops/_op_impl/tbe/inv.py index e2b749a5aa..a838a31d4c 100644 --- a/mindspore/ops/_op_impl/tbe/inv.py +++ b/mindspore/ops/_op_impl/tbe/inv.py @@ -25,9 +25,10 @@ inv_op_info = TBERegOp("Inv") \ .partial_flag(True) \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.I32_Default, DataType.I32_Default) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ + .op_pattern("formatAgnostic") \ + .dtype_format(DataType.I32_None, DataType.I32_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/inv_grad.py b/mindspore/ops/_op_impl/tbe/inv_grad.py index 70626b8808..a81c1df624 100644 --- a/mindspore/ops/_op_impl/tbe/inv_grad.py +++ b/mindspore/ops/_op_impl/tbe/inv_grad.py @@ -26,10 +26,11 @@ inv_grad_op_info = TBERegOp("InvGrad") \ .input(0, "x", False, "required", "all") \ .input(1, "grad", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.I32_Default, DataType.I32_Default, DataType.I32_Default) \ - .dtype_format(DataType.I8_Default, DataType.I8_Default, DataType.I8_Default) \ + .op_pattern("broadcast") \ + .dtype_format(DataType.F16_None, DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None, DataType.F32_None) \ + .dtype_format(DataType.I32_None, DataType.I32_None, DataType.I32_None) \ + .dtype_format(DataType.I8_None, DataType.I8_None, DataType.I8_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/invert.py b/mindspore/ops/_op_impl/tbe/invert.py index 887eee45e7..73b33a5764 100644 --- a/mindspore/ops/_op_impl/tbe/invert.py +++ b/mindspore/ops/_op_impl/tbe/invert.py @@ -25,8 +25,9 @@ invert_op_info = TBERegOp("Invert") \ .partial_flag(True) \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.I16_Default, DataType.I16_Default) \ - .dtype_format(DataType.U16_Default, DataType.U16_Default) \ + .op_pattern("formatAgnostic") \ + .dtype_format(DataType.I16_None, DataType.I16_None) \ + .dtype_format(DataType.U16_None, DataType.U16_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/layer_norm.py b/mindspore/ops/_op_impl/tbe/layer_norm.py index 03ddd2dc6c..2414b9bcb3 100644 --- a/mindspore/ops/_op_impl/tbe/layer_norm.py +++ b/mindspore/ops/_op_impl/tbe/layer_norm.py @@ -33,18 +33,10 @@ layer_norm_op_info = TBERegOp("LayerNorm") \ .output(1, "mean", False, "required", "all") \ .output(2, "variance", False, "required", "all") \ .op_pattern("dynamicFormat") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.F16_Default, DataType.F16_Default, - DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD, DataType.F16_5HD, DataType.F16_5HD, - DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F16_FracNZ, DataType.F16_Default, DataType.F16_Default, DataType.F16_FracNZ, - DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, - DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD, - DataType.F32_5HD, DataType.F32_5HD) \ - .dtype_format(DataType.F32_FracNZ, DataType.F32_Default, DataType.F32_Default, DataType.F32_FracNZ, - DataType.F32_Default, DataType.F32_Default) \ + .dtype_format(DataType.F16_None, DataType.F16_None, DataType.F16_None, DataType.F16_None, + DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None, DataType.F32_None, DataType.F32_None, + DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/layer_norm_beta_gamma_backprop.py b/mindspore/ops/_op_impl/tbe/layer_norm_beta_gamma_backprop.py index deca384032..8fec87d6d7 100644 --- a/mindspore/ops/_op_impl/tbe/layer_norm_beta_gamma_backprop.py +++ b/mindspore/ops/_op_impl/tbe/layer_norm_beta_gamma_backprop.py @@ -31,18 +31,10 @@ layer_norm_beta_gamma_backprop_op_info = TBERegOp("LayerNormBetaGammaBackprop") .output(0, "pd_gamma", False, "required", "all") \ .output(1, "pd_beta", False, "required", "all") \ .op_pattern("dynamicFormat") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.F16_Default, DataType.F16_Default, - DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD, DataType.F16_5HD, DataType.F16_5HD, - DataType.F32_5HD, DataType.F32_5HD) \ - .dtype_format(DataType.F16_FracNZ, DataType.F16_FracNZ, DataType.F16_Default, DataType.F16_Default, - DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, - DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD, - DataType.F32_5HD, DataType.F32_5HD) \ - .dtype_format(DataType.F32_FracNZ, DataType.F32_FracNZ, DataType.F32_Default, DataType.F32_Default, - DataType.F32_Default, DataType.F32_Default) \ + .dtype_format(DataType.F16_None, DataType.F16_None, DataType.F16_None, DataType.F16_None, + DataType.F32_None, DataType.F32_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None, DataType.F32_None, DataType.F32_None, + DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/layer_norm_x_backprop.py b/mindspore/ops/_op_impl/tbe/layer_norm_x_backprop.py index 1d4f1ef231..ac2d28e9b8 100644 --- a/mindspore/ops/_op_impl/tbe/layer_norm_x_backprop.py +++ b/mindspore/ops/_op_impl/tbe/layer_norm_x_backprop.py @@ -30,18 +30,10 @@ layer_norm_x_backprop_op_info = TBERegOp("LayerNormXBackprop") \ .input(4, "gamma", False, "required", "all") \ .output(0, "pd_x", False, "required", "all") \ .op_pattern("dynamicFormat") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.F16_Default, DataType.F16_Default, - DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD, DataType.F16_5HD, DataType.F16_5HD, - DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F16_FracNZ, DataType.F16_FracNZ, DataType.F16_Default, DataType.F16_Default, - DataType.F16_Default, DataType.F16_FracNZ) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, - DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD, - DataType.F32_5HD, DataType.F32_5HD) \ - .dtype_format(DataType.F32_FracNZ, DataType.F32_FracNZ, DataType.F32_Default, DataType.F32_Default, - DataType.F32_Default, DataType.F32_FracNZ) \ + .dtype_format(DataType.F16_None, DataType.F16_None, DataType.F16_None, DataType.F16_None, + DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None, DataType.F32_None, DataType.F32_None, + DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/less.py b/mindspore/ops/_op_impl/tbe/less.py index 947c40b949..9197e831f1 100644 --- a/mindspore/ops/_op_impl/tbe/less.py +++ b/mindspore/ops/_op_impl/tbe/less.py @@ -26,16 +26,12 @@ less_op_info = TBERegOp("Less") \ .input(0, "x1", False, "required", "all") \ .input(1, "x2", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.I8_Default, DataType.I8_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.I8_5HD, DataType.I8_5HD, DataType.BOOL_5HD) \ - .dtype_format(DataType.U8_Default, DataType.U8_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.U8_5HD, DataType.U8_5HD, DataType.BOOL_5HD) \ - .dtype_format(DataType.I32_Default, DataType.I32_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.I32_5HD, DataType.I32_5HD, DataType.BOOL_5HD) \ - .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD, DataType.BOOL_5HD) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD, DataType.BOOL_5HD) \ + .op_pattern("broadcast") \ + .dtype_format(DataType.I8_None, DataType.I8_None, DataType.BOOL_None) \ + .dtype_format(DataType.U8_None, DataType.U8_None, DataType.BOOL_None) \ + .dtype_format(DataType.I32_None, DataType.I32_None, DataType.BOOL_None) \ + .dtype_format(DataType.F16_None, DataType.F16_None, DataType.BOOL_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None, DataType.BOOL_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/less_equal.py b/mindspore/ops/_op_impl/tbe/less_equal.py index 14cf7c8906..83ee8a1c36 100644 --- a/mindspore/ops/_op_impl/tbe/less_equal.py +++ b/mindspore/ops/_op_impl/tbe/less_equal.py @@ -28,16 +28,12 @@ less_equal_op_info = TBERegOp("LessEqual") \ .input(0, "x1", False, "required", "all") \ .input(1, "x2", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.I8_Default, DataType.I8_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.I8_5HD, DataType.I8_5HD, DataType.BOOL_5HD) \ - .dtype_format(DataType.U8_Default, DataType.U8_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.U8_5HD, DataType.U8_5HD, DataType.BOOL_5HD) \ - .dtype_format(DataType.I32_Default, DataType.I32_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.I32_5HD, DataType.I32_5HD, DataType.BOOL_5HD) \ - .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD, DataType.BOOL_5HD) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD, DataType.BOOL_5HD) \ + .op_pattern("broadcast") \ + .dtype_format(DataType.I8_None, DataType.I8_None, DataType.BOOL_None) \ + .dtype_format(DataType.U8_None, DataType.U8_None, DataType.BOOL_None) \ + .dtype_format(DataType.I32_None, DataType.I32_None, DataType.BOOL_None) \ + .dtype_format(DataType.F16_None, DataType.F16_None, DataType.BOOL_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None, DataType.BOOL_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/lin_space.py b/mindspore/ops/_op_impl/tbe/lin_space.py index aed41e80d4..6e474c50ea 100644 --- a/mindspore/ops/_op_impl/tbe/lin_space.py +++ b/mindspore/ops/_op_impl/tbe/lin_space.py @@ -29,8 +29,8 @@ lin_space_op_info = TBERegOp("LinSpace") \ .input(2, "stop", False, "required", "all") \ .input(3, "num", False, "required", "all") \ .output(0, "output", False, "required", "all") \ - .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, DataType.I32_Default, - DataType.F32_Default,) \ + .dtype_format(DataType.F32_None, DataType.F32_None, DataType.F32_None, DataType.I32_None, + DataType.F32_None,) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/log.py b/mindspore/ops/_op_impl/tbe/log.py index b7da647248..e5895066d0 100644 --- a/mindspore/ops/_op_impl/tbe/log.py +++ b/mindspore/ops/_op_impl/tbe/log.py @@ -25,10 +25,9 @@ log_op_info = TBERegOp("Log") \ .partial_flag(True) \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD) \ + .op_pattern("formatAgnostic") \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/log1p.py b/mindspore/ops/_op_impl/tbe/log1p.py index a4c6377b6a..6ae151ebf9 100644 --- a/mindspore/ops/_op_impl/tbe/log1p.py +++ b/mindspore/ops/_op_impl/tbe/log1p.py @@ -25,10 +25,9 @@ log1p_op_info = TBERegOp("Log1p") \ .partial_flag(True) \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD) \ + .op_pattern("formatAgnostic") \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/logical_and.py b/mindspore/ops/_op_impl/tbe/logical_and.py index 925a4e82d8..8347d0a03f 100644 --- a/mindspore/ops/_op_impl/tbe/logical_and.py +++ b/mindspore/ops/_op_impl/tbe/logical_and.py @@ -26,10 +26,8 @@ logical_and_op_info = TBERegOp("LogicalAnd") \ .input(0, "x1", False, "required", "all") \ .input(1, "x2", False, "required", "all") \ .output(0, "y", True, "required", "all") \ - .dtype_format(DataType.BOOL_Default, DataType.BOOL_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.BOOL_FracZ, DataType.BOOL_FracZ, DataType.BOOL_FracZ) \ - .dtype_format(DataType.BOOL_C1HWNCoC0, DataType.BOOL_C1HWNCoC0, DataType.BOOL_C1HWNCoC0) \ - .dtype_format(DataType.BOOL_5HD, DataType.BOOL_5HD, DataType.BOOL_5HD) \ + .op_pattern("broadcast") \ + .dtype_format(DataType.BOOL_None, DataType.BOOL_None, DataType.BOOL_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/logical_not.py b/mindspore/ops/_op_impl/tbe/logical_not.py index 3d40441156..17c26b5cfd 100644 --- a/mindspore/ops/_op_impl/tbe/logical_not.py +++ b/mindspore/ops/_op_impl/tbe/logical_not.py @@ -25,10 +25,8 @@ logical_not_op_info = TBERegOp("LogicalNot") \ .partial_flag(True) \ .input(0, "x", False, "required", "all") \ .output(0, "y", True, "required", "all") \ - .dtype_format(DataType.BOOL_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.BOOL_FracZ, DataType.BOOL_FracZ) \ - .dtype_format(DataType.BOOL_C1HWNCoC0, DataType.BOOL_C1HWNCoC0) \ - .dtype_format(DataType.BOOL_5HD, DataType.BOOL_5HD) \ + .op_pattern("formatAgnostic") \ + .dtype_format(DataType.BOOL_None, DataType.BOOL_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/logical_or.py b/mindspore/ops/_op_impl/tbe/logical_or.py index bf8d82c656..c99a31f534 100644 --- a/mindspore/ops/_op_impl/tbe/logical_or.py +++ b/mindspore/ops/_op_impl/tbe/logical_or.py @@ -26,10 +26,8 @@ logical_or_op_info = TBERegOp("LogicalOr") \ .input(0, "x1", False, "required", "all") \ .input(1, "x2", False, "required", "all") \ .output(0, "y", True, "required", "all") \ - .dtype_format(DataType.BOOL_Default, DataType.BOOL_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.BOOL_FracZ, DataType.BOOL_FracZ, DataType.BOOL_FracZ) \ - .dtype_format(DataType.BOOL_C1HWNCoC0, DataType.BOOL_C1HWNCoC0, DataType.BOOL_C1HWNCoC0) \ - .dtype_format(DataType.BOOL_5HD, DataType.BOOL_5HD, DataType.BOOL_5HD) \ + .op_pattern("broadcast") \ + .dtype_format(DataType.BOOL_None, DataType.BOOL_None, DataType.BOOL_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/lrn.py b/mindspore/ops/_op_impl/tbe/lrn.py new file mode 100644 index 0000000000..2f22684f09 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/lrn.py @@ -0,0 +1,41 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""LRN op""" +from mindspore.ops.op_info_register import op_info_register, TBERegOp, DataType + +lrn_op_info = TBERegOp("LRN") \ + .fusion_type("OPAQUE") \ + .async_flag(False) \ + .binfile_name("lrn.so") \ + .compute_cost(10) \ + .kernel_name("lrn") \ + .partial_flag(True) \ + .attr("depth_radius", "optional", "int", "all", "5") \ + .attr("bias", "optional", "float", "all", "1.0") \ + .attr("alpha", "optional", "float", "all", "1.0") \ + .attr("beta", "optional", "float", "all", "0.5") \ + .attr("norm_region", "optional", "str", "all", "ACROSS_CHANNELS") \ + .input(0, "x", False, "required", "all") \ + .output(0, "y", False, "required", "all") \ + .dtype_format(DataType.F16_NCHW, DataType.F16_NCHW) \ + .dtype_format(DataType.F32_NCHW, DataType.F32_NCHW) \ + .get_op_info() + + +@op_info_register(lrn_op_info) +def _lrn_tbe(): + """LRN TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/lrn_grad.py b/mindspore/ops/_op_impl/tbe/lrn_grad.py new file mode 100644 index 0000000000..4d37cf741b --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/lrn_grad.py @@ -0,0 +1,42 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""LRNGrad op""" +from mindspore.ops.op_info_register import op_info_register, TBERegOp, DataType + +lrn_grad_op_info = TBERegOp("LRNGrad") \ + .fusion_type("OPAQUE") \ + .async_flag(False) \ + .binfile_name("lrn_grad.so") \ + .compute_cost(10) \ + .kernel_name("lrn_grad") \ + .partial_flag(True) \ + .attr("depth_radius", "optional", "int", "all") \ + .attr("bias", "optional", "float", "all") \ + .attr("alpha", "optional", "float", "all") \ + .attr("beta", "optional", "float", "all") \ + .input(0, "grads", False, "required", "all") \ + .input(1, "x", False, "required", "all") \ + .input(2, "y", False, "required", "all") \ + .output(0, "z", False, "required", "all") \ + .dtype_format(DataType.F16_NCHW, DataType.F16_NCHW, DataType.F16_NCHW, DataType.F16_NCHW) \ + .dtype_format(DataType.F32_NCHW, DataType.F32_NCHW, DataType.F32_NCHW, DataType.F32_NCHW) \ + .get_op_info() + + +@op_info_register(lrn_grad_op_info) +def _lrn_grad_tbe(): + """LRNGrad TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/max_pool_grad_grad.py b/mindspore/ops/_op_impl/tbe/max_pool_grad_grad.py new file mode 100644 index 0000000000..cdaceb2b0e --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/max_pool_grad_grad.py @@ -0,0 +1,41 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""MaxPoolGradGrad op""" +from mindspore.ops.op_info_register import op_info_register, TBERegOp, DataType + +max_pool_grad_grad_op_info = TBERegOp("MaxPoolGradGrad") \ + .fusion_type("OPAQUE") \ + .async_flag(False) \ + .binfile_name("max_pool_grad_grad.so") \ + .compute_cost(10) \ + .kernel_name("max_pool_grad_grad") \ + .partial_flag(True) \ + .attr("ksize", "required", "listInt", "all") \ + .attr("strides", "required", "listInt", "all") \ + .attr("padding", "required", "str", "all") \ + .attr("data_format", "optional", "str", "all") \ + .input(0, "x1", False, "required", "all") \ + .input(1, "x2", False, "required", "all") \ + .input(2, "grad", False, "required", "all") \ + .output(0, "y", False, "required", "all") \ + .dtype_format(DataType.F16_5HD, DataType.F16_5HD, DataType.F16_5HD, DataType.F16_5HD) \ + .get_op_info() + + +@op_info_register(max_pool_grad_grad_op_info) +def _max_pool_grad_grad_tbe(): + """MaxPoolGradGrad TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/max_pool_grad_grad_with_argmax.py b/mindspore/ops/_op_impl/tbe/max_pool_grad_grad_with_argmax.py new file mode 100644 index 0000000000..52a9392f3e --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/max_pool_grad_grad_with_argmax.py @@ -0,0 +1,41 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""MaxPoolGradGradWithArgmax op""" +from mindspore.ops.op_info_register import op_info_register, TBERegOp, DataType + +max_pool_grad_grad_with_argmax_op_info = TBERegOp("MaxPoolGradGradWithArgmax") \ + .fusion_type("OPAQUE") \ + .async_flag(False) \ + .binfile_name("max_pool_grad_grad_with_argmax.so") \ + .compute_cost(10) \ + .kernel_name("max_pool_grad_grad_with_argmax") \ + .partial_flag(True) \ + .attr("ksize", "required", "listInt", "all") \ + .attr("strides", "required", "listInt", "all") \ + .attr("padding", "required", "str", "all") \ + .input(0, "x", False, "required", "all") \ + .input(1, "grad", False, "required", "all") \ + .input(2, "argmax", False, "optional", "all") \ + .output(0, "y", False, "required", "all") \ + .dtype_format(DataType.F16_5HD, DataType.F16_5HD, DataType.U16_5HD, DataType.F16_5HD) \ + .dtype_format(DataType.F16_5HD, DataType.F16_5HD, DataType.I64_5HD, DataType.F16_5HD) \ + .get_op_info() + + +@op_info_register(max_pool_grad_grad_with_argmax_op_info) +def _max_pool_grad_grad_with_argmax_tbe(): + """MaxPoolGradGradWithArgmax TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/maximum.py b/mindspore/ops/_op_impl/tbe/maximum.py index 6fb7d05e03..b4740ca55d 100644 --- a/mindspore/ops/_op_impl/tbe/maximum.py +++ b/mindspore/ops/_op_impl/tbe/maximum.py @@ -26,12 +26,10 @@ maximum_op_info = TBERegOp("Maximum") \ .input(0, "x1", False, "required", "all") \ .input(1, "x2", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.I32_Default, DataType.I32_Default, DataType.I32_Default) \ - .dtype_format(DataType.I32_5HD, DataType.I32_5HD, DataType.I32_5HD) \ - .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD) \ + .op_pattern("broadcast") \ + .dtype_format(DataType.I32_None, DataType.I32_None, DataType.I32_None) \ + .dtype_format(DataType.F16_None, DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/maximum_grad.py b/mindspore/ops/_op_impl/tbe/maximum_grad.py index b9bc9c09f8..341b7dafbd 100644 --- a/mindspore/ops/_op_impl/tbe/maximum_grad.py +++ b/mindspore/ops/_op_impl/tbe/maximum_grad.py @@ -30,18 +30,13 @@ maximum_grad_op_info = TBERegOp("MaximumGrad") \ .input(2, "x2", False, "required", "all") \ .output(0, "y1", False, "required", "all") \ .output(1, "y2", False, "required", "all") \ - .dtype_format(DataType.I32_Default, DataType.I32_Default, DataType.I32_Default, DataType.I32_Default, - DataType.I32_Default) \ - .dtype_format(DataType.I32_5HD, DataType.I32_5HD, DataType.I32_5HD, DataType.I32_5HD, - DataType.I32_5HD) \ - .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.F16_Default, DataType.F16_Default, - DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD, DataType.F16_5HD, DataType.F16_5HD, - DataType.F16_5HD) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, - DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD, - DataType.F32_5HD) \ + .op_pattern("broadcast") \ + .dtype_format(DataType.I32_None, DataType.I32_None, DataType.I32_None, DataType.I32_None, + DataType.I32_None) \ + .dtype_format(DataType.F16_None, DataType.F16_None, DataType.F16_None, DataType.F16_None, + DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None, DataType.F32_None, DataType.F32_None, + DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/minimum.py b/mindspore/ops/_op_impl/tbe/minimum.py index 1cebfd3dad..cc1347b616 100644 --- a/mindspore/ops/_op_impl/tbe/minimum.py +++ b/mindspore/ops/_op_impl/tbe/minimum.py @@ -27,12 +27,10 @@ minimum_op_info = TBERegOp("Minimum") \ .input(0, "x1", False, "required", "all") \ .input(1, "x2", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.I32_Default, DataType.I32_Default, DataType.I32_Default) \ - .dtype_format(DataType.I32_5HD, DataType.I32_5HD, DataType.I32_5HD) \ - .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD) \ + .op_pattern("broadcast") \ + .dtype_format(DataType.I32_None, DataType.I32_None, DataType.I32_None) \ + .dtype_format(DataType.F16_None, DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/minimum_grad.py b/mindspore/ops/_op_impl/tbe/minimum_grad.py index c3ea1c3a56..8ad459af17 100644 --- a/mindspore/ops/_op_impl/tbe/minimum_grad.py +++ b/mindspore/ops/_op_impl/tbe/minimum_grad.py @@ -30,18 +30,13 @@ minimum_grad_op_info = TBERegOp("MinimumGrad") \ .input(2, "x2", False, "required", "all") \ .output(0, "y1", False, "required", "all") \ .output(1, "y2", False, "required", "all") \ - .dtype_format(DataType.I32_Default, DataType.I32_Default, DataType.I32_Default, DataType.I32_Default, - DataType.I32_Default) \ - .dtype_format(DataType.I32_5HD, DataType.I32_5HD, DataType.I32_5HD, DataType.I32_5HD, - DataType.I32_5HD) \ - .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.F16_Default, DataType.F16_Default, - DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD, DataType.F16_5HD, DataType.F16_5HD, - DataType.F16_5HD) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, - DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD, - DataType.F32_5HD) \ + .op_pattern("broadcast") \ + .dtype_format(DataType.I32_None, DataType.I32_None, DataType.I32_None, DataType.I32_None, + DataType.I32_None) \ + .dtype_format(DataType.F16_None, DataType.F16_None, DataType.F16_None, DataType.F16_None, + DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None, DataType.F32_None, DataType.F32_None, + DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/mod.py b/mindspore/ops/_op_impl/tbe/mod.py new file mode 100644 index 0000000000..334a3e3820 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/mod.py @@ -0,0 +1,41 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Mod op""" +from mindspore.ops.op_info_register import op_info_register, TBERegOp, DataType + +mod_op_info = TBERegOp("Mod") \ + .fusion_type("ELEMWISE") \ + .async_flag(False) \ + .binfile_name("mod.so") \ + .compute_cost(10) \ + .kernel_name("mod") \ + .partial_flag(True) \ + .input(0, "x1", False, "required", "all") \ + .input(1, "x2", False, "required", "all") \ + .output(0, "y", False, "required", "all") \ + .op_pattern("broadcast") \ + .dtype_format(DataType.I8_None, DataType.I8_None, DataType.I8_None) \ + .dtype_format(DataType.U8_None, DataType.U8_None, DataType.U8_None) \ + .dtype_format(DataType.I32_None, DataType.I32_None, DataType.I32_None) \ + .dtype_format(DataType.F16_None, DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None, DataType.F32_None) \ + .get_op_info() + + +@op_info_register(mod_op_info) +def _mod_tbe(): + """Mod TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/neg.py b/mindspore/ops/_op_impl/tbe/neg.py index feb648f056..ee281a3bb2 100644 --- a/mindspore/ops/_op_impl/tbe/neg.py +++ b/mindspore/ops/_op_impl/tbe/neg.py @@ -25,14 +25,10 @@ neg_op_info = TBERegOp("Neg") \ .partial_flag(True) \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.I8_Default, DataType.I8_Default) \ - .dtype_format(DataType.I8_5HD, DataType.I8_5HD) \ - .dtype_format(DataType.I32_Default, DataType.I32_Default) \ - .dtype_format(DataType.I32_5HD, DataType.I32_5HD) \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD) \ + .op_pattern("formatAgnostic") \ + .dtype_format(DataType.I32_None, DataType.I32_None) \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/not_equal.py b/mindspore/ops/_op_impl/tbe/not_equal.py index bd801d9a40..80993a905e 100644 --- a/mindspore/ops/_op_impl/tbe/not_equal.py +++ b/mindspore/ops/_op_impl/tbe/not_equal.py @@ -26,16 +26,12 @@ not_equal_op_info = TBERegOp("NotEqual") \ .input(0, "x1", False, "required", "all") \ .input(1, "x2", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.I8_Default, DataType.I8_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.I8_5HD, DataType.I8_5HD, DataType.BOOL_5HD) \ - .dtype_format(DataType.U8_Default, DataType.U8_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.U8_5HD, DataType.U8_5HD, DataType.BOOL_5HD) \ - .dtype_format(DataType.I32_Default, DataType.I32_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.I32_5HD, DataType.I32_5HD, DataType.BOOL_5HD) \ - .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD, DataType.BOOL_5HD) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD, DataType.BOOL_5HD) \ + .op_pattern("broadcast") \ + .dtype_format(DataType.I8_None, DataType.I8_None, DataType.BOOL_None) \ + .dtype_format(DataType.U8_None, DataType.U8_None, DataType.BOOL_None) \ + .dtype_format(DataType.I32_None, DataType.I32_None, DataType.BOOL_None) \ + .dtype_format(DataType.F16_None, DataType.F16_None, DataType.BOOL_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None, DataType.BOOL_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/ones_like.py b/mindspore/ops/_op_impl/tbe/ones_like.py index ae6871cac5..9b31d6d4fa 100644 --- a/mindspore/ops/_op_impl/tbe/ones_like.py +++ b/mindspore/ops/_op_impl/tbe/ones_like.py @@ -25,11 +25,12 @@ ones_like_op_info = TBERegOp("OnesLike") \ .partial_flag(True) \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.U8_Default, DataType.U8_Default) \ - .dtype_format(DataType.I8_Default, DataType.I8_Default) \ - .dtype_format(DataType.I32_Default, DataType.I32_Default) \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ + .op_pattern("formatAgnostic") \ + .dtype_format(DataType.U8_None, DataType.U8_None) \ + .dtype_format(DataType.I8_None, DataType.I8_None) \ + .dtype_format(DataType.I32_None, DataType.I32_None) \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/pow.py b/mindspore/ops/_op_impl/tbe/pow.py index 223a139252..39e55aab23 100644 --- a/mindspore/ops/_op_impl/tbe/pow.py +++ b/mindspore/ops/_op_impl/tbe/pow.py @@ -26,11 +26,12 @@ pow_op_info = TBERegOp("Pow") \ .input(0, "x1", False, "required", "all") \ .input(1, "x2", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.I8_Default, DataType.I8_Default, DataType.I8_Default) \ - .dtype_format(DataType.U8_Default, DataType.U8_Default, DataType.U8_Default) \ - .dtype_format(DataType.I32_Default, DataType.I32_Default, DataType.I32_Default) \ - .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default) \ + .op_pattern("broadcast") \ + .dtype_format(DataType.I8_None, DataType.I8_None, DataType.I8_None) \ + .dtype_format(DataType.U8_None, DataType.U8_None, DataType.U8_None) \ + .dtype_format(DataType.I32_None, DataType.I32_None, DataType.I32_None) \ + .dtype_format(DataType.F16_None, DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/reciprocal.py b/mindspore/ops/_op_impl/tbe/reciprocal.py index 77f3bfac27..c620fb17a6 100644 --- a/mindspore/ops/_op_impl/tbe/reciprocal.py +++ b/mindspore/ops/_op_impl/tbe/reciprocal.py @@ -26,12 +26,7 @@ reciprocal_op_info = TBERegOp("Reciprocal") \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ .op_pattern("dynamicFormat") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F16_NHWC, DataType.F16_NHWC) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD) \ - .dtype_format(DataType.F32_NHWC, DataType.F32_NHWC) \ + .dtype_format(DataType.None_None, DataType.None_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/reduce_all.py b/mindspore/ops/_op_impl/tbe/reduce_all.py index a372dcf6b7..17f5574819 100644 --- a/mindspore/ops/_op_impl/tbe/reduce_all.py +++ b/mindspore/ops/_op_impl/tbe/reduce_all.py @@ -27,8 +27,8 @@ reduce_all_op_info = TBERegOp("ReduceAll") \ .attr("keep_dims", "optional", "bool", "all") \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.BOOL_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.BOOL_FracZ, DataType.BOOL_FracZ) \ + .op_pattern("reduce") \ + .dtype_format(DataType.BOOL_None, DataType.BOOL_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/reduce_max.py b/mindspore/ops/_op_impl/tbe/reduce_max.py index ab0e766f59..a5d5ce3e8c 100644 --- a/mindspore/ops/_op_impl/tbe/reduce_max.py +++ b/mindspore/ops/_op_impl/tbe/reduce_max.py @@ -27,12 +27,13 @@ reduce_max_d_op_info = TBERegOp("ReduceMax") \ .attr("keep_dims", "optional", "bool", "all") \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.BOOL_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.I8_Default, DataType.I8_Default) \ - .dtype_format(DataType.U8_Default, DataType.U8_Default) \ - .dtype_format(DataType.I32_Default, DataType.I32_Default) \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ + .op_pattern("reduce") \ + .dtype_format(DataType.BOOL_None, DataType.BOOL_None) \ + .dtype_format(DataType.I8_None, DataType.I8_None) \ + .dtype_format(DataType.U8_None, DataType.U8_None) \ + .dtype_format(DataType.I32_None, DataType.I32_None) \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/reduce_mean_d.py b/mindspore/ops/_op_impl/tbe/reduce_mean_d.py index e427b34869..a0890816d2 100644 --- a/mindspore/ops/_op_impl/tbe/reduce_mean_d.py +++ b/mindspore/ops/_op_impl/tbe/reduce_mean_d.py @@ -27,10 +27,11 @@ reduce_mean_d_op_info = TBERegOp("ReduceMeanD") \ .attr("keep_dims", "optional", "bool", "all") \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.I8_Default, DataType.I8_Default) \ - .dtype_format(DataType.U8_Default, DataType.U8_Default) \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ + .op_pattern("reduce") \ + .dtype_format(DataType.I8_None, DataType.I8_None) \ + .dtype_format(DataType.U8_None, DataType.U8_None) \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/reduce_min.py b/mindspore/ops/_op_impl/tbe/reduce_min.py index f1601ebc94..d64dd7ba5c 100644 --- a/mindspore/ops/_op_impl/tbe/reduce_min.py +++ b/mindspore/ops/_op_impl/tbe/reduce_min.py @@ -27,14 +27,11 @@ reduce_min_op_info = TBERegOp("ReduceMin") \ .attr("keep_dims", "required", "bool", "all") \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.I8_Default, DataType.I8_Default) \ - .dtype_format(DataType.I8_FracZ, DataType.I8_FracZ) \ - .dtype_format(DataType.U8_Default, DataType.U8_Default) \ - .dtype_format(DataType.U8_FracZ, DataType.U8_FracZ) \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_FracZ, DataType.F16_FracZ) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_FracZ, DataType.F32_FracZ) \ + .op_pattern("reduce") \ + .dtype_format(DataType.I8_None, DataType.I8_None) \ + .dtype_format(DataType.U8_None, DataType.U8_None) \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/reduce_prod.py b/mindspore/ops/_op_impl/tbe/reduce_prod.py index 416418e140..b6f1386ce2 100644 --- a/mindspore/ops/_op_impl/tbe/reduce_prod.py +++ b/mindspore/ops/_op_impl/tbe/reduce_prod.py @@ -27,14 +27,11 @@ reduce_prod_op_info = TBERegOp("ReduceProd") \ .attr("keep_dims", "optional", "bool", "all") \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.I8_Default, DataType.I8_Default) \ - .dtype_format(DataType.I8_FracZ, DataType.I8_FracZ) \ - .dtype_format(DataType.U8_Default, DataType.U8_Default) \ - .dtype_format(DataType.U8_FracZ, DataType.U8_FracZ) \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_FracZ, DataType.F16_FracZ) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_FracZ, DataType.F32_FracZ) \ + .op_pattern("reduce") \ + .dtype_format(DataType.I8_None, DataType.I8_None) \ + .dtype_format(DataType.U8_None, DataType.U8_None) \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/reduce_sum.py b/mindspore/ops/_op_impl/tbe/reduce_sum.py index 2f76f74562..72b7803f94 100644 --- a/mindspore/ops/_op_impl/tbe/reduce_sum.py +++ b/mindspore/ops/_op_impl/tbe/reduce_sum.py @@ -27,8 +27,9 @@ reduce_sum_op_info = TBERegOp("ReduceSum") \ .attr("keep_dims", "optional", "bool", "all") \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ + .op_pattern("reduce") \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/relu.py b/mindspore/ops/_op_impl/tbe/relu.py index 8a7d023afd..1a586e02a8 100644 --- a/mindspore/ops/_op_impl/tbe/relu.py +++ b/mindspore/ops/_op_impl/tbe/relu.py @@ -25,15 +25,11 @@ relu_op_info = TBERegOp("ReLU") \ .partial_flag(True) \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.I8_Default, DataType.I8_Default) \ - .dtype_format(DataType.I8_5HD, DataType.I8_5HD) \ - .dtype_format(DataType.I32_Default, DataType.I32_Default) \ - .dtype_format(DataType.I32_5HD, DataType.I32_5HD) \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD) \ - .dtype_format(DataType.F16_FracNZ, DataType.F16_FracNZ) \ + .op_pattern("formatAgnostic") \ + .dtype_format(DataType.I8_None, DataType.I8_None) \ + .dtype_format(DataType.I32_None, DataType.I32_None) \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/relu6.py b/mindspore/ops/_op_impl/tbe/relu6.py index d9bd7f9f8e..9a68774d63 100644 --- a/mindspore/ops/_op_impl/tbe/relu6.py +++ b/mindspore/ops/_op_impl/tbe/relu6.py @@ -25,12 +25,10 @@ relu6_op_info = TBERegOp("ReLU6") \ .partial_flag(True) \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD) \ - .dtype_format(DataType.I32_Default, DataType.I32_Default) \ - .dtype_format(DataType.I32_5HD, DataType.I32_5HD) \ + .op_pattern("formatAgnostic") \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ + .dtype_format(DataType.I32_None, DataType.I32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/relu_grad.py b/mindspore/ops/_op_impl/tbe/relu_grad.py index 0f9e962cfd..040294f973 100644 --- a/mindspore/ops/_op_impl/tbe/relu_grad.py +++ b/mindspore/ops/_op_impl/tbe/relu_grad.py @@ -26,16 +26,12 @@ relugrad_op_info = TBERegOp("ReluGrad") \ .input(0, "gradients", False, "required", "all") \ .input(1, "features", False, "required", "all") \ .output(0, "backprops", True, "required", "all") \ - .dtype_format(DataType.I8_Default, DataType.I8_Default, DataType.I8_Default) \ - .dtype_format(DataType.I8_5HD, DataType.I8_5HD, DataType.I8_5HD) \ - .dtype_format(DataType.U8_Default, DataType.U8_Default, DataType.U8_Default) \ - .dtype_format(DataType.U8_5HD, DataType.U8_5HD, DataType.U8_5HD) \ - .dtype_format(DataType.I32_Default, DataType.I32_Default, DataType.I32_Default) \ - .dtype_format(DataType.I32_5HD, DataType.I32_5HD, DataType.I32_5HD) \ - .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD) \ + .op_pattern("broadcast") \ + .dtype_format(DataType.I8_None, DataType.I8_None, DataType.I8_None) \ + .dtype_format(DataType.U8_None, DataType.U8_None, DataType.U8_None) \ + .dtype_format(DataType.I32_None, DataType.I32_None, DataType.I32_None) \ + .dtype_format(DataType.F16_None, DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/round.py b/mindspore/ops/_op_impl/tbe/round.py index 4559a3def0..64e94e8c42 100644 --- a/mindspore/ops/_op_impl/tbe/round.py +++ b/mindspore/ops/_op_impl/tbe/round.py @@ -25,12 +25,10 @@ round_op_info = TBERegOp("Round") \ .partial_flag(True) \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F16_FracZ, DataType.F16_FracZ) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD) \ - .dtype_format(DataType.F32_FracZ, DataType.F32_FracZ) \ + .op_pattern("formatAgnostic") \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ + .dtype_format(DataType.I32_None, DataType.I32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/rsqrt.py b/mindspore/ops/_op_impl/tbe/rsqrt.py index b0830cf484..e080bf9772 100644 --- a/mindspore/ops/_op_impl/tbe/rsqrt.py +++ b/mindspore/ops/_op_impl/tbe/rsqrt.py @@ -25,14 +25,9 @@ rsqrt_op_info = TBERegOp("Rsqrt") \ .partial_flag(True) \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F16_FracZ, DataType.F16_FracZ) \ - .dtype_format(DataType.F16_C1HWNCoC0, DataType.F16_C1HWNCoC0) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD) \ - .dtype_format(DataType.F32_FracZ, DataType.F32_FracZ) \ - .dtype_format(DataType.F32_C1HWNCoC0, DataType.F32_C1HWNCoC0) \ + .op_pattern("formatAgnostic") \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/scatter_add.py b/mindspore/ops/_op_impl/tbe/scatter_add.py index ea54719d4d..0202cdf457 100644 --- a/mindspore/ops/_op_impl/tbe/scatter_add.py +++ b/mindspore/ops/_op_impl/tbe/scatter_add.py @@ -31,6 +31,8 @@ scatter_add_op_info = TBERegOp("ScatterAdd") \ .dtype_format(DataType.F16_Default, DataType.I32_Default, DataType.F16_Default, DataType.F16_Default) \ .dtype_format(DataType.F32_Default, DataType.I32_Default, DataType.F32_Default, DataType.F32_Default) \ .dtype_format(DataType.I32_Default, DataType.I32_Default, DataType.I32_Default, DataType.I32_Default) \ + .dtype_format(DataType.I8_Default, DataType.I32_Default, DataType.I8_Default, DataType.I8_Default) \ + .dtype_format(DataType.U8_Default, DataType.I32_Default, DataType.U8_Default, DataType.U8_Default) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/scatter_div.py b/mindspore/ops/_op_impl/tbe/scatter_div.py new file mode 100644 index 0000000000..5c6572dfd7 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/scatter_div.py @@ -0,0 +1,42 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""ScatterDiv op""" +from mindspore.ops.op_info_register import op_info_register, TBERegOp, DataType + +scatter_div_op_info = TBERegOp("ScatterDiv") \ + .fusion_type("ELEMWISE") \ + .async_flag(False) \ + .binfile_name("scatter_div.so") \ + .compute_cost(10) \ + .kernel_name("scatter_div") \ + .partial_flag(True) \ + .attr("use_locking", "optional", "bool", "all") \ + .input(0, "var", False, "required", "all") \ + .input(1, "indices", False, "required", "all") \ + .input(2, "updates", False, "required", "all") \ + .output(0, "var", False, "required", "all") \ + .dtype_format(DataType.F16_Default, DataType.I32_Default, DataType.F16_Default, DataType.F16_Default) \ + .dtype_format(DataType.F32_Default, DataType.I32_Default, DataType.F32_Default, DataType.F32_Default) \ + .dtype_format(DataType.I32_Default, DataType.I32_Default, DataType.I32_Default, DataType.I32_Default) \ + .dtype_format(DataType.I8_Default, DataType.I32_Default, DataType.I8_Default, DataType.I8_Default) \ + .dtype_format(DataType.U8_Default, DataType.I32_Default, DataType.U8_Default, DataType.U8_Default) \ + .get_op_info() + + +@op_info_register(scatter_div_op_info) +def _scatter_div_tbe(): + """ScatterDiv TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/scatter_max.py b/mindspore/ops/_op_impl/tbe/scatter_max.py new file mode 100644 index 0000000000..ba85d63d35 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/scatter_max.py @@ -0,0 +1,40 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""ScatterMax op""" +from mindspore.ops.op_info_register import op_info_register, TBERegOp, DataType + +scatter_max_op_info = TBERegOp("ScatterMax") \ + .fusion_type("ELEMWISE") \ + .async_flag(False) \ + .binfile_name("scatter_max.so") \ + .compute_cost(10) \ + .kernel_name("scatter_max") \ + .partial_flag(True) \ + .attr("use_locking", "optional", "bool", "all") \ + .input(0, "var", False, "required", "all") \ + .input(1, "indices", False, "required", "all") \ + .input(2, "updates", False, "required", "all") \ + .output(0, "var", False, "required", "all") \ + .dtype_format(DataType.F16_Default, DataType.I32_Default, DataType.F16_Default, DataType.F16_Default) \ + .dtype_format(DataType.F32_Default, DataType.I32_Default, DataType.F32_Default, DataType.F32_Default) \ + .dtype_format(DataType.I32_Default, DataType.I32_Default, DataType.I32_Default, DataType.I32_Default) \ + .get_op_info() + + +@op_info_register(scatter_max_op_info) +def _scatter_max_tbe(): + """ScatterMax TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/scatter_min.py b/mindspore/ops/_op_impl/tbe/scatter_min.py new file mode 100644 index 0000000000..a4ab87a0d7 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/scatter_min.py @@ -0,0 +1,40 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""ScatterMin op""" +from mindspore.ops.op_info_register import op_info_register, TBERegOp, DataType + +scatter_min_op_info = TBERegOp("ScatterMin") \ + .fusion_type("ELEMWISE") \ + .async_flag(False) \ + .binfile_name("scatter_min.so") \ + .compute_cost(10) \ + .kernel_name("scatter_min") \ + .partial_flag(True) \ + .attr("use_locking", "optional", "bool", "all") \ + .input(0, "var", False, "required", "all") \ + .input(1, "indices", False, "required", "all") \ + .input(2, "updates", False, "required", "all") \ + .output(0, "var", False, "required", "all") \ + .dtype_format(DataType.F16_Default, DataType.I32_Default, DataType.F16_Default, DataType.F16_Default) \ + .dtype_format(DataType.F32_Default, DataType.I32_Default, DataType.F32_Default, DataType.F32_Default) \ + .dtype_format(DataType.I32_Default, DataType.I32_Default, DataType.I32_Default, DataType.I32_Default) \ + .get_op_info() + + +@op_info_register(scatter_min_op_info) +def _scatter_min_tbe(): + """ScatterMin TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/scatter_mul.py b/mindspore/ops/_op_impl/tbe/scatter_mul.py new file mode 100644 index 0000000000..dd77e33f6d --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/scatter_mul.py @@ -0,0 +1,42 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""ScatterMul op""" +from mindspore.ops.op_info_register import op_info_register, TBERegOp, DataType + +scatter_mul_op_info = TBERegOp("ScatterMul") \ + .fusion_type("ELEMWISE") \ + .async_flag(False) \ + .binfile_name("scatter_mul.so") \ + .compute_cost(10) \ + .kernel_name("scatter_mul") \ + .partial_flag(True) \ + .attr("use_locking", "optional", "bool", "all") \ + .input(0, "var", False, "required", "all") \ + .input(1, "indices", False, "required", "all") \ + .input(2, "updates", False, "required", "all") \ + .output(0, "var", False, "required", "all") \ + .dtype_format(DataType.F16_Default, DataType.I32_Default, DataType.F16_Default, DataType.F16_Default) \ + .dtype_format(DataType.F32_Default, DataType.I32_Default, DataType.F32_Default, DataType.F32_Default) \ + .dtype_format(DataType.I32_Default, DataType.I32_Default, DataType.I32_Default, DataType.I32_Default) \ + .dtype_format(DataType.I8_Default, DataType.I32_Default, DataType.I8_Default, DataType.I8_Default) \ + .dtype_format(DataType.U8_Default, DataType.I32_Default, DataType.U8_Default, DataType.U8_Default) \ + .get_op_info() + + +@op_info_register(scatter_mul_op_info) +def _scatter_mul_tbe(): + """ScatterMul TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/scatter_sub.py b/mindspore/ops/_op_impl/tbe/scatter_sub.py new file mode 100644 index 0000000000..2e5e01389c --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/scatter_sub.py @@ -0,0 +1,42 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""ScatterSub op""" +from mindspore.ops.op_info_register import op_info_register, TBERegOp, DataType + +scatter_sub_op_info = TBERegOp("ScatterSub") \ + .fusion_type("ELEMWISE") \ + .async_flag(False) \ + .binfile_name("scatter_sub.so") \ + .compute_cost(10) \ + .kernel_name("scatter_sub") \ + .partial_flag(True) \ + .attr("use_locking", "optional", "bool", "all") \ + .input(0, "var", False, "required", "all") \ + .input(1, "indices", False, "required", "all") \ + .input(2, "updates", False, "required", "all") \ + .output(0, "var", False, "required", "all") \ + .dtype_format(DataType.F16_Default, DataType.I32_Default, DataType.F16_Default, DataType.F16_Default) \ + .dtype_format(DataType.F32_Default, DataType.I32_Default, DataType.F32_Default, DataType.F32_Default) \ + .dtype_format(DataType.I32_Default, DataType.I32_Default, DataType.I32_Default, DataType.I32_Default) \ + .dtype_format(DataType.I8_Default, DataType.I32_Default, DataType.I8_Default, DataType.I8_Default) \ + .dtype_format(DataType.U8_Default, DataType.I32_Default, DataType.U8_Default, DataType.U8_Default) \ + .get_op_info() + + +@op_info_register(scatter_sub_op_info) +def _scatter_sub_tbe(): + """ScatterSub TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/select.py b/mindspore/ops/_op_impl/tbe/select.py index e924f05021..97e6baa58e 100644 --- a/mindspore/ops/_op_impl/tbe/select.py +++ b/mindspore/ops/_op_impl/tbe/select.py @@ -28,16 +28,7 @@ select_op_info = TBERegOp("Select") \ .input(2, "x2", False, "required", "all") \ .output(0, "y", False, "required", "all") \ .op_pattern("dynamicFormat") \ - .dtype_format(DataType.BOOL_Default, DataType.I8_Default, DataType.I8_Default, DataType.I8_Default) \ - .dtype_format(DataType.BOOL_Default, DataType.U8_Default, DataType.U8_Default, DataType.U8_Default) \ - .dtype_format(DataType.BOOL_Default, DataType.I32_Default, DataType.I32_Default, DataType.I32_Default) \ - .dtype_format(DataType.BOOL_Default, DataType.F16_Default, DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.BOOL_Default, DataType.F32_Default, DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.BOOL_5HD, DataType.I8_5HD, DataType.I8_5HD, DataType.I8_5HD) \ - .dtype_format(DataType.BOOL_5HD, DataType.U8_5HD, DataType.U8_5HD, DataType.U8_5HD) \ - .dtype_format(DataType.BOOL_5HD, DataType.I32_5HD, DataType.I32_5HD, DataType.I32_5HD) \ - .dtype_format(DataType.BOOL_5HD, DataType.F16_5HD, DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.BOOL_5HD, DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD) \ + .dtype_format(DataType.None_None, DataType.None_None, DataType.None_None, DataType.None_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/sigmoid.py b/mindspore/ops/_op_impl/tbe/sigmoid.py index 38413c0432..312fdda8d0 100644 --- a/mindspore/ops/_op_impl/tbe/sigmoid.py +++ b/mindspore/ops/_op_impl/tbe/sigmoid.py @@ -25,16 +25,9 @@ sigmoid_op_info = TBERegOp("Sigmoid") \ .partial_flag(True) \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F16_FracZ, DataType.F16_FracZ) \ - .dtype_format(DataType.F16_FracNZ, DataType.F16_FracNZ) \ - .dtype_format(DataType.F16_C1HWNCoC0, DataType.F16_C1HWNCoC0) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD) \ - .dtype_format(DataType.F32_FracZ, DataType.F32_FracZ) \ - .dtype_format(DataType.F32_FracNZ, DataType.F32_FracNZ) \ - .dtype_format(DataType.F32_C1HWNCoC0, DataType.F32_C1HWNCoC0) \ + .op_pattern("formatAgnostic") \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/sign.py b/mindspore/ops/_op_impl/tbe/sign.py index 99f7970316..7a7e290649 100644 --- a/mindspore/ops/_op_impl/tbe/sign.py +++ b/mindspore/ops/_op_impl/tbe/sign.py @@ -26,9 +26,9 @@ sign_op_info = TBERegOp("Sign") \ .op_pattern("formatAgnostic") \ .input(0, "x", None, "required", None) \ .output(0, "y", True, "required", "all") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.I32_Default, DataType.I32_Default) \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ + .dtype_format(DataType.I32_None, DataType.I32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/sin.py b/mindspore/ops/_op_impl/tbe/sin.py index f01f687926..6d32e13750 100644 --- a/mindspore/ops/_op_impl/tbe/sin.py +++ b/mindspore/ops/_op_impl/tbe/sin.py @@ -26,10 +26,8 @@ sin_op_info = TBERegOp("Sin") \ .op_pattern("formatAgnostic") \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD) \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/sinh.py b/mindspore/ops/_op_impl/tbe/sinh.py index 27eb66d274..6bff4c9ca2 100644 --- a/mindspore/ops/_op_impl/tbe/sinh.py +++ b/mindspore/ops/_op_impl/tbe/sinh.py @@ -26,8 +26,8 @@ sinh_op_info = TBERegOp("Sinh") \ .op_pattern("formatAgnostic") \ .input(0, "x", False, "required", "all") \ .output(0, "y", True, "required", "all") \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD) \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/softmax_grad_ext.py b/mindspore/ops/_op_impl/tbe/softmax_grad_ext.py index d43183dcb7..dcbe47cbff 100644 --- a/mindspore/ops/_op_impl/tbe/softmax_grad_ext.py +++ b/mindspore/ops/_op_impl/tbe/softmax_grad_ext.py @@ -31,14 +31,8 @@ softmax_grad_ext_op_info = TBERegOp("SoftmaxGradExt") \ .input(2, "x2", False, "required", "all") \ .output(0, "y", True, "required", "all") \ .op_pattern("dynamicFormat") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default, - DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD, - DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default, - DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD, - DataType.F32_5HD, DataType.F32_5HD) \ + .dtype_format(DataType.None_None, DataType.None_None, + DataType.None_None, DataType.None_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/softplus.py b/mindspore/ops/_op_impl/tbe/softplus.py index 92261d91ef..1678b00474 100644 --- a/mindspore/ops/_op_impl/tbe/softplus.py +++ b/mindspore/ops/_op_impl/tbe/softplus.py @@ -26,8 +26,8 @@ softplus_op_info = TBERegOp("Softplus") \ .op_pattern("formatAgnostic") \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/softplus_grad.py b/mindspore/ops/_op_impl/tbe/softplus_grad.py index 3dc0e7ee0c..3260fb7ad9 100644 --- a/mindspore/ops/_op_impl/tbe/softplus_grad.py +++ b/mindspore/ops/_op_impl/tbe/softplus_grad.py @@ -27,8 +27,8 @@ softplus_grad_op_info = TBERegOp("SoftplusGrad") \ .input(0, "gradients", False, "required", "all") \ .input(1, "features", False, "required", "all") \ .output(0, "backprops", False, "required", "all") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default) \ + .dtype_format(DataType.F16_None, DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/softsign.py b/mindspore/ops/_op_impl/tbe/softsign.py new file mode 100644 index 0000000000..97f560c939 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/softsign.py @@ -0,0 +1,37 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Softsign op""" +from mindspore.ops.op_info_register import op_info_register, TBERegOp, DataType + +softsign_op_info = TBERegOp("Softsign") \ + .fusion_type("OPAQUE") \ + .async_flag(False) \ + .binfile_name("softsign.so") \ + .compute_cost(10) \ + .kernel_name("softsign") \ + .partial_flag(True) \ + .op_pattern("formatAgnostic") \ + .input(0, "x", False, "required", "all") \ + .output(0, "y", False, "required", "all") \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ + .get_op_info() + + +@op_info_register(softsign_op_info) +def _softsign_tbe(): + """Softsign TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/space_to_batch_nd.py b/mindspore/ops/_op_impl/tbe/space_to_batch_nd.py index 3a50b56a24..c1094cb55c 100644 --- a/mindspore/ops/_op_impl/tbe/space_to_batch_nd.py +++ b/mindspore/ops/_op_impl/tbe/space_to_batch_nd.py @@ -25,8 +25,8 @@ space_to_batch_nd_op_info = TBERegOp("SpaceToBatchND") \ .partial_flag(True) \ .attr("block_shape", "required", "listInt", "all") \ .attr("paddings", "required", "listListInt", "all") \ - .input(0, "x", False, "required", "all") \ - .output(0, "y", False, "required", "all") \ + .input(0, "x", False, "required", "all", reshape_type="NH") \ + .output(0, "y", False, "required", "all", reshape_type="NH") \ .dtype_format(DataType.F16_5HD, DataType.F16_5HD) \ .dtype_format(DataType.F32_5HD, DataType.F32_5HD) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/sparse_apply_adagrad_v2.py b/mindspore/ops/_op_impl/tbe/sparse_apply_adagrad_v2.py new file mode 100644 index 0000000000..088edb60d3 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/sparse_apply_adagrad_v2.py @@ -0,0 +1,48 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""SparseApplyAdagradV2D op""" +from mindspore.ops.op_info_register import op_info_register, TBERegOp, DataType + +sparse_apply_adagrad_v2_d_op_info = TBERegOp("SparseApplyAdagradV2") \ + .fusion_type("OPAQUE") \ + .async_flag(False) \ + .binfile_name("sparse_apply_adagrad_v2_d.so") \ + .compute_cost(10) \ + .kernel_name("sparse_apply_adagrad_v2_d") \ + .partial_flag(True) \ + .attr("lr", "required", "float", "all") \ + .attr("epsilon", "required", "float", "all") \ + .attr("use_locking", "optional", "bool", "all") \ + .attr("update_slots", "optional", "bool", "all") \ + .input(0, "var", False, "required", "all") \ + .input(1, "accum", False, "required", "all") \ + .input(2, "grad", False, "required", "all") \ + .input(3, "indices", False, "required", "all") \ + .output(0, "var", False, "required", "all") \ + .output(1, "accum", False, "required", "all") \ + .dtype_format(DataType.F32_NCHW, DataType.F32_NCHW, DataType.F32_NCHW, DataType.I32_NCHW, + DataType.F32_NCHW, DataType.F32_NCHW) \ + .dtype_format(DataType.F32_NHWC, DataType.F32_NHWC, DataType.F32_NHWC, DataType.I32_NHWC, + DataType.F32_NHWC, DataType.F32_NHWC) \ + .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, DataType.I32_Default, + DataType.F32_Default, DataType.F32_Default) \ + .get_op_info() + + +@op_info_register(sparse_apply_adagrad_v2_d_op_info) +def _sparse_apply_adagrad_v2_tbe(): + """SparseApplyAdagradV2D TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/sparse_apply_ftrl_v2.py b/mindspore/ops/_op_impl/tbe/sparse_apply_ftrl_v2.py new file mode 100644 index 0000000000..518c524010 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/sparse_apply_ftrl_v2.py @@ -0,0 +1,52 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""SparseApplyFtrlV2D op""" +from mindspore.ops.op_info_register import op_info_register, TBERegOp, DataType + +sparse_apply_ftrl_v2_d_op_info = TBERegOp("SparseApplyFtrlV2") \ + .fusion_type("OPAQUE") \ + .async_flag(False) \ + .binfile_name("sparse_apply_ftrl_v2_d.so") \ + .compute_cost(10) \ + .kernel_name("sparse_apply_ftrl_v2_d") \ + .partial_flag(True) \ + .attr("lr", "required", "float", "all") \ + .attr("l1", "required", "float", "all") \ + .attr("l2", "required", "float", "all") \ + .attr("l2_shrinkage", "required", "float", "all") \ + .attr("lr_power", "required", "float", "all") \ + .attr("use_locking", "optional", "bool", "true,false", "false") \ + .input(0, "var", False, "required", "all") \ + .input(1, "accum", False, "required", "all") \ + .input(2, "linear", False, "required", "all") \ + .input(3, "grad", False, "required", "all") \ + .input(4, "indices", False, "required", "all") \ + .output(0, "var", False, "required", "all") \ + .output(1, "accum", False, "required", "all") \ + .output(2, "linear", False, "required", "all") \ + .dtype_format(DataType.F32_NCHW, DataType.F32_NCHW, DataType.F32_NCHW, DataType.F32_NCHW, + DataType.I32_NCHW, DataType.F32_NCHW, DataType.F32_NCHW, DataType.F32_NCHW) \ + .dtype_format(DataType.F32_NHWC, DataType.F32_NHWC, DataType.F32_NHWC, DataType.F32_NHWC, + DataType.I32_NHWC, DataType.F32_NHWC, DataType.F32_NHWC, DataType.F32_NHWC) \ + .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, + DataType.I32_Default, DataType.F32_Default, DataType.F32_Default, DataType.F32_Default) \ + .get_op_info() + + +@op_info_register(sparse_apply_ftrl_v2_d_op_info) +def _sparse_apply_ftrl_v2_tbe(): + """SparseApplyFtrlV2D TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/split_d.py b/mindspore/ops/_op_impl/tbe/split_d.py index d2faf31096..80557dd701 100644 --- a/mindspore/ops/_op_impl/tbe/split_d.py +++ b/mindspore/ops/_op_impl/tbe/split_d.py @@ -28,28 +28,7 @@ split_d_op_info = TBERegOp("Split") \ .input(0, "value", False, "required", "all") \ .output(0, "output", False, "dynamic", "all") \ .op_pattern("dynamicFormat") \ - .dtype_format(DataType.BOOL_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.BOOL_NHWC, DataType.BOOL_NHWC) \ - .dtype_format(DataType.I8_Default, DataType.I8_Default) \ - .dtype_format(DataType.I8_NHWC, DataType.I8_NHWC) \ - .dtype_format(DataType.U8_Default, DataType.U8_Default) \ - .dtype_format(DataType.U8_NHWC, DataType.U8_NHWC) \ - .dtype_format(DataType.I16_Default, DataType.I16_Default) \ - .dtype_format(DataType.I16_NHWC, DataType.I16_NHWC) \ - .dtype_format(DataType.U16_Default, DataType.U16_Default) \ - .dtype_format(DataType.U16_NHWC, DataType.U16_NHWC) \ - .dtype_format(DataType.I32_Default, DataType.I32_Default) \ - .dtype_format(DataType.I32_NHWC, DataType.I32_NHWC) \ - .dtype_format(DataType.U32_Default, DataType.U32_Default) \ - .dtype_format(DataType.U32_NHWC, DataType.U32_NHWC) \ - .dtype_format(DataType.I64_Default, DataType.I64_Default) \ - .dtype_format(DataType.I64_NHWC, DataType.I64_NHWC) \ - .dtype_format(DataType.U64_Default, DataType.U64_Default) \ - .dtype_format(DataType.U64_NHWC, DataType.U64_NHWC) \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_NHWC, DataType.F16_NHWC) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_NHWC, DataType.F32_NHWC) \ + .dtype_format(DataType.None_None, DataType.None_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/splitv.py b/mindspore/ops/_op_impl/tbe/splitv.py index 29f65c7e87..6948524bd7 100644 --- a/mindspore/ops/_op_impl/tbe/splitv.py +++ b/mindspore/ops/_op_impl/tbe/splitv.py @@ -29,28 +29,7 @@ split_v_op_info = TBERegOp("SplitV") \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "dynamic", "all") \ .op_pattern("dynamicFormat") \ - .dtype_format(DataType.BOOL_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.BOOL_NHWC, DataType.BOOL_NHWC) \ - .dtype_format(DataType.I8_Default, DataType.I8_Default) \ - .dtype_format(DataType.I8_NHWC, DataType.I8_NHWC) \ - .dtype_format(DataType.U8_Default, DataType.U8_Default) \ - .dtype_format(DataType.U8_NHWC, DataType.U8_NHWC) \ - .dtype_format(DataType.I16_Default, DataType.I16_Default) \ - .dtype_format(DataType.I16_NHWC, DataType.I16_NHWC) \ - .dtype_format(DataType.U16_Default, DataType.U16_Default) \ - .dtype_format(DataType.U16_NHWC, DataType.U16_NHWC) \ - .dtype_format(DataType.I32_Default, DataType.I32_Default) \ - .dtype_format(DataType.I32_NHWC, DataType.I32_NHWC) \ - .dtype_format(DataType.U32_Default, DataType.U32_Default) \ - .dtype_format(DataType.U32_NHWC, DataType.U32_NHWC) \ - .dtype_format(DataType.I64_Default, DataType.I64_Default) \ - .dtype_format(DataType.I64_NHWC, DataType.I64_NHWC) \ - .dtype_format(DataType.U64_Default, DataType.U64_Default) \ - .dtype_format(DataType.U64_NHWC, DataType.U64_NHWC) \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_NHWC, DataType.F16_NHWC) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_NHWC, DataType.F32_NHWC) \ + .dtype_format(DataType.None_None, DataType.None_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/sqrt.py b/mindspore/ops/_op_impl/tbe/sqrt.py index f9e339713b..0c2610f8d1 100644 --- a/mindspore/ops/_op_impl/tbe/sqrt.py +++ b/mindspore/ops/_op_impl/tbe/sqrt.py @@ -25,12 +25,9 @@ sqrt_op_info = TBERegOp("Sqrt") \ .partial_flag(True) \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F16_NHWC, DataType.F16_NHWC) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD) \ - .dtype_format(DataType.F32_NHWC, DataType.F32_NHWC) \ + .op_pattern("formatAgnostic") \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/square.py b/mindspore/ops/_op_impl/tbe/square.py index c3eeb12780..aa1d9875cb 100644 --- a/mindspore/ops/_op_impl/tbe/square.py +++ b/mindspore/ops/_op_impl/tbe/square.py @@ -25,15 +25,10 @@ square_op_info = TBERegOp("Square") \ .partial_flag(True) \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.I32_Default, DataType.I32_Default) \ - .dtype_format(DataType.I32_5HD, DataType.I32_5HD) \ - .dtype_format(DataType.I32_NHWC, DataType.I32_NHWC) \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F16_NHWC, DataType.F16_NHWC) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD) \ - .dtype_format(DataType.F32_NHWC, DataType.F32_NHWC) \ + .op_pattern("formatAgnostic") \ + .dtype_format(DataType.I32_None, DataType.I32_None) \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/sub.py b/mindspore/ops/_op_impl/tbe/sub.py index 8e97681c6b..e2454acd35 100644 --- a/mindspore/ops/_op_impl/tbe/sub.py +++ b/mindspore/ops/_op_impl/tbe/sub.py @@ -26,12 +26,10 @@ sub_op_info = TBERegOp("Sub") \ .input(0, "x1", False, "required", "all") \ .input(1, "x2", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.I32_Default, DataType.I32_Default, DataType.I32_Default) \ - .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.I32_5HD, DataType.I32_5HD, DataType.I32_5HD) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD) \ + .op_pattern("broadcast") \ + .dtype_format(DataType.I32_None, DataType.I32_None, DataType.I32_None) \ + .dtype_format(DataType.F16_None, DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/tanh.py b/mindspore/ops/_op_impl/tbe/tanh.py index c5b1caf1dd..7dff31518f 100644 --- a/mindspore/ops/_op_impl/tbe/tanh.py +++ b/mindspore/ops/_op_impl/tbe/tanh.py @@ -25,12 +25,9 @@ tanh_op_info = TBERegOp("Tanh") \ .partial_flag(True) \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD) \ - .dtype_format(DataType.F32_FracNZ, DataType.F32_FracNZ) \ - .dtype_format(DataType.F16_FracNZ, DataType.F16_FracNZ) \ + .op_pattern("formatAgnostic") \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/trans_data.py b/mindspore/ops/_op_impl/tbe/trans_data.py index c0cce302cd..91b4bc85f5 100644 --- a/mindspore/ops/_op_impl/tbe/trans_data.py +++ b/mindspore/ops/_op_impl/tbe/trans_data.py @@ -50,7 +50,7 @@ trans_data_op_info = TBERegOp("TransData") \ .dtype_format(DataType.F16_HWCN, DataType.F16_FracZ) \ .dtype_format(DataType.F16_FracZ, DataType.F16_HWCN) \ .dtype_format(DataType.F16_C1HWNCoC0, DataType.F16_HWCN) \ - .dtype_format(DataType.F16_HWCN, DataType.F16_5HD) \ + .dtype_format(DataType.F16_HWCN, DataType.F16_C1HWNCoC0) \ .dtype_format(DataType.F16_Default, DataType.F16_FracNZ) \ .dtype_format(DataType.F32_Default, DataType.F32_FracNZ) \ .dtype_format(DataType.F16_FracNZ, DataType.F16_Default) \ diff --git a/mindspore/ops/_op_impl/tbe/unsorted_segment_sum.py b/mindspore/ops/_op_impl/tbe/unsorted_segment_sum.py index b1f81b72b0..10cc3db0b0 100644 --- a/mindspore/ops/_op_impl/tbe/unsorted_segment_sum.py +++ b/mindspore/ops/_op_impl/tbe/unsorted_segment_sum.py @@ -28,16 +28,7 @@ unsorted_segment_sum_op_info = TBERegOp("UnsortedSegmentSum") \ .input(1, "segment_ids", False, "required", "all") \ .output(0, "y", False, "required", "all") \ .op_pattern("dynamicFormat") \ - .dtype_format(DataType.I8_Default, DataType.I32_Default, DataType.I8_Default) \ - .dtype_format(DataType.I8_5HD, DataType.I32_5HD, DataType.I8_5HD) \ - .dtype_format(DataType.U8_Default, DataType.I32_Default, DataType.U8_Default) \ - .dtype_format(DataType.U8_5HD, DataType.I32_5HD, DataType.U8_5HD) \ - .dtype_format(DataType.I32_Default, DataType.I32_Default, DataType.I32_Default) \ - .dtype_format(DataType.I32_5HD, DataType.I32_5HD, DataType.I32_5HD) \ - .dtype_format(DataType.F16_Default, DataType.I32_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.I32_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F32_Default, DataType.I32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.I32_5HD, DataType.F32_5HD) \ + .dtype_format(DataType.None_None, DataType.None_None, DataType.None_None) \ .get_op_info() diff --git a/mindspore/ops/composite/base.py b/mindspore/ops/composite/base.py index e283867684..b0f16d82bf 100644 --- a/mindspore/ops/composite/base.py +++ b/mindspore/ops/composite/base.py @@ -30,16 +30,16 @@ from ...common.parameter import Parameter __all__ = [EnvInstance_, TupleAdd_, TupleSlice_, UnpackCall_, TupleGetItemTensor_] -def add_flags(fn, **flags): +def add_flags(fn=None, **flags): """ - An interface to add flag for a function. + An decorator to add flag for a function. Note: Only supports bool value. Args: - fn (Function): Function or cell to add flag. - flags (bool): Flags use kwargs. + fn (Function): Function or cell to add flag. Default: None. + flags (dict): Flags use kwargs. Default: None. Returns: Function, the fn added flags. @@ -47,11 +47,17 @@ def add_flags(fn, **flags): Examples: >>> add_flags(net, predit=True) """ - # need set the attr and access on c++ - if not hasattr(fn, "_mindspore_flags"): - fn._mindspore_flags = {} - fn._mindspore_flags.update({**flags}) - return fn + def deco(fn): + # need set the attr and access on c++ + if not hasattr(fn, "_mindspore_flags"): + fn._mindspore_flags = {} + + fn._mindspore_flags.update({**flags}) + return fn + ret = deco + if fn is not None: + ret = deco(fn) + return ret def core(fn=None, **flags): diff --git a/mindspore/ops/composite/multitype_ops/_compile_utils.py b/mindspore/ops/composite/multitype_ops/_compile_utils.py index 906d74948a..a02a8c8809 100644 --- a/mindspore/ops/composite/multitype_ops/_compile_utils.py +++ b/mindspore/ops/composite/multitype_ops/_compile_utils.py @@ -65,9 +65,9 @@ def _generate_indices_from_tuple_of_mixed_tensors(data, tuple_index, op_name): tuple_len = len(tuple_index) for i in range(tuple_len): if i in int_positions: - tuple_index_new = tuple_index_new + (F.scalar_to_tensor(tuple_index[i], mstype.int32),) + tuple_index_new += (F.scalar_to_tensor(tuple_index[i], mstype.int32),) else: - tuple_index_new = tuple_index_new + (tuple_index[i],) + tuple_index_new += (tuple_index[i],) indexes_types = hyper_map(F.typeof, tuple_index_new) tensor_positions, slice_positions, ellipsis_position = \ const_utils.separate_mixed_tensors_index(indexes_types, op_name) @@ -167,12 +167,13 @@ def _tensor_getitem(self, index): return tensor_index_by_tensor(self, index) if isinstance(index, tuple): return tensor_index_by_tuple(self, index) + # bool type should be judged before int + if isinstance(index, bool): + return _tensor_index_by_bool(self, index) if isinstance(index, int): return _tensor_index_by_integer(self, index) if isinstance(index, slice): return tensor_index_by_slice(self, index) - if isinstance(index, bool): - return _tensor_index_by_bool(self, index) if index is None: return F.expand_dims(self, 0) if index is ...: @@ -206,7 +207,8 @@ def tensor_index_by_slice(data, slice_index): """Tensor getitem by a single slice""" shape = F.shape(data) if not shape: - const_utils.raise_index_error("When tensor is indexed by a slice, the dimension of the tensor cannot be 0.") + const_utils.raise_index_error("When tensor is indexed by a slice, the dimension of the tensor" + "cannot be 0.") begin_strides, end_strides, step_strides = const_utils.get_stride_info_from_slice(shape, slice_index) return F.strided_slice(data, begin_strides, end_strides, step_strides) @@ -215,7 +217,11 @@ def _tensor_index_by_integer(data, number): """Tensor getitem by a single integer number""" shape = F.shape(data) if not shape: - const_utils.raise_index_error("When tensor is indexed by an integer, the dimension of the tensor cannot be 0.") + return const_utils.raise_type_error("When tensor is indexed by an integer," + "the dimension of the tensor cannot be 0.") + if number >= shape[0]: + return const_utils.raise_index_error("index {} is out of bounds for axis 0 with size {}".format( + number, shape[0])) begin_strides, end_strides, step_strides = const_utils.get_stride_info_from_integer(shape, number) shrink_axis_mask = 1 return P.StridedSlice(0, 0, 0, 0, shrink_axis_mask)(data, begin_strides, end_strides, step_strides) @@ -261,6 +267,8 @@ def _tensor_index_by_tuple_slice(data, t): def tensor_index_by_tuple(data, tuple_index): """Tensor getitem by tuple of various types""" + if len(tuple_index) == 1: + return data[tuple_index[0]] indexes_types = hyper_map(F.typeof, tuple_index) index_elements_type = const_utils.tuple_index_elements_type(indexes_types, const_utils.TENSOR_GETITEM) if index_elements_type == const_utils.NO_TENSOR: @@ -424,11 +432,12 @@ def tensor_setitem_by_slice_with_number(data, input_slice, value): def tensor_setitem_by_tuple_with_number(data, tuple_index, value): """Assigns the tensor by tuple with number value.""" + if len(tuple_index) == 1: + data[tuple_index[0]] = value + return data indexes_types = hyper_map(F.typeof, tuple_index) index_elements_type = const_utils.tuple_index_elements_type(indexes_types, const_utils.TENSOR_SETITEM) - if index_elements_type == const_utils.NO_TENSOR: - return tensor_setitem_by_slice_with_number(data, tuple_index, value) if index_elements_type == const_utils.ALL_TENSOR: indices = _generate_indices_from_tuple_of_tensor(data, tuple_index, @@ -485,11 +494,12 @@ def tensor_setitem_by_slice_with_tensor(data, input_slice, value): def tensor_setitem_by_tuple_with_tensor(data, tuple_index, value): """Assigns the tensor by tuple with tensor value.""" + if len(tuple_index) == 1: + data[tuple_index[0]] = value + return data indexes_types = hyper_map(F.typeof, tuple_index) index_elements_type = const_utils.tuple_index_elements_type(indexes_types, const_utils.TENSOR_SETITEM) - if index_elements_type == const_utils.NO_TENSOR: - return tensor_setitem_by_slice_with_tensor(data, tuple_index, value) if index_elements_type == const_utils.ALL_TENSOR: indices = _generate_indices_from_tuple_of_tensor(data, tuple_index, @@ -507,6 +517,9 @@ def tensor_setitem_by_tuple_with_tensor(data, tuple_index, value): def tensor_setitem_by_tuple_with_tuple(data, tuple_index, value): """Assigns the tensor by tuple with tuple of value.""" + if len(tuple_index) == 1: + data[tuple_index[0]] = value + return data indexes_types = hyper_map(F.typeof, tuple_index) index_elements_type = const_utils.tuple_index_elements_type(indexes_types, const_utils.TENSOR_SETITEM) diff --git a/mindspore/ops/composite/multitype_ops/_constexpr_utils.py b/mindspore/ops/composite/multitype_ops/_constexpr_utils.py index 02756ffe56..88a900407a 100644 --- a/mindspore/ops/composite/multitype_ops/_constexpr_utils.py +++ b/mindspore/ops/composite/multitype_ops/_constexpr_utils.py @@ -44,6 +44,10 @@ SET_ITEM_BY_TUPLE_OF_TENSOR = 1 def raise_index_error(msg): raise IndexError(msg) +@constexpr +def raise_type_error(msg): + raise TypeError(msg) + @constexpr def check_equal(param1, param2, msg="{},{}"): @@ -339,6 +343,8 @@ def check_tensors_dtype_same(data_dtype, value_dtype, op_name): @constexpr def generate_broadcast_shape(shapes, op_name): """Generate broadcast shape for a tuple of shape.""" + if not shapes: + return () broadcast_shape = shapes[0] for i, shape in enumerate(shapes): logger.debug(f"Broadcasts the {i}th tensor, the shape is {shape}.") @@ -541,6 +547,11 @@ def generate_index_info_from_tuple_of_mixed_tensors(data_shape, slice_indexes[slice_count].step) # Use list to represent slicing result. indexes_info[pos] = list(range(data_shape[pos]))[slice_obj] + if not indexes_info[pos]: + raise IndexError("An empty slice is not supported, got {}:{}:{}".format( + slice_indexes[slice_count].start, + slice_indexes[slice_count].stop, + slice_indexes[slice_count].step)) slice_count += 1 elif isinstance(ele_type, mstype.ellipsis_type): if ellipsis_num != 0: diff --git a/mindspore/ops/composite/multitype_ops/setitem_impl.py b/mindspore/ops/composite/multitype_ops/setitem_impl.py index 38cf0141f0..30c943b69f 100644 --- a/mindspore/ops/composite/multitype_ops/setitem_impl.py +++ b/mindspore/ops/composite/multitype_ops/setitem_impl.py @@ -85,6 +85,22 @@ def _list_setitem_with_List(data, number_index, value): return F.list_setitem(data, number_index, value) +@setitem.register("List", "Number", "Tuple") +def _list_setitem_with_Tuple(data, number_index, value): + """ + Assigns value to list. + + Inputs: + data (list): Data of type lis. + number_index (Number): Index of data. + value (list): Value given. + + Outputs: + list, type is same as the element type of data. + """ + return F.list_setitem(data, number_index, value) + + @setitem.register("Dictionary", "String", "Tensor") def _dict_setitem_with_tensor(data, key, value): """ diff --git a/mindspore/ops/functional.py b/mindspore/ops/functional.py index 5637274bfb..a5c3165ab1 100644 --- a/mindspore/ops/functional.py +++ b/mindspore/ops/functional.py @@ -153,10 +153,20 @@ shape_mul = Primitive("shape_mul") # a primitive to compare between tuple. stop_gradient = Primitive("stop_gradient") + +make_indexed_slices = Primitive('MakeIndexedSlices') +indexed_slices_get_values = Primitive('IndexedSlicesGetValues') +indexed_slices_get_indices = Primitive('IndexedSlicesGetIndices') +indexed_slices_get_dense_shape = Primitive('IndexedSlicesGetDenseShape') +is_indexed_slices = Primitive('IsIndexedSlices') + + tensor_operator_registry.register('__add__', tensor_add) tensor_operator_registry.register('__sub__', tensor_sub) tensor_operator_registry.register('__mul__', tensor_mul) tensor_operator_registry.register('__truediv__', tensor_div) +tensor_operator_registry.register('__mod__', tensor_mod) +tensor_operator_registry.register('__floordiv__', tensor_floordiv) #ms cannot support Tensor(True) compare tensor_operator_registry.register('__eq__', equal) tensor_operator_registry.register('__ne__', not_equal) diff --git a/mindspore/ops/operations/__init__.py b/mindspore/ops/operations/__init__.py index ad83ac346a..487b42c099 100644 --- a/mindspore/ops/operations/__init__.py +++ b/mindspore/ops/operations/__init__.py @@ -25,9 +25,9 @@ from .array_ops import (Argmax, Argmin, Cast, Concat, Pack, Unpack, Fill, GatherNd, GatherV2, SparseGatherV2, InvertPermutation, IsInstance, IsSubClass, ArgMaxWithValue, OnesLike, ZerosLike, Rank, Reshape, ResizeNearestNeighbor, ArgMinWithValue, - SameTypeShape, ScatterAdd, ScatterMax, ScatterUpdate, - ScalarToArray, ScalarToTensor, ScatterNd, ScatterNdUpdate, Select, - Shape, Size, Slice, Split, EmbeddingLookup, + SameTypeShape, ScatterAdd, ScatterSub, ScatterMul, ScatterDiv, ScatterMax, ScatterMin, + ScatterUpdate, ScalarToArray, ScalarToTensor, ScatterNd, ScatterNdUpdate, Select, + Shape, Size, Slice, Split, TransShape, EmbeddingLookup, Squeeze, StridedSlice, Tile, TensorScatterUpdate, Transpose, TruncatedNormal, TupleToArray, UnsortedSegmentMin, UnsortedSegmentSum, SpaceToDepth, DepthToSpace, SpaceToBatch, BatchToSpace, @@ -35,9 +35,9 @@ from .array_ops import (Argmax, Argmin, Cast, Concat, Pack, Unpack, from .comm_ops import (AllGather, AllReduce, _AlltoAll, ReduceScatter, Broadcast, _MirrorOperator, ReduceOp, _VirtualDataset, _VirtualDiv, _GetTensorSlice, - HostAllGather, HostReduceScatter) + _HostAllGather, _HostReduceScatter) from .debug_ops import (ImageSummary, InsertGradientOf, HookBackward, ScalarSummary, - TensorSummary, HistogramSummary, Print) + TensorSummary, HistogramSummary, Debug, Print) from .control_ops import ControlDepend, GeSwitch, Merge from .inner_ops import ScalarCast @@ -45,7 +45,7 @@ from .math_ops import (Abs, ACos, Asin, Asinh, AddN, AccumulateNV2, AssignAdd, A BitwiseXor, Inv, Invert, ApproximateEqual, InplaceAdd, InplaceSub, ReduceMax, ReduceMin, ReduceMean, ReduceSum, ReduceAll, ReduceProd, CumProd, Cos, Div, DivNoNan, Equal, EqualCount, Exp, Expm1, Erf, Erfc, Floor, FloorDiv, FloorMod, Ceil, - Acosh, Greater, GreaterEqual, Less, LessEqual, Log, Log1p, LogicalAnd, + Acosh, Greater, GreaterEqual, Less, LessEqual, Log, Log1p, LogicalAnd, Mod, LogicalNot, LogicalOr, MatMul, Maximum, Minimum, Mul, Neg, NMSWithMask, NotEqual, NPUAllocFloatStatus, NPUClearFloatStatus, @@ -69,13 +69,13 @@ from .nn_ops import (LSTM, SGD, Adam, SparseApplyAdam, SparseApplyLazyAdam, Appl MaxPoolWithArgmax, OneHot, Pad, MirrorPad, PReLU, ReLU, ReLU6, ReLUV2, HSwish, HSigmoid, ResizeBilinear, Sigmoid, SigmoidCrossEntropyWithLogits, - SmoothL1Loss, Softmax, Softplus, - RNNTLoss, + SmoothL1Loss, Softmax, Softsign, Softplus, LRN, RNNTLoss, SoftmaxCrossEntropyWithLogits, ROIAlign, SparseSoftmaxCrossEntropyWithLogits, Tanh, TopK, BinaryCrossEntropy, SparseApplyAdagrad, LARSUpdate, ApplyFtrl, SparseApplyFtrl, - ApplyProximalAdagrad, SparseApplyProximalAdagrad, + ApplyProximalAdagrad, SparseApplyProximalAdagrad, SparseApplyAdagradV2, SparseApplyFtrlV2, ApplyAdaMax, ApplyAdadelta, ApplyAdagrad, ApplyAdagradV2, + ApplyAddSign, ApplyPowerSign, ApplyGradientDescent, ApplyProximalGradientDescent, ApplyRMSProp, ApplyCenteredRMSProp, BasicLSTMCell, InTopK) from .other_ops import (Assign, IOU, BoundingBoxDecode, BoundingBoxEncode, CheckValid, MakeRefKey, Partial, Depend, CheckBprop) @@ -118,6 +118,7 @@ __all__ = [ 'SparseApplyLazyAdam', 'Softplus', 'Softmax', + 'Softsign', 'LogSoftmax', 'SoftmaxCrossEntropyWithLogits', 'ROIAlign', @@ -184,6 +185,7 @@ __all__ = [ 'ImageSummary', 'TensorSummary', 'HistogramSummary', + "Debug", "Print", 'InsertGradientOf', 'HookBackward', @@ -224,8 +226,12 @@ __all__ = [ 'BoundingBoxDecode', 'L2Normalize', 'ScatterAdd', + 'ScatterSub', + 'ScatterMul', + 'ScatterDiv', 'ScatterNd', 'ScatterMax', + 'ScatterMin', 'ResizeNearestNeighbor', 'HistogramFixedWidth', 'Pad', @@ -255,10 +261,8 @@ __all__ = [ 'UnsortedSegmentSum', 'UnsortedSegmentMin', "AllGather", - "HostAllGather", "AllReduce", "ReduceScatter", - "HostReduceScatter", "Broadcast", "ReduceOp", 'ScalarCast', @@ -292,6 +296,7 @@ __all__ = [ "Abs", "BinaryCrossEntropy", "SparseApplyAdagrad", + "SparseApplyAdagradV2", "SpaceToDepth", "DepthToSpace", "Conv2DBackpropInput", @@ -302,12 +307,17 @@ __all__ = [ "ApplyFtrl", "SpaceToBatch", "SparseApplyFtrl", + "SparseApplyFtrlV2", "ApplyProximalAdagrad", "SparseApplyProximalAdagrad", "ApplyAdaMax", "ApplyAdadelta", "ApplyAdagrad", "ApplyAdagradV2", + "ApplyAddSign", + "ApplyPowerSign", + "ApplyGradientDescent", + "ApplyProximalGradientDescent", "BatchToSpace", "Atan2", "ApplyRMSProp", @@ -329,7 +339,9 @@ __all__ = [ "ApproximateEqual", "InplaceUpdate", "InTopK", - "CropAndResize" + "CropAndResize", + "LRN", + "Mod" ] __all__.sort() diff --git a/mindspore/ops/operations/_grad_ops.py b/mindspore/ops/operations/_grad_ops.py index 7520212b77..5e5e56f708 100644 --- a/mindspore/ops/operations/_grad_ops.py +++ b/mindspore/ops/operations/_grad_ops.py @@ -237,6 +237,7 @@ class Conv2DBackpropFilter(PrimitiveWithInfer): self.add_prim_attr('stride', self.stride) self.dilation = dilation self.group = group + self.add_prim_attr('groups', group) self.add_prim_attr('data_format', "NCHW") def __infer__(self, doutput, x, w_size): @@ -535,6 +536,51 @@ class MaxPoolGrad(_PoolGrad): return x1_dtype +class MaxPoolGradGrad(_PoolGrad): + r""" + Performs gradients of the MaxPoolGrad operation. + + Args: + ksize (Union[int, tuple[int]]): The size of kernel used to take the maximum value, + is an int number that represents height and width are both ksize, or a tuple + of two int numbers that represent height and width respectively. Default: 1. + strides (Union[int, tuple[int]]): The distance of kernel moving, an int number that represents + the height and width of movement are both strides, or a tuple of two int numbers that + represent height and width of movement respectively. Default: 1. + padding (str): The optional values for pad mode, is "same" or "valid", not case sensitive. + Default: "valid". + + - same: Adopts the way of completion. Output height and width will be the same as + the input. Total number of padding will be calculated for horizontal and vertical + direction and evenly distributed to top and bottom, left and right if possible. + Otherwise, the last extra padding will be done from the bottom and the right side. + + - valid: Adopts the way of discarding. The possibly largest height and width of output + will be return without padding. Extra pixels will be discarded. + + Inputs: + - **origin_input** (Tensor) - Tensor with data format "NCHW", data type should be float16. + - **origin_output** (Tensor) - Data type same as `origin_input`. + - **grad** (Tensor) - Data type same as `origin_input`. + + Outputs: + Tensor, With data type same as `origin_input`. + + """ + + @prim_attr_register + def __init__(self, ksize=1, strides=1, padding="VALID"): + super(MaxPoolGradGrad, self).__init__(ksize, strides, padding) + + def infer_shape(self, x1_shape, x2_shape, grad_shape): + return x1_shape + + def infer_dtype(self, x1_dtype, x2_dtype, grad_dtype): + args = {'x1_dtype': x1_dtype, 'x2_dtype': x2_dtype, 'grad_dtype': grad_dtype} + validator.check_tensor_type_same(args, [mstype.float16], self.name) + return x1_dtype + + class MaximumGrad(Primitive): """Grad for maximum.""" @@ -563,6 +609,54 @@ class MaxPoolGradWithArgmax(_PoolGrad): return grad_dtype +class MaxPoolGradGradWithArgmax(_PoolGrad): + r""" + Computes the gradients of MaxPoolGradWithArgmax. + + Args: + ksize (Union[int, tuple[int]]): The size of kernel used to take the maximum value, + is an int number that represents height and width are both ksize, or a tuple + of two int numbers that represent height and width respectively. Default: 1. + strides (Union[int, tuple[int]]): The distance of kernel moving, an int number that represents + the height and width of movement are both strides, or a tuple of two int numbers that + represent height and width of movement respectively. Default: 1. + padding (str): The optional values for pad mode, is "same" or "valid", not case sensitive. + Default: "valid". + + - same: Adopts the way of completion. Output height and width will be the same as + the input. Total number of padding will be calculated for horizontal and vertical + direction and evenly distributed to top and bottom, left and right if possible. + Otherwise, the last extra padding will be done from the bottom and the right side. + + - valid: Adopts the way of discarding. The possibly largest height and width of output + will be return without padding. Extra pixels will be discarded. + + Inputs: + - **x** (Tensor) - Tensor with data format "NCHW", data type should be float16. + - **grad** (Tensor) - Data type same as `x`. + - **argmax** (Tensor) - Data type should be uint16 or int64. + + Outputs: + Tensor, With data type same as `x`. + + """ + + @prim_attr_register + def __init__(self, ksize=1, strides=1, padding="VALID"): + self.init_prim_io_names(inputs=['x', 'grad', 'argmax'], outputs=['output']) + super(MaxPoolGradGradWithArgmax, self).__init__(ksize, strides, padding) + + def infer_shape(self, x_shape, grad_shape, argmax_shape): + if not grad_shape: + raise TypeError("The dout of MaxPoolGradGradWithArgmax should be a Tensor.") + return x_shape + + def infer_dtype(self, x_dtype, grad_dtype, argmax_dtype): + args = {'x_dtype': x_dtype, 'grad_dtype': grad_dtype} + validator.check_tensor_type_same(args, [mstype.float16], self.name) + return grad_dtype + + class MinimumGrad(Primitive): """Grad for minimum.""" @@ -1223,7 +1317,7 @@ class EmbeddingLookupCommGrad(PrimitiveWithInfer): Perform the gradient for the communication part of EmbeddingLookup operator. This works ONLY when 'reduce_scatter_flag' is True in 'EmbeddingLookup'. Roughly speaking, - this primitive is implemented by StridedSlice --> HostAllGather --> Concat. This primitive runs on host. + this primitive is implemented by StridedSlice --> _HostAllGather --> Concat. This primitive runs on host. """ @prim_attr_register def __init__(self): @@ -1234,8 +1328,8 @@ class EmbeddingLookupCommGrad(PrimitiveWithInfer): """ This primitive is implemented by three steps: 1) Split the 'dy' along dimension 0 into 'split_num' parts. - 2) For each part, perform HostAllGather((0, 1, 2, 3, 4, 5, 6, 7)) on the host. - 3) After HostAllGather, there are still 'split_num' parts in each process. Then, perform Concat on them + 2) For each part, perform _HostAllGather((0, 1, 2, 3, 4, 5, 6, 7)) on the host. + 3) After _HostAllGather, there are still 'split_num' parts in each process. Then, perform Concat on them along dimension 0. The output shape of this primitive: shape(output)[0] == shape(dy)[0] * 8 @@ -1421,3 +1515,22 @@ class InvGrad(PrimitiveWithInfer): validator.check_type_name("dgate", x, [mstype.float16, mstype.float32, mstype.int32, mstype.int8], self.name) validator.check_type_name("grad", grad, [mstype.float16, mstype.float32, mstype.int32, mstype.int8], self.name) return x + + +class LRNGrad(PrimitiveWithInfer): + """Computes gradients for LRN operation.""" + @prim_attr_register + def __init__(self, depth_radius=5, bias=1.0, alpha=1.0, beta=0.5): + self.init_prim_io_names(inputs=['grads', 'x', 'y'], outputs=['z']) + validator.check_value_type("depth_radius", depth_radius, [int], self.name) + validator.check_value_type("bias", bias, [float], self.name) + validator.check_value_type("alpha", alpha, [float], self.name) + validator.check_value_type("beta", beta, [float], self.name) + + def infer_dtype(self, grads, x, y): + args = {"grads": grads, "x": x, "y": y} + validator.check_tensor_type_same(args, (mstype.float16, mstype.float32,), self.name) + return x + + def infer_shape(self, grads, x, y): + return x diff --git a/mindspore/ops/operations/_inner_ops.py b/mindspore/ops/operations/_inner_ops.py index 2ace10a6b4..be7e901757 100644 --- a/mindspore/ops/operations/_inner_ops.py +++ b/mindspore/ops/operations/_inner_ops.py @@ -18,6 +18,9 @@ from ..._checkparam import Rel from ..._checkparam import Validator as validator from ...common import dtype as mstype +from ..._c_expression import signature_rw as sig_rw +from ..._c_expression import signature_kind as sig_kind +from ..._c_expression import signature_dtype as sig_dtype from ..primitive import PrimitiveWithInfer, prim_attr_register @@ -461,6 +464,183 @@ class EmbeddingLookup(PrimitiveWithInfer): return out +class SparseApplyFtrlNoReturn(PrimitiveWithInfer): + """ + Update relevant entries according to the FTRL-proximal scheme. + + Args: + lr (float): The learning rate value, must be positive. + l1 (float): l1 regularization strength, must be greater than or equal to zero. + l2 (float): l2 regularization strength, must be greater than or equal to zero. + lr_power (float): Learning rate power controls how the learning rate decreases during training, + must be less than or equal to zero. Use fixed learning rate if `lr_power` is zero. + use_locking (bool): Use locks for update operation if True . Default: False. + + Inputs: + - **var** (Parameter): The variable to be updated. The data type must be float32. + - **accum** (Parameter): The accum to be updated, must be same type and shape as `var`. + - **linear** (Parameter): The linear to be updated, must be same type and shape as `var`. + - **grad** (Tensor): A tensor of the same type as `var`, for the gradient. + - **indices** (Tensor): A vector of indices into the first dimension of `var` and `accum`. The shape + of `indices` must be the same as `grad` in first dimension. The type must be int32. + + Outputs: + Tuple of 3 Tensor, this operator will update the input parameters directly, the outputs are useless. + + - **var** (Tensor) - A Tensor with shape (1,). + - **accum** (Tensor) - A Tensor with shape (1,). + - **linear** (Tensor) - A Tensor with shape (1,). + + Examples: + >>> import mindspore + >>> import mindspore.nn as nn + >>> import numpy as np + >>> from mindspore import Parameter + >>> from mindspore import Tensor + >>> from mindspore.ops import operations as P + >>> class SparseApplyFtrlNet(nn.Cell): + >>> def __init__(self): + >>> super(SparseApplyFtrlNet, self).__init__() + >>> self.sparse_apply_ftrl = P.SparseApplyFtrlV2(lr=0.01, l1=0.0, l2=0.0, lr_power=-0.5) + >>> self.var = Parameter(Tensor(np.random.rand(3, 1, 2).astype(np.float32)), name="var") + >>> self.accum = Parameter(Tensor(np.random.rand(3, 1, 2).astype(np.float32)), name="accum") + >>> self.linear = Parameter(Tensor(np.random.rand(3, 1, 2).astype(np.float32)), name="linear") + >>> + >>> def construct(self, grad, indices): + >>> out = self.sparse_apply_ftrl(self.var, self.accum, self.linear, grad, indices) + >>> return out + >>> + >>> net = SparseApplyFtrlNet() + >>> grad = Tensor(np.random.rand(2, 1, 2).astype(np.float32)) + >>> indices = Tensor(np.array([0, 1]).astype(np.int32)) + >>> output = net(grad, indices) + """ + __mindspore_signature__ = ( + ('var', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('accum', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('linear', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('grad', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('indices', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T1) + ) + + @prim_attr_register + def __init__(self, lr, l1, l2, lr_power, use_locking=False): + self.init_prim_io_names(inputs=['var', 'accum', 'linear', 'grad', 'indices'], + outputs=['output']) + validator.check_value_type("lr", lr, [float], self.name) + validator.check_value_type("l1", l1, [float], self.name) + validator.check_value_type("l2", l2, [float], self.name) + validator.check_value_type("lr_power", lr_power, [float], self.name) + self.lr = validator.check_number_range("lr", lr, 0.0, float("inf"), Rel.INC_NEITHER, self.name) + self.l1 = validator.check_number_range("l1", l1, 0.0, float("inf"), Rel.INC_LEFT, self.name) + self.l2 = validator.check_number_range("l2", l2, 0.0, float("inf"), Rel.INC_LEFT, self.name) + self.lr_power = validator.check_number("lr_power", lr_power, 0, Rel.LE, self.name) + self.use_locking = validator.check_value_type("use_locking", use_locking, [bool], self.name) + self.add_prim_attr('primitive_target', 'CPU') + + def infer_shape(self, var_shape, accum_shape, linear_shape, grad_shape, indices_shape): + validator.check('var shape', var_shape, 'accum shape', accum_shape, Rel.EQ, self.name) + validator.check('var shape', var_shape, 'linear shape', linear_shape, Rel.EQ, self.name) + if len(var_shape) > 1: + validator.check('var_shape[1:]', var_shape[1:], 'grad_shape[1:]', grad_shape[1:], Rel.EQ, self.name) + validator.check_integer("indices rank", len(indices_shape), 1, Rel.EQ, self.name) + validator.check('grad_shape[0]', grad_shape[0], 'indices_shape[0]', indices_shape[0], Rel.EQ, self.name) + return [1], [1], [1] + + def infer_dtype(self, var_dtype, accum_dtype, linear_dtype, grad_dtype, indices_dtype): + args = {"var_dtype": var_dtype, "accum_dtype": accum_dtype, + "linear_dtype": linear_dtype, "grad_dtype": grad_dtype} + validator.check_tensor_type_same(args, [mstype.float32], self.name) + validator.check_tensor_type_same({"indices_dtype": indices_dtype}, [mstype.int32], self.name) + return var_dtype, accum_dtype, linear_dtype + + +class SparseApplyProximalAdagradNoReturn(PrimitiveWithInfer): + r""" + Updates relevant entries according to the proximal adagrad algorithm. + + .. math:: + accum += grad * grad + .. math:: + \text{prox_v} = var - lr * grad * \frac{1}{\sqrt{accum}} + .. math:: + var = \frac{sign(\text{prox_v})}{1 + lr * l2} * \max(\left| \text{prox_v} \right| - lr * l1, 0) + + Args: + use_locking (bool): If True, updating of the var and accum tensors will be protected. Default: False. + + Inputs: + - **var** (Parameter) - Variable tensor to be updated. The data type must be float32. + - **accum** (Parameter) - Variable tensor to be updated. Has the same dtype as `var`. + - **lr** (Tensor): The learning rate value. The data type must be float32. + - **l1** (Tensor): l1 regularization strength. The data type must be float32. + - **l2** (Tensor): l2 regularization strength. The data type must be float32. + - **grad** (Tensor) - A tensor of the same type as `var`, for the gradient. The data type must be float32. + - **indices** (Tensor) - A vector of indices into the first dimension of `var` and `accum`. The data type + must be int32. + + Outputs: + Tuple of 2 Tensor, this operator will update the input parameters directly, the outputs are useless. + + - **var** (Tensor) - A Tensor with shape (1,). + - **accum** (Tensor) - A Tensor with shape (1,). + + Examples: + >>> import numpy as np + >>> import mindspore.nn as nn + >>> from mindspore import Tensor, Parameter + >>> from mindspore.ops import operations as P + >>> class Net(nn.Cell): + >>> def __init__(self): + >>> super(Net, self).__init__() + >>> self.sparse_apply_proximal_adagrad = P.SparseApplyProximalAdagradV2() + >>> self.var = Parameter(Tensor(np.random.rand(3, 1, 2).astype(np.float32)), name="var") + >>> self.accum = Parameter(Tensor(np.random.rand(3, 1, 2).astype(np.float32)), name="accum") + >>> self.lr = Tensor(0.01, mstype.float32) + >>> self.l1 = Tensor(0.0, mstype.float32) + >>> self.l2 = Tensor(0.0, mstype.float32) + >>> def construct(self, grad, indices): + >>> out = self.sparse_apply_proximal_adagrad(self.var, self.accum, self.lr, self.l1, + >>> self.l2, grad, indices) + >>> return out + >>> net = Net() + >>> grad = Tensor(np.random.rand(2, 1, 2).astype(np.float32)) + >>> indices = Tensor(np.array([0, 1]).astype(np.int32)) + >>> output = net(grad, indices) + """ + __mindspore_signature__ = ( + ('var', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('accum', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('lr', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('l1', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('l2', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('grad', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('indices', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T1) + ) + + @prim_attr_register + def __init__(self, use_locking=False): + self.init_prim_io_names(inputs=['var', 'accum', 'lr', 'l1', 'l2', 'grad', 'indices'], + outputs=['output']) + self.use_locking = validator.check_value_type("use_locking", use_locking, [bool], self.name) + self.add_prim_attr('primitive_target', 'CPU') + + def infer_shape(self, var_shape, accum_shape, lr_shape, l1_shape, l2_shape, grad_shape, indices_shape): + validator.check_integer("indices rank", len(indices_shape), 1, Rel.EQ, self.name) + return [1], [1] + + def infer_dtype(self, var_dtype, accum_dtype, lr_dtype, l1_dtype, l2_dtype, grad_dtype, indices_dtype): + args = {'var': var_dtype, 'accum': accum_dtype, 'grad': grad_dtype} + validator.check_tensor_type_same(args, [mstype.float32], self.name) + validator.check_scalar_or_tensor_type_same({"lr": lr_dtype}, [mstype.float32], self.name) + validator.check_scalar_or_tensor_type_same({"l1": l1_dtype}, [mstype.float32], self.name) + validator.check_scalar_or_tensor_type_same({"l2": l2_dtype}, [mstype.float32], self.name) + valid_types = [mstype.int16, mstype.int32, mstype.int64, + mstype.uint16, mstype.uint32, mstype.uint64] + validator.check_tensor_type_same({'indices': indices_dtype}, valid_types, self.name) + return var_dtype, accum_dtype + + class LinSpace(PrimitiveWithInfer): r""" Generates values in an interval. And return the corresponding interpolation accroding to assist. diff --git a/mindspore/ops/operations/_quant_ops.py b/mindspore/ops/operations/_quant_ops.py index 42c2406906..1f4de03d3c 100644 --- a/mindspore/ops/operations/_quant_ops.py +++ b/mindspore/ops/operations/_quant_ops.py @@ -106,7 +106,7 @@ class MinMaxUpdatePerChannel(PrimitiveWithInfer): Args: ema (bool): Use EMA algorithm update value min and max. Default: False. ema_decay (int) : EMA algorithm decay parameter. Default: 0.999. - channel_axis (int): Channel asis for per channel compute. Default: 1. + channel_axis (int): Quantization by channel axis. Ascend backend only supports 0 or 1. Default: 1. Inputs: - **x** (Tensor) : float32 Tensor representing the shape of the output tensor. @@ -123,11 +123,13 @@ class MinMaxUpdatePerChannel(PrimitiveWithInfer): >>> output_tensor = MinMaxUpdatePerChannel(num_bits=8)(x, min, max) """ support_quant_bit = [4, 7, 8] + ascend_support_x_rank = [2, 4] @prim_attr_register def __init__(self, ema=False, ema_decay=0.999, channel_axis=1): """init FakeQuantPerChannelUpdate OP for Ascend""" - if context.get_context('device_target') == "Ascend": + self.is_ascend = context.get_context('device_target') == "Ascend" + if self.is_ascend: from mindspore.ops._op_impl._custom_op import minmax_update_perchannel if ema and not ema_decay: raise ValueError( @@ -136,13 +138,18 @@ class MinMaxUpdatePerChannel(PrimitiveWithInfer): self.ema = validator.check_value_type('ema', ema, (bool,), self.name) self.ema_decay = validator.check_number_range( 'ema_decay', ema_decay, 0, 1, Rel.INC_BOTH, self.name) - self.channel_axis = validator.check_integer( - 'channel axis', channel_axis, 0, Rel.GE, self.name) + if self.is_ascend: + self.channel_axis = validator.check_int_range('channel_axis', channel_axis, 0, 1, Rel.INC_BOTH, self.name) + else: + self.channel_axis = validator.check_integer('channel_axis', channel_axis, 0, Rel.GE, self.name) self.init_prim_io_names( inputs=['x', 'min', 'max'], outputs=['min_up', 'max_up']) def infer_shape(self, x_shape, min_shape, max_shape): - validator.check_integer("x rank", len(x_shape), 1, Rel.GT, self.name) + if self.is_ascend and len(x_shape) not in self.ascend_support_x_rank: + raise ValueError(f"For '{self.name}' x rank should be in '{self.ascend_support_x_rank}'") + if not self.is_ascend: + validator.check_integer("x rank", len(x_shape), 1, Rel.GE, self.name) validator.check("min shape", min_shape, "max shape", max_shape, Rel.EQ, self.name) validator.check_integer("min shape", len( @@ -221,8 +228,8 @@ class FakeQuantPerLayer(PrimitiveWithInfer): 'ema_decay', ema_decay, 0, 1, Rel.INC_BOTH, self.name) self.num_bits = validator.check_integer( 'num_bits', num_bits, 0, Rel.GT, self.name) - self.quant_delay = validator.check_value_type( - 'quant_delay', quant_delay, (int,), self.name) + self.quant_delay = validator.check_integer( + 'quant_delay', quant_delay, 0, Rel.GE, self.name) self.init_prim_io_names(inputs=['x', 'min', 'max'], outputs=['out']) @@ -314,6 +321,7 @@ class FakeQuantPerChannel(PrimitiveWithInfer): symmetric (bool): Quantization algorithm use symmetric or not. Default: False. narrow_range (bool): Quantization algorithm use narrow range or not. Default: False. training (bool): Training the network or not. Default: True. + channel_axis (int): Quantization by channel axis. Ascend backend only supports 0 or 1. Default: 1. Inputs: - **x** (Tensor) : 4-D float32 Tensor representing the shape of the output tensor. @@ -331,6 +339,7 @@ class FakeQuantPerChannel(PrimitiveWithInfer): >>> result = fake_quant(input_x, _min, _max) """ support_quant_bit = [4, 7, 8] + ascend_support_x_rank = [2, 4] @prim_attr_register def __init__(self, @@ -343,7 +352,8 @@ class FakeQuantPerChannel(PrimitiveWithInfer): training=True, channel_axis=1): """init FakeQuantPerChannel OP""" - if context.get_context('device_target') == "Ascend": + self.is_ascend = context.get_context('device_target') == "Ascend" + if self.is_ascend: from mindspore.ops._op_impl._custom_op import fake_quant_perchannel if num_bits not in self.support_quant_bit: raise ValueError( @@ -363,14 +373,19 @@ class FakeQuantPerChannel(PrimitiveWithInfer): 'ema_decay', ema_decay, 0, 1, Rel.INC_BOTH, self.name) self.num_bits = validator.check_integer( 'num_bits', num_bits, 0, Rel.GT, self.name) - self.quant_delay = validator.check_value_type( - 'quant_delay', quant_delay, (int,), self.name) - self.channel_axis = validator.check_integer( - 'channel_axis', channel_axis, 0, Rel.GE, self.name) + self.quant_delay = validator.check_integer( + 'quant_delay', quant_delay, 0, Rel.GE, self.name) + if self.is_ascend: + self.channel_axis = validator.check_int_range('channel_axis', channel_axis, 0, 1, Rel.INC_BOTH, self.name) + else: + self.channel_axis = validator.check_integer('channel_axis', channel_axis, 0, Rel.GE, self.name) self.init_prim_io_names(inputs=['x', 'min', 'max'], outputs=['out']) def infer_shape(self, x_shape, min_shape, max_shape): - validator.check_integer("x rank", len(x_shape), 1, Rel.GE, self.name) + if self.is_ascend and len(x_shape) not in self.ascend_support_x_rank: + raise ValueError(f"For '{self.name}' x rank should be in '{self.ascend_support_x_rank}'") + if not self.is_ascend: + validator.check_integer("x rank", len(x_shape), 1, Rel.GE, self.name) validator.check("min shape", min_shape, "max shape", max_shape, Rel.EQ, self.name) validator.check_integer( "min shape", min_shape[0], x_shape[self.channel_axis], Rel.EQ, self.name) diff --git a/mindspore/ops/operations/array_ops.py b/mindspore/ops/operations/array_ops.py index 72533791c6..7b7e8b2b64 100644 --- a/mindspore/ops/operations/array_ops.py +++ b/mindspore/ops/operations/array_ops.py @@ -37,6 +37,40 @@ from ..._c_expression import signature_kind as sig_kind from ..._c_expression import signature_dtype as sig_dtype from ..._c_expression import typing + +class _ScatterOp(PrimitiveWithInfer): + """ + Define Scatter operators + """ + __mindspore_signature__ = ( + ('x', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('indices', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T1), + ('updates', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T) + ) + @staticmethod + def _check_scatter_shape(x_shape, indices_shape, updates_shape, prim_name): + if updates_shape and updates_shape != indices_shape + x_shape[1:]: + raise ValueError(f"For '{prim_name}', the shape of updates should be [] or " + f"updates_shape = indices_shape + x_shape[1:], but got x_shape: {x_shape}, " + f"indices_shape: {indices_shape}, updates_shape: {updates_shape}.") + + @prim_attr_register + def __init__(self, use_locking=False): + """Init _ScatterOp""" + validator.check_value_type('use_locking', use_locking, [bool], self.name) + self.init_prim_io_names(inputs=['x', 'indices', 'updates'], outputs=['y']) + + def infer_shape(self, x_shape, indices_shape, updates_shape): + _ScatterOp._check_scatter_shape(x_shape, indices_shape, updates_shape, self.name) + return x_shape + + def infer_dtype(self, x_dtype, indices_dtype, updates_dtype): + validator.check_tensor_type_same({'indices': indices_dtype}, [mstype.int32], self.name) + args = {"x": x_dtype, "updates": updates_dtype} + validator.check_tensor_type_same(args, mstype.number_type, self.name) + return x_dtype + + def _check_infer_attr_reduce(axis, keep_dims, prim_name): validator.check_value_type('keep_dims', keep_dims, [bool], prim_name) validator.check_value_type('axis', axis, [int, tuple], prim_name) @@ -193,7 +227,7 @@ class Cast(PrimitiveWithInfer): self.init_prim_io_names(inputs=['x', 'dst_type'], outputs=['output']) def check_elim(self, x, dtype): - if isinstance(x, (Tensor, numbers.Number, Parameter)): + if isinstance(x, (Tensor, numbers.Number, Parameter)): if isinstance(x, Tensor) and x.dtype == dtype: return (True, x) if isinstance(x, numbers.Number): @@ -564,7 +598,7 @@ class SparseGatherV2(GatherV2): >>> input_params = Tensor(np.array([[1, 2, 7, 42], [3, 4, 54, 22], [2, 2, 55, 3]]), mindspore.float32) >>> input_indices = Tensor(np.array([1, 2]), mindspore.int32) >>> axis = 1 - >>> out = P.GatherV2()(input_params, input_indices, axis) + >>> out = P.SparseGatherV2()(input_params, input_indices, axis) """ class EmbeddingLookup(PrimitiveWithInfer): @@ -997,11 +1031,10 @@ class InvertPermutation(PrimitiveWithInfer): values can not be negative. Inputs: - - **input_x** (Union(tuple[int], Tensor[int])) - The input tuple is constructed by multiple + - **input_x** (Union(tuple[int]) - The input tuple is constructed by multiple integers, i.e., :math:`(y_1, y_2, ..., y_S)` representing the indices. The values must include 0. There can be no duplicate values or negative values. - If the input is Tensor, it must be 1-d and the dtype is int. Only constant value is allowed. - + Only constant value is allowed. Outputs: tuple[int]. the lenth is same as input. @@ -1016,6 +1049,7 @@ class InvertPermutation(PrimitiveWithInfer): @prim_attr_register def __init__(self): """init InvertPermutation""" + self.const_value = True def __infer__(self, x): x_shp = x['shape'] @@ -1024,17 +1058,15 @@ class InvertPermutation(PrimitiveWithInfer): raise ValueError(f'For \'{self.name}\' the input value must be const.') validator.check_value_type("shape", x_shp, [tuple, list], self.name) if mstype.issubclass_(x['dtype'], mstype.tensor): - validator.check('x dimension', len(x_shp), '', 1, Rel.EQ, self.name) - validator.check_tensor_type_same({'x dtype': x['dtype']}, mstype.int_type, self.name) - x_value = [int(i) for i in x_value.asnumpy()] + raise ValueError(f'For \'{self.name}\' the input value must be non-Tensor.') z = [x_value[i] for i in range(len(x_value))] z.sort() for i in range(1, len(z)): - if z[i-1] == z[i]: + if z[i - 1] == z[i]: raise ValueError(f"For {self.name}, {z[i]} is duplicated in the input.") validator.check(f'value min', min(x_value), '', 0, Rel.EQ, self.name) - validator.check(f'value max', max(x_value), '', len(x_value)-1, Rel.EQ, self.name) + validator.check(f'value max', max(x_value), '', len(x_value) - 1, Rel.EQ, self.name) y = [None] * len(x_value) for i, value in enumerate(x_value): @@ -1168,7 +1200,7 @@ class ArgMaxWithValue(PrimitiveWithInfer): - output_x (Tensor) - The maximum value of input tensor, the shape same as index. Examples: - >>> input_x = Tensor(np.random.rand(5)) + >>> input_x = Tensor(np.random.rand(5), mindspore.float32) >>> index, output = P.ArgMaxWithValue()(input_x) """ @@ -1381,11 +1413,9 @@ class UnsortedSegmentMin(PrimitiveWithInfer): """ Computes the minimum along segments of a tensor. - If the given segment_ids is negative, the value will be ignored. - Inputs: - **input_x** (Tensor) - The shape is :math:`(x_1, x_2, ..., x_R)`. - - **segment_ids** (Tensor) - A `1-D` tensor whose shape is :math:`(x_1)`. + - **segment_ids** (Tensor) - A `1-D` tensor whose shape is :math:`(x_1)`, the value should be >= 0. - **num_segments** (int) - The value spcifies the number of distinct `segment_ids`. Outputs: @@ -1481,7 +1511,7 @@ class Concat(PrimitiveWithInfer): def _get_pack_shape(x_shape, x_type, axis, prim_name): """for pack output shape""" validator.check_value_type("shape", x_shape, [tuple, list], prim_name) - validator.check_integer("len of input_x", len(x_shape), 1, Rel.GT, prim_name) + validator.check_integer("len of input_x", len(x_shape), 1, Rel.GE, prim_name) validator.check_subclass("input_x[0]", x_type[0], mstype.tensor, prim_name) rank_base = len(x_shape[0]) N = len(x_shape) @@ -1737,6 +1767,57 @@ class Select(PrimitiveWithInfer): return None +def _compute_slicing_length(begin, end, stride, x_shape, i): + """Compute the length of the slicing.""" + if i >= len(x_shape): + raise ValueError(f"For 'StridedSlice', When their is no new axis, the index length must be less or " + f"equal than the dim of x.") + x_dim = x_shape[i] + if stride > 0: + # When slicing forward, convert begin and end to positive numbers. + if begin >= x_dim or end < -x_dim: + # When slicing forward, if begin >= x_dim or end < -x_dim, the length of the slicing is 0. + slicing_length = 0 + else: + if -x_dim <= begin < 0: + begin += x_dim + if begin < -x_dim: + # When slicing forward, if begin < -x_dim, set begin = 0, which means start from the 0th element. + begin = 0 + if -x_dim <= end < 0: + end += x_dim + if end > x_dim: + # When slicing forward, if end > x_dim, set end = x_dims, which means slice to the last element. + end = x_dim + if begin >= end: + # When slicing forward, if begin >= end, the length of the slicing is 0. + slicing_length = 0 + else: + slicing_length = 1 + (end - 1 - begin) // stride + else: + # When slicing backward, convert begin and end to negative numbers. + if begin < -x_dim or end >= x_dim: + # When slicing backward, if begin < -x_dim or end >= x_dim, the length of the slicing is 0. + slicing_length = 0 + else: + if 0 <= begin < x_dim: + begin += -x_dim + if begin >= x_dim: + # When slicing backward, if begin >= x_dim, set begin = -1, which means start from the last element. + begin = -1 + if 0 < end < x_dim: + end += -x_dim + if end < -x_dim - 1: + # When slicing backward, if end < -x_dim - 1, set end = -x_dim - 1, which means + # slicing to the 0th element. + end = -x_dim - 1 + if begin <= end: + # When slicing backward, if begin <= end, the length of the slicing is 0. + slicing_length = 0 + else: + slicing_length = 1 + (end + 1 - begin) // stride + return slicing_length + class StridedSlice(PrimitiveWithInfer): r""" @@ -1800,13 +1881,15 @@ class StridedSlice(PrimitiveWithInfer): ellipsis_mask=0, new_axis_mask=0, shrink_axis_mask=0): - """init StrideSlice""" + """Init StrideSlice""" self.init_prim_io_names(inputs=['x', 'begin', 'end', 'strides'], outputs=['output']) - validator.check_value_type('begin_mask', begin_mask, [int], self.name) - validator.check_value_type('end_mask', end_mask, [int], self.name) - validator.check_value_type('ellipsis_mask', ellipsis_mask, [int], self.name) - validator.check_value_type('new_axis_mask', new_axis_mask, [int], self.name) - validator.check_value_type('shrink_axis_mask', shrink_axis_mask, [int], self.name) + validator.check_integer('begin_mask', begin_mask, 0, Rel.GE, self.name) + validator.check_integer('end_mask', end_mask, 0, Rel.GE, self.name) + validator.check_integer('ellipsis_mask', ellipsis_mask, 0, Rel.GE, self.name) + if len(tuple(filter(lambda x: x == '1', bin(ellipsis_mask)[-1:1:-1]))) > 1: + raise ValueError(f"For '{self.name}', only support one ellipsis in the index, but got {end_mask}.") + validator.check_integer('new_axis_mask', new_axis_mask, 0, Rel.GE, self.name) + validator.check_integer('shrink_axis_mask', shrink_axis_mask, 0, Rel.GE, self.name) def __infer__(self, x, begin, end, strides): begin_v, end_v, strides_v = begin['value'], end['value'], strides['value'] @@ -1814,58 +1897,103 @@ class StridedSlice(PrimitiveWithInfer): validator.check_value_type("end", end_v, [tuple], self.name) validator.check_value_type("strides", strides_v, [tuple], self.name) - x_shape = x['shape'] - x_shp_len = len(x_shape) - if len(begin_v) != x_shp_len or len(end_v) != x_shp_len or len(strides_v) != x_shp_len: - raise ValueError(f"For \'{self.name}\' the length of begin index{begin_v}, end index{end_v} and " - f"strides{strides_v} must be equal to the dims({x_shp_len}) of input.") + if tuple(filter(lambda x: not isinstance(x, int), begin_v + end_v + strides_v)): + raise ValueError(f"For {self.name}, both the begins, ends, and strides must be a tuple of int, " + f"but got begins: {begin_v}, ends: {end_v}, strides: {strides_v}.") - ret_shape = [] - append_dimensions = [] - shrink_pos = bin(self.shrink_axis_mask)[::-1] - new_pos = bin(self.new_axis_mask)[::-1] - for i in range(x_shp_len): - # After the integer is converted to binary, it is a str and the first two chars are the flag char '0b' - if i < (len(new_pos) - 2) and new_pos[i] == '1': - ret_shape.append(1) - append_dimensions.append(x_shape[x_shp_len - 1 - len(append_dimensions)]) - continue - if i < (len(shrink_pos) - 2) and shrink_pos[i] == '1': - validator.check_integer(f'begin[{i}]', begin_v[i], -x_shape[i], Rel.GE, self.name) - validator.check_integer(f'begin[{i}]', begin_v[i], x_shape[i], Rel.LT, self.name) - continue - - begin_idx = begin_v[i] - end_idx = end_v[i] - strides_idx = strides_v[i] - if self.begin_mask: - begin_idx = 0 - if self.end_mask: - end_idx = x_shape[i] - validator.check_integer(f'begin[{i}]', begin_idx, x_shape[i], Rel.LE, self.name) - validator.check_integer(f'end[{i}]', end_idx, x_shape[i], Rel.LE, self.name) - validator.check_integer(f'strides[{i}]', strides_idx, 0, Rel.NE, self.name) - if strides_idx > 0: - # If sliced forward , end_idx >= begin_idx - validator.check(f'begin[{i}]', begin_idx, f'end[{i}]', end_idx, Rel.LE) - if begin_idx < 0 < end_idx: - # Turn negative begin_idx into positive values - begin_idx = x_shape[i] + begin_idx - num_elems = (end_idx - begin_idx + strides_idx - 1) // strides_idx - else: - # If sliced backwards, end_idx <= begin_idx - validator.check(f'begin[{i}]', begin_idx, f'end[{i}]', end_idx, Rel.GE) - if end_idx < 0 < begin_idx: - # Turn negative end_idx into positive values - end_idx = x_shape[i] + end_idx - num_elems = (end_idx - begin_idx + strides_idx + 1) // strides_idx - - ret_shape.append(num_elems) - if append_dimensions: - ret_shape += append_dimensions[::-1] + if tuple(filter(lambda x: x == 0, strides_v)): + raise ValueError(f"For '{self.name}', the strides cannot contain 0, but got strides: {strides_v}.") + + if len(end_v) != len(begin_v) or len(strides_v) != len(begin_v): + raise ValueError(f"For '{self.name}' the length of begin index: {begin_v}, end index: {end_v} and " + f"strides: {strides_v} must be equal.") + + ret_shape = self._compute_slicing_shape(x['shape'], begin_v, end_v, strides_v) + + value = None if all(ret_shape) else Tensor(np.array([]).reshape(ret_shape), x['dtype'].element_type()) return {'shape': ret_shape, 'dtype': x['dtype'], - 'value': None} + 'value': value} + + def _compute_slicing_shape(self, x_shape, begin_v, end_v, strides_v): + """Compute the shape of the slicing.""" + x_rank = len(x_shape) + slice_len = len(begin_v) + + # After the integer is converted to binary, it is a str and the first two chars are the flag char '0b'. + begin_pos = bin(self.begin_mask)[-1:1:-1] + end_pos = bin(self.end_mask)[-1:1:-1] + ellipsis_pos = bin(self.ellipsis_mask)[-1:1:-1] + new_axis_pos = bin(self.new_axis_mask)[-1:1:-1] + shrink_axis_pos = bin(self.shrink_axis_mask)[-1:1:-1] + + ret_shape = [] + i, j = 0, 0 + has_ellipsis = False + while i < x_rank or j < slice_len: + if j < slice_len: + begin, end, stride = begin_v[j], end_v[j], strides_v[j] + + if j < len(ellipsis_pos) and ellipsis_pos[j] == '1': + # When there is ellipsis, the latter part of the ellipsis will be processed separately. + has_ellipsis = True + break + if j < len(begin_pos) and begin_pos[j] == '1': + begin = -1 if strides_v[j] < 0 else 0 + if j < len(end_pos) and end_pos[j] == '1': + end = -(x_shape[i] + 1) if strides_v[j] < 0 else x_shape[i] + if j < len(new_axis_pos) and new_axis_pos[j] == '1': + ret_shape.append(1) + j += 1 + continue + if j < len(shrink_axis_pos) and shrink_axis_pos[j] == '1': + if (not -x_shape[i] <= begin < x_shape[i]) or stride < 0: + raise ValueError(f"For {self.name}, when shrink axis, the stride cannot be negative number, " + f"and begin should be in [-{x_shape[i]}, {x_shape[i]}), " + f"but got stride: {stride}, begin: {begin}.") + j += 1 + i += 1 + continue + else: + begin, end, stride = 0, x_shape[i], 1 + + slicing_length = _compute_slicing_length(begin, end, stride, x_shape, i) + ret_shape.append(slicing_length) + i += 1 + j += 1 + if has_ellipsis: + # When there is ellipsis, handle the second half of the ellipsis split. + ellipsis_occupied_dims = x_rank - i - (slice_len - (j + 1)) + \ + len(tuple(filter(lambda x: x == '1', new_axis_pos[j + 1:slice_len]))) + ret_shape.extend(x_shape[i:i + ellipsis_occupied_dims]) + j += 1 + i += ellipsis_occupied_dims + + while i < x_rank or j < slice_len: + begin, end, stride = begin_v[j], end_v[j], strides_v[j] + + if j < len(begin_pos) and begin_pos[j] == '1': + begin = -1 if strides_v[j] < 0 else 0 + if j < len(end_pos) and end_pos[j] == '1': + end = -(x_shape[i] + 1) if strides_v[j] < 0 else x_shape[i] + if j < len(new_axis_pos) and new_axis_pos[j] == '1': + ret_shape.append(1) + j += 1 + continue + if j < len(shrink_axis_pos) and shrink_axis_pos[j] == '1': + if (not -x_shape[i] <= begin < x_shape[i]) or stride < 0: + raise ValueError(f"For {self.name}, when shrink axis, the stride cannot be negative number, " + f"and begin should be in [-{x_shape[i]}, {x_shape[i]}), " + f"but got stride: {stride}, begin: {begin}.") + j += 1 + i += 1 + continue + + slicing_length = _compute_slicing_length(begin, end, stride, x_shape, i) + ret_shape.append(slicing_length) + i += 1 + j += 1 + return ret_shape class Diag(PrimitiveWithInfer): @@ -2146,6 +2274,7 @@ class TensorScatterUpdate(PrimitiveWithInfer): >>> op = P.TensorScatterUpdate() >>> output = op(input_x, indices, update) """ + @prim_attr_register def __init__(self): """Init TensorScatterUpdate""" @@ -2165,7 +2294,7 @@ class TensorScatterUpdate(PrimitiveWithInfer): return x_dtype -class ScatterUpdate(PrimitiveWithInfer): +class ScatterUpdate(_ScatterOp): """ Update tensor value by using input indices and value. @@ -2177,8 +2306,8 @@ class ScatterUpdate(PrimitiveWithInfer): Inputs: - **input_x** (Parameter) - The target tensor, with data type of Parameter. - **indices** (Tensor) - The index of input tensor. With int32 data type. - - **update** (Tensor) - The tensor to update the input tensor, has the same type as input, - and update.shape = indices.shape + input_x.shape[1:]. + - **updates** (Tensor) - The tensor to update the input tensor, has the same type as input, + and updates.shape = indices.shape + input_x.shape[1:]. Outputs: Tensor, has the same shape and type as `input_x`. @@ -2187,26 +2316,17 @@ class ScatterUpdate(PrimitiveWithInfer): >>> np_x = np.array([[-0.1, 0.3, 3.6], [0.4, 0.5, -3.2]]) >>> input_x = mindspore.Parameter(Tensor(np_x, mindspore.float32), name="x") >>> indices = Tensor(np.array([[0, 0], [1, 1]]), mindspore.int32) - >>> np_update = np.array([[[1.0, 2.2, 1.0], [2.0, 1.2, 1.0]], [[2.0, 2.2, 1.0], [3.0, 1.2, 1.0]]]) - >>> update = Tensor(np_update, mindspore.float32) + >>> np_updates = np.array([[[1.0, 2.2, 1.0], [2.0, 1.2, 1.0]], [[2.0, 2.2, 1.0], [3.0, 1.2, 1.0]]]) + >>> updates = Tensor(np_updates, mindspore.float32) >>> op = P.ScatterUpdate() - >>> output = op(input_x, indices, update) + >>> output = op(input_x, indices, updates) """ - __mindspore_signature__ = ( - ('x', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), - ('indices', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T1), - ('value', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T) - ) + @prim_attr_register def __init__(self, use_locking=True): """Init ScatterUpdate""" validator.check_value_type('use_locking', use_locking, [bool], self.name) - self.init_prim_io_names(inputs=['x', 'indices', 'value'], outputs=['y']) - - def infer_shape(self, x_shape, indices_shape, value_shape): - if indices_shape + x_shape[1:] != value_shape: - raise ValueError("For 'ScatterUpdate', input value are not match with input indices.") - return x_shape + self.init_prim_io_names(inputs=['x', 'indices', 'updates'], outputs=['y']) def infer_dtype(self, x_dtype, indices_dtype, value_dtype): validator.check_tensor_type_same({'indices': indices_dtype}, [mstype.int32], self.name) @@ -2245,6 +2365,7 @@ class ScatterNdUpdate(PrimitiveWithInfer): ('indices', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T1), ('value', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T) ) + @prim_attr_register def __init__(self, use_locking=True): """Init ScatterNdUpdate""" @@ -2264,14 +2385,8 @@ class ScatterNdUpdate(PrimitiveWithInfer): validator.check_tensor_type_same(args, (mstype.bool_,) + mstype.number_type, self.name) return x_dtype -def _check_scatter_shape(x_shape, indices_shape, updates_shape, prim_name): - if updates_shape and updates_shape != indices_shape + x_shape[1:]: - raise ValueError(f"For '{prim_name}', the shape of updates should be [] or " - f"updates_shape = indices_shape + x_shape[1:], but got x_shape: {x_shape}, " - f"indices_shape: {indices_shape}, updates_shape: {updates_shape}.") - -class ScatterMax(PrimitiveWithInfer): +class ScatterMax(_ScatterOp): """ Update the value of the input tensor through the max operation. @@ -2305,18 +2420,37 @@ class ScatterMax(PrimitiveWithInfer): self.init_prim_io_names(inputs=['x', 'indices', 'updates'], outputs=['y']) validator.check_value_type('use_locking', use_locking, (bool,), self.name) - def infer_shape(self, x_shape, indices_shape, updates_shape): - _check_scatter_shape(x_shape, indices_shape, updates_shape, self.name) - return x_shape - def infer_dtype(self, x_dtype, indices_dtype, updates_dtype): - validator.check_tensor_type_same({'indices': indices_dtype}, (mstype.int32,), self.name) - args = {"x": x_dtype, "updates": updates_dtype} - validator.check_tensor_type_same(args, mstype.number_type, self.name) - return x_dtype +class ScatterMin(_ScatterOp): + """ + Update the value of the input tensor through the min operation. + + Using given values to update tensor value through the min operation, along with the input indices. + This operation outputs the `input_x` after the update is done, which makes it convenient to use the updated value. + + Args: + use_locking (bool): Whether protect the assignment by a lock. Default: False. + + Inputs: + - **input_x** (Parameter) - The target parameter. + - **indices** (Tensor) - The index to do min operation whose data type should be mindspore.int32. + - **updates** (Tensor) - The tensor doing the min operation with `input_x`, + the data type is same as `input_x`, the shape is `indices_shape + x_shape[1:]`. + + Outputs: + Parameter, the updated `input_x`. + + Examples: + >>> input_x = Parameter(Tensor(np.array([[0.0, 1.0, 2.0], [0.0, 0.0, 0.0]]), mindspore.float32), name="input_x") + >>> indices = Tensor(np.array([[0, 0], [1, 1]]), mindspore.int32) + >>> update = Tensor(np.ones([2, 2, 3]), mindspore.float32) + >>> scatter_min = P.ScatterMin() + >>> output = scatter_min(input_x, indices, update) + [[0.0, 1.0, 1.0], [0.0, 0.0, 0.0]] + """ -class ScatterAdd(PrimitiveWithInfer): +class ScatterAdd(_ScatterOp): """ Update the value of the input tensor through the add operation. @@ -2344,20 +2478,92 @@ class ScatterAdd(PrimitiveWithInfer): [[1.0, 1.0, 1.0], [3.0, 3.0, 3.0]] """ - @prim_attr_register - def __init__(self, use_locking=False): - """Init ScatterAdd""" - validator.check_value_type('use_locking', use_locking, (bool,), self.name) - def infer_shape(self, x_shape, indices_shape, updates_shape): - _check_scatter_shape(x_shape, indices_shape, updates_shape, self.name) - return x_shape +class ScatterSub(_ScatterOp): + """ + Update the value of the input tensor through the sub operation. - def infer_dtype(self, x_dtype, indices_dtype, updates_dtype): - validator.check_tensor_type_same({'indices': indices_dtype}, (mstype.int32,), self.name) - args = {'x': x_dtype, 'updates': updates_dtype} - validator.check_tensor_type_same(args, mstype.number_type, self.name) - return x_dtype + Using given values to update tensor value through the sub operation, along with the input indices. + This operation outputs the `input_x` after the update is done, which makes it convenient to use the updated value. + + Args: + use_locking (bool): Whether protect the assignment by a lock. Default: False. + + Inputs: + - **input_x** (Parameter) - The target parameter. + - **indices** (Tensor) - The index to do sub operation whose data type should be mindspore.int32. + - **updates** (Tensor) - The tensor doing the sub operation with `input_x`, + the data type is same as `input_x`, the shape is `indices_shape + x_shape[1:]`. + + Outputs: + Parameter, the updated `input_x`. + + Examples: + >>> input_x = Parameter(Tensor(np.array([[0.0, 0.0, 0.0], [1.0, 1.0, 1.0]]), mindspore.float32), name="x") + >>> indices = Tensor(np.array([[0, 1]]), mindspore.int32) + >>> updates = Tensor(np.array([[1.0, 1.0, 1.0], [2.0, 2.0, 2.0]]), mindspore.float32) + >>> scatter_sub = P.ScatterSub() + >>> output = scatter_sub(input_x, indices, updates) + [[-1.0, -1.0, -1.0], [-1.0, -1.0, -1.0]] + """ + + +class ScatterMul(_ScatterOp): + """ + Update the value of the input tensor through the mul operation. + + Using given values to update tensor value through the mul operation, along with the input indices. + This operation outputs the `input_x` after the update is done, which makes it convenient to use the updated value. + + Args: + use_locking (bool): Whether protect the assignment by a lock. Default: False. + + Inputs: + - **input_x** (Parameter) - The target parameter. + - **indices** (Tensor) - The index to do mul operation whose data type should be mindspore.int32. + - **updates** (Tensor) - The tensor doing the mul operation with `input_x`, + the data type is same as `input_x`, the shape is `indices_shape + x_shape[1:]`. + + Outputs: + Parameter, the updated `input_x`. + + Examples: + >>> input_x = Parameter(Tensor(np.array([[1.0, 1.0, 1.0], [2.0, 2.0, 2.0]]), mindspore.float32), name="x") + >>> indices = Tensor(np.array([0, 1]), mindspore.int32) + >>> updates = Tensor(np.ones([[2.0, 2.0, 2.0], [2.0, 2.0, 2.0]]), mindspore.float32) + >>> scatter_mul = P.ScatterMul() + >>> output = scatter_mul(input_x, indices, updates) + [[2.0, 2.0, 2.0], [4.0, 4.0, 4.0]] + """ + + +class ScatterDiv(_ScatterOp): + """ + Update the value of the input tensor through the div operation. + + Using given values to update tensor value through the div operation, along with the input indices. + This operation outputs the `input_x` after the update is done, which makes it convenient to use the updated value. + + Args: + use_locking (bool): Whether protect the assignment by a lock. Default: False. + + Inputs: + - **input_x** (Parameter) - The target parameter. + - **indices** (Tensor) - The index to do div operation whose data type should be mindspore.int32. + - **updates** (Tensor) - The tensor doing the div operation with `input_x`, + the data type is same as `input_x`, the shape is `indices_shape + x_shape[1:]`. + + Outputs: + Parameter, the updated `input_x`. + + Examples: + >>> input_x = Parameter(Tensor(np.array([[6.0, 6.0, 6.0], [2.0, 2.0, 2.0]]), mindspore.float32), name="x") + >>> indices = Tensor(np.array([0, 1]), mindspore.int32) + >>> updates = Tensor(np.ones([[2.0, 2.0, 2.0], [2.0, 2.0, 2.0]]), mindspore.float32) + >>> scatter_div = P.ScatterDiv() + >>> output = scatter_div(input_x, indices, updates) + [[3.0, 3.0, 3.0], [1.0, 1.0, 1.0]] + """ class SpaceToDepth(PrimitiveWithInfer): @@ -2680,16 +2886,20 @@ class SpaceToBatchND(PrimitiveWithInfer): def infer_shape(self, x_shape): x_rank = len(x_shape) + validator.check_integer('x_shape rank', x_rank, 4, Rel.EQ, self.name) out_shape = copy.deepcopy(x_shape) block_shape_prod = 1 - for i in range(x_rank - 2): - padded = out_shape[i + 2] + self.paddings[i][0] + \ + offset = 2 + if x_rank < 4: + offset = 1 + for i in range(len(self.block_shape)): + padded = out_shape[i + offset] + self.paddings[i][0] + \ self.paddings[i][1] if padded % self.block_shape[i] != 0: raise ValueError(f'For \'{self.name}\' padded[{i}] {padded} should be divisible by ' f'block_shape[{i}] {self.block_shape[i]}') - out_shape[i + 2] = padded // self.block_shape[i] + out_shape[i + offset] = padded // self.block_shape[i] block_shape_prod = block_shape_prod * self.block_shape[i] out_shape[0] *= block_shape_prod return out_shape @@ -2760,15 +2970,19 @@ class BatchToSpaceND(PrimitiveWithInfer): def infer_shape(self, x_shape): x_rank = len(x_shape) + validator.check_integer('x_shape rank', x_rank, 4, Rel.EQ, self.name) out_shape = copy.deepcopy(x_shape) block_shape_prod = 1 - for i in range(x_rank - 2): + offset = 2 + if x_rank < 4: + offset = 1 + for i in range(len(self.block_shape)): block_shape_prod = block_shape_prod * self.block_shape[i] - x_block_prod = out_shape[i + 2] * self.block_shape[i] + x_block_prod = out_shape[i + offset] * self.block_shape[i] crops_sum = self.crops[i][0] + self.crops[i][1] validator.check("x block shape prod", x_block_prod, 'crops sum', crops_sum, Rel.GT, self.name) - out_shape[i + 2] = x_block_prod - crops_sum + out_shape[i + offset] = x_block_prod - crops_sum if out_shape[0] % block_shape_prod != 0: raise ValueError(f'For \'{self.name}\' input_x dimension 0 {out_shape[0]} should be divisible by ' @@ -2859,36 +3073,34 @@ class InplaceUpdate(PrimitiveWithInfer): [ 4. 5.] [ 6. 7.]]] """ + @prim_attr_register def __init__(self, indices): """Init InplaceUpdate""" - self.init_prim_io_names(inputs=['x', 'indices', 'v'], outputs=['y']) + self.init_prim_io_names(inputs=['x', 'v'], outputs=['y']) + self.indices = indices validator.check_value_type("indices", indices, [int, tuple], self.name) if isinstance(indices, int): - self.add_prim_attr('indices', (indices,)) + self.indices = (indices,) for item in self.indices: validator.check_value_type("item of indices", item, [int], self.name) def infer_dtype(self, x_dtype, v_dtype): + args = {'x': x_dtype, 'v': v_dtype} valid_type = [mstype.int32, mstype.float16, mstype.float32] - validator.check_tensor_type_same( - { - "x": x_dtype, - "v": v_dtype - }, valid_type, self.name) - + validator.check_tensor_type_same(args, valid_type, self.name) return x_dtype def infer_shape(self, x_shape, v_shape): validator.check("x", len(x_shape), "v", len(v_shape), Rel.EQ, self.name) - - x_rank = len(x_shape) - for idx in range(x_rank)[1:]: - validator.check("x dim %d" % idx, x_shape[idx], 'v dim %d' % idx, v_shape[idx], Rel.EQ, self.name) - validator.check("size of indices", len(self.indices), "v's first dimension", v_shape[0], Rel.EQ, self.name) - + for i in self.indices: + if i < 0 or i >= x_shape[0]: + raise ValueError(f'The value of indices must be in [0, {x_shape[0]}), but got {i}.') + x_rank = len(x_shape) + for idx in range(x_rank)[1:]: + validator.check('v dim %d' % idx, v_shape[idx], "x dim %d" % idx, x_shape[idx], Rel.EQ, self.name) return x_shape @@ -2939,3 +3151,28 @@ class ReverseSequence(PrimitiveWithInfer): validator.check_tensor_type_same({"x_dtype": x}, mstype.number_type + (mstype.bool_,), self.name) validator.check_tensor_type_same({"seq_lengths_dtype": seq_lengths}, [mstype.int32, mstype.int64], self.name) return x + + +class TransShape(PrimitiveWithInfer): + """ + Transform the shape of input tensor to target shape. + + Inputs: + - **input_x** (Tensor) - A input tensor. + - **out_shape** (tuple[int]) - The shape of output data. + + Outputs: + Tensor, a tensor whose data type is same as 'input_x', and the shape is same as the `out_shape`. + """ + @prim_attr_register + def __init__(self): + self.__setattr_flag__ = True + + def __infer__(self, x, shape): + shp = shape['value'] + dtype = x['dtype'] + validator.check_tensor_type_same({'x': dtype}, mstype.number_type + (mstype.bool_,), self.name) + self.add_prim_attr('out_shape', tuple(shp)) + return {'shape': shp, + 'dtype': dtype, + 'value': None} diff --git a/mindspore/ops/operations/comm_ops.py b/mindspore/ops/operations/comm_ops.py index dc690b5f6e..f8b47a28c3 100644 --- a/mindspore/ops/operations/comm_ops.py +++ b/mindspore/ops/operations/comm_ops.py @@ -98,6 +98,7 @@ class AllReduce(PrimitiveWithInfer): self.op = op self.add_prim_attr('group', _get_group(group)) self.add_prim_attr('fusion', 0) + self.add_prim_attr('index', 0) def vm_impl(self, x): """Implement by vm mode.""" @@ -176,13 +177,13 @@ class AllGather(PrimitiveWithInfer): raise NotImplementedError -class HostAllGather(PrimitiveWithInfer): +class _HostAllGather(PrimitiveWithInfer): """ Gathers tensors from the specified communication group on host. Note: Tensor must have the same shape and format in all processes participating in the collective. - HostAllGather is a host-side operator, it depends on OpenMPI and must use build option -M on + _HostAllGather is a host-side operator, it depends on OpenMPI and must use build option -M on to enable it. Using mpirun command to run it: mpirun -output-filename log -merge-stderr-to-stdout -np 3 python test_host_all_gather.py @@ -199,27 +200,6 @@ class HostAllGather(PrimitiveWithInfer): Outputs: Tensor. If the number of devices in the group is N, then the shape of output is :math:`(N, x_1, x_2, ..., x_R)`. - - Examples: - >>> import mindspore.nn as nn - >>> import mindspore.context as context - >>> import mindspore.ops.operations as P - >>> from mindspore import Tensor - >>> - >>> context.set_context(mode=context.GRAPH_MODE, device_target='CPU') - >>> context.set_mpi_config(enable_mpi=True) - >>> - >>> class Net(nn.Cell): - >>> def __init__(self): - >>> super(Net, self).__init__() - >>> self.hostallgather = P.HostAllGather(group=(0, 1, 2, 3)) - >>> - >>> def construct(self, x): - >>> return self.hostallgather(x) - >>> - >>> input_ = Tensor(np.ones([2, 8]).astype(np.float32)) - >>> net = Net() - >>> output = net(input_) """ @prim_attr_register @@ -308,13 +288,13 @@ class ReduceScatter(PrimitiveWithInfer): raise NotImplementedError -class HostReduceScatter(PrimitiveWithInfer): +class _HostReduceScatter(PrimitiveWithInfer): """ Reduces and scatters tensors from the specified communication group on host. Note: Tensor must have the same shape and format in all processes participating in the collective. - HostReduceScatter is a host-side operator, it depends on OpenMPI and must use build option + _HostReduceScatter is a host-side operator, it depends on OpenMPI and must use build option -M on to enable it. Using mpirun command to run it: mpirun -output-filename log -merge-stderr-to-stdout -np 3 python test_host_reduce_scatter.py @@ -328,28 +308,6 @@ class HostReduceScatter(PrimitiveWithInfer): or elements of group are not int. ValueError: If the first dimension of input can not be divided by group size, or group is not set, or rank_id not in [0, 7]. - - Examples: - >>> import mindspore.nn as nn - >>> import mindspore.context as context - >>> import mindspore.ops.operations as P - >>> from mindspore import Tensor - >>> from mindspore.ops.operations.comm_ops import ReduceOp - >>> - >>> context.set_context(mode=context.GRAPH_MODE, device_target='CPU') - >>> context.set_mpi_config(enable_mpi=True) - >>> - >>> class Net(nn.Cell): - >>> def __init__(self): - >>> super(Net, self).__init__() - >>> self.hostreducescatter = P.HostReduceScatter(ReduceOp.SUM, group=[0, 1, 2, 3]) - >>> - >>> def construct(self, x): - >>> return self.hostreducescatter(x) - >>> - >>> input_ = Tensor(np.ones([8, 8]).astype(np.float32)) - >>> net = Net() - >>> output = net(input_) """ @prim_attr_register def __init__(self, op=ReduceOp.SUM, group=None): diff --git a/mindspore/ops/operations/control_ops.py b/mindspore/ops/operations/control_ops.py index e7ac4572ce..7ef6da2e91 100644 --- a/mindspore/ops/operations/control_ops.py +++ b/mindspore/ops/operations/control_ops.py @@ -30,6 +30,8 @@ class ControlDepend(Primitive): tells the engine that the destination operations should depend on the source operation which means the source operations should be executed before the destination. + Note: + This operation does not work in `PYNATIVE_MODE`. Args: depend_mode (int): Use 0 for normal depend, 1 for depend on operations that used the parameter. Default: 0. @@ -47,8 +49,6 @@ class ControlDepend(Primitive): Bool. This operation has no actual data output, it will be used to setup the order of relative operations. Examples: - >>> # In the following example, the data calculation uses original global_step. After the calculation the global - >>> # step should be increased, so the add operation should depend on the data calculation operation. >>> class Net(nn.Cell): >>> def __init__(self): >>> super(Net, self).__init__() diff --git a/mindspore/ops/operations/debug_ops.py b/mindspore/ops/operations/debug_ops.py index c6b635a69f..066791d4df 100644 --- a/mindspore/ops/operations/debug_ops.py +++ b/mindspore/ops/operations/debug_ops.py @@ -17,7 +17,7 @@ from types import FunctionType, MethodType from ..._checkparam import Validator as validator from ...common import dtype as mstype -from ..primitive import prim_attr_register, PrimitiveWithInfer +from ..primitive import prim_attr_register, PrimitiveWithInfer, Primitive def _check_summary_param(name, value, class_name): @@ -32,6 +32,13 @@ def _check_summary_param(name, value, class_name): validator.check_value_type('value', v_type, [type(mstype.tensor)], class_name) +# Note: The return value of the summary operator is not used, +# so there's nothing special about the return `dtype` or `shape`, any value is ok. +# The `value` should be set to None, else summary operators may be optimized at compile graph phase, +# it cause summary operators can not record data in constant folding scene. +SUMMARY_RETURN_VALUE = {'dtype': mstype.int32, 'shape': [1], 'value': None} + + class ScalarSummary(PrimitiveWithInfer): """ Output scalar to protocol buffer through scalar summary operator. @@ -67,7 +74,7 @@ class ScalarSummary(PrimitiveWithInfer): raise ValueError(f"For 'value' the type should be scalar, " f"shape should be [] or [1] in {self.__class__.__name__}, but got {v_shape}.") - return value + return SUMMARY_RETURN_VALUE class ImageSummary(PrimitiveWithInfer): @@ -104,7 +111,7 @@ class ImageSummary(PrimitiveWithInfer): raise ValueError(f"For 'value' the dim should be {image_dim} in {self.__class__.__name__}," f" but got {len(v_shape)}.") - return value + return SUMMARY_RETURN_VALUE class TensorSummary(PrimitiveWithInfer): @@ -142,7 +149,7 @@ class TensorSummary(PrimitiveWithInfer): raise ValueError(f"For 'value' the type should be tensor in {self.__class__.__name__}, " f"shape should not be [].") - return value + return SUMMARY_RETURN_VALUE class HistogramSummary(PrimitiveWithInfer): @@ -180,7 +187,7 @@ class HistogramSummary(PrimitiveWithInfer): raise ValueError(f"For 'value' the type should be tensor in {self.__class__.__name__}, " f"shape should not be [].") - return value + return SUMMARY_RETURN_VALUE class InsertGradientOf(PrimitiveWithInfer): @@ -302,12 +309,6 @@ class Print(PrimitiveWithInfer): Output tensor or string to stdout. Note: - The print operation cannot support the following cases currently. - - 1. The type of tensor is float64 or bool. - - 2. The data of tensor is a scalar type. - In pynative mode, please use python print function. Inputs: @@ -327,7 +328,7 @@ class Print(PrimitiveWithInfer): @prim_attr_register def __init__(self): - pass + self.add_prim_attr("_side_effect", True) def __call__(self, *args): for arg in args: @@ -340,3 +341,29 @@ class Print(PrimitiveWithInfer): for dtype in inputs: validator.check_subclass("input", dtype, (mstype.tensor, mstype.string), self.name) return mstype.int32 + + +class Debug(Primitive): + """ + Print tensor value. + + Inputs: + - **value** (Tensor) - The value of tensor. + + Examples: + >>> class DebugNN(nn.Cell): + >>> def __init__(self,): + >>> self.debug = nn.Debug() + >>> + >>> def construct(self, x, y): + >>> x = self.add(x, y) + >>> self.debug(x) + >>> return x + """ + + @prim_attr_register + def __init__(self): + """init""" + + def __call__(self, *args, **kwargs): + pass diff --git a/mindspore/ops/operations/image_ops.py b/mindspore/ops/operations/image_ops.py index 68dae34530..1e366b5ea6 100644 --- a/mindspore/ops/operations/image_ops.py +++ b/mindspore/ops/operations/image_ops.py @@ -34,21 +34,20 @@ class CropAndResize(PrimitiveWithInfer): Inputs: - **x** (Tensor) - The input image must be a 4-D tensor of shape [batch, image_height, image_width, depth]. - Types allowed: int8, int16, int32, int64, float16, float32, float64, uint8, uint16. + Types allowed: int8, int16, int32, int64, float16, float32, float64, uint8, uint16. - **boxes** (Tensor) - A 2-D tensor of shape [num_boxes, 4]. - The i-th row of the tensor specifies the coordinates of a box in the box_ind[i] image - and is specified in normalized coordinates [y1, x1, y2, x2]. A normalized coordinate value of y is mapped to - the image coordinate at y * (image_height - 1), so as the [0, 1] interval of normalized image height is - mapped to [0, image_height - 1] in image height coordinates. We do allow y1 > y2, in which case the sampled - crop is an up-down flipped version of the original image. The width dimension is treated similarly. - Normalized coordinates outside the [0, 1] range are allowed, in which case we use extrapolation_value to - extrapolate the input image values. Types allowd: float32. + The i-th row of the tensor specifies the coordinates of a box in the box_ind[i] image + and is specified in normalized coordinates [y1, x1, y2, x2]. A normalized coordinate value of y is mapped to + the image coordinate at y * (image_height - 1), so as the [0, 1] interval of normalized image height is + mapped to [0, image_height - 1] in image height coordinates. We do allow y1 > y2, in which case the sampled + crop is an up-down flipped version of the original image. The width dimension is treated similarly. + Normalized coordinates outside the [0, 1] range are allowed, in which case we use extrapolation_value to + extrapolate the input image values. Types allowd: float32. - **box_index** (Tensor) - A 1-D tensor of shape [num_boxes] with int32 values in [0, batch). - The value of box_ind[i] specifies the image that the i-th box refers to. Types allowd: int32. - - **crop_size** (Tensor) - Only constant value is allowd. Types allowed: int32. - A 1-D tensor of 2 elements, size = [crop_height, crop_width]. - All cropped image patches are resized to this size. The aspect ratio of the image content is not preserved. - Both crop_height and crop_width need to be positive. + The value of box_ind[i] specifies the image that the i-th box refers to. Types allowd: int32. + - **crop_size** (Tuple[int]) - A tuple of two int32 elements: (crop_height, crop_width). + Only constant value is allowed. All cropped image patches are resized to this size. + The aspect ratio of the image content is not preserved. Both crop_height and crop_width need to be positive. Outputs: A 4-D tensor of shape [num_boxes, crop_height, crop_width, depth] with type: float32. @@ -68,10 +67,10 @@ class CropAndResize(PrimitiveWithInfer): >>> IMAGE_WIDTH = 256 >>> CHANNELS = 3 >>> image = np.random.normal(size=[BATCH_SIZE, IMAGE_HEIGHT, IMAGE_WIDTH, CHANNELS]).astype(np.float32) - >>> boxes = np.random.uniform(shape=[NUM_BOXES, 4]).astype(np.float32) - >>> box_index = np.random.uniform(shape=[NUM_BOXES], low=0, high=BATCH_SIZE).astype(np.int32) - >>> crop_size = np.array([24, 24]).astype(np.int32) - >>> crop_and_resize = CropAndResizeNet(crop_size=Tensor(crop_size)) + >>> boxes = np.random.uniform(size=[NUM_BOXES, 4]).astype(np.float32) + >>> box_index = np.random.uniform(size=[NUM_BOXES], low=0, high=BATCH_SIZE).astype(np.int32) + >>> crop_size = (24, 24) + >>> crop_and_resize = CropAndResizeNet(crop_size=crop_size) >>> output = crop_and_resize(Tensor(image), Tensor(boxes), Tensor(box_index)) >>> print(output.asnumpy()) """ @@ -91,11 +90,10 @@ class CropAndResize(PrimitiveWithInfer): x_shape = list(x['shape']) boxes_shape = list(boxes['shape']) box_index_shape = list(box_index['shape']) - crop_size_shape = list(crop_size['shape']) # get value if crop_size['value'] is None: - raise ValueError(f"For {self.name}, crop_size must be const.") - crop_size_value = crop_size['value'].asnumpy() + raise ValueError(f"For {self.name}, crop_size must be constant.") + crop_size_value = crop_size['value'] # get dtype x_dtype = x['dtype'] boxes_dtype = boxes['dtype'] @@ -107,15 +105,20 @@ class CropAndResize(PrimitiveWithInfer): mstype.float32, mstype.float64, mstype.uint8, mstype.uint16], self.name) validator.check_tensor_type_same({"boxes": boxes_dtype}, [mstype.float32], self.name) validator.check_tensor_type_same({"box_index": box_index_dtype}, [mstype.int32], self.name) - validator.check_tensor_type_same({"crop_size": crop_size_dtype}, [mstype.int32], self.name) + validator.check_value_type("crop_size", crop_size_value, [tuple], self.name) # check input shape rank validator.check("x rank", len(x_shape), "expected", 4, Rel.EQ, self.name) validator.check("boxes rank", len(boxes_shape), "expected", 2, Rel.EQ, self.name) validator.check("box_index rank", len(box_index_shape), "expected", 1, Rel.EQ, self.name) - validator.check("crop_size rank", len(crop_size_shape), "expected", 1, Rel.EQ, self.name) - + validator.check("crop_size size", len(crop_size_value), "expected", 2, Rel.EQ, self.name) validator.check("boxes dim_0", boxes_shape[0], "box_index dim_0", box_index_shape[0], Rel.EQ, self.name) validator.check("boxes dim_1", boxes_shape[1], "expected", 4, Rel.EQ, self.name) + # check crop_size_value + validator.check("crop_height", crop_size_value[0], "minimum", 0, Rel.GT, self.name) + validator.check("crop_width", crop_size_value[1], "minimum", 0, Rel.GT, self.name) + # check crop_size element type + validator.check("crop_height dtype", crop_size_dtype[0], mstype.int32, self.name) + validator.check("crop_width dtype", crop_size_dtype[1], mstype.int32, self.name) num_boxes = boxes_shape[0] crop_height = crop_size_value[0] diff --git a/mindspore/ops/operations/math_ops.py b/mindspore/ops/operations/math_ops.py index 08cd481582..b4a684d2f7 100644 --- a/mindspore/ops/operations/math_ops.py +++ b/mindspore/ops/operations/math_ops.py @@ -888,7 +888,8 @@ class Neg(PrimitiveWithInfer): def infer_value(self, input_x): if input_x is not None: input_x = input_x.asnumpy() - return Tensor(-input_x) + out = np.array(-input_x, input_x.dtype) + return Tensor(out) return None @@ -925,32 +926,31 @@ class InplaceAdd(PrimitiveWithInfer): """init InplaceAdd""" self.init_prim_io_names(inputs=['x', 'v'], outputs=['y']) self.indices = indices - - def infer_shape(self, x_shape, v_shape): - validator.check("x", len(x_shape), "v", len(v_shape), Rel.EQ, self.name) - if isinstance(self.indices, int): - validator.check("size of indices", 1, "v's first dimension", v_shape[0], - Rel.EQ, self.name) - if self.indices < 0 or self.indices >= x_shape[0]: - raise ValueError(f'The value of indices must be in [0, {x_shape[0]}), but got {self.indices}.') - else: - validator.check("size of indices", len(self.indices), "v's first dimension", v_shape[0], - Rel.EQ, self.name) - for i in self.indices: - if i < 0 or i >= x_shape[0]: - raise ValueError(f'The value of indices must be in [0, {x_shape[0]}), but got {i}.') - if len(x_shape) > 1: - validator.check("x's ith dimension", x_shape[1:], "v's ith dimension", v_shape[1:], - Rel.EQ, self.name) - return x_shape + validator.check_value_type('indices', indices, [tuple, int], self.name) + if isinstance(indices, int): + self.indices = (indices,) + for item in self.indices: + validator.check_value_type("item of indices", item, [int], self.name) def infer_dtype(self, x_dtype, v_dtype): args = {'x': x_dtype, 'v': v_dtype} valid_type = [mstype.int32, mstype.float16, mstype.float32] validator.check_tensor_type_same(args, valid_type, self.name) - validator.check_value_type('indices', self.indices, [tuple, int], self.name) return x_dtype + def infer_shape(self, x_shape, v_shape): + validator.check("x", len(x_shape), "v", len(v_shape), Rel.EQ, self.name) + validator.check("size of indices", len(self.indices), "v's first dimension", v_shape[0], + Rel.EQ, self.name) + for i in self.indices: + if i < 0 or i >= x_shape[0]: + raise ValueError(f'The value of indices must be in [0, {x_shape[0]}), but got {i}.') + x_rank = len(x_shape) + for idx in range(x_rank)[1:]: + validator.check('v dim %d' % idx, v_shape[idx], "x dim %d" % idx, x_shape[idx], Rel.EQ, self.name) + + return x_shape + class InplaceSub(PrimitiveWithInfer): """ @@ -984,32 +984,31 @@ class InplaceSub(PrimitiveWithInfer): """init InplaceSub""" self.init_prim_io_names(inputs=['x', 'v'], outputs=['y']) self.indices = indices - - def infer_shape(self, x_shape, v_shape): - validator.check("x", len(x_shape), "v", len(v_shape), Rel.EQ, self.name) - if isinstance(self.indices, int): - validator.check("size of indices", 1, "v's first dimension", v_shape[0], - Rel.EQ, self.name) - if self.indices < 0 or self.indices >= x_shape[0]: - raise ValueError(f'The value of indices must be in [0, {x_shape[0]}), but got {self.indices}.') - else: - validator.check("size of indices", len(self.indices), "v's first dimension", v_shape[0], - Rel.EQ, self.name) - for i in self.indices: - if i < 0 or i >= x_shape[0]: - raise ValueError(f'The value of indices must be in [0, {x_shape[0]}), but got {i}.') - if len(x_shape) > 1: - validator.check("x's ith dimension", x_shape[1:], "v's ith dimension", v_shape[1:], - Rel.EQ, self.name) - return x_shape + validator.check_value_type('indices', indices, [tuple, int], self.name) + if isinstance(indices, int): + self.indices = (indices,) + for item in self.indices: + validator.check_value_type("item of indices", item, [int], self.name) def infer_dtype(self, x_dtype, v_dtype): args = {'x': x_dtype, 'v': v_dtype} valid_type = [mstype.int32, mstype.float16, mstype.float32] validator.check_tensor_type_same(args, valid_type, self.name) - validator.check_value_type('indices', self.indices, [tuple, int], self.name) return x_dtype + def infer_shape(self, x_shape, v_shape): + validator.check("x", len(x_shape), "v", len(v_shape), Rel.EQ, self.name) + validator.check("size of indices", len(self.indices), "v's first dimension", v_shape[0], + Rel.EQ, self.name) + for i in self.indices: + if i < 0 or i >= x_shape[0]: + raise ValueError(f'The value of indices must be in [0, {x_shape[0]}), but got {i}.') + x_rank = len(x_shape) + for idx in range(x_rank)[1:]: + validator.check('v dim %d' % idx, v_shape[idx], "x dim %d" % idx, x_shape[idx], Rel.EQ, self.name) + + return x_shape + class Sub(_MathBinaryOp): """ @@ -1357,12 +1356,12 @@ class HistogramFixedWidth(PrimitiveWithInfer): Args: dtype (string): An optional attribute. Must be one of the following types: "int32", "int64". Default: "int32". - nbins (Tensor): Number of histogram bins, the type is int32. + nbins (int): Number of histogram bins, the type is positive integer. Inputs: - **x** (Tensor) - Numeric Tensor. Must be one of the following types: int32, float32, float16. - **range** (Tensor) - Must have the same type as x. Shape [2] Tensor of same dtype as x. - x <= range[0] will be mapped to hist[0], x >= range[1] will be mapped to hist[-1]. + x <= range[0] will be mapped to hist[0], x >= range[1] will be mapped to hist[-1]. Outputs: Tensor, the type is int32. @@ -1378,6 +1377,7 @@ class HistogramFixedWidth(PrimitiveWithInfer): @prim_attr_register def __init__(self, nbins, dtype='int32'): self.nbins = validator.check_value_type("nbins", nbins, [int], self.name) + validator.check_integer("nbins", nbins, 1, Rel.GE, self.name) valid_values = ['int32', 'int64'] self.dtype = validator.check_string("dtype", dtype, valid_values, self.name) self.init_prim_io_names(inputs=['x', 'range'], outputs=['y']) @@ -1646,8 +1646,9 @@ class Div(_MathBinaryOp): Inputs: - **input_x** (Union[Tensor, Number, bool]) - The first input is a number or a bool or a tensor whose data type is number or bool. - - **input_y** (Union[Tensor, Number, bool]) - The second input is a number or - a bool when the first input is a tensor or a tensor whose data type is number or bool. + - **input_y** (Union[Tensor, Number, bool]) - When the first input is a tensor, The second input + could be a number or a bool, or a tensor whose data type is number or bool. When the first input + is a number or a bool, the second input should be a tensor whose data type is number or bool. Outputs: Tensor, the shape is same as the shape after broadcasting, @@ -1667,7 +1668,8 @@ class Div(_MathBinaryOp): if x is not None and y is not None: x = x.asnumpy() y = y.asnumpy() - return Tensor(x / y) + out = np.array(x / y, x.dtype) + return Tensor(out) return None @@ -1742,6 +1744,42 @@ class FloorDiv(_MathBinaryOp): """ +class Mod(_MathBinaryOp): + """ + Computes the remainder of dividing the first input tensor by the second input tensor element-wise. + + The inputs must be two tensors or one tensor and one scalar. When the inputs are two tensors, + both dtypes cannot be bool, and the shapes of them could be broadcast. When the inputs are one tensor + and one scalar, the scalar only could be a constant. + + Inputs: + - **input_x** (Union[Tensor, Number]) - The first input is a number or a tensor whose data type is number. + - **input_y** (Union[Tensor, Number]) - When the first input is a tensor, The second input + could be a number or a tensor whose data type is number. When the first input is a number, + the second input should be a tensor whose data type is number. + + Outputs: + Tensor, the shape is same as the shape after broadcasting, + and the data type is the one with high precision or high digits among the two inputs. + + Raises: + ValueError: When `input_x` and `input_y` are not the same dtype. + + Examples: + >>> input_x = Tensor(np.array([-4.0, 5.0, 6.0]), mindspore.float32) + >>> input_y = Tensor(np.array([3.0, 2.0, 3.0]), mindspore.float32) + >>> mod = P.Mod() + >>> mod(input_x, input_y) + """ + + def infer_value(self, x, y): + if x is not None and y is not None: + x = x.asnumpy() + y = y.asnumpy() + return Tensor(np.fmod(x, y)) + return None + + class Floor(PrimitiveWithInfer): """ Round a tensor down to the closest integer element-wise. @@ -1888,7 +1926,7 @@ class Cosh(PrimitiveWithInfer): class Asinh(PrimitiveWithInfer): """ - Compute inverse hyperbolic cosine of x element-wise. + Compute inverse hyperbolic sine of x element-wise. Inputs: - **input_x** (Tensor) - The shape of tensor is :math:`(x_1, x_2, ..., x_R)`. @@ -2645,7 +2683,7 @@ class Sin(PrimitiveWithInfer): class Asin(PrimitiveWithInfer): """ - Computes arccosine of input element-wise. + Computes arcsine of input element-wise. Inputs: - **input_x** (Tensor) - The shape of tensor is :math:`(x_1, x_2, ..., x_R)`. diff --git a/mindspore/ops/operations/nn_ops.py b/mindspore/ops/operations/nn_ops.py index 9ff20fecc0..35860b314e 100644 --- a/mindspore/ops/operations/nn_ops.py +++ b/mindspore/ops/operations/nn_ops.py @@ -224,6 +224,41 @@ class Softplus(PrimitiveWithInfer): return input_x +class Softsign(PrimitiveWithInfer): + r""" + Softsign activation function. + + The function is shown as follows: + + .. math:: + \text{output} = \frac{\text{input_x}}{1 + \abs{\text{input_x}}}, + + Inputs: + - **input_x** (Tensor) - The input tensor whose data type should be float. + + Outputs: + Tensor, with the same type and shape as the `input_x`. + + Examples: + >>> input_x = Tensor(np.array([0, -1, 2, 30, -30]), mindspore.float32) + >>> softsign = P.Softsign() + >>> softsign(input_x) + [0. -0.5 0.6666667 0.9677419 -0.9677419] + """ + + @prim_attr_register + def __init__(self): + """init Softsign""" + self.init_prim_io_names(inputs=['x'], outputs=['output']) + + def infer_shape(self, input_x): + return input_x + + def infer_dtype(self, input_x): + validator.check_tensor_type_same({'input_x': input_x}, mstype.float_type, self.name) + return input_x + + class ReLU(PrimitiveWithInfer): r""" Computes ReLU(Rectified Linear Unit) of input tensor element-wise. @@ -1669,7 +1704,7 @@ class DataFormatDimMap(PrimitiveWithInfer): Inputs: - **input_x** (Tensor) - A Tensor with each element as a dimension index in source data format. - Must be in the range [-4, 4). It's type is int32. + Must be in the range [-4, 4). It's type is int32. Outputs: Tensor, has the same type as the `input_x`. @@ -1793,6 +1828,8 @@ class SGD(PrimitiveWithInfer): @prim_attr_register def __init__(self, dampening=0.0, weight_decay=0.0, nesterov=False): validator.check_value_type("nesterov", nesterov, [bool], self.name) + if nesterov and dampening != 0: + raise ValueError(f"Nesterov need zero dampening!") self.init_prim_io_names(inputs=['parameters', 'gradient', 'learning_rate', 'accum', 'momentum', 'stat'], outputs=['output']) @@ -1960,7 +1997,7 @@ class ApplyCenteredRMSProp(PrimitiveWithInfer): >>> mean_grad = Tensor(np.arange(12).astype(np.float32).reshape(2, 3, 2), mindspore.float32) >>> mean_square = Tensor(np.arange(-8, 4).astype(np.float32).reshape(2, 3, 2), mindspore.float32) >>> moment = Tensor(np.arange(12).astype(np.float32).reshape(2, 3, 2), mindspore.float32) - >>> grad = Tensor(np.arange(12).astype(np.float32).rehspae(2, 3, 2), mindspore.float32) + >>> grad = Tensor(np.arange(12).astype(np.float32).reshape(2, 3, 2), mindspore.float32) >>> learning_rate = Tensor(0.9, mindspore.float32) >>> decay = 0.0 >>> momentum = 1e-10 @@ -2206,7 +2243,8 @@ class ResizeBilinear(PrimitiveWithInfer): rescale by `new_height / height`. Default: False. Inputs: - - **input** (Tensor) - Image to be resized. Tensor of shape `(N_i, ..., N_n, height, width)`. + - **input** (Tensor) - Image to be resized. Tensor of shape `(N_i, ..., N_n, height, width)`, + with data type of float32 or float16. Outputs: Tensor, resized image. Tensor of shape `(N_i, ..., N_n, new_height, new_width)` in `float32`. @@ -2890,11 +2928,11 @@ class SparseApplyAdam(PrimitiveWithInfer): - **indices** (Tensor) - Gradient indices. With int32 data type. Outputs: - Tuple of 3 Tensor, the updated parameters. + Tuple of 3 Tensor, this operator will update the input parameters directly, the outputs are useless. - - **var** (Tensor) - The same shape and data type as `var`. - - **m** (Tensor) - The same shape and data type as `m`. - - **v** (Tensor) - The same shape and data type as `v`. + - **var** (Tensor) - A Tensor with shape (1,). + - **m** (Tensor) - A Tensor with shape (1,). + - **v** (Tensor) - A Tensor with shape (1,). Examples: >>> import numpy as np @@ -2951,6 +2989,7 @@ class SparseApplyAdam(PrimitiveWithInfer): self.init_prim_io_names(inputs=['var', 'm', 'v', 'beta1_power', 'beta2_power', 'lr', 'beta1', 'beta2', 'epsilon', 'grad', 'indices'], outputs=['var', 'm', 'v']) + self.add_prim_attr('primitive_target', 'CPU') def infer_shape(self, var_shape, m_shape, v_shape, beta1_power_shape, beta2_power_shape, lr_shape, beta1_shape, beta2_shape, epsilon_shape, grad_shape, indices_shape): @@ -2962,7 +3001,7 @@ class SparseApplyAdam(PrimitiveWithInfer): raise ValueError(f"For '{self.name}', the shape of updates should be [] or " f"grad_shape = indices_shape + var_shape[1:], but got var_shape: {var_shape}, " f"indices_shape: {indices_shape}, grad_shape: {grad_shape}.") - return var_shape, m_shape, v_shape + return [1], [1], [1] def infer_dtype(self, var_dtype, m_dtype, v_dtype, beta1_power_dtype, beta2_power_dtype, lr_dtype, beta1_dtype, beta2_dtype, epsilon_dtype, grad_dtype, indices_dtype): @@ -3024,11 +3063,11 @@ class SparseApplyLazyAdam(PrimitiveWithInfer): - **indices** (Tensor) - Gradient indices. With int32 data type. Outputs: - Tuple of 3 Tensor, the updated parameters. + Tuple of 3 Tensor, this operator will update the input parameters directly, the outputs are useless. - - **var** (Tensor) - The same shape and data type as `var`. - - **m** (Tensor) - The same shape and data type as `m`. - - **v** (Tensor) - The same shape and data type as `v`. + - **var** (Tensor) - A Tensor with shape (1,). + - **m** (Tensor) - A Tensor with shape (1,). + - **v** (Tensor) - A Tensor with shape (1,). Examples: >>> import numpy as np @@ -3085,6 +3124,7 @@ class SparseApplyLazyAdam(PrimitiveWithInfer): self.init_prim_io_names(inputs=['var', 'm', 'v', 'beta1_power', 'beta2_power', 'lr', 'beta1', 'beta2', 'epsilon', 'grad', 'indices'], outputs=['var', 'm', 'v']) + self.add_prim_attr('primitive_target', 'CPU') def infer_shape(self, var_shape, m_shape, v_shape, beta1_power_shape, beta2_power_shape, lr_shape, beta1_shape, beta2_shape, epsilon_shape, grad_shape, indices_shape): @@ -3096,7 +3136,7 @@ class SparseApplyLazyAdam(PrimitiveWithInfer): raise ValueError(f"For '{self.name}', the shape of updates should be [] or " f"grad_shape = indices_shape + var_shape[1:], but got var_shape: {var_shape}, " f"indices_shape: {indices_shape}, grad_shape: {grad_shape}.") - return var_shape, m_shape, v_shape + return [1], [1], [1] def infer_dtype(self, var_dtype, m_dtype, v_dtype, beta1_power_dtype, beta2_power_dtype, lr_dtype, beta1_dtype, beta2_dtype, epsilon_dtype, grad_dtype, indices_dtype): @@ -3212,16 +3252,23 @@ class ApplyAdaMax(PrimitiveWithInfer): :math:`\epsilon` represents `epsilon`. Inputs: - - **var** (Parameter) - Variable to be updated. + - **var** (Parameter) - Variable to be updated. With float32 or float16 data type. - **m** (Parameter) - The 1st moment vector in the updating formula. Has the same shape and type as `var`. + With float32 or float16 data type. - **v** (Parameter) - The 2nd moment vector in the updating formula. Mean square gradients, - has the same shape and type as `var`. - - **beta1_power** (float) - :math:`beta_1^t` in the updating formula. - - **lr** (float) - Learning rate, :math:`l` in the updating formula. Has the same type as `var`. - - **beta1** (float) - The exponential decay rate for the 1st moment estimates. - - **beta2** (float) - The exponential decay rate for the 2nd moment estimates. - - **epsilon** (float) - A small value added for numerical stability. + has the same shape and type as `var`. With float32 or float16 data type. + - **beta1_power** (Union[Number, Tensor]) - :math:`beta_1^t` in the updating formula, should be scalar. + With float32 or float16 data type. + - **lr** (Union[Number, Tensor]) - Learning rate, :math:`l` in the updating formula, should be scalar. + With float32 or float16 data type. + - **beta1** (Union[Number, Tensor]) - The exponential decay rate for the 1st moment estimates, + should be scalar. With float32 or float16 data type. + - **beta2** (Union[Number, Tensor]) - The exponential decay rate for the 2nd moment estimates, + should be scalar. With float32 or float16 data type. + - **epsilon** (Union[Number, Tensor]) - A small value added for numerical stability, should be scalar. + With float32 or float16 data type. - **grad** (Tensor) - A tensor for gradient. Has the same shape and type as `var`. + With float32 or float16 data type. Outputs: Tuple of 3 Tensor, the updated parameters. @@ -3231,17 +3278,29 @@ class ApplyAdaMax(PrimitiveWithInfer): - **v** (Tensor) - The same shape and data type as `v`. Examples: - >>> var = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="var") - >>> m = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="m") - >>> v = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="v") + >>> import numpy as np + >>> import mindspore.nn as nn + >>> from mindspore import Tensor, Parameter + >>> from mindspore.ops import operations as P + >>> import mindspore.common.dtype as mstype + >>> class Net(nn.Cell): + >>> def __init__(self): + >>> super(Net, self).__init__() + >>> self.apply_ada_max = P.ApplyAdaMax() + >>> self.var = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="var") + >>> self.m = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="m") + >>> self.v = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="v") + >>> def construct(self, beta1_power, lr, beta1, beta2, epsilon, grad): + >>> out = self.apply_ada_max(self.var, self.m, self.v, beta1_power, lr, beta1, beta2, epsilon, grad) + >>> return out + >>> net = Net() + >>> beta1_power =Tensor(0.9, mstype.float32) + >>> lr = Tensor(0.001, mstype.float32) + >>> beta1 = Tensor(0.9, mstype.float32) + >>> beta2 = Tensor(0.99, mstype.float32) + >>> epsilon = Tensor(1e-10, mstype.float32) >>> grad = Tensor(np.random.rand(3, 3).astype(np.float32)) - >>> beta1_power = 0.9 - >>> lr = 0.001 - >>> beta1 = 0.9 - >>> beta2 = 0.99 - >>> epsilon = 1e-10 - >>> apply_ada_max = P.ApplyAdaMax() - >>> output = apply_ada_max(var, m, v, beta1_power, lr, beta1, beta2, epsilon, grad) + >>> result = net(beta1_power, lr, beta1, beta2, epsilon, grad) """ __mindspore_signature__ = ( @@ -3249,11 +3308,11 @@ class ApplyAdaMax(PrimitiveWithInfer): ('m', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), ('v', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), ('beta1_power', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, - sig_dtype.T), - ('lr', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), - ('beta1', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), - ('beta2', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), - ('epsilon', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + sig_dtype.T1), + ('lr', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T2), + ('beta1', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T3), + ('beta2', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T4), + ('epsilon', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T5), ('grad', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T) ) @@ -3263,19 +3322,41 @@ class ApplyAdaMax(PrimitiveWithInfer): def infer_shape(self, var_shape, m_shape, v_shape, beta1_power_shape, lr_shape, beta1_shape, beta2_shape, epsilon_shape, grad_shape): - validator.check("var_shape", var_shape, "m_shape", m_shape, Rel.EQ, self.name) - validator.check("var_shape", var_shape, "v_shape", v_shape, Rel.EQ, self.name) - validator.check("var_shape", var_shape, "grad_shape", grad_shape, Rel.EQ, self.name) + validator.check("m_shape", m_shape, "var_shape", var_shape, Rel.EQ, self.name) + validator.check("v_shape", v_shape, "var_shape", var_shape, Rel.EQ, self.name) + validator.check("grad_shape", grad_shape, "var_shape", var_shape, Rel.EQ, self.name) + beta1_power_shp_len = len(beta1_power_shape) + validator.check_integer("beta1 power's rank", beta1_power_shp_len, 1, Rel.LE, self.name) + if beta1_power_shp_len == 1: + validator.check_integer("beta1_power_shape[0]", beta1_power_shape[0], 1, Rel.EQ, self.name) + lr_shp_len = len(lr_shape) + validator.check_integer("lr's rank", lr_shp_len, 1, Rel.LE, self.name) + if lr_shp_len == 1: + validator.check_integer("lr_shape[0]", lr_shape[0], 1, Rel.EQ, self.name) + beta1_shp_len = len(beta1_shape) + validator.check_integer("beta1's rank", beta1_shp_len, 1, Rel.LE, self.name) + if beta1_shp_len == 1: + validator.check_integer("beta1_shape[0]", beta1_shape[0], 1, Rel.EQ, self.name) + beta2_shp_len = len(beta2_shape) + validator.check_integer("beta2's rank", beta2_shp_len, 1, Rel.LE, self.name) + if beta2_shp_len == 1: + validator.check_integer("beta2_shape[0]", beta2_shape[0], 1, Rel.EQ, self.name) + epsilon_shp_len = len(epsilon_shape) + validator.check_integer("epsilon's rank", epsilon_shp_len, 1, Rel.LE, self.name) + if epsilon_shp_len == 1: + validator.check_integer("epsilon_shape[0]", epsilon_shape[0], 1, Rel.EQ, self.name) return var_shape, m_shape, v_shape def infer_dtype(self, var_dtype, m_dtype, v_dtype, beta1_power_dtype, lr_dtype, beta1_dtype, beta2_dtype, epsilon_dtype, grad_dtype): + valid_types = [mstype.float16, mstype.float32] args = {"var": var_dtype, "m": m_dtype, "v": v_dtype, "grad": grad_dtype} - validator.check_tensor_type_same(args, mstype.number_type, self.name) - - scalar_args = {"beta1_power": beta1_power_dtype, 'lr': lr_dtype, "beta1": beta1_dtype, - "beta2": beta2_dtype, "epsilon": epsilon_dtype} - validator.check_scalar_or_tensor_type_same(scalar_args, [mstype.float16, mstype.float32], self.name, True) + validator.check_tensor_type_same(args, valid_types, self.name) + validator.check_scalar_or_tensor_type_same({"beta1_power": beta1_power_dtype}, valid_types, self.name) + validator.check_scalar_or_tensor_type_same({"lr": lr_dtype}, valid_types, self.name) + validator.check_scalar_or_tensor_type_same({"beta1": beta1_dtype}, valid_types, self.name) + validator.check_scalar_or_tensor_type_same({"beta2": beta2_dtype}, valid_types, self.name) + validator.check_scalar_or_tensor_type_same({"epsilon": epsilon_dtype}, valid_types, self.name) return var_dtype, m_dtype, v_dtype @@ -3293,13 +3374,16 @@ class ApplyAdadelta(PrimitiveWithInfer): var -= lr * update Inputs: - - **var** (Parameter) - Weights to be updated. + - **var** (Parameter) - Weights to be updated. With float32 or float16 data type. - **accum** (Parameter) - Accum to be updated, has the same shape and type as `var`. + With float32 or float16 data type. - **accum_update** (Parameter) - Accum_update to be updated, has the same shape and type as `var`. - - **lr** (float) - Learning rate, has the same type as `var`. - - **rho** (float) - Decay rate. - - **epsilon** (float) - A small value added for numerical stability. - - **grad** (Tensor) - Gradients, has the same shape and type as `var`. + With float32 or float16 data type. + - **lr** (Union[Number, Tensor]) - Learning rate, must be scalar. With float32 or float16 data type. + - **rho** (Union[Number, Tensor]) - Decay rate, must be scalar. With float32 or float16 data type. + - **epsilon** (Union[Number, Tensor]) - A small value added for numerical stability, must be scalar. + With float32 or float16 data type. + - **grad** (Tensor) - Gradients, has the same shape and type as `var`. With float32 or float16 data type. Outputs: Tuple of 3 Tensor, the updated parameters. @@ -3309,15 +3393,27 @@ class ApplyAdadelta(PrimitiveWithInfer): - **accum_update** (Tensor) - The same shape and data type as `accum_update`. Examples: - >>> var = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="var") - >>> accum = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="accum") - >>> accum_update = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="accum_update") + >>> import numpy as np + >>> import mindspore.nn as nn + >>> from mindspore import Tensor, Parameter + >>> from mindspore.ops import operations as P + >>> import mindspore.common.dtype as mstype + >>> class Net(nn.Cell): + >>> def __init__(self): + >>> super(Net, self).__init__() + >>> self.apply_adadelta = P.ApplyAdadelta() + >>> self.var = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="var") + >>> self.accum = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="accum") + >>> self.accum_update = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="accum_update") + >>> def construct(self, lr, rho, epsilon, grad): + >>> out = self.apply_adadelta(self.var, self.accum, self.accum_update, lr, rho, epsilon, grad) + >>> return out + >>> net = Net() + >>> lr = Tensor(0.001, mstype.float32) + >>> rho = Tensor(0.0, mstype.float32) + >>> epsilon = Tensor(1e-6, mstype.float32) >>> grad = Tensor(np.random.rand(3, 3).astype(np.float32)) - >>> lr = 0.001 - >>> rho = 0.0 - >>> epsilon = 1e-6 - >>> apply_adadelta = P.ApplyAdadelta() - >>> output = apply_adadelta(var, accum, accum_update, lr, rho, epsilon, grad) + >>> result = net(lr, rho, epsilon, grad) """ __mindspore_signature__ = ( @@ -3325,9 +3421,9 @@ class ApplyAdadelta(PrimitiveWithInfer): ('accum', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), ('accum_update', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), - ('lr', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), - ('rho', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), - ('epsilon', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('lr', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T1), + ('rho', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T2), + ('epsilon', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T3), ('grad', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T) ) @@ -3337,18 +3433,31 @@ class ApplyAdadelta(PrimitiveWithInfer): def infer_shape(self, var_shape, accum_shape, accum_update_shape, lr_shape, rho_shape, epsilon_shape, grad_shape): - validator.check("var_shape", var_shape, "accum_shape", accum_shape, Rel.EQ, self.name) - validator.check("var_shape", var_shape, "accum_update_shape", accum_update_shape, Rel.EQ, self.name) - validator.check("var_shape", var_shape, "grad_shape", grad_shape, Rel.EQ, self.name) + validator.check("accum_shape", accum_shape, "var_shape", var_shape, Rel.EQ, self.name) + validator.check("accum_update_shape", accum_update_shape, "var_shape", var_shape, Rel.EQ, self.name) + validator.check("grad_shape", grad_shape, "var_shape", var_shape, Rel.EQ, self.name) + lr_shp_len = len(lr_shape) + validator.check_integer("lr's rank", lr_shp_len, 1, Rel.LE, self.name) + if lr_shp_len == 1: + validator.check_integer("lr_shape[0]", lr_shape[0], 1, Rel.EQ, self.name) + rho_shp_len = len(rho_shape) + validator.check_integer("rho's rank", rho_shp_len, 1, Rel.LE, self.name) + if rho_shp_len == 1: + validator.check_integer("rho_shape[0]", rho_shape[0], 1, Rel.EQ, self.name) + epsilon_shp_len = len(epsilon_shape) + validator.check_integer("lepsilon's rank", epsilon_shp_len, 1, Rel.LE, self.name) + if epsilon_shp_len == 1: + validator.check_integer("epsilon_shape[0]", epsilon_shape[0], 1, Rel.EQ, self.name) return var_shape, accum_shape, accum_update_shape - def infer_dtype(self, var_dtype, accum_dtype, accum_update_dtype, lr_dtype, rho_shape, + def infer_dtype(self, var_dtype, accum_dtype, accum_update_dtype, lr_dtype, rho_dtype, epsilon_dtype, grad_dtype): + valid_types = [mstype.float16, mstype.float32] args = {"var": var_dtype, "accum": accum_dtype, "accum_update": accum_update_dtype, "grad": grad_dtype} - validator.check_tensor_type_same(args, mstype.number_type, self.name) - - scalar_args = {"lr": lr_dtype, "rho": rho_shape, "epsilon": epsilon_dtype} - validator.check_scalar_or_tensor_type_same(scalar_args, [mstype.float16, mstype.float32], self.name, True) + validator.check_tensor_type_same(args, valid_types, self.name) + validator.check_scalar_or_tensor_type_same({"lr": lr_dtype}, valid_types, self.name) + validator.check_scalar_or_tensor_type_same({"rho": rho_dtype}, valid_types, self.name) + validator.check_scalar_or_tensor_type_same({"epsilon": epsilon_dtype}, valid_types, self.name) return var_dtype, accum_dtype, accum_update_dtype @@ -3365,10 +3474,12 @@ class ApplyAdagrad(PrimitiveWithInfer): update_slots (bool): If `True`, `accum` will be updated. Default: True. Inputs: - - **var** (Parameter) - Variable to be updated. + - **var** (Parameter) - Variable to be updated. With float32 or float16 data type. - **accum** (Parameter) - Accum to be updated. The shape and dtype should be the same as `var`. - - **lr** (float): The learning rate value, has the same type as `var`. + With float32 or float16 data type. + - **lr** (Union[Number, Tensor]) - The learning rate value, should be scalar. With float32 or float16 data type. - **grad** (Tensor) - A tensor for gradient. The shape and dtype should be the same as `var`. + With float32 or float16 data type. Outputs: Tuple of 2 Tensor, the updated parameters. @@ -3377,18 +3488,30 @@ class ApplyAdagrad(PrimitiveWithInfer): - **accum** (Tensor) - The same shape and data type as `accum`. Examples: - >>> var = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="var") - >>> accum = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="accum") + >>> import numpy as np + >>> import mindspore.nn as nn + >>> from mindspore import Tensor, Parameter + >>> from mindspore.ops import operations as P + >>> import mindspore.common.dtype as mstype + >>> class Net(nn.Cell): + >>> def __init__(self): + >>> super(Net, self).__init__() + >>> self.apply_adagrad = P.ApplyAdagrad() + >>> self.var = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="var") + >>> self.accum = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="accum") + >>> def construct(self, lr, grad): + >>> out = self.apply_adagrad(self.var, self.accum, lr, grad) + >>> return out + >>> net = Net() + >>> lr = Tensor(0.001, mstype.float32) >>> grad = Tensor(np.random.rand(3, 3).astype(np.float32)) - >>> lr = 0.01 - >>> apply_adagrad = P.ApplyAdagrad() - >>> output = apply_adagrad(var, accum, lr, grad) + >>> result = net(lr, grad) """ __mindspore_signature__ = ( ('var', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), ('accum', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), - ('lr', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('lr', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T1), ('grad', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T) ) @@ -3397,14 +3520,18 @@ class ApplyAdagrad(PrimitiveWithInfer): validator.check_value_type("update_slots", update_slots, [bool], self.name) def infer_shape(self, var_shape, accum_shape, lr_shape, grad_shape): - validator.check('var shape', var_shape, 'accum shape', accum_shape, Rel.EQ, self.name) - validator.check('var shape', var_shape, 'grad shape', grad_shape, Rel.EQ, self.name) + validator.check('accum shape', accum_shape, 'var shape', var_shape, Rel.EQ, self.name) + validator.check('grad shape', grad_shape, 'var shape', var_shape, Rel.EQ, self.name) + lr_shp_len = len(lr_shape) + validator.check_integer("lr's rank", lr_shp_len, 1, Rel.LE, self.name) + if lr_shp_len == 1: + validator.check_integer("lr_shape[0]", lr_shape[0], 1, Rel.EQ, self.name) return var_shape, accum_shape def infer_dtype(self, var_dtype, accum_dtype, lr_dtype, grad_dtype): args = {'var': var_dtype, 'accum': accum_dtype, 'grad': grad_dtype} - validator.check_tensor_type_same(args, mstype.number_type, self.name) valid_types = [mstype.float16, mstype.float32] + validator.check_tensor_type_same(args, valid_types, self.name) validator.check_scalar_or_tensor_type_same({'lr': lr_dtype}, valid_types, self.name) return var_dtype, accum_dtype @@ -3423,10 +3550,12 @@ class ApplyAdagradV2(PrimitiveWithInfer): update_slots (bool): If `True`, `accum` will be updated. Default: True. Inputs: - - **var** (Parameter) - Variable to be updated. + - **var** (Parameter) - Variable to be updated. With float32 or float16 data type. - **accum** (Parameter) - Accum to be updated. The shape and dtype should be the same as `var`. - - **lr** (float): The learning rate value, has the same type as `var`. + With float32 or float16 data type. + - **lr** (Union[Number, Tensor]) - The learning rate value, should be scalar. With float32 or float16 data type. - **grad** (Tensor) - A tensor for gradient. The shape and dtype should be the same as `var`. + With float32 or float16 data type. Outputs: Tuple of 2 Tensor, the updated parameters. @@ -3435,18 +3564,30 @@ class ApplyAdagradV2(PrimitiveWithInfer): - **accum** (Tensor) - The same shape and data type as `m`. Examples: - >>> var = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="var") - >>> accum = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="accum") + >>> import numpy as np + >>> import mindspore.nn as nn + >>> from mindspore import Tensor, Parameter + >>> from mindspore.ops import operations as P + >>> import mindspore.common.dtype as mstype + >>> class Net(nn.Cell): + >>> def __init__(self): + >>> super(Net, self).__init__() + >>> self.apply_adagrad_v2 = P.ApplyAdagradV2(epsilon=1e-6) + >>> self.var = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="var") + >>> self.accum = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="accum") + >>> def construct(self, lr, grad): + >>> out = self.apply_adagrad_v2(self.var, self.accum, lr, grad) + >>> return out + >>> net = Net() + >>> lr = Tensor(0.001, mstype.float32) >>> grad = Tensor(np.random.rand(3, 3).astype(np.float32)) - >>> lr = 0.01 - >>> apply_adagrad_v2 = P.ApplyAdagradV2(epsilon=1e-6) - >>> output = apply_adagrad_v2(var, accum, lr, grad) + >>> result = net(lr, grad) """ __mindspore_signature__ = ( ('var', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), ('accum', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), - ('lr', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('lr', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T1), ('grad', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T) ) @@ -3458,12 +3599,16 @@ class ApplyAdagradV2(PrimitiveWithInfer): def infer_shape(self, var_shape, accum_shape, lr_shape, grad_shape): validator.check('var shape', var_shape, 'accum shape', accum_shape, Rel.EQ, self.name) validator.check('var shape', var_shape, 'grad shape', grad_shape, Rel.EQ, self.name) + lr_shp_len = len(lr_shape) + validator.check_integer("lr's rank", lr_shp_len, 1, Rel.LE, self.name) + if lr_shp_len == 1: + validator.check_integer("lr_shape[0]", lr_shape[0], 1, Rel.EQ, self.name) return var_shape, accum_shape def infer_dtype(self, var_dtype, accum_dtype, lr_dtype, grad_dtype): args = {'var': var_dtype, 'accum': accum_dtype, 'grad': grad_dtype} - validator.check_tensor_type_same(args, mstype.number_type, self.name) valid_types = [mstype.float16, mstype.float32] + validator.check_tensor_type_same(args, valid_types, self.name) validator.check_scalar_or_tensor_type_same({'lr': lr_dtype}, valid_types, self.name) return var_dtype, accum_dtype @@ -3548,6 +3693,88 @@ class SparseApplyAdagrad(PrimitiveWithInfer): return var_type, accum_type +class SparseApplyAdagradV2(PrimitiveWithInfer): + r""" + Update relevant entries according to the adagrad scheme. + + .. math:: + accum += grad * grad + .. math:: + var -= lr * grad * \frac{1}{\sqrt{accum} + \epsilon} + + Args: + lr (float): Learning rate. + epsilon (float): A small value added for numerical stability. + use_locking (bool): If `True`, updating of the var and accum tensors will be protected. Default: False. + update_slots (bool): If `True`, the computation logic will be different to `False`. Default: True. + + Inputs: + - **var** (Parameter) - Variable to be updated. The type must be float32. + - **accum** (Parameter) - Accum to be updated. The shape must be the same as `var`'s shape, + the type must be float32. + - **grad** (Tensor) - Gradient. The shape must be the same as `var`'s shape except first dimension, + the type must be float32. + - **indices** (Tensor) - A vector of indices into the first dimension of `var` and `accum`. + The shape of `indices` must be the same as `grad` in first dimension, the type must be int32. + + Outputs: + Tuple of 2 Tensor, the updated parameters. + + - **var** (Tensor) - The same shape and data type as `var`. + - **accum** (Tensor) - The same shape and data type as `accum`. + + Examples: + >>> import numpy as np + >>> import mindspore.nn as nn + >>> from mindspore import Tensor, Parameter + >>> from mindspore.ops import operations as P + >>> import mindspore.common.dtype as mstype + >>> class Net(nn.Cell): + >>> def __init__(self): + >>> super(Net, self).__init__() + >>> self.sparse_apply_adagrad_v2 = P.SparseApplyAdagradV2(lr=1e-8, epsilon=1e-6) + >>> self.var = Parameter(Tensor(np.ones([3, 3, 3]).astype(np.float32)), name="var") + >>> self.accum = Parameter(Tensor(np.ones([3, 3, 3]).astype(np.float32)), name="accum") + >>> + >>> def construct(self, grad, indices): + >>> out = self.sparse_apply_adagrad_v2(self.var, self.accum, grad, indices) + >>> return out + >>> net = Net() + >>> grad = Tensor(np.random.rand(3, 3, 3).astype(np.float32)) + >>> indices = Tensor([0, 1, 2], mstype.int32) + >>> result = net(grad, indices) + """ + + __mindspore_signature__ = ( + ('var', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('accum', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('grad', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('indices', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T1) + ) + + @prim_attr_register + def __init__(self, lr, epsilon, use_locking=False, update_slots=True): + self.lr = validator.check_value_type("lr", lr, [float], self.name) + self.epsilon = validator.check_value_type("epsilon", epsilon, [float], self.name) + self.use_locking = validator.check_value_type("update_slots", update_slots, [bool], self.name) + self.update_slots = validator.check_value_type("use_locking", use_locking, [bool], self.name) + + def infer_shape(self, var_shape, accum_shape, grad_shape, indices_shape): + validator.check('var shape', var_shape, 'accum shape', accum_shape, Rel.EQ, self.name) + validator.check('len of var shape', len(var_shape), 'len of grad shape', len(grad_shape), Rel.EQ, self.name) + if len(var_shape) > 1: + validator.check('var_shape[1:]', var_shape[1:], 'grad_shape[1:]', grad_shape[1:], Rel.EQ, self.name) + validator.check_integer("indices rank", len(indices_shape), 1, Rel.EQ, self.name) + validator.check('grad_shape[0]', grad_shape[0], 'indices_shape[0]', indices_shape[0], Rel.EQ, self.name) + return var_shape, accum_shape + + def infer_dtype(self, var_type, accum_type, grad_type, indices_type): + args = {'var': var_type, 'accum': accum_type, 'grad': grad_type} + validator.check_tensor_type_same(args, [mstype.float32], self.name) + validator.check_tensor_type_same({'indices': indices_type}, [mstype.int32], self.name) + return var_type, accum_type + + class ApplyProximalAdagrad(PrimitiveWithInfer): r""" Update relevant entries according to the proximal adagrad algorithm. @@ -3563,14 +3790,14 @@ class ApplyProximalAdagrad(PrimitiveWithInfer): use_locking (bool): If True, updating of the var and accum tensors will be protected. Default: False. Inputs: - - **var** (Parameter) - Variable to be updated. The data type should be float. + - **var** (Parameter) - Variable to be updated. The data type should be float16 or float32. - **accum** (Parameter) - Accum to be updated. Must has the same shape and dtype as `var`. - - **lr** (Union[Number, Tensor]): The learning rate value. It should be a scalar tensor or number. - The data type should be float. - - **l1** (Union[Number, Tensor]): l1 regularization strength, must be greater than or equal to zero. - It should be a scalar tensor or number. The data type should be float. - - **l2** (Union[Number, Tensor]): l2 regularization strength, must be greater than or equal to zero. - It should be a scalar tensor or number. The data type should be float. + - **lr** (Union[Number, Tensor]) - The learning rate value, should be scalar. The data type should be + float16 or float32. + - **l1** (Union[Number, Tensor]) - l1 regularization strength, should be scalar. The data type should be + float16 or float32. + - **l2** (Union[Number, Tensor]) - l2 regularization strength, should be scalar. The data type should be + float16 or float32. - **grad** (Tensor) - Gradient. Must has the same shape and dtype as `var`. Outputs: @@ -3604,28 +3831,42 @@ class ApplyProximalAdagrad(PrimitiveWithInfer): __mindspore_signature__ = ( ('var', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), ('accum', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), - ('lr', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), - ('l1', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), - ('l2', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('lr', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T1), + ('l1', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T2), + ('l2', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T3), ('grad', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T) ) @prim_attr_register def __init__(self, use_locking=False): - self.init_prim_io_names(inputs=['var', 'accum', 'lr', 'l1', 'l2', 'grad'], outputs=['output']) + self.init_prim_io_names(inputs=['var', 'accum', 'lr', 'l1', 'l2', 'grad'], + outputs=['var', 'accum']) self.use_locking = validator.check_value_type("use_locking", use_locking, [bool], self.name) def infer_shape(self, var_shape, accum_shape, lr_shape, l1_shape, l2_shape, grad_shape): - validator.check('var shape', var_shape, 'accum shape', accum_shape, Rel.EQ, self.name) - validator.check('var shape', var_shape, 'grad shape', grad_shape, Rel.EQ, self.name) + validator.check('accum shape', accum_shape, 'var shape', var_shape, Rel.EQ, self.name) + validator.check('grad shape', grad_shape, 'var shape', var_shape, Rel.EQ, self.name) + lr_shp_len = len(lr_shape) + validator.check_integer("lr's rank", lr_shp_len, 1, Rel.LE, self.name) + if lr_shp_len == 1: + validator.check_integer("lr_shape[0]", lr_shape[0], 1, Rel.EQ, self.name) + l1_shp_len = len(l1_shape) + validator.check_integer("l1's rank", l1_shp_len, 1, Rel.LE, self.name) + if l1_shp_len == 1: + validator.check_integer("l1_shape[0]", l1_shape[0], 1, Rel.EQ, self.name) + l2_shp_len = len(l2_shape) + validator.check_integer("l2's rank", l2_shp_len, 1, Rel.LE, self.name) + if l2_shp_len == 1: + validator.check_integer("l2_shape[0]", l2_shape[0], 1, Rel.EQ, self.name) return var_shape, accum_shape def infer_dtype(self, var_dtype, accum_dtype, lr_dtype, l1_dtype, l2_dtype, grad_dtype): valid_types = [mstype.float16, mstype.float32] args = {'var': var_dtype, 'accum': accum_dtype, 'grad': grad_dtype} validator.check_tensor_type_same(args, valid_types, self.name) - scalar_args = {"lr": lr_dtype, "l1": l1_dtype, "l2": l2_dtype} - validator.check_scalar_or_tensor_type_same(scalar_args, valid_types, self.name) + validator.check_scalar_or_tensor_type_same({"lr": lr_dtype}, valid_types, self.name) + validator.check_scalar_or_tensor_type_same({"l1": l1_dtype}, valid_types, self.name) + validator.check_scalar_or_tensor_type_same({"l2": l2_dtype}, valid_types, self.name) return var_dtype, accum_dtype @@ -3647,12 +3888,9 @@ class SparseApplyProximalAdagrad(PrimitiveWithInfer): Inputs: - **var** (Parameter) - Variable tensor to be updated. The data type must be float32. - **accum** (Parameter) - Variable tensor to be updated. Has the same dtype as `var`. - - **lr** (Union[Number, Tensor]): The learning rate value. It should be a scalar tensor or number. - The data type must be float32. - - **l1** (Union[Number, Tensor]): l1 regularization strength, must be greater than or equal to zero. - It should be a scalar tensor or number. The data type must be float32. - - **l2** (Union[Number, Tensor]): l2 regularization strength, must be greater than or equal to zero. - It should be a scalar tensor or number. The data type must be float32. + - **lr** (Union[Number, Tensor]) - The learning rate value. The data type must be float32. + - **l1** (Union[Number, Tensor]) - l1 regularization strength. The data type must be float32. + - **l2** (Union[Number, Tensor]) - l2 regularization strength. The data type must be float32. - **grad** (Tensor) - A tensor of the same type as `var`, for the gradient. The data type must be float32. - **indices** (Tensor) - A vector of indices into the first dimension of `var` and `accum`. @@ -3689,17 +3927,17 @@ class SparseApplyProximalAdagrad(PrimitiveWithInfer): __mindspore_signature__ = ( ('var', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), ('accum', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), - ('lr', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), - ('l1', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), - ('l2', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('lr', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T1), + ('l1', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T2), + ('l2', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T3), ('grad', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), - ('indices', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T1) + ('indices', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T4) ) @prim_attr_register def __init__(self, use_locking=False): self.init_prim_io_names(inputs=['var', 'accum', 'lr', 'l1', 'l2', 'grad', 'indices'], - outputs=['output']) + outputs=['var', 'accum']) self.use_locking = validator.check_value_type("use_locking", use_locking, [bool], self.name) def infer_shape(self, var_shape, accum_shape, lr_shape, l1_shape, l2_shape, grad_shape, indices_shape): @@ -3709,14 +3947,358 @@ class SparseApplyProximalAdagrad(PrimitiveWithInfer): def infer_dtype(self, var_dtype, accum_dtype, lr_dtype, l1_dtype, l2_dtype, grad_dtype, indices_dtype): args = {'var': var_dtype, 'accum': accum_dtype, 'grad': grad_dtype} validator.check_tensor_type_same(args, [mstype.float32], self.name) - scalar_args = {"lr": lr_dtype, "l1": l1_dtype, "l2": l2_dtype} - validator.check_scalar_or_tensor_type_same(scalar_args, [mstype.float32], self.name) + validator.check_scalar_or_tensor_type_same({"lr": lr_dtype}, [mstype.float32], self.name) + validator.check_scalar_or_tensor_type_same({"l1": l1_dtype}, [mstype.float32], self.name) + validator.check_scalar_or_tensor_type_same({"l2": l2_dtype}, [mstype.float32], self.name) valid_types = [mstype.int16, mstype.int32, mstype.int64, mstype.uint16, mstype.uint32, mstype.uint64] validator.check_tensor_type_same({'indices': indices_dtype}, valid_types, self.name) return var_dtype, accum_dtype +class ApplyAddSign(PrimitiveWithInfer): + r""" + Update relevant entries according to the AddSign algorithm. + + .. math:: + \begin{array}{ll} \\ + m_{t} = \beta * m_{t-1} + (1 - \beta) * g \\ + \text{update} = (\alpha + \text{sign_decay} * sign(g) * sign(m)) * g \\ + var = var - lr_{t} * \text{update} + \end{array} + + :math:`t` represents updating step while, :math:`m` represents the 1st moment vector, :math:`m_{t-1}` + is the last momentent of :math:`m_{t}`, :math:`lr` represents scaling factor `lr`, :math:`g` represents `grad`. + + Inputs: + - **var** (Parameter) - Variable tensor to be updated. With float32 or float16 data type. + - **m** (Parameter) - Variable tensor to be updated. Has the same dtype as `var`. + - **lr** (Union[Number, Tensor]) - The learning rate value, should be a scalar. + With float32 or float16 data type. + - **alpha** (Union[Number, Tensor]) - Should be a scalar. With float32 or float16 data type. + - **sign_decay** (Union[Number, Tensor]) - Should be a scalar. With float32 or float16 data type. + - **beta** (Union[Number, Tensor]) - The exponential decay rate, should be a scalar. + With float32 or float16 data type. + - **grad** (Tensor) - A tensor of the same type as `var`, for the gradient. + + Outputs: + Tuple of 2 Tensor, the updated parameters. + + - **var** (Tensor) - The same shape and data type as `var`. + - **m** (Tensor) - The same shape and data type as `m`. + + Examples: + >>> import numpy as np + >>> import mindspore.nn as nn + >>> from mindspore import Tensor, Parameter + >>> from mindspore.ops import operations as P + >>> class Net(nn.Cell): + >>> def __init__(self): + >>> super(Net, self).__init__() + >>> self.apply_add_sign = P.ApplyAddSign() + >>> self.var = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="var") + >>> self.m = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="m") + >>> self.lr = 0.001 + >>> self.alpha = 1.0 + >>> self.sign_decay = 0.99 + >>> self.beta = 0.9 + >>> def construct(self, grad): + >>> out = self.apply_add_sign(self.var, self.m, self.lr, self.alpha, self.sign_decay, self.beta, grad) + >>> return out + >>> net = Net() + >>> grad = Tensor(np.random.rand(3, 3).astype(np.float32)) + >>> output = net(grad) + """ + + __mindspore_signature__ = ( + ('var', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('m', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('lr', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T1), + ('alpha', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T2), + ('sign_decay', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, + sig_dtype.T3), + ('beta', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T4), + ('grad', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T) + ) + + @prim_attr_register + def __init__(self): + "init ApplyAddSign" + + def infer_shape(self, var_shape, m_shape, lr_shape, alpha_shape, sign_decay_shape, beta_shape, grad_shape): + validator.check('m_shape', m_shape, 'var_shape', var_shape, Rel.EQ, self.name) + validator.check('grad_shape', grad_shape, 'var_shape', var_shape, Rel.EQ, self.name) + lr_shape_len = len(lr_shape) + validator.check_integer("lr's rank", lr_shape_len, 1, Rel.LE, self.name) + if lr_shape_len == 1: + validator.check_integer("lr_shape[0]", lr_shape[0], 1, Rel.EQ, self.name) + alpha_shape_len = len(alpha_shape) + validator.check_integer("alpha's rank", alpha_shape_len, 1, Rel.LE, self.name) + if alpha_shape_len == 1: + validator.check_integer("alpha_shape[0]", alpha_shape[0], 1, Rel.EQ, self.name) + sign_decay_shape_len = len(sign_decay_shape) + validator.check_integer("sign_decay's rank", sign_decay_shape_len, 1, Rel.LE, self.name) + if sign_decay_shape_len == 1: + validator.check_integer("sign_decay_shape[0]", sign_decay_shape[0], 1, Rel.EQ, self.name) + beta_shape_len = len(beta_shape) + validator.check_integer("beta's rank", beta_shape_len, 1, Rel.LE, self.name) + if beta_shape_len == 1: + validator.check_integer("beta_shape[0]", beta_shape[0], 1, Rel.EQ, self.name) + return var_shape, m_shape + + def infer_dtype(self, var_dtype, m_dtype, lr_dtype, alpha_dtype, sign_decay_dtype, beta_dtype, grad_dtype): + valid_types = [mstype.float16, mstype.float32] + args = {'var': var_dtype, 'm': m_dtype, 'grad': grad_dtype} + validator.check_tensor_type_same(args, valid_types, self.name) + validator.check_scalar_or_tensor_type_same({"lr": lr_dtype}, valid_types, self.name) + validator.check_scalar_or_tensor_type_same({"alpha": alpha_dtype}, valid_types, self.name) + validator.check_scalar_or_tensor_type_same({"sign_decay": sign_decay_dtype}, valid_types, self.name) + validator.check_scalar_or_tensor_type_same({"beta": beta_dtype}, valid_types, self.name) + return var_dtype, m_dtype + + +class ApplyPowerSign(PrimitiveWithInfer): + r""" + Update relevant entries according to the AddSign algorithm. + + .. math:: + \begin{array}{ll} \\ + m_{t} = \beta * m_{t-1} + (1 - \beta) * g \\ + \text{update} = \exp(\text{logbase} * \text{sign_decay} * sign(g) * sign(m)) * g \\ + var = var - lr_{t} * \text{update} + \end{array} + + :math:`t` represents updating step while, :math:`m` represents the 1st moment vector, :math:`m_{t-1}` + is the last momentent of :math:`m_{t}`, :math:`lr` represents scaling factor `lr`, :math:`g` represents `grad`. + + Inputs: + - **var** (Parameter) - Variable tensor to be updated. With float32 or float16 data type. + - **m** (Parameter) - Variable tensor to be updated. Has the same dtype as `var`. + - **lr** (Union[Number, Tensor]) - The learning rate value, should be a scalar. + With float32 or float16 data type. + - **logbase** (Union[Number, Tensor]) - Should be a scalar. With float32 or float16 data type. + - **sign_decay** (Union[Number, Tensor]) - Should be a scalar. With float32 or float16 data type. + - **beta** (Union[Number, Tensor]) - The exponential decay rate, should be a scalar. + With float32 or float16 data type. + - **grad** (Tensor) - A tensor of the same type as `var`, for the gradient. + + Outputs: + Tuple of 2 Tensor, the updated parameters. + + - **var** (Tensor) - The same shape and data type as `var`. + - **m** (Tensor) - The same shape and data type as `m`. + + Examples: + >>> import numpy as np + >>> import mindspore.nn as nn + >>> from mindspore import Tensor, Parameter + >>> from mindspore.ops import operations as P + >>> class Net(nn.Cell): + >>> def __init__(self): + >>> super(Net, self).__init__() + >>> self.apply_power_sign = P.ApplyPowerSign() + >>> self.var = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="var") + >>> self.m = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="m") + >>> self.lr = 0.001 + >>> self.logbase = np.e + >>> self.sign_decay = 0.99 + >>> self.beta = 0.9 + >>> def construct(self, grad): + >>> out = self.apply_power_sign(self.var, self.m, self.lr, self.logbase, + self.sign_decay, self.beta, grad) + >>> return out + >>> net = Net() + >>> grad = Tensor(np.random.rand(3, 3).astype(np.float32)) + >>> output = net(grad) + """ + + __mindspore_signature__ = ( + ('var', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('m', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('lr', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T1), + ('logbase', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T2), + ('sign_decay', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, + sig_dtype.T3), + ('beta', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T4), + ('grad', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T) + ) + + @prim_attr_register + def __init__(self): + "init ApplyPowerSign" + + def infer_shape(self, var_shape, m_shape, lr_shape, logbase_shape, sign_decay_shape, beta_shape, grad_shape): + validator.check('m_shape', m_shape, 'var_shape', var_shape, Rel.EQ, self.name) + validator.check('grad_shape', grad_shape, 'var_shape', var_shape, Rel.EQ, self.name) + lr_shape_len = len(lr_shape) + validator.check_integer("lr's rank", lr_shape_len, 1, Rel.LE, self.name) + if lr_shape_len == 1: + validator.check_integer("lr_shape[0]", lr_shape[0], 1, Rel.EQ, self.name) + logbase_shape_len = len(logbase_shape) + validator.check_integer("logbase's rank", logbase_shape_len, 1, Rel.LE, self.name) + if logbase_shape_len == 1: + validator.check_integer("logbase_shape[0]", logbase_shape[0], 1, Rel.EQ, self.name) + sign_decay_shape_len = len(sign_decay_shape) + validator.check_integer("sign_decay's rank", sign_decay_shape_len, 1, Rel.LE, self.name) + if sign_decay_shape_len == 1: + validator.check_integer("sign_decay_shape[0]", sign_decay_shape[0], 1, Rel.EQ, self.name) + beta_shape_len = len(beta_shape) + validator.check_integer("beta's rank", beta_shape_len, 1, Rel.LE, self.name) + if beta_shape_len == 1: + validator.check_integer("beta_shape[0]", beta_shape[0], 1, Rel.EQ, self.name) + return var_shape, m_shape + + def infer_dtype(self, var_dtype, m_dtype, lr_dtype, logbase_dtype, sign_decay_dtype, beta_dtype, grad_dtype): + valid_types = [mstype.float16, mstype.float32] + args = {'var': var_dtype, 'm': m_dtype, 'grad': grad_dtype} + validator.check_tensor_type_same(args, valid_types, self.name) + validator.check_scalar_or_tensor_type_same({"lr": lr_dtype}, valid_types, self.name) + validator.check_scalar_or_tensor_type_same({"logbase": logbase_dtype}, valid_types, self.name) + validator.check_scalar_or_tensor_type_same({"sign_decay": sign_decay_dtype}, valid_types, self.name) + validator.check_scalar_or_tensor_type_same({"beta": beta_dtype}, valid_types, self.name) + return var_dtype, m_dtype + + +class ApplyGradientDescent(PrimitiveWithInfer): + r""" + Update relevant entries according to the following formula. + + .. math:: + var = var - \alpha * \delta + + Inputs: + - **var** (Parameter) - Variable tensor to be updated. With float32 or float16 data type. + - **alpha** (Union[Number, Tensor]) - Scaling factor, should be a scalar. With float32 or float16 data type. + - **delta** (Tensor) - A tensor for the change. Has the same type as `var`. + + Outputs: + Tensor, representing the updated var. + + Examples: + >>> import numpy as np + >>> import mindspore.nn as nn + >>> from mindspore import Tensor, Parameter + >>> from mindspore.ops import operations as P + >>> class Net(nn.Cell): + >>> def __init__(self): + >>> super(Net, self).__init__() + >>> self.apply_gradient_descent = P.ApplyGradientDescent() + >>> self.var = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="var") + >>> self.alpha = 0.001 + >>> def construct(self, delta): + >>> out = self.apply_gradient_descent(self.var, self.alpha, delta) + >>> return out + >>> net = Net() + >>> delta = Tensor(np.random.rand(3, 3).astype(np.float32)) + >>> output = net(delta) + """ + + __mindspore_signature__ = ( + ('var', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('alpha', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T1), + ('delta', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T) + ) + + @prim_attr_register + def __init__(self): + "init ApplyGradientDescent" + + def infer_shape(self, var_shape, alpha_shape, delta_shape): + validator.check('delta shape', delta_shape, 'var shape', var_shape, Rel.EQ, self.name) + alpha_shape_len = len(alpha_shape) + validator.check_integer("alpha's rank", alpha_shape_len, 1, Rel.LE, self.name) + if alpha_shape_len == 1: + validator.check_integer("alpha_shape[0]", alpha_shape[0], 1, Rel.EQ, self.name) + return var_shape + + def infer_dtype(self, var_dtype, alpha_dtype, delta_dtype): + valid_types = [mstype.float16, mstype.float32] + args = {'var': var_dtype, 'delta': delta_dtype} + validator.check_tensor_type_same(args, valid_types, self.name) + validator.check_scalar_or_tensor_type_same({"alpha": alpha_dtype}, valid_types, self.name) + return var_dtype + + +class ApplyProximalGradientDescent(PrimitiveWithInfer): + r""" + Update relevant entries according to the FOBOS(Forward Backward Splitting) algorithm. + + .. math:: + \text{prox_v} = var - \alpha * \delta + .. math:: + var = \frac{sign(\text{prox_v})}{1 + \alpha * l2} * \max(\left| \text{prox_v} \right| - alpha * l1, 0) + + Inputs: + - **var** (Parameter) - Variable tensor to be updated. With float32 or float16 data type. + - **alpha** (Union[Number, Tensor]) - Saling factor, should be a scalar. With float32 or float16 data type. + - **l1** (Union[Number, Tensor]) - l1 regularization strength, should be scalar. + With float32 or float16 data type. + - **l2** (Union[Number, Tensor]) - l2 regularization strength, should be scalar. + With float32 or float16 data type. + - **delta** (Tensor) - A tensor for the change. Has the same type as `var`. + + Outputs: + Tensor, representing the updated var. + + Examples: + >>> import numpy as np + >>> import mindspore.nn as nn + >>> from mindspore import Tensor, Parameter + >>> from mindspore.ops import operations as P + >>> class Net(nn.Cell): + >>> def __init__(self): + >>> super(Net, self).__init__() + >>> self.apply_proximal_gradient_descent = P.ApplyProximalGradientDescent() + >>> self.var = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="var") + >>> self.alpha = 0.001 + >>> self.l1 = 0.0 + >>> self.l2 = 0.0 + >>> def construct(self, delta): + >>> out = self.apply_proximal_gradient_descent(self.var, self.alpha, self.l1, self.l2, delta) + >>> return out + >>> net = Net() + >>> delta = Tensor(np.random.rand(3, 3).astype(np.float32)) + >>> output = net(delta) + """ + + __mindspore_signature__ = ( + ('var', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('alpha', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T1), + ('l1', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T2), + ('l2', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T3), + ('delta', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T) + ) + + @prim_attr_register + def __init__(self): + "init ApplyGradientDescent" + + def infer_shape(self, var_shape, alpha_shape, l1_shape, l2_shape, delta_shape): + validator.check('delta shape', delta_shape, 'var shape', var_shape, Rel.EQ, self.name) + alpha_shape_len = len(alpha_shape) + validator.check_integer("alpha's rank", alpha_shape_len, 1, Rel.LE, self.name) + if alpha_shape_len == 1: + validator.check_integer("alpha_shape[0]", alpha_shape[0], 1, Rel.EQ, self.name) + l1_shape_len = len(l1_shape) + validator.check_integer("l1's rank", l1_shape_len, 1, Rel.LE, self.name) + if l1_shape_len == 1: + validator.check_integer("l1_shape[0]", l1_shape[0], 1, Rel.EQ, self.name) + l2_shape_len = len(l2_shape) + validator.check_integer("l2's rank", l2_shape_len, 1, Rel.LE, self.name) + if l2_shape_len == 1: + validator.check_integer("l2_shape[0]", l2_shape[0], 1, Rel.EQ, self.name) + return var_shape + + def infer_dtype(self, var_dtype, alpha_dtype, l1_dtype, l2_dtype, delta_dtype): + valid_types = [mstype.float16, mstype.float32] + args = {'var': var_dtype, 'delta': delta_dtype} + validator.check_tensor_type_same(args, valid_types, self.name) + validator.check_scalar_or_tensor_type_same({"alpha": alpha_dtype}, valid_types, self.name) + validator.check_scalar_or_tensor_type_same({"l1": l1_dtype}, valid_types, self.name) + validator.check_scalar_or_tensor_type_same({"l2": l2_dtype}, valid_types, self.name) + return var_dtype + + class LARSUpdate(PrimitiveWithInfer): """ Conduct lars (layer-wise adaptive rate scaling) update on the square sum of gradient. @@ -3891,9 +4473,9 @@ class SparseApplyFtrl(PrimitiveWithInfer): use_locking (bool): Use locks for update operation if True . Default: False. Inputs: - - **var** (Tensor): The variable to be updated. - - **accum** (Tensor): The accum to be updated, must be same type and shape as `var`. - - **linear** (Tensor): The linear to be updated, must be same type and shape as `var`. + - **var** (Parameter): The variable to be updated. The data type must be float32. + - **accum** (Parameter): The accum to be updated, must be same type and shape as `var`. + - **linear** (Parameter): The linear to be updated, must be same type and shape as `var`. - **grad** (Tensor): A tensor of the same type as `var`, for the gradient. - **indices** (Tensor): A vector of indices into the first dimension of `var` and `accum`. The shape of `indices` must be the same as `grad` in first dimension. The type must be int32. @@ -3928,6 +4510,14 @@ class SparseApplyFtrl(PrimitiveWithInfer): >>> output = net(grad, indices) """ + __mindspore_signature__ = ( + ('var', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('accum', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('linear', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('grad', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('indices', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T1) + ) + @prim_attr_register def __init__(self, lr, l1, l2, lr_power, use_locking=False): validator.check_value_type("lr", lr, [float], self.name) @@ -3957,6 +4547,98 @@ class SparseApplyFtrl(PrimitiveWithInfer): return var_dtype, accum_dtype, linear_dtype +class SparseApplyFtrlV2(PrimitiveWithInfer): + """ + Update relevant entries according to the FTRL-proximal scheme. + + Args: + lr (float): The learning rate value, must be positive. + l1 (float): l1 regularization strength, must be greater than or equal to zero. + l2 (float): l2 regularization strength, must be greater than or equal to zero. + l2_shrinkage (float): L2 shrinkage regularization. + lr_power (float): Learning rate power controls how the learning rate decreases during training, + must be less than or equal to zero. Use fixed learning rate if `lr_power` is zero. + use_locking (bool): If `True`, updating of the var and accum tensors will be protected. Default: False. + + Inputs: + - **var** (Parameter) - The variable to be updated. The data type must be float32. + - **accum** (Parameter) - The accum to be updated, must be same type and shape as `var`. + - **linear** (Parameter) - The linear to be updated, must be same type and shape as `var`. + - **grad** (Tensor) - A tensor of the same type as `var`, for the gradient. + - **indices** (Tensor) - A vector of indices into the first dimension of `var` and `accum`. + The shape of `indices` must be the same as `grad` in first dimension. The type must be int32. + + Outputs: + Tuple of 3 Tensor, the updated parameters. + + - **var** (Tensor): Tensor, has the same shape and type as `var`. + - **accum** (Tensor): Tensor, has the same shape and type as `accum`. + - **linear** (Tensor): Tensor, has the same shape and type as `linear`. + + Examples: + >>> import mindspore + >>> import mindspore.nn as nn + >>> import numpy as np + >>> from mindspore import Parameter + >>> from mindspore import Tensor + >>> from mindspore.ops import operations as P + >>> class SparseApplyFtrlV2Net(nn.Cell): + >>> def __init__(self): + >>> super(SparseApplyFtrlV2Net, self).__init__() + >>> self.sparse_apply_ftrl_v2 = P.SparseApplyFtrlV2(lr=0.01, l1=0.0, l2=0.0, + l2_shrinkage=0.0, lr_power=-0.5) + >>> self.var = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="var") + >>> self.accum = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="accum") + >>> self.linear = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="linear") + >>> + >>> def construct(self, grad, indices): + >>> out = self.sparse_apply_ftrl_v2(self.var, self.accum, self.linear, grad, indices) + >>> return out + >>> + >>> net = SparseApplyFtrlV2Net() + >>> grad = Tensor(np.random.rand(3, 3).astype(np.float32)) + >>> indices = Tensor(np.ones([3]), mindspore.int32) + >>> output = net(grad, indices) + """ + + __mindspore_signature__ = ( + ('var', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('accum', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('linear', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('grad', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('indices', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T1) + ) + + @prim_attr_register + def __init__(self, lr, l1, l2, l2_shrinkage, lr_power, use_locking=False): + validator.check_value_type("lr", lr, [float], self.name) + validator.check_value_type("l1", l1, [float], self.name) + validator.check_value_type("l2", l2, [float], self.name) + validator.check_value_type("lr_power", lr_power, [float], self.name) + self.lr = validator.check_number_range("lr", lr, 0.0, float("inf"), Rel.INC_NEITHER, self.name) + self.l1 = validator.check_number_range("l1", l1, 0.0, float("inf"), Rel.INC_LEFT, self.name) + self.l2 = validator.check_number_range("l2", l2, 0.0, float("inf"), Rel.INC_LEFT, self.name) + self.lr_power = validator.check_number("lr_power", lr_power, 0, Rel.LE, self.name) + self.l2_shrinkage = validator.check_value_type("l2_shrinkage", l2_shrinkage, [float], self.name) + self.use_locking = validator.check_value_type("use_locking", use_locking, [bool], self.name) + + def infer_shape(self, var_shape, accum_shape, linear_shape, grad_shape, indices_shape): + validator.check('var shape', var_shape, 'accum shape', accum_shape, Rel.EQ, self.name) + validator.check('var shape', var_shape, 'linear shape', linear_shape, Rel.EQ, self.name) + if len(var_shape) > 1: + validator.check('var_shape[1:]', var_shape[1:], 'grad_shape[1:]', grad_shape[1:], Rel.EQ, self.name) + validator.check_integer("indices rank", len(indices_shape), 1, Rel.EQ, self.name) + validator.check('grad_shape[0]', grad_shape[0], 'indices_shape[0]', indices_shape[0], Rel.EQ, self.name) + return var_shape, accum_shape, linear_shape + + def infer_dtype(self, var_dtype, accum_dtype, linear_dtype, grad_dtype, indices_dtype): + args = {"var_dtype": var_dtype, "accum_dtype": accum_dtype, + "linear_dtype": linear_dtype, "grad_dtype": grad_dtype} + validator.check_tensor_type_same(args, [mstype.float32], self.name) + validator.check_tensor_type_same({"indices_dtype": indices_dtype}, [mstype.int32], self.name) + return var_dtype, accum_dtype, linear_dtype + + class ConfusionMulGrad(PrimitiveWithInfer): """ `output0` is the result of which input0 dot multily input1. @@ -4048,6 +4730,7 @@ class Dropout(PrimitiveWithInfer): def infer_dtype(self, x_dtype): valid_types = (mstype.float16, mstype.float32) + validator.check_subclass("x", x_dtype, mstype.tensor, self.name) validator.check_tensor_type_same({"x_dtype": x_dtype}, valid_types, self.name) return x_dtype, x_dtype @@ -4082,6 +4765,8 @@ class DropoutGrad(PrimitiveWithInfer): def infer_dtype(self, dy_dtype, mask_dtype): valid_types = (mstype.float16, mstype.float32) + validator.check_subclass("dy", dy_dtype, mstype.tensor, self.name) + validator.check_subclass("mask", mask_dtype, mstype.tensor, self.name) validator.check_tensor_type_same({"dy_dtype": dy_dtype}, valid_types, self.name) return dy_dtype @@ -4307,3 +4992,44 @@ class InTopK(PrimitiveWithInfer): validator.check("x2", len(x2_shape), "", 1, Rel.EQ, self.name) validator.check("size of x2", x2_shape[0], "x1's first dimension", x1_shape[0], Rel.EQ, self.name) return x2_shape + + +class LRN(PrimitiveWithInfer): + r""" + Local Response Normalization + + Args: + depth_radius (int): Half-width of the 1-D normalization window. Shape of 0-D. + bias (float): An offset (usually positive to avoid dividing by 0). + alpha (float): A scale factor, usually positive. + beta (float): An exponent. + norm_region (str): Specify normalization region. Options: "ACROSS_CHANNELS", "WITHIN_CHANNEL". + Default: "ACROSS_CHANNELS". + + Inputs: + - **x** (Tensor) - A 4D Tensor with float16 or float32 data type. + + Outputs: + Tensor, With shape and data type same as the input tensor. + + Examples: + >>> x = Tensor(np.random.rand(1, 10, 4, 4)), mindspore.float32) + >>> lrn = P.LRN() + >>> lrn(x) + """ + @prim_attr_register + def __init__(self, depth_radius=5, bias=1.0, alpha=1.0, beta=0.5, norm_region="ACROSS_CHANNELS"): + """Init LRN""" + self.init_prim_io_names(inputs=['x'], outputs=['y']) + validator.check_value_type("depth_radius", depth_radius, [int], self.name) + validator.check_value_type("bias", bias, [float], self.name) + validator.check_value_type("alpha", alpha, [float], self.name) + validator.check_value_type("beta", beta, [float], self.name) + validator.check_value_type("norm_region", norm_region, [str], self.name) + + def infer_dtype(self, x_dtype): + validator.check_tensor_type_same({"x": x_dtype}, (mstype.float16, mstype.float32,), self.name) + return x_dtype + + def infer_shape(self, x_shape): + return x_shape diff --git a/mindspore/ops/operations/other_ops.py b/mindspore/ops/operations/other_ops.py index 74c6080ab4..b6b938d800 100644 --- a/mindspore/ops/operations/other_ops.py +++ b/mindspore/ops/operations/other_ops.py @@ -59,8 +59,7 @@ class Assign(PrimitiveWithInfer): return variable def infer_dtype(self, variable, value): - args = {"variable": variable, "value": value} - validator.check_tensor_type_same(args, (mstype.bool_,) + mstype.number_type, self.name) + # Add a type validation later when we don't have to assign a value to RefKey. return variable diff --git a/mindspore/parallel/_auto_parallel_context.py b/mindspore/parallel/_auto_parallel_context.py index 74250f12e5..93fe233855 100644 --- a/mindspore/parallel/_auto_parallel_context.py +++ b/mindspore/parallel/_auto_parallel_context.py @@ -400,6 +400,23 @@ class _AutoParallelContext: self.check_context_handle() return self._context_handle.get_global_rank_is_set() + def set_enable_parallel_optimizer(self, enable_parallel_optimizer): + """ + Set enable/disable parallel optimizer. + + Args: + set_enable_parallel_optimizer (bool): Enable/disable parallel optimizer. + """ + self.check_context_handle() + if not isinstance(enable_parallel_optimizer, bool): + raise TypeError('enable_parallel_optimizer is invalid type') + self._context_handle.set_enable_parallel_optimizer(enable_parallel_optimizer) + + def get_enable_parallel_optimizer(self): + """Get parallel optimizer flag.""" + self.check_context_handle() + return self._context_handle.get_enable_parallel_optimizer() + def reset(self): """Reset all settings.""" self.check_context_handle() @@ -433,7 +450,8 @@ _set_auto_parallel_context_func_map = { "parameter_broadcast": auto_parallel_context().set_parameter_broadcast, "strategy_ckpt_load_file": auto_parallel_context().set_strategy_ckpt_load_file, "strategy_ckpt_save_file": auto_parallel_context().set_strategy_ckpt_save_file, - "full_batch": auto_parallel_context().set_full_batch} + "full_batch": auto_parallel_context().set_full_batch, + "enable_parallel_optimizer": auto_parallel_context().set_enable_parallel_optimizer} _get_auto_parallel_context_func_map = { @@ -447,13 +465,15 @@ _get_auto_parallel_context_func_map = { "parameter_broadcast": auto_parallel_context().get_parameter_broadcast, "strategy_ckpt_load_file": auto_parallel_context().get_strategy_ckpt_load_file, "strategy_ckpt_save_file": auto_parallel_context().get_strategy_ckpt_save_file, - "full_batch": auto_parallel_context().get_full_batch} + "full_batch": auto_parallel_context().get_full_batch, + "enable_parallel_optimizer": auto_parallel_context().get_enable_parallel_optimizer} @args_type_check(device_num=int, global_rank=int, mirror_mean=bool, cast_before_mirror=bool, loss_repeated_mean=bool, parallel_mode=str, auto_parallel_search_mode=str, parameter_broadcast=bool, strategy_ckpt_load_file=str, - strategy_ckpt_save_file=str, full_batch=bool) + strategy_ckpt_save_file=str, full_batch=bool, enable_parallel_optimizer=bool) + def _set_auto_parallel_context(**kwargs): """ Set auto parallel context. @@ -493,6 +513,7 @@ def _set_auto_parallel_context(**kwargs): strategy_ckpt_load_file (str): The path to load parallel strategy checkpoint. Default: '' strategy_ckpt_save_file (str): The path to save parallel strategy checkpoint. Default: '' full_batch (bool): Whether to load the whole batch on each device. Default: False. + enable_parallel_optimizer (bool): Enable using optimizer segmentation or noe. Default: False. Raises: ValueError: If input key is not attribute in auto parallel context. @@ -535,5 +556,6 @@ def _reset_auto_parallel_context(): - parameter_broadcast: False. - strategy_ckpt_load_file: "" - strategy_ckpt_save_file: "" + - enable_parallel_optimizer: False """ auto_parallel_context().reset() diff --git a/mindspore/parallel/mpi/_mpi_config.py b/mindspore/parallel/mpi/_mpi_config.py index e43305fb76..811066ca5d 100644 --- a/mindspore/parallel/mpi/_mpi_config.py +++ b/mindspore/parallel/mpi/_mpi_config.py @@ -104,7 +104,7 @@ def _get_mpi_config(attr_key): Object, The value of given attribute key. Raises: - ValueError: If input key is not an attribute in context. + ValueError: If input key is not an attribute in config. """ if not hasattr(_mpi_config(), attr_key): raise ValueError("Get context keyword %s is not recognized!" % attr_key) diff --git a/mindspore/train/callback/_checkpoint.py b/mindspore/train/callback/_checkpoint.py index d185377c83..e0048ad713 100644 --- a/mindspore/train/callback/_checkpoint.py +++ b/mindspore/train/callback/_checkpoint.py @@ -24,14 +24,13 @@ from mindspore import log as logger from mindspore._checkparam import check_bool, check_int_non_negative from mindspore.train._utils import _make_directory from mindspore.train.serialization import _exec_save_checkpoint, _save_graph - from ._callback import Callback, set_cur_net + _cur_dir = os.getcwd() _save_dir = _cur_dir - def _check_file_name_prefix(file_name_prefix): """ Check file name valid or not. diff --git a/mindspore/train/callback/_loss_monitor.py b/mindspore/train/callback/_loss_monitor.py index 3c1da218c2..766777e878 100644 --- a/mindspore/train/callback/_loss_monitor.py +++ b/mindspore/train/callback/_loss_monitor.py @@ -76,7 +76,7 @@ class LossMonitor(Callback): step_loss = np.mean(step_loss.asnumpy()) self.losses.append(step_loss) - cur_step_in_epoch = int((cb_params.cur_step_num - 1) % cb_params.batch_num) + cur_step_in_epoch = int((cb_params.cur_step_num - 1) % cb_params.batch_num) + 1 if isinstance(step_loss, float) and (np.isnan(step_loss) or np.isinf(step_loss)): raise ValueError("Epoch: [{:3d}/{:3d}], step: [{:5d}/{:5d}]. " @@ -86,8 +86,8 @@ class LossMonitor(Callback): if self._per_print_times != 0 and cb_params.cur_step_num % self._per_print_times == 0: print("Epoch: [{:3d}/{:3d}], step: [{:5d}/{:5d}], " - "loss: [{:5.4f}/{:5.4f}], time: [{:5.4f}]".format( - cb_params.cur_epoch_num - 1, cb_params.epoch_num, + "loss: [{:5.4f}], avg los: [{:5.4f}], time: [{:5.4f}ms]".format( + cb_params.cur_epoch_num, cb_params.epoch_num, cur_step_in_epoch, int(cb_params.batch_num), step_loss, np.mean(self.losses), step_mseconds), flush=True) diff --git a/mindspore/train/callback/_summary_collector.py b/mindspore/train/callback/_summary_collector.py index e2e4a9cc2d..1550c3c55c 100644 --- a/mindspore/train/callback/_summary_collector.py +++ b/mindspore/train/callback/_summary_collector.py @@ -73,7 +73,7 @@ class SummaryCollector(Callback): summary_dir (str): The collected data will be persisted to this directory. If the directory does not exist, it will be created automatically. collect_freq (int): Set the frequency of data collection, it should be greater then zero, - and the unit is `step`. Default: 10. + and the unit is `step`. Default: 10. The first step will be recorded at any time. It is important to note that if the data sink mode is used, the unit will become the `epoch`. It is not recommended to collect data too frequently, which can affect performance. collect_specified_data (Union[None, dict]): Perform custom operations on the collected data. Default: None. @@ -83,6 +83,7 @@ class SummaryCollector(Callback): The data that supports control is shown below. - collect_metric: Whether to collect training metrics, currently only loss is collected. + The first output will be treated as loss, and it will be averaged. Optional: True/False. Default: True. - collect_graph: Whether to collect computational graph, currently only training computational graph is collected. Optional: True/False. Default: True. @@ -142,9 +143,6 @@ class SummaryCollector(Callback): 'histogram_regular': None } - # _OPTIMIZER_FAILED means find optimizer failed, so we will not collect data about optimizer. - _OPTIMIZER_FAILED = 'Failed' - def __init__(self, summary_dir, collect_freq=10, collect_specified_data=None, keep_default_action=True, custom_lineage_data=None): super(SummaryCollector, self).__init__() @@ -158,17 +156,22 @@ class SummaryCollector(Callback): self._check_action(keep_default_action) self._collect_specified_data = self._process_specified_data(collect_specified_data, keep_default_action) - logger.info(f"For `collect_specified_data` the value after processing is: {self._collect_specified_data}.") + msg = f"For 'collect_specified_data' the value after processing is: {self._collect_specified_data}." + logger.info(msg) self._check_custom_lineage_data(custom_lineage_data) self._custom_lineage_data = custom_lineage_data - self._optimizer = None + self._temp_optimizer = None self._has_saved_train_network = False self._has_saved_custom_data = False self._is_parse_loss_success = True + self._first_step = True + self._dataset_sink_mode = True def __enter__(self): + self._first_step = True + self._dataset_sink_mode = True self._record = SummaryRecord(log_dir=self._summary_dir) return self @@ -228,7 +231,7 @@ class SummaryCollector(Callback): if specified_data is None: if action: return self._DEFAULT_SPECIFIED_DATA - return None + return dict() check_value_type('collect_specified_data', specified_data, [dict, type(None)]) @@ -237,7 +240,8 @@ class SummaryCollector(Callback): unexpected_params = set(specified_data) - set(self._DEFAULT_SPECIFIED_DATA) if unexpected_params: - raise ValueError(f'For `collect_specified_data` the keys {unexpected_params} are unsupported.') + raise ValueError(f'For `collect_specified_data` the keys {unexpected_params} are unsupported, ' + f'expect the follow keys: {list(self._DEFAULT_SPECIFIED_DATA.keys())}') if 'histogram_regular' in specified_data: check_value_type('histogram_regular', specified_data.get('histogram_regular'), (str, type(None))) @@ -248,7 +252,8 @@ class SummaryCollector(Callback): check_value_type(item, specified_data.get(item), bool) if action: - result = dict(self._DEFAULT_SPECIFIED_DATA).update(specified_data) + result = dict(self._DEFAULT_SPECIFIED_DATA) + result.update(specified_data) else: result = specified_data return result @@ -280,9 +285,13 @@ class SummaryCollector(Callback): def step_end(self, run_context): cb_params = run_context.original_args() + if self._first_step: + # Notice: This way of determining whether dataset sink mode is True does not work in the eval scenario + self._dataset_sink_mode = bool(cb_params.cur_step_num == cb_params.batch_num) if cb_params.mode == ModeEnum.TRAIN.value: - if cb_params.cur_step_num % self._collect_freq: + + if not self._is_collect_this_step(cb_params): return if not self._has_saved_train_network: @@ -292,6 +301,7 @@ class SummaryCollector(Callback): self._collect_metric(cb_params) self._collect_histogram(cb_params) + self._first_step = False self._record.record(cb_params.cur_step_num) def end(self, run_context): @@ -301,6 +311,8 @@ class SummaryCollector(Callback): else: self._collect_eval_lineage(cb_params) + # This is a workaround to avoid record '_summary_tensor_cache'. + self._record.set_mode('eval') # There's nothing special about setting step to 0 here, just to satisfy the interface call self._record.record(step=0) @@ -317,6 +329,18 @@ class SummaryCollector(Callback): raise ValueError(f"There are more than one {self.__class__.__name__} instance in callback list," f"but expected only one {self.__class__.__name__} instance.") + def _is_collect_this_step(self, cb_params): + """Decide whether to collect data for the current step.""" + # Make sure the first step data is recorded + if not self._first_step: + if self._dataset_sink_mode: + if cb_params.cur_epoch_num % self._collect_freq: + return False + else: + if cb_params.cur_step_num % self._collect_freq: + return False + return True + @staticmethod def _package_custom_lineage_data(custom_lineage_data): """ @@ -350,7 +374,7 @@ class SummaryCollector(Callback): input_data = getattr(cb_params, 'train_dataset_element', None) if input_data is None: self._collect_specified_data['collect_input_data'] = False - logger.info("There is not a `train_dataset_element` in cb_params.") + logger.info("The 'train_dataset_element' in cb_params is None, maybe there is dataset sink mode.") return if isinstance(input_data, (list, tuple)): @@ -358,6 +382,7 @@ class SummaryCollector(Callback): try: self._record.add_value(PluginEnum.IMAGE.value, 'input_data/auto', input_data) except ValueError: + logger.warning('The input data of network are not image, so will not collect by SummaryCollector.') self._collect_specified_data['collect_input_data'] = False return @@ -395,7 +420,12 @@ class SummaryCollector(Callback): loss = self._get_loss(cb_params) if loss is None: return - self._record.add_value(PluginEnum.SCALAR.value, 'loss/auto', loss) + + try: + self._record.add_value(PluginEnum.SCALAR.value, 'loss/auto', loss) + except ValueError: + logger.warning("The output of network is not a scalar, so will not collect loss in SummaryCollector.") + self._collect_specified_data['collect_metric'] = False def _get_loss(self, cb_params): """ @@ -413,19 +443,16 @@ class SummaryCollector(Callback): output = cb_params.net_outputs if output is None: - logger.warning("Can not find any output by this network.") + logger.warning("Can not find any output by this network, so will not collect loss in SummaryCollector.") self._is_parse_loss_success = False return None - if isinstance(output, (int, float)): + if isinstance(output, (int, float, Tensor)): loss = output - elif isinstance(output, (list, tuple)): + elif isinstance(output, (list, tuple)) and output: # If the output is a list, since the default network returns loss first, # we assume that the first one is loss. loss = output[0] - elif isinstance(output, Tensor) and (not output.shape or output.shape == [1]): - loss_numpy = output.asnumpy() - loss = float(np.atleast_1d(loss_numpy)[0]) else: logger.warning("The output type could not be identified, so no loss was recorded in SummaryCollector.") self._is_parse_loss_success = False @@ -434,6 +461,8 @@ class SummaryCollector(Callback): if not isinstance(loss, Tensor): loss = Tensor(loss) + precision = 4 + loss = Tensor(round(np.mean(loss.asnumpy()), precision)) return loss def _get_optimizer(self, cb_params): @@ -446,21 +475,25 @@ class SummaryCollector(Callback): Returns: Union[Optimizer, None], if parse optimizer success, will return a optimizer, else return None. """ - if self._optimizer == self._OPTIMIZER_FAILED: + # 'optimizer_failed' means find optimizer failed, so we will not collect data about optimizer. + optimizer_failed = 'Failed' + if self._temp_optimizer == optimizer_failed: return None - if self._optimizer is not None: - return self._optimizer + if self._temp_optimizer is not None: + return self._temp_optimizer optimizer = cb_params.optimizer if optimizer is None: - network = cb_params.train_network if cb_params.mode == 'train' else cb_params.eval_work + network = cb_params.train_network if cb_params.mode == 'train' else cb_params.eval_network optimizer = self._parse_optimizer_by_network(network) if optimizer is None or not isinstance(optimizer, Optimizer): - logger.warning("Can not find optimizer in network, or the optimizer does not inherit Mindpore's optimizer, " - "so we will not collect data about optimizer in SummaryCollector.") - optimizer = self._OPTIMIZER_FAILED + logger.warning("Can not find optimizer in network, or the optimizer does not inherit MindSpore's " + "optimizer, so we will not collect data about optimizer in SummaryCollector.") + optimizer = None + + self._temp_optimizer = optimizer if optimizer is not None else optimizer_failed return optimizer @@ -469,6 +502,8 @@ class SummaryCollector(Callback): """Parse optimizer from network, if parse success will return a optimizer, else return None.""" optimizer = None for _, cell in network.cells_and_names(): + if isinstance(cell, Optimizer): + return cell try: optimizer = getattr(cell, 'optimizer') except AttributeError: @@ -489,11 +524,11 @@ class SummaryCollector(Callback): if 'histogram_regular' not in self._collect_specified_data: return - self._optimizer = self._get_optimizer(cb_params) - if self._optimizer is None: + optimizer = self._get_optimizer(cb_params) + if optimizer is None: return - parameters = self._optimizer.parameters + parameters = optimizer.parameters regular = self._collect_specified_data.get('histogram_regular') if regular is not None: for parameter in parameters: @@ -538,7 +573,7 @@ class SummaryCollector(Callback): train_lineage[LineageMetadata.loss] = None optimizer = self._get_optimizer(cb_params) - learning_rate = self._get_learning_rate(optimizer) + learning_rate = self._get_learning_rate(optimizer) if optimizer is not None else None if learning_rate is not None: train_lineage[LineageMetadata.learning_rate] = list(np.atleast_1d(learning_rate.asnumpy()))[0] @@ -554,7 +589,6 @@ class SummaryCollector(Callback): train_lineage[LineageMetadata.step_num] = cb_params.cur_step_num train_lineage[LineageMetadata.parallel_mode] = cb_params.parallel_mode train_lineage[LineageMetadata.device_num] = cb_params.device_number - train_lineage[LineageMetadata.batch_size] = cb_params.batch_num ckpt_file_path = self._get_ckpt_file_path(cb_params) train_lineage[LineageMetadata.model_path] = json.dumps(dict(ckpt=ckpt_file_path)) @@ -640,6 +674,8 @@ class SummaryCollector(Callback): batch_size = dataset.get_batch_size() dataset_size = int(batch_num * batch_size) + lineage_dict[LineageMetadata.batch_size] = batch_size + if cb_params.mode == ModeEnum.TRAIN.value: lineage_dict[LineageMetadata.train_dataset_path] = dataset_dir lineage_dict[LineageMetadata.train_dataset_size] = dataset_size @@ -677,7 +713,7 @@ class SummaryCollector(Callback): return output_dataset.dataset_dir if isinstance(output_dataset, dataset_files_set): return output_dataset.dataset_files[0] - return self._get_dataset_path(output_dataset.input[0]) + return self._get_dataset_path(output_dataset.children[0]) @staticmethod def _get_ckpt_file_path(cb_params): @@ -734,7 +770,7 @@ class SummaryCollector(Callback): cb_params (_InternalCallbackParam): Callback parameters. Returns: - Union[Loss_fn, None], a Cell object, if parse failed, will return None. + Union[Cell, None], a Cell object, if parse failed, will return None. """ loss_fn = cb_params.loss_fn if loss_fn is not None: diff --git a/mindspore/train/quant/quant.py b/mindspore/train/quant/quant.py index 937e54a7e4..bc44ba22c2 100644 --- a/mindspore/train/quant/quant.py +++ b/mindspore/train/quant/quant.py @@ -33,7 +33,6 @@ from ...ops.operations import _inner_ops as inner from ...train import serialization from . import quant_utils - _ACTIVATION_MAP = {nn.ReLU: quant.ReLUQuant, nn.ReLU6: quant.ReLU6Quant, nn.HSigmoid: quant.HSigmoidQuant, @@ -42,15 +41,14 @@ _ACTIVATION_MAP = {nn.ReLU: quant.ReLUQuant, class _AddFakeQuantInput(nn.Cell): """ - Add FakeQuant at input and output of the Network. Only support one input and one output case. + Add FakeQuant OP at input of the network. Only support one input case. """ def __init__(self, network, quant_delay=0): super(_AddFakeQuantInput, self).__init__(auto_prefix=False) + self.fake_quant_input = quant.FakeQuantWithMinMax(min_init=-6, max_init=6, quant_delay=quant_delay, ema=True) + self.fake_quant_input.update_parameters_name('fake_quant_input.') self.network = network - self.fake_quant_input = quant.FakeQuantWithMinMax( - min_init=-6, max_init=6, quant_delay=quant_delay, ema=True) - self.fake_quant_input.update_parameters_name('fake_quant_input') def construct(self, data): data = self.fake_quant_input(data) @@ -60,7 +58,7 @@ class _AddFakeQuantInput(nn.Cell): class _AddFakeQuantAfterSubCell(nn.Cell): """ - Add FakeQuant after of the sub Cell. + Add FakeQuant OP after of the sub Cell. """ def __init__(self, subcell, **kwargs): @@ -115,11 +113,12 @@ class ConvertToQuantNetwork: self.network.update_cell_prefix() network = self._convert_subcells2quant(self.network) network = _AddFakeQuantInput(network) + self.network.update_cell_type("quant") return network def _convert_subcells2quant(self, network): """ - convet sub cell to quant cell + convert sub cell like `Conv2dBnAct` and `DenseBnAct` to quant cell """ cells = network.name_cells() change = False @@ -138,23 +137,25 @@ class ConvertToQuantNetwork: if isinstance(network, nn.SequentialCell) and change: network.cell_list = list(network.cells()) - # tensoradd to tensoradd quant + # add FakeQuant OP after OP in while list add_list = [] for name in network.__dict__: if name[0] == '_': continue attr = network.__dict__[name] - if isinstance(attr, ops.Primitive) and attr.name in ConvertToQuantNetwork.__quant_op_name__: + if isinstance(attr, ops.Primitive) and attr.name in self.__quant_op_name__: add_list.append((name, attr)) for name, prim_op in add_list: prefix = name add_quant = _AddFakeQuantAfterSubCell(prim_op, num_bits=self.act_bits, - quant_delay=self.act_delay, + quant_delay=self.act_qdelay, per_channel=self.act_channel, symmetric=self.act_symmetric, narrow_range=self.act_range) - prefix = '.'.join([network.param_prefix, self._convert_op_name(prim_op.name)]) + prefix = self._convert_op_name(prim_op.name) + if network.param_prefix: + prefix = '.'.join([network.param_prefix, self._convert_op_name(prim_op.name)]) add_quant.update_parameters_name(prefix + '.') del network.__dict__[name] network.insert_child_to_cell(name, add_quant) @@ -162,11 +163,11 @@ class ConvertToQuantNetwork: def _convert_conv(self, subcell): """ - convet conv cell to quant cell + convert Conv2d cell to quant cell """ conv_inner = subcell.conv - bn_inner = subcell.batchnorm - if subcell.batchnorm is not None and self.bn_fold: + if subcell.has_bn and self.bn_fold: + bn_inner = subcell.batchnorm conv_inner = quant.Conv2dBatchNormQuant(conv_inner.in_channels, conv_inner.out_channels, kernel_size=conv_inner.kernel_size, @@ -176,7 +177,6 @@ class ConvertToQuantNetwork: dilation=conv_inner.dilation, group=conv_inner.group, eps=bn_inner.eps, - momentum=bn_inner.momentum, quant_delay=self.weight_qdelay, freeze_bn=self.freeze_bn, per_channel=self.weight_channel, @@ -184,6 +184,11 @@ class ConvertToQuantNetwork: fake=True, symmetric=self.weight_symmetric, narrow_range=self.weight_range) + # change original network BatchNormal OP parameters to quant network + conv_inner.gamma = subcell.batchnorm.gamma + conv_inner.beta = subcell.batchnorm.beta + conv_inner.moving_mean = subcell.batchnorm.moving_mean + conv_inner.moving_variance = subcell.batchnorm.moving_variance del subcell.batchnorm subcell.batchnorm = None subcell.has_bn = False @@ -202,6 +207,10 @@ class ConvertToQuantNetwork: num_bits=self.weight_bits, symmetric=self.weight_symmetric, narrow_range=self.weight_range) + # change original network Conv2D OP parameters to quant network + conv_inner.weight = subcell.conv.weight + if subcell.conv.has_bias: + conv_inner.bias = subcell.conv.bias subcell.conv = conv_inner if subcell.has_act and subcell.activation is not None: subcell.activation = self._convert_activation(subcell.activation) @@ -228,6 +237,10 @@ class ConvertToQuantNetwork: per_channel=self.weight_channel, symmetric=self.weight_symmetric, narrow_range=self.weight_range) + # change original network Dense OP parameters to quant network + dense_inner.weight = subcell.dense.weight + if subcell.dense.has_bias: + dense_inner.bias = subcell.dense.bias subcell.dense = dense_inner if subcell.has_act and subcell.activation is not None: subcell.activation = self._convert_activation(subcell.activation) @@ -245,24 +258,24 @@ class ConvertToQuantNetwork: act_class = activation.__class__ if act_class not in _ACTIVATION_MAP: raise ValueError( - "Unsupported activation in auto Quant: ", act_class) + "Unsupported activation in auto quant: ", act_class) return _ACTIVATION_MAP[act_class](num_bits=self.act_bits, quant_delay=self.act_qdelay, per_channel=self.act_channel, - symmetric=self.weight_symmetric, - narrow_range=self.weight_range) + symmetric=self.act_symmetric, + narrow_range=self.act_range) -class ExportQuantNetworkDeploy: +class ExportToQuantInferNetwork: """ - Convert quantization aware network to deploy network. + Convert quantization aware network to infer network. Args: - network (Cell): MindSpore network produced by `convert_quant_network`. - inputs (Tensor): Inputs of the `network`. + network (Cell): MindSpore network API `convert_quant_network`. + inputs (Tensor): Input tensors of the `quantization aware training network`. Returns: - Cell, converted network. + Cell, GEIR backend Infer network. """ __quant_op_name__ = ["TensorAdd", "Sub", "Mul", "RealDiv"] @@ -272,7 +285,7 @@ class ExportQuantNetworkDeploy: network = validator.check_isinstance('network', network, (nn.Cell,)) self.data_type = mstype.int8 self.network = copy.deepcopy(network) - self.all_paramters = {p.name: p for p in self.network.get_parameters()} + self.all_parameters = {p.name: p for p in self.network.get_parameters()} self.get_inputs_table(inputs) def get_inputs_table(self, inputs): @@ -300,8 +313,8 @@ class ExportQuantNetworkDeploy: info = self.quant_info_table.get(w_minq_name, None) if info: fack_quant_a_in_op, minq_name = info - maxq = self.all_paramters[minq_name[:-4] + "maxq"] - minq = self.all_paramters[minq_name] + maxq = self.all_parameters[minq_name[:-4] + "maxq"] + minq = self.all_parameters[minq_name] scale_a_in, zp_a_in = quant_utils.scale_zp_from_data(fack_quant_a_in_op, maxq, minq, np_type) else: logger.warning(f"Do not find `fake_quant` from input with `fack_quant.minq` {w_minq_name}") @@ -342,7 +355,7 @@ class ExportQuantNetworkDeploy: return block def _convert_quant2deploy(self, network): - """Convet network's all quant subcell to deploy subcell.""" + """Convert network's all quant subcell to deploy subcell.""" cells = network.name_cells() change = False for name in cells: @@ -380,18 +393,26 @@ class ExportQuantNetworkDeploy: return network -def export_geir(network, *inputs, file_name): +def export(network, *inputs, file_name, file_format='GEIR'): """ - Exports MindSpore quant predict model to deploy with GEIR. + Exports MindSpore quantization predict model to deploy with GEIR. Args: network (Cell): MindSpore network produced by `convert_quant_network`. - inputs (Tensor): Inputs of the `network`. + inputs (Tensor): Inputs of the `quantization aware training network`. file_name (str): File name of model to export. + file_format (str): MindSpore currently supports 'GEIR' format for exported quantization aware model. + - GEIR: Graph Engine Intermediate Representation. An Intermediate representation format of Ascend model. """ - exporter = ExportQuantNetworkDeploy(network, *inputs) - deploy_net = exporter.run() - serialization.export(deploy_net, *inputs, file_name=file_name, file_format="GEIR") + supported_formats = ['GEIR'] + + if file_format not in supported_formats: + raise ValueError('Illegal file format {}.'.format(file_format)) + + if file_format == 'GEIR': + exporter = ExportToQuantInferNetwork(network, *inputs) + deploy_net = exporter.run() + serialization.export(deploy_net, *inputs, file_name=file_name, file_format=file_format) def convert_quant_network(network, @@ -408,19 +429,19 @@ def convert_quant_network(network, Args: network (Cell): Obtain a pipeline through network for saving graph summary. - quant_delay (int or tuple): Number of steps after which weights and activations are quantized during - eval. The first element represent weights and second element represent data flow. Default: (0, 0) bn_fold (bool): Flag to used bn fold ops for simulation inference operation. Default: False. freeze_bn (int): Number of steps after which BatchNorm OP parameters used total mean and variance. Default: 0. - num_bits (int or tuple): Number of bits to use for quantizing weights and activations. The first + quant_delay (int, list or tuple): Number of steps after which weights and activations are quantized during + eval. The first element represent weights and second element represent data flow. Default: (0, 0) + num_bits (int, list or tuple): Number of bits to use for quantizing weights and activations. The first element represent weights and second element represent data flow. Default: (8, 8) - per_channel (int or tuple): Quantization granularity based on layer or on channel. If `True` + per_channel (bool, list or tuple): Quantization granularity based on layer or on channel. If `True` then base on per channel otherwise base on per layer. The first element represent weights and second element represent data flow. Default: (False, False) - symmetric (int or tuple): Quantization algorithm use symmetric or not. If `True` then base on - symmetric otherwise base on assymmetric. The first element represent weights and second + symmetric (bool, list or tuple): Quantization algorithm use symmetric or not. If `True` then base on + symmetric otherwise base on asymmetric. The first element represent weights and second element represent data flow. Default: (False, False) - narrow_range (int or tuple): Quantization algorithm use narrow range or not. If `True` then base + narrow_range (bool, list or tuple): Quantization algorithm use narrow range or not. If `True` then base on narrow range otherwise base on off narrow range. The first element represent weights and second element represent data flow. Default: (False, False) @@ -428,6 +449,7 @@ def convert_quant_network(network, Cell, Network which has change to quantization aware training network cell. """ support_device = ["Ascend", "GPU"] + def convert2list(name, value): if not isinstance(value, list) and not isinstance(value, tuple): value = [value] @@ -442,7 +464,7 @@ def convert_quant_network(network, narrow_range = convert2list("narrow range", narrow_range) if context.get_context('device_target') not in support_device: - raise KeyError("Not support {} backend.".format(context.get_context('device_target'))) + raise KeyError("Unsupported {} device target.".format(context.get_context('device_target'))) net = ConvertToQuantNetwork(network=network, quant_delay=quant_delay, diff --git a/mindspore/train/quant/quant_utils.py b/mindspore/train/quant/quant_utils.py index c9e6ac92e1..c4a8004012 100644 --- a/mindspore/train/quant/quant_utils.py +++ b/mindspore/train/quant/quant_utils.py @@ -125,7 +125,7 @@ def scale_zp_from_fack_quant_cell(cell, data_type): """ minq = cell.minq.data.asnumpy() maxq = cell.maxq.data.asnumpy() - op = cell.fake_quant + op = cell.fake_quant_infer scale, zp = cal_quantization_params( minq, maxq, data_type, diff --git a/mindspore/train/serialization.py b/mindspore/train/serialization.py index c39104c6ff..d74bee2706 100644 --- a/mindspore/train/serialization.py +++ b/mindspore/train/serialization.py @@ -29,7 +29,7 @@ from mindspore.common.api import _executor from mindspore.common import dtype as mstype from mindspore._checkparam import check_input_data -__all__ = ["save_checkpoint", "load_checkpoint", "load_param_into_net", "export"] +__all__ = ["save_checkpoint", "load_checkpoint", "load_param_into_net", "export", "parse_print"] tensor_to_ms_type = {"Int8": mstype.int8, "Uint8": mstype.uint8, "Int16": mstype.int16, "Uint16": mstype.uint16, "Int32": mstype.int32, "Uint32": mstype.uint32, "Int64": mstype.int64, "Uint64": mstype.uint64, @@ -101,14 +101,14 @@ def _update_param(param, new_param): param.set_parameter_data(type(param.data)(new_param.data)) -def save_checkpoint(parameter_list, ckpoint_file_name): +def save_checkpoint(parameter_list, ckpt_file_name): """ Saves checkpoint info to a specified file. Args: parameter_list (list): Parameters list, each element is a dict like {"name":xx, "type":xx, "shape":xx, "data":xx}. - ckpoint_file_name (str): Checkpoint file name. + ckpt_file_name (str): Checkpoint file name. Raises: RuntimeError: Failed to save the Checkpoint file. @@ -133,22 +133,22 @@ def save_checkpoint(parameter_list, ckpoint_file_name): for dim in param['data'].shape: param_tensor.dims.append(dim) - with open(ckpoint_file_name, "wb") as f: + with open(ckpt_file_name, "wb") as f: f.write(checkpoint_list.SerializeToString()) - os.chmod(ckpoint_file_name, stat.S_IRUSR) + os.chmod(ckpt_file_name, stat.S_IRUSR) except BaseException as e: - logger.error("Failed to save the checkpoint file %s.", ckpoint_file_name) + logger.error("Failed to save the checkpoint file %s.", ckpt_file_name) raise RuntimeError(e.__str__()) logger.info("Save checkpoint process finish.") -def load_checkpoint(ckpoint_file_name, net=None): +def load_checkpoint(ckpt_file_name, net=None): """ Loads checkpoint info from a specified file. Args: - ckpoint_file_name (str): Checkpoint file name. + ckpt_file_name (str): Checkpoint file name. net (Cell): Cell network. Default: None Returns: @@ -157,28 +157,30 @@ def load_checkpoint(ckpoint_file_name, net=None): Raises: ValueError: Checkpoint file is incorrect. """ - if not isinstance(ckpoint_file_name, str): - raise ValueError("The ckpoint_file_name must be String.") + if not isinstance(ckpt_file_name, str): + raise ValueError("The ckpt_file_name must be string.") - if not os.path.exists(ckpoint_file_name) or ckpoint_file_name[-5:] != ".ckpt": + if not os.path.exists(ckpt_file_name): + raise ValueError("The checkpoint file is not exist.") + + if ckpt_file_name[-5:] != ".ckpt": raise ValueError("Please input the correct checkpoint file name.") - if os.path.getsize(ckpoint_file_name) == 0: + if os.path.getsize(ckpt_file_name) == 0: raise ValueError("The checkpoint file may be empty, please make sure enter the correct file name.") logger.info("Execute load checkpoint process.") checkpoint_list = Checkpoint() try: - with open(ckpoint_file_name, "rb") as f: + with open(ckpt_file_name, "rb") as f: pb_content = f.read() checkpoint_list.ParseFromString(pb_content) except BaseException as e: - logger.error("Failed to read the checkpoint file %s, please check the correct of the file.", ckpoint_file_name) + logger.error("Failed to read the checkpoint file `%s`, please check the correct of the file.", ckpt_file_name) raise ValueError(e.__str__()) parameter_dict = {} - try: for element in checkpoint_list.value: data = element.tensor.tensor_content @@ -206,7 +208,7 @@ def load_checkpoint(ckpoint_file_name, net=None): logger.info("Load checkpoint process finish.") except BaseException as e: - logger.error("Failed to load the checkpoint file %s.", ckpoint_file_name) + logger.error("Failed to load the checkpoint file `%s`.", ckpt_file_name) raise RuntimeError(e.__str__()) if net: @@ -303,14 +305,14 @@ def _save_graph(network, file_name): os.chmod(file_name, stat.S_IWUSR | stat.S_IRUSR) -def _exec_save_checkpoint(train_network, ckpoint_file_name, integrated_save=True): +def _exec_save_checkpoint(train_network, ckpt_file_name, integrated_save=True): """ Saves checkpoint for 'ms' backend. Args: train_network (Network): The train network for training. - ckpoint_file_name (str): The name of checkpoint file. - integrated_save (bool): Whether to intergrated save in automatic model parallel scene. + ckpt_file_name (str): The name of checkpoint file. + integrated_save (bool): Whether to integrated save in automatic model parallel scene. """ param_dict = {} @@ -334,7 +336,7 @@ def _exec_save_checkpoint(train_network, ckpoint_file_name, integrated_save=True each_param["data"] = param_data param_list.append(each_param) - save_checkpoint(param_list, ckpoint_file_name) + save_checkpoint(param_list, ckpt_file_name) def _get_merged_param_data(net, param_name, param_data): @@ -501,6 +503,13 @@ def parse_print(print_file_name): tensor_list.append(Tensor(param_value, ms_type)) # Scale type else: + data_type_ = data_type.lower() + if 'float' in data_type_: + param_data = float(param_data[0]) + elif 'int' in data_type_: + param_data = int(param_data[0]) + elif 'bool' in data_type_: + param_data = bool(param_data[0]) tensor_list.append(Tensor(param_data, ms_type)) except BaseException as e: diff --git a/mindspore/train/summary/_summary_adapter.py b/mindspore/train/summary/_summary_adapter.py index 1ae5bdd2d5..4cbb7b8fd5 100644 --- a/mindspore/train/summary/_summary_adapter.py +++ b/mindspore/train/summary/_summary_adapter.py @@ -91,7 +91,7 @@ def package_graph_event(data): return graph_event -def package_summary_event(data_list, step): +def package_summary_event(data_list, step, wall_time): """ Package the summary to event protobuffer. @@ -105,7 +105,7 @@ def package_summary_event(data_list, step): # create the event of summary summary_event = Event() summary = summary_event.summary - summary_event.wall_time = time.time() + summary_event.wall_time = wall_time summary_event.step = int(step) for value in data_list: diff --git a/mindspore/train/summary/_writer_pool.py b/mindspore/train/summary/_writer_pool.py index 2d219743de..d9cdfd3c8c 100644 --- a/mindspore/train/summary/_writer_pool.py +++ b/mindspore/train/summary/_writer_pool.py @@ -14,15 +14,16 @@ # ============================================================================ """Write events to disk in a base directory.""" import os +import time from collections import deque from multiprocessing import Pool, Process, Queue, cpu_count from ._lineage_adapter import serialize_to_lineage_event from ._summary_adapter import package_graph_event, package_summary_event -from ._summary_writer import SummaryWriter, LineageWriter +from ._summary_writer import LineageWriter, SummaryWriter -def _pack_data(datadict): +def _pack_data(datadict, wall_time): """Pack data according to which plugin.""" result = [] summaries, step, mode = [], None, None @@ -37,7 +38,7 @@ def _pack_data(datadict): step = data.get('step') mode = data.get('mode') if summaries: - result.append(['summary', mode, package_summary_event(summaries, step).SerializeToString()]) + result.append(['summary', mode, package_summary_event(summaries, step, wall_time).SerializeToString()]) return result @@ -59,7 +60,7 @@ class WriterPool(Process): def run(self): writers = self._get_writers() - with Pool() as pool: + with Pool(min(cpu_count(), 32)) as pool: deq = deque() while True: while deq and deq[0].ready(): @@ -70,7 +71,7 @@ class WriterPool(Process): if not self._queue.empty(): action, data = self._queue.get() if action == 'WRITE': - deq.append(pool.apply_async(_pack_data, (data,))) + deq.append(pool.apply_async(_pack_data, (data, time.time()))) elif action == 'FLUSH': for writer in writers: writer.flush() diff --git a/mindspore/train/summary/summary_record.py b/mindspore/train/summary/summary_record.py index 61c2c8adeb..21c8c58d3b 100644 --- a/mindspore/train/summary/summary_record.py +++ b/mindspore/train/summary/summary_record.py @@ -13,6 +13,7 @@ # limitations under the License. # ============================================================================ """Record the summary event.""" +import atexit import os import re import threading @@ -21,7 +22,7 @@ from mindspore import log as logger from ..._c_expression import Tensor from ..._checkparam import _check_str_by_regular -from .._utils import _make_directory, _check_to_numpy, _check_lineage_value +from .._utils import _check_lineage_value, _check_to_numpy, _make_directory from ._summary_adapter import get_event_file_name, package_graph_event from ._writer_pool import WriterPool @@ -62,11 +63,15 @@ class SummaryRecord: """ SummaryRecord is used to record the summary data and lineage data. + The API will create a summary file and lineage files lazily in a given directory and writes data to them. + It writes the data to files by executing the 'record' method. In addition to record the data bubbled up from + the network by defining the summary operators, SummaryRecord also supports to record extra data which + can be added by calling add_value. + Note: - The API will create a summary file and a lineage file lazily in a given directory and writes data to them. - It writes the data to files by executing the record method. In addition to record the data bubbled up from - the network by defining the summary operators, SummaryRecord also supports to record extra data which - can be added by calling add_value. Finally, make sure to close the SummaryRecord object at the end. + 1. Make sure to close the SummaryRecord at the end, or the process will not exit. + Please see the Example section below on how to properly close with two ways. + 2. The SummaryRecord instance can only allow one at a time, otherwise it will cause problems with data writes. Args: log_dir (str): The log_dir is a directory location to save the summary. @@ -81,8 +86,15 @@ class SummaryRecord: RuntimeError: If the log_dir can not be resolved to a canonicalized absolute pathname. Examples: - >>> with SummaryRecord(log_dir="/opt/log", file_prefix="xxx_", file_suffix="_yyy") as summary_record: + >>> # use in with statement to auto close + >>> with SummaryRecord(log_dir="./summary_dir") as summary_record: >>> pass + >>> + >>> # use in try .. finally .. to ensure closing + >>> try: + >>> summary_record = SummaryRecord(log_dir="./summary_dir") + >>> finally: + >>> summary_record.close() """ def __init__(self, @@ -93,8 +105,8 @@ class SummaryRecord: file_suffix="_MS", network=None): - self._closed, self._mode = False, 'train' - self._data_pool = _dictlist() + self._closed, self._event_writer = False, None + self._mode, self._data_pool = 'train', _dictlist() _check_str_by_regular(file_prefix) _check_str_by_regular(file_suffix) @@ -132,6 +144,7 @@ class SummaryRecord: self._event_writer = WriterPool(log_dir, summary=self.full_file_name, lineage=get_event_file_name('events', '_lineage')) + atexit.register(self.close) def __enter__(self): """Enter the context manager.""" @@ -154,7 +167,7 @@ class SummaryRecord: ValueError: When the mode is not recognized. Examples: - >>> with SummaryRecord(log_dir="/opt/log", file_prefix="xxx_", file_suffix="_yyy") as summary_record: + >>> with SummaryRecord(log_dir="./summary_dir", file_prefix="xxx_", file_suffix="_yyy") as summary_record: >>> summary_record.set_mode('eval') """ mode_spec = 'train', 'eval' @@ -193,7 +206,7 @@ class SummaryRecord: TypeError: When the value is not a Tensor. Examples: - >>> with SummaryRecord(log_dir="/opt/log", file_prefix="xxx_", file_suffix="_yyy") as summary_record: + >>> with SummaryRecord(log_dir="./summary_dir", file_prefix="xxx_", file_suffix="_yyy") as summary_record: >>> summary_record.add_value('scalar', 'loss', Tensor(0.1)) """ if plugin in ('tensor', 'scalar', 'image', 'histogram'): @@ -202,6 +215,9 @@ class SummaryRecord: if not isinstance(value, Tensor): raise TypeError(f'Expect the value to be Tensor, but got {type(value).__name__}') np_value = _check_to_numpy(plugin, value) + if name in {item['tag'] for item in self._data_pool[plugin]}: + entry = repr(f'{name}/{plugin}') + logger.warning(f'{entry} has duplicate values. Only the newest one will be recorded.') self._data_pool[plugin].append(dict(tag=name, mode=self._mode, value=np_value)) elif plugin in ('train_lineage', 'eval_lineage', 'dataset_graph', 'custom_lineage_data'): @@ -225,10 +241,10 @@ class SummaryRecord: bool, whether the record process is successful or not. Examples: - >>> with SummaryRecord(log_dir="/opt/log", file_prefix="xxx_", file_suffix="_yyy") as summary_record: + >>> with SummaryRecord(log_dir="./summary_dir", file_prefix="xxx_", file_suffix="_yyy") as summary_record: >>> summary_record.record(step=2) """ - logger.info("SummaryRecord step is %r.", step) + logger.debug("SummaryRecord step is %r.", step) if self._closed: logger.error("The record writer is closed.") return False @@ -282,7 +298,7 @@ class SummaryRecord: str, the full path of log file. Examples: - >>> with SummaryRecord(log_dir="/opt/log", file_prefix="xxx_", file_suffix="_yyy") as summary_record: + >>> with SummaryRecord(log_dir="./summary_dir", file_prefix="xxx_", file_suffix="_yyy") as summary_record: >>> print(summary_record.log_dir) """ return self.full_file_name @@ -294,7 +310,7 @@ class SummaryRecord: Call it to make sure that all pending events have been written to disk. Examples: - >>> with SummaryRecord(log_dir="/opt/log", file_prefix="xxx_", file_suffix="_yyy") as summary_record: + >>> with SummaryRecord(log_dir="./summary_dir", file_prefix="xxx_", file_suffix="_yyy") as summary_record: >>> summary_record.flush() """ if self._closed: @@ -307,18 +323,18 @@ class SummaryRecord: Flush all events and close summary records. Please use with statement to autoclose. Examples: - >>> with SummaryRecord(log_dir="/opt/log", file_prefix="xxx_", file_suffix="_yyy") as summary_record: - >>> pass # summary_record autoclosed + >>> try: + >>> summary_record = SummaryRecord(log_dir="./summary_dir") + >>> finally: + >>> summary_record.close() """ if not self._closed and self._event_writer: # event writer flush and close logger.info('Please wait it may take quite some time to finish writing and closing.') + atexit.unregister(self.close) self._event_writer.close() self._closed = True - def __del__(self) -> None: - self.close() - @staticmethod def _parse_from(name: str = None): """Parse the tag and type from name.""" diff --git a/model_zoo/README.md b/model_zoo/README.md index 24be683b22..2dde985679 100644 --- a/model_zoo/README.md +++ b/model_zoo/README.md @@ -32,6 +32,7 @@ In order to facilitate developers to enjoy the benefits of MindSpore framework a - [Natural Language Processing](#natural-language-processing) - [BERT](#bert) - [MASS](#mass) + - [Transformer](#transformer) # Announcements @@ -301,6 +302,26 @@ In order to facilitate developers to enjoy the benefits of MindSpore framework a | Model for inference | | | Scripts | | +#### [Transformer](#table-of-contents) + +| Parameters | Transformer | +| -------------------------- | -------------------------------------------------------------- | +| Published Year | 2017 | +| Paper | [Attention Is All You Need ](https://arxiv.org/abs/1706.03762) | +| Resource | Ascend 910 | +| Features | • Multi-GPU training support with Ascend | +| MindSpore Version | 0.5.0-beta | +| Dataset | WMT Englis-German | +| Training Parameters | epoch=52, batch_size=96 | +| Optimizer | Adam | +| Loss Function | Softmax Cross Entropy | +| BLEU Score | 28.7 | +| Speed | 410ms/step (8pcs) | +| Loss | 2.8 | +| Params (M) | 213.7 | +| Checkpoint for inference | 2.4G (.ckpt file) | +| Scripts | https://gitee.com/mindspore/mindspore/tree/master/model_zoo/Transformer | + #### License [Apache License 2.0](https://github.com/mindspore-ai/mindspore/blob/master/LICENSE) diff --git a/model_zoo/Transformer/create_data.py b/model_zoo/Transformer/create_data.py index af941623cb..fc81d3c232 100644 --- a/model_zoo/Transformer/create_data.py +++ b/model_zoo/Transformer/create_data.py @@ -37,13 +37,13 @@ class SampleInstance(): def __str__(self): s = "" s += "source sos tokens: %s\n" % (" ".join( - [tokenization.printable_text(x) for x in self.source_sos_tokens])) + [tokenization.convert_to_printable(x) for x in self.source_sos_tokens])) s += "source eos tokens: %s\n" % (" ".join( - [tokenization.printable_text(x) for x in self.source_eos_tokens])) + [tokenization.convert_to_printable(x) for x in self.source_eos_tokens])) s += "target sos tokens: %s\n" % (" ".join( - [tokenization.printable_text(x) for x in self.target_sos_tokens])) + [tokenization.convert_to_printable(x) for x in self.target_sos_tokens])) s += "target eos tokens: %s\n" % (" ".join( - [tokenization.printable_text(x) for x in self.target_eos_tokens])) + [tokenization.convert_to_printable(x) for x in self.target_eos_tokens])) s += "\n" return s @@ -185,9 +185,9 @@ def main(): if total_written <= 20: logging.info("*** Example ***") logging.info("source tokens: %s", " ".join( - [tokenization.printable_text(x) for x in instance.source_eos_tokens])) + [tokenization.convert_to_printable(x) for x in instance.source_eos_tokens])) logging.info("target tokens: %s", " ".join( - [tokenization.printable_text(x) for x in instance.target_sos_tokens])) + [tokenization.convert_to_printable(x) for x in instance.target_sos_tokens])) for feature_name in features.keys(): feature = features[feature_name] diff --git a/model_zoo/Transformer/scripts/replace-quote.perl b/model_zoo/Transformer/scripts/replace-quote.perl index 95f9abcc91..5e6e715daf 100644 --- a/model_zoo/Transformer/scripts/replace-quote.perl +++ b/model_zoo/Transformer/scripts/replace-quote.perl @@ -1,4 +1,19 @@ #!/usr/bin/env perl +#!/bin/bash +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ use warnings; use strict; diff --git a/model_zoo/Transformer/src/tokenization.py b/model_zoo/Transformer/src/tokenization.py index fd0fc97955..6c4f4fec20 100644 --- a/model_zoo/Transformer/src/tokenization.py +++ b/model_zoo/Transformer/src/tokenization.py @@ -1,193 +1,158 @@ -# coding=utf-8 -# Copyright 2018 The Google AI Language Team Authors. +# Copyright 2020 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -############################################################################### -# Modified by Huawei Technologies Co., Ltd, May, 2020, with following changes: -# - Remove some unused classes and functions -# - Modify load_vocab, convert_to_unicode, printable_text function -# - Modify BasicTokenizer class -# - Add WhiteSpaceTokenizer class -############################################################################### - +# ============================================================================ """Tokenization utilities.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - +import sys import collections import unicodedata -import six -def convert_to_unicode(text): - """Converts `text` to Unicode (if it's not already), assuming utf-8 input.""" - if six.PY3: +def convert_to_printable(text): + """ + Converts `text` to a printable coding format. + """ + if sys.version_info[0] == 3: if isinstance(text, str): return text if isinstance(text, bytes): return text.decode("utf-8", "ignore") - raise ValueError("Unsupported string type: %s" % (type(text))) - if six.PY2: + raise ValueError("Only support type `str` or `bytes`, while text type is `%s`" % (type(text))) + if sys.version_info[0] == 2: if isinstance(text, str): - return text.decode("utf-8", "ignore") - if isinstance(text, unicode): return text - raise ValueError("Unsupported string type: %s" % (type(text))) - raise ValueError("Not running on Python2 or Python 3?") - + if isinstance(text, unicode): + return text.encode("utf-8") + raise ValueError("Only support type `str` or `unicode`, while text type is `%s`" % (type(text))) + raise ValueError("Only supported when running on Python2 or Python3.") -def printable_text(text): - """Returns text encoded in a way suitable for print or `logging`.""" - # These functions want `str` for both Python2 and Python3, but in one case - # it's a Unicode string and in the other it's a byte string. - if six.PY3: +def convert_to_unicode(text): + """ + Converts `text` to Unicode format. + """ + if sys.version_info[0] == 3: if isinstance(text, str): return text if isinstance(text, bytes): return text.decode("utf-8", "ignore") - raise ValueError("Unsupported string type: %s" % (type(text))) - if six.PY2: + raise ValueError("Only support type `str` or `bytes`, while text type is `%s`" % (type(text))) + if sys.version_info[0] == 2: if isinstance(text, str): - return text + return text.decode("utf-8", "ignore") if isinstance(text, unicode): - return text.encode("utf-8") - raise ValueError("Unsupported string type: %s" % (type(text))) - raise ValueError("Not running on Python2 or Python 3?") + return text + raise ValueError("Only support type `str` or `unicode`, while text type is `%s`" % (type(text))) + raise ValueError("Only supported when running on Python2 or Python3.") -def load_vocab(vocab_file): - """Loads a vocabulary file into a dictionary.""" - vocab = collections.OrderedDict() +def load_vocab_file(vocab_file): + """ + Loads a vocabulary file and turns into a {token:id} dictionary. + """ + vocab_dict = collections.OrderedDict() index = 0 - with open(vocab_file, "r") as reader: + with open(vocab_file, "r") as vocab: while True: - token = convert_to_unicode(reader.readline()) + token = convert_to_unicode(vocab.readline()) if not token: break token = token.strip() - vocab[token] = index + vocab_dict[token] = index index += 1 - return vocab + return vocab_dict -def convert_by_vocab(vocab, items): - """Converts a sequence of [tokens|ids] using the vocab.""" +def convert_by_vocab_dict(vocab_dict, items): + """ + Converts a sequence of [tokens|ids] according to the vocab dict. + """ output = [] for item in items: - if item in vocab: - output.append(vocab[item]) + if item in vocab_dict: + output.append(vocab_dict[item]) else: - output.append(vocab[""]) + output.append(vocab_dict[""]) return output -def convert_tokens_to_ids(vocab, tokens): - return convert_by_vocab(vocab, tokens) - - -def convert_ids_to_tokens(inv_vocab, ids): - return convert_by_vocab(inv_vocab, ids) - - -def whitespace_tokenize(text): - """Runs basic whitespace cleaning and splitting on a piece of text.""" - text = text.strip() - if not text: - return [] - tokens = text.split() - return tokens - - class WhiteSpaceTokenizer(): - """Runs end-to-end tokenziation.""" + """ + Whitespace tokenizer. + """ def __init__(self, vocab_file): - self.vocab = load_vocab(vocab_file) - self.inv_vocab = {v: k for k, v in self.vocab.items()} - self.basic_tokenizer = BasicTokenizer() - - def tokenize(self, text): - return self.basic_tokenizer.tokenize(text) - - def convert_tokens_to_ids(self, tokens): - return convert_by_vocab(self.vocab, tokens) - - def convert_ids_to_tokens(self, ids): - return convert_by_vocab(self.inv_vocab, ids) - - -class BasicTokenizer(): - """Runs basic tokenization (punctuation splitting, lower casing, etc.).""" - - def __init__(self): - """Constructs a BasicTokenizer.""" + self.vocab_dict = load_vocab_file(vocab_file) + self.inv_vocab_dict = {index: token for token, index in self.vocab_dict.items()} + + def _is_whitespace_char(self, char): + """ + Checks if it is a whitespace character(regard "\t", "\n", "\r" as whitespace here). + """ + if char in (" ", "\t", "\n", "\r"): + return True + uni = unicodedata.category(char) + if uni == "Zs": + return True + return False - def tokenize(self, text): - """Tokenizes a piece of text.""" - text = convert_to_unicode(text) - text = self._clean_text(text) - return whitespace_tokenize(text) + def _is_control_char(self, char): + """ + Checks if it is a control character. + """ + if char in ("\t", "\n", "\r"): + return False + uni = unicodedata.category(char) + if uni in ("Cc", "Cf"): + return True + return False def _clean_text(self, text): - """Performs invalid character removal and whitespace cleanup on text.""" + """ + Remove invalid characters and cleanup whitespace. + """ output = [] for char in text: cp = ord(char) - if cp == 0 or cp == 0xfffd or _is_control(char): + if cp == 0 or cp == 0xfffd or self._is_control_char(char): continue - if _is_whitespace(char): + if self._is_whitespace_char(char): output.append(" ") else: output.append(char) return "".join(output) + def _whitespace_tokenize(self, text): + """ + Clean whitespace and split text into tokens. + """ + text = text.strip() + if not text: + tokens = [] + else: + tokens = text.split() + return tokens -def _is_whitespace(char): - """Checks whether `chars` is a whitespace character.""" - # \t, \n, and \r are technically contorl characters but we treat them - # as whitespace since they are generally considered as such. - if char in (" ", "\t", "\n", "\r"): - return True - cat = unicodedata.category(char) - if cat == "Zs": - return True - return False + def tokenize(self, text): + """ + Tokenizes text. + """ + text = convert_to_unicode(text) + text = self._clean_text(text) + tokens = self._whitespace_tokenize(text) + return tokens + def convert_tokens_to_ids(self, tokens): + return convert_by_vocab_dict(self.vocab_dict, tokens) -def _is_control(char): - """Checks whether `chars` is a control character.""" - # These are technically control characters but we count them as whitespace - # characters. - if char in ("\t", "\n", "\r"): - return False - cat = unicodedata.category(char) - if cat in ("Cc", "Cf"): - return True - return False - - -def _is_punctuation(char): - """Checks whether `chars` is a punctuation character.""" - cp = ord(char) - # We treat all non-letter/number ASCII as punctuation. - # Characters such as "^", "$", and "`" are not in the Unicode - # Punctuation class but we treat them as punctuation anyways, for - # consistency. - if ((33 <= cp <= 47) or (58 <= cp <= 64) or (91 <= cp <= 96) or (123 <= cp <= 126)): - return True - cat = unicodedata.category(char) - if cat.startswith("P"): - return True - return False + def convert_ids_to_tokens(self, ids): + return convert_by_vocab_dict(self.inv_vocab_dict, ids) diff --git a/model_zoo/Transformer/src/transformer_for_train.py b/model_zoo/Transformer/src/transformer_for_train.py index 758ac65ab5..76237bee96 100644 --- a/model_zoo/Transformer/src/transformer_for_train.py +++ b/model_zoo/Transformer/src/transformer_for_train.py @@ -277,8 +277,8 @@ class TransformerTrainOneStepWithLossScaleCell(nn.Cell): if scale_update_cell: self.loss_scale = Parameter(Tensor(scale_update_cell.get_loss_scale(), dtype=mstype.float32), name="loss_scale") - self.add_flags(has_effect=True) + @C.add_flags(has_effect=True) def construct(self, source_eos_ids, source_eos_mask, diff --git a/model_zoo/bert/README.md b/model_zoo/bert/README.md index 1fa755e72f..3ed2bf6783 100644 --- a/model_zoo/bert/README.md +++ b/model_zoo/bert/README.md @@ -89,7 +89,7 @@ config.py: optimizer optimizer used in the network: AdamWerigtDecayDynamicLR | Lamb | Momentum, default is "Lamb" finetune_config.py: - task task type: NER | SQUAD | OTHERS + task task type: SeqLabeling | Regression | Classification | COLA | SQUAD num_labels number of labels to do classification data_file dataset file to load: PATH, default is "/your/path/train.tfrecord" schema_file dataset schema file to load: PATH, default is "/your/path/schema.json" @@ -101,7 +101,7 @@ finetune_config.py: optimizer optimizer used in fine-tune network: AdamWeigtDecayDynamicLR | Lamb | Momentum, default is "Lamb" evaluation_config.py: - task task type: NER | SQUAD | OTHERS + task task type: SeqLabeling | Regression | Classification | COLA num_labels number of labels to do classsification data_file dataset file to load: PATH, default is "/your/path/evaluation.tfrecord" schema_file dataset schema file to load: PATH, default is "/your/path/schema.json" diff --git a/model_zoo/bert/evaluation.py b/model_zoo/bert/evaluation.py index 4877b60cef..4e8b2a3aea 100644 --- a/model_zoo/bert/evaluation.py +++ b/model_zoo/bert/evaluation.py @@ -19,6 +19,7 @@ Bert evaluation script. import os import argparse +import math import numpy as np import mindspore.common.dtype as mstype from mindspore import context @@ -29,19 +30,24 @@ import mindspore.dataset.transforms.c_transforms as C from mindspore.train.model import Model from mindspore.train.serialization import load_checkpoint, load_param_into_net from src.evaluation_config import cfg, bert_net_cfg -from src.utils import BertNER, BertCLS +from src.utils import BertNER, BertCLS, BertReg from src.CRF import postprocess from src.cluener_evaluation import submit from src.finetune_config import tag_to_index + class Accuracy(): - ''' + """ calculate accuracy - ''' + """ def __init__(self): self.acc_num = 0 self.total_num = 0 + def update(self, logits, labels): + """ + Update accuracy + """ labels = labels.asnumpy() labels = np.reshape(labels, -1) logits = logits.asnumpy() @@ -50,18 +56,20 @@ class Accuracy(): self.total_num += len(labels) print("=========================accuracy is ", self.acc_num / self.total_num) + class F1(): - ''' + """ calculate F1 score - ''' + """ def __init__(self): self.TP = 0 self.FP = 0 self.FN = 0 + def update(self, logits, labels): - ''' + """ update F1 score - ''' + """ labels = labels.asnumpy() labels = np.reshape(labels, -1) if cfg.use_crf: @@ -80,10 +88,76 @@ class F1(): self.FP += np.sum(pos_eva&(~pos_label)) self.FN += np.sum((~pos_eva)&pos_label) + +class MCC(): + """ + Calculate Matthews Correlation Coefficient. + """ + def __init__(self): + self.TP = 0 + self.FP = 0 + self.FN = 0 + self.TN = 0 + + def update(self, logits, labels): + """ + Update MCC score + """ + labels = labels.asnumpy() + labels = np.reshape(labels, -1) + labels = labels.astype(np.bool) + logits = logits.asnumpy() + logit_id = np.argmax(logits, axis=-1) + logit_id = np.reshape(logit_id, -1) + logit_id = logit_id.astype(np.bool) + ornot = logit_id ^ labels + + self.TP += (~ornot & labels).sum() + self.FP += (ornot & ~labels).sum() + self.FN += (ornot & labels).sum() + self.TN += (~ornot & ~labels).sum() + + +class Spearman_Correlation(): + """ + calculate Spearman Correlation coefficient + """ + def __init__(self): + self.label = [] + self.logit = [] + + def update(self, logits, labels): + """ + Update Spearman Correlation + """ + labels = labels.asnumpy() + labels = np.reshape(labels, -1) + logits = logits.asnumpy() + logits = np.reshape(logits, -1) + self.label.append(labels) + self.logit.append(logits) + + def cal(self): + """ + Calculate Spearman Correlation + """ + label = np.concatenate(self.label) + logit = np.concatenate(self.logit) + sort_label = label.argsort()[::-1] + sort_logit = logit.argsort()[::-1] + n = len(label) + d_acc = 0 + for i in range(n): + d = np.where(sort_label == i)[0] - np.where(sort_logit == i)[0] + d_acc += d**2 + ps = 1 - 6*d_acc/n/(n**2-1) + return ps + + def get_dataset(batch_size=1, repeat_count=1, distribute_file=''): - ''' + """ get dataset - ''' + """ _ = distribute_file ds = de.TFRecordDataset([cfg.data_file], cfg.schema_file, columns_list=["input_ids", "input_mask", @@ -92,7 +166,11 @@ def get_dataset(batch_size=1, repeat_count=1, distribute_file=''): ds = ds.map(input_columns="segment_ids", operations=type_cast_op) ds = ds.map(input_columns="input_mask", operations=type_cast_op) ds = ds.map(input_columns="input_ids", operations=type_cast_op) - ds = ds.map(input_columns="label_ids", operations=type_cast_op) + if cfg.task == "Regression": + type_cast_op_float = C.TypeCast(mstype.float32) + ds = ds.map(input_columns="label_ids", operations=type_cast_op_float) + else: + ds = ds.map(input_columns="label_ids", operations=type_cast_op) ds = ds.repeat(repeat_count) # apply shuffle operation @@ -103,10 +181,11 @@ def get_dataset(batch_size=1, repeat_count=1, distribute_file=''): ds = ds.batch(batch_size, drop_remainder=True) return ds + def bert_predict(Evaluation): - ''' + """ prediction function - ''' + """ target = args_opt.device_target if target == "Ascend": devid = int(os.getenv('DEVICE_ID')) @@ -131,15 +210,33 @@ def bert_predict(Evaluation): return model, dataset def test_eval(): - ''' + """ evaluation function - ''' - task_type = BertNER if cfg.task == "NER" else BertCLS + """ + if cfg.task == "SeqLabeling": + task_type = BertNER + elif cfg.task == "Regression": + task_type = BertReg + elif cfg.task == "Classification": + task_type = BertCLS + elif cfg.task == "COLA": + task_type = BertCLS + else: + raise ValueError("Task not supported.") model, dataset = bert_predict(task_type) + if cfg.clue_benchmark: submit(model, cfg.data_file, bert_net_cfg.seq_length) else: - callback = F1() if cfg.task == "NER" else Accuracy() + if cfg.task == "SeqLabeling": + callback = F1() + elif cfg.task == "COLA": + callback = MCC() + elif cfg.task == "Regression": + callback = Spearman_Correlation() + else: + callback = Accuracy() + columns_list = ["input_ids", "input_mask", "segment_ids", "label_ids"] for data in dataset.create_dict_iterator(): input_data = [] @@ -149,10 +246,19 @@ def test_eval(): logits = model.predict(input_ids, input_mask, token_type_id, label_ids) callback.update(logits, label_ids) print("==============================================================") - if cfg.task == "NER": + if cfg.task == "SeqLabeling": print("Precision {:.6f} ".format(callback.TP / (callback.TP + callback.FP))) print("Recall {:.6f} ".format(callback.TP / (callback.TP + callback.FN))) print("F1 {:.6f} ".format(2*callback.TP / (2*callback.TP + callback.FP + callback.FN))) + elif cfg.task == "COLA": + TP = callback.TP + TN = callback.TN + FP = callback.FP + FN = callback.FN + mcc = (TP*TN-FP*FN)/math.sqrt((TP+FP)*(TP+FN)*(TN+FP)*(TN+FN)) + print("MCC: {:.6f}".format(mcc)) + elif cfg.task == "Regression": + print("Spearman Correlation is {:.6f}".format(callback.cal()[0])) else: print("acc_num {} , total_num {}, accuracy {:.6f}".format(callback.acc_num, callback.total_num, callback.acc_num / callback.total_num)) diff --git a/model_zoo/bert/finetune.py b/model_zoo/bert/finetune.py index df16e3c91d..eb1880b9cc 100644 --- a/model_zoo/bert/finetune.py +++ b/model_zoo/bert/finetune.py @@ -13,13 +13,13 @@ # limitations under the License. # ============================================================================ -''' +""" Bert finetune script. -''' +""" import os import argparse -from src.utils import BertFinetuneCell, BertCLS, BertNER, BertSquad, BertSquadCell +from src.utils import BertFinetuneCell, BertCLS, BertNER, BertSquad, BertSquadCell, BertReg from src.finetune_config import cfg, bert_net_cfg, tag_to_index import mindspore.common.dtype as mstype from mindspore import context @@ -34,14 +34,14 @@ from mindspore.train.callback import CheckpointConfig, ModelCheckpoint from mindspore.train.serialization import load_checkpoint, load_param_into_net class LossCallBack(Callback): - ''' + """ Monitor the loss in training. If the loss is NAN or INF, terminate training. Note: If per_print_times is 0, do not print loss. Args: per_print_times (int): Print loss every times. Default: 1. - ''' + """ def __init__(self, per_print_times=1): super(LossCallBack, self).__init__() if not isinstance(per_print_times, int) or per_print_times < 0: @@ -56,16 +56,20 @@ class LossCallBack(Callback): f.write("\n") def get_dataset(batch_size=1, repeat_count=1, distribute_file=''): - ''' + """ get dataset - ''' + """ ds = de.TFRecordDataset([cfg.data_file], cfg.schema_file, columns_list=["input_ids", "input_mask", "segment_ids", "label_ids"]) type_cast_op = C.TypeCast(mstype.int32) ds = ds.map(input_columns="segment_ids", operations=type_cast_op) ds = ds.map(input_columns="input_mask", operations=type_cast_op) ds = ds.map(input_columns="input_ids", operations=type_cast_op) - ds = ds.map(input_columns="label_ids", operations=type_cast_op) + if cfg.task == "Regression": + type_cast_op_float = C.TypeCast(mstype.float32) + ds = ds.map(input_columns="label_ids", operations=type_cast_op_float) + else: + ds = ds.map(input_columns="label_ids", operations=type_cast_op) ds = ds.repeat(repeat_count) # apply shuffle operation @@ -77,9 +81,9 @@ def get_dataset(batch_size=1, repeat_count=1, distribute_file=''): return ds def get_squad_dataset(batch_size=1, repeat_count=1, distribute_file=''): - ''' + """ get SQuAD dataset - ''' + """ ds = de.TFRecordDataset([cfg.data_file], cfg.schema_file, columns_list=["input_ids", "input_mask", "segment_ids", "start_positions", "end_positions", "unique_ids", "is_impossible"]) @@ -97,9 +101,9 @@ def get_squad_dataset(batch_size=1, repeat_count=1, distribute_file=''): return ds def test_train(): - ''' + """ finetune function - ''' + """ target = args_opt.device_target if target == "Ascend": devid = int(os.getenv('DEVICE_ID')) @@ -113,7 +117,7 @@ def test_train(): raise Exception("Target error, GPU or Ascend is supported.") #BertCLSTrain for classification #BertNERTrain for sequence labeling - if cfg.task == 'NER': + if cfg.task == 'SeqLabeling': if cfg.use_crf: netwithloss = BertNER(bert_net_cfg, True, num_labels=len(tag_to_index), use_crf=True, tag_to_index=tag_to_index, dropout_prob=0.1) @@ -121,8 +125,12 @@ def test_train(): netwithloss = BertNER(bert_net_cfg, True, num_labels=cfg.num_labels, dropout_prob=0.1) elif cfg.task == 'SQUAD': netwithloss = BertSquad(bert_net_cfg, True, 2, dropout_prob=0.1) - else: + elif cfg.task == 'Regression': + netwithloss = BertReg(bert_net_cfg, True, num_labels=cfg.num_labels, dropout_prob=0.1) + elif cfg.task == 'Classification': netwithloss = BertCLS(bert_net_cfg, True, num_labels=cfg.num_labels, dropout_prob=0.1) + else: + raise Exception("Target error, GPU or Ascend is supported.") if cfg.task == 'SQUAD': dataset = get_squad_dataset(bert_net_cfg.batch_size, cfg.epoch_num) else: diff --git a/model_zoo/bert/src/bert_for_pre_training.py b/model_zoo/bert/src/bert_for_pre_training.py index 5e014f02ba..802391ee86 100644 --- a/model_zoo/bert/src/bert_for_pre_training.py +++ b/model_zoo/bert/src/bert_for_pre_training.py @@ -132,9 +132,9 @@ class GetNextSentenceOutput(nn.Cell): def __init__(self, config): super(GetNextSentenceOutput, self).__init__() self.log_softmax = _selected_ops.LogSoftmax() - self.weight_init = TruncatedNormal(config.initializer_range) + weight_init = TruncatedNormal(config.initializer_range) self.dense = nn.Dense(config.hidden_size, 2, - weight_init=self.weight_init, has_bias=True).to_float(config.compute_type) + weight_init=weight_init, has_bias=True).to_float(config.compute_type) self.dtype = config.dtype self.cast = P.Cast() @@ -321,7 +321,6 @@ class BertTrainOneStepCell(nn.Cell): if self.reducer_flag: # apply grad reducer on grads grads = self.grad_reducer(grads) - succ = self.optimizer(grads) return F.depend(loss, succ) @@ -380,8 +379,8 @@ class BertTrainOneStepWithLossScaleCell(nn.Cell): if scale_update_cell: self.loss_scale = Parameter(Tensor(scale_update_cell.get_loss_scale(), dtype=mstype.float32), name="loss_scale") - self.add_flags(has_effect=True) + @C.add_flags(has_effect=True) def construct(self, input_ids, input_mask, diff --git a/model_zoo/bert/src/utils.py b/model_zoo/bert/src/utils.py index 9b5383877b..ec5651b205 100644 --- a/model_zoo/bert/src/utils.py +++ b/model_zoo/bert/src/utils.py @@ -13,9 +13,9 @@ # limitations under the License. # ============================================================================ -''' +""" Functional Cells used in Bert finetune and evaluation. -''' +""" import mindspore.nn as nn from mindspore.common.initializer import TruncatedNormal @@ -245,6 +245,32 @@ class BertSquadCell(nn.Cell): ret = (loss, cond) return F.depend(ret, succ) + +class BertRegressionModel(nn.Cell): + """ + Bert finetune model for regression task + """ + def __init__(self, config, is_training, num_labels=2, dropout_prob=0.0, use_one_hot_embeddings=False): + super(BertRegressionModel, self).__init__() + self.bert = BertModel(config, is_training, use_one_hot_embeddings) + self.cast = P.Cast() + self.weight_init = TruncatedNormal(config.initializer_range) + self.log_softmax = P.LogSoftmax(axis=-1) + self.dtype = config.dtype + self.num_labels = num_labels + self.dropout = nn.Dropout(1 - dropout_prob) + self.dense_1 = nn.Dense(config.hidden_size, 1, weight_init=self.weight_init, + has_bias=True).to_float(mstype.float16) + + def construct(self, input_ids, input_mask, token_type_id): + _, pooled_output, _ = self.bert(input_ids, token_type_id, input_mask) + cls = self.cast(pooled_output, self.dtype) + cls = self.dropout(cls) + logits = self.dense_1(cls) + logits = self.cast(logits, self.dtype) + return logits + + class BertCLSModel(nn.Cell): """ This class is responsible for classification task evaluation, i.e. XNLI(num_labels=3), @@ -274,9 +300,9 @@ class BertCLSModel(nn.Cell): return log_probs class BertSquadModel(nn.Cell): - ''' - This class is responsible for SQuAD - ''' + """ + Bert finetune model for SQuAD v1.1 task + """ def __init__(self, config, is_training, num_labels=2, dropout_prob=0.0, use_one_hot_embeddings=False): super(BertSquadModel, self).__init__() self.bert = BertModel(config, is_training, use_one_hot_embeddings) @@ -401,9 +427,9 @@ class BertNER(nn.Cell): return loss class BertSquad(nn.Cell): - ''' + """ Train interface for SQuAD finetuning task. - ''' + """ def __init__(self, config, is_training, num_labels=2, dropout_prob=0.0, use_one_hot_embeddings=False): super(BertSquad, self).__init__() self.bert = BertSquadModel(config, is_training, num_labels, dropout_prob, use_one_hot_embeddings) @@ -432,3 +458,24 @@ class BertSquad(nn.Cell): end_logits = self.squeeze(logits[:, :, 1:2]) total_loss = (unique_id, start_logits, end_logits) return total_loss + + +class BertReg(nn.Cell): + """ + Bert finetune model with loss for regression task + """ + def __init__(self, config, is_training, num_labels=2, dropout_prob=0.0, use_one_hot_embeddings=False): + super(BertReg, self).__init__() + self.bert = BertRegressionModel(config, is_training, num_labels, dropout_prob, use_one_hot_embeddings) + self.loss = nn.MSELoss() + self.is_training = is_training + self.sigmoid = P.Sigmoid() + self.cast = P.Cast() + self.mul = P.Mul() + def construct(self, input_ids, input_mask, token_type_id, labels): + logits = self.bert(input_ids, input_mask, token_type_id) + if self.is_training: + loss = self.loss(logits, labels) + else: + loss = logits + return loss diff --git a/model_zoo/deepfm/scripts/run_distribute_train.sh b/model_zoo/deepfm/scripts/run_distribute_train.sh index a19c857936..fb2c3db17c 100644 --- a/model_zoo/deepfm/scripts/run_distribute_train.sh +++ b/model_zoo/deepfm/scripts/run_distribute_train.sh @@ -21,7 +21,7 @@ echo "After running the script, the network runs in the background, The log will export RANK_SIZE=$1 DATA_URL=$2 -export MINDSPORE_HCCL_CONFIG_PAHT=$3 +export MINDSPORE_HCCL_CONFIG_PATH=$3 for ((i=0; i>> net = Net() - >>> loss_fn = nn.SoftmaxCrossEntropyWithLogits() - >>> optim = nn.Momentum(net.trainable_params(), learning_rate=0.1, momentum=0.9) - >>> loss_net = nn.WithLossCell(net, loss_fn) - >>> train_net = nn.TrainOneStepCell(loss_net, optim) - """ - - def __init__(self, network, optimizer, sens=1.0): - super(TrainOneStepCell, self).__init__(auto_prefix=False) - self.network = network - self.network.add_flags(defer_inline=True) - self.weights = ParameterTuple(network.trainable_params()) - self.optimizer = optimizer - self.grad = C.GradOperation('grad', get_by_list=True, sens_param=True) - self.sens = sens - - def construct(self): - weights = self.weights - loss = self.network() - sens = P.Fill()(P.DType()(loss), P.Shape()(loss), self.sens) - grads = self.grad(self.network, weights)(sens) - return F.depend(loss, self.optimizer(grads)) - - -class TrainNetWrapper(nn.Cell): - """ - Wraps the GCN model with optimizer. - - Args: - network (Cell): GCN network. - label (numpy.ndarray): Dataset labels. - mask (numpy.ndarray): Mask for training, evaluation or test. - config (ConfigGCN): Configuration for GCN. - """ - - def __init__(self, network, label, mask, config): - super(TrainNetWrapper, self).__init__(auto_prefix=True) - self.network = network - loss_net = LossWrapper(network, label, mask, config.weight_decay) - optimizer = nn.Adam(loss_net.trainable_params(), - learning_rate=config.learning_rate) - self.loss_train_net = TrainOneStepCell(loss_net, optimizer) - self.accuracy = Accuracy(label, mask) - - def construct(self): - loss = self.loss_train_net() - accuracy = self.accuracy(self.network()) - return loss, accuracy diff --git a/model_zoo/gcn/src/metrics.py b/model_zoo/gcn/src/metrics.py index 5930956776..e923f17e39 100644 --- a/model_zoo/gcn/src/metrics.py +++ b/model_zoo/gcn/src/metrics.py @@ -17,6 +17,9 @@ from mindspore import nn from mindspore import Tensor from mindspore.common import dtype as mstype from mindspore.ops import operations as P +from mindspore.common.parameter import ParameterTuple +from mindspore.ops import composite as C +from mindspore.ops import functional as F class Loss(nn.Cell): @@ -68,3 +71,116 @@ class Accuracy(nn.Cell): mask = mask / mask_reduce accuracy_all *= mask return self.mean(accuracy_all) + + +class LossAccuracyWrapper(nn.Cell): + """ + Wraps the GCN model with loss and accuracy cell. + + Args: + network (Cell): GCN network. + label (numpy.ndarray): Dataset labels. + mask (numpy.ndarray): Mask for training, evaluation or test. + weight_decay (float): Weight decay parameter for weight of the first convolution layer. + """ + + def __init__(self, network, label, mask, weight_decay): + super(LossAccuracyWrapper, self).__init__() + self.network = network + self.loss = Loss(label, mask, weight_decay, network.trainable_params()[0]) + self.accuracy = Accuracy(label, mask) + + def construct(self): + preds = self.network() + loss = self.loss(preds) + accuracy = self.accuracy(preds) + return loss, accuracy + + +class LossWrapper(nn.Cell): + """ + Wraps the GCN model with loss. + + Args: + network (Cell): GCN network. + label (numpy.ndarray): Dataset labels. + mask (numpy.ndarray): Mask for training. + weight_decay (float): Weight decay parameter for weight of the first convolution layer. + """ + + def __init__(self, network, label, mask, weight_decay): + super(LossWrapper, self).__init__() + self.network = network + self.loss = Loss(label, mask, weight_decay, network.trainable_params()[0]) + + def construct(self): + preds = self.network() + loss = self.loss(preds) + return loss + + +class TrainOneStepCell(nn.Cell): + r""" + Network training package class. + + Wraps the network with an optimizer. The resulting Cell be trained without inputs. + Backward graph will be created in the construct function to do parameter updating. Different + parallel modes are available to run the training. + + Args: + network (Cell): The training network. + optimizer (Cell): Optimizer for updating the weights. + sens (Number): The scaling number to be filled as the input of backpropagation. Default value is 1.0. + + Outputs: + Tensor, a scalar Tensor with shape :math:`()`. + + Examples: + >>> net = Net() + >>> loss_fn = nn.SoftmaxCrossEntropyWithLogits() + >>> optim = nn.Momentum(net.trainable_params(), learning_rate=0.1, momentum=0.9) + >>> loss_net = nn.WithLossCell(net, loss_fn) + >>> train_net = nn.TrainOneStepCell(loss_net, optim) + """ + + def __init__(self, network, optimizer, sens=1.0): + super(TrainOneStepCell, self).__init__(auto_prefix=False) + self.network = network + self.network.add_flags(defer_inline=True) + self.weights = ParameterTuple(network.trainable_params()) + self.optimizer = optimizer + self.grad = C.GradOperation('grad', get_by_list=True, sens_param=True) + self.sens = sens + + def construct(self): + weights = self.weights + loss = self.network() + sens = P.Fill()(P.DType()(loss), P.Shape()(loss), self.sens) + grads = self.grad(self.network, weights)(sens) + return F.depend(loss, self.optimizer(grads)) + + +class TrainNetWrapper(nn.Cell): + """ + Wraps the GCN model with optimizer. + + Args: + network (Cell): GCN network. + label (numpy.ndarray): Dataset labels. + mask (numpy.ndarray): Mask for training, evaluation or test. + config (ConfigGCN): Configuration for GCN. + """ + + def __init__(self, network, label, mask, config): + super(TrainNetWrapper, self).__init__(auto_prefix=True) + self.network = network + loss_net = LossWrapper(network, label, mask, config.weight_decay) + optimizer = nn.Adam(loss_net.trainable_params(), + learning_rate=config.learning_rate) + self.loss_train_net = TrainOneStepCell(loss_net, optimizer) + self.accuracy = Accuracy(label, mask) + + def construct(self): + loss = self.loss_train_net() + accuracy = self.accuracy(self.network()) + return loss, accuracy diff --git a/model_zoo/gcn/train.py b/model_zoo/gcn/train.py index 220d2ecd6b..a5e8094804 100644 --- a/model_zoo/gcn/train.py +++ b/model_zoo/gcn/train.py @@ -26,9 +26,10 @@ from matplotlib import animation from sklearn import manifold from mindspore import context -from model_zoo.gcn.src.gcn import GCN, LossAccuracyWrapper, TrainNetWrapper -from model_zoo.gcn.src.config import ConfigGCN -from model_zoo.gcn.src.dataset import get_adj_features_labels, get_mask +from src.gcn import GCN +from src.metrics import LossAccuracyWrapper, TrainNetWrapper +from src.config import ConfigGCN +from src.dataset import get_adj_features_labels, get_mask def t_SNE(out_feature, dim): diff --git a/model_zoo/lenet/eval.py b/model_zoo/lenet/eval.py index a9842f4426..bcd5503c39 100644 --- a/model_zoo/lenet/eval.py +++ b/model_zoo/lenet/eval.py @@ -20,16 +20,14 @@ python eval.py --data_path /YourDataPath --ckpt_path Your.ckpt import os import argparse -from src.dataset import create_dataset -from src.config import mnist_cfg as cfg -from src.lenet import LeNet5 import mindspore.nn as nn from mindspore import context from mindspore.train.serialization import load_checkpoint, load_param_into_net -from mindspore.train.callback import ModelCheckpoint, CheckpointConfig from mindspore.train import Model from mindspore.nn.metrics import Accuracy - +from src.dataset import create_dataset +from src.config import mnist_cfg as cfg +from src.lenet import LeNet5 if __name__ == "__main__": parser = argparse.ArgumentParser(description='MindSpore Lenet Example') @@ -49,9 +47,6 @@ if __name__ == "__main__": net_loss = nn.SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True, reduction="mean") repeat_size = cfg.epoch_size net_opt = nn.Momentum(network.trainable_params(), cfg.lr, cfg.momentum) - config_ck = CheckpointConfig(save_checkpoint_steps=cfg.save_checkpoint_steps, - keep_checkpoint_max=cfg.keep_checkpoint_max) - ckpoint_cb = ModelCheckpoint(prefix="checkpoint_lenet", config=config_ck) model = Model(network, net_loss, net_opt, metrics={"Accuracy": Accuracy()}) print("============== Starting Testing ==============") diff --git a/model_zoo/lenet_quant/README.md b/model_zoo/lenet_quant/README.md index 26cdcc3ecd..2fd3e129a2 100644 --- a/model_zoo/lenet_quant/README.md +++ b/model_zoo/lenet_quant/README.md @@ -33,7 +33,7 @@ Then you will get the following display ```bash >>> Found existing installation: mindspore-ascend >>> Uninstalling mindspore-ascend: ->>> Successfully uninstalled mindspore-ascend. +>>> Successfully uninstalled mindspore-ascend. ``` ### Prepare Dataset @@ -128,9 +128,9 @@ After all the following we will get the loss value of each step as following: ```bash >>> Epoch: [ 1/ 10] step: [ 1/ 900], loss: [2.3040/2.5234], time: [1.300234] >>> ... ->>> Epoch: [ 10/ 10] step: [887/ 900], loss: [0.0113/0.0223], time: [1.300234] ->>> Epoch: [ 10/ 10] step: [888/ 900], loss: [0.0334/0.0223], time: [1.300234] ->>> Epoch: [ 10/ 10] step: [889/ 900], loss: [0.0233/0.0223], time: [1.300234] +>>> Epoch: [ 9/ 10] step: [887/ 900], loss: [0.0113/0.0223], time: [1.300234] +>>> Epoch: [ 9/ 10] step: [888/ 900], loss: [0.0334/0.0223], time: [1.300234] +>>> Epoch: [ 9/ 10] step: [889/ 900], loss: [0.0233/0.0223], time: [1.300234] ``` Also, you can just run this command instead. @@ -186,7 +186,7 @@ model = Model(network, net_loss, net_opt, metrics={"Accuracy": Accuracy()}) ### train quantization aware model -Also, you can just run this command instread. +Also, you can just run this command instead. ```python python train_quant.py --data_path MNIST_Data --device_target Ascend --ckpt_path checkpoint_lenet.ckpt @@ -197,9 +197,9 @@ After all the following we will get the loss value of each step as following: ```bash >>> Epoch: [ 1/ 10] step: [ 1/ 900], loss: [2.3040/2.5234], time: [1.300234] >>> ... ->>> Epoch: [ 10/ 10] step: [887/ 900], loss: [0.0113/0.0223], time: [1.300234] ->>> Epoch: [ 10/ 10] step: [888/ 900], loss: [0.0334/0.0223], time: [1.300234] ->>> Epoch: [ 10/ 10] step: [889/ 900], loss: [0.0233/0.0223], time: [1.300234] +>>> Epoch: [ 9/ 10] step: [887/ 900], loss: [0.0113/0.0223], time: [1.300234] +>>> Epoch: [ 9/ 10] step: [888/ 900], loss: [0.0334/0.0223], time: [1.300234] +>>> Epoch: [ 9/ 10] step: [889/ 900], loss: [0.0233/0.0223], time: [1.300234] ``` ### Evaluate quantization aware model @@ -215,7 +215,7 @@ param_dict = load_checkpoint(args.ckpt_path) load_param_into_net(network, param_dict) # convert funsion netwrok to quantization aware network -network = quant.convert_quant_network(network +network = quant.convert_quant_network(network) ``` Also, you can just run this command insread. @@ -235,7 +235,7 @@ The top1 accuracy would display on shell. Here are some optional parameters: ```bash ---device_target {Ascend,GPU,CPU} +--device_target {Ascend,GPU} device where the code will be implemented (default: Ascend) --data_path DATA_PATH path where the dataset is saved diff --git a/model_zoo/lenet_quant/eval.py b/model_zoo/lenet_quant/eval.py index c1e3a5fd8c..df9a5b123b 100644 --- a/model_zoo/lenet_quant/eval.py +++ b/model_zoo/lenet_quant/eval.py @@ -23,7 +23,6 @@ import argparse import mindspore.nn as nn from mindspore import context from mindspore.train.serialization import load_checkpoint, load_param_into_net -from mindspore.train.callback import ModelCheckpoint, CheckpointConfig from mindspore.train import Model from mindspore.nn.metrics import Accuracy from src.dataset import create_dataset @@ -32,7 +31,7 @@ from src.lenet_fusion import LeNet5 as LeNet5Fusion parser = argparse.ArgumentParser(description='MindSpore MNIST Example') parser.add_argument('--device_target', type=str, default="Ascend", - choices=['Ascend', 'GPU', 'CPU'], + choices=['Ascend', 'GPU'], help='device where the code will be implemented (default: Ascend)') parser.add_argument('--data_path', type=str, default="./MNIST_Data", help='path where the dataset is saved') @@ -47,15 +46,17 @@ if __name__ == "__main__": ds_eval = create_dataset(os.path.join(args.data_path, "test"), cfg.batch_size, 1) step_size = ds_eval.get_dataset_size() + # define fusion network network = LeNet5Fusion(cfg.num_classes) + # define loss net_loss = nn.SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True, reduction="mean") - repeat_size = cfg.epoch_size + # define network optimization net_opt = nn.Momentum(network.trainable_params(), cfg.lr, cfg.momentum) - config_ck = CheckpointConfig(save_checkpoint_steps=cfg.epoch_size * step_size, - keep_checkpoint_max=cfg.keep_checkpoint_max) - ckpoint_cb = ModelCheckpoint(prefix="checkpoint_lenet", config=config_ck) + + # call back and monitor model = Model(network, net_loss, net_opt, metrics={"Accuracy": Accuracy()}) + # load check point into network param_dict = load_checkpoint(args.ckpt_path) load_param_into_net(network, param_dict) diff --git a/model_zoo/lenet_quant/eval_quant.py b/model_zoo/lenet_quant/eval_quant.py index 492f6d36b2..96740dc51f 100644 --- a/model_zoo/lenet_quant/eval_quant.py +++ b/model_zoo/lenet_quant/eval_quant.py @@ -23,7 +23,6 @@ import argparse import mindspore.nn as nn from mindspore import context from mindspore.train.serialization import load_checkpoint, load_param_into_net -from mindspore.train.callback import ModelCheckpoint, CheckpointConfig from mindspore.train import Model from mindspore.nn.metrics import Accuracy from mindspore.train.quant import quant @@ -33,7 +32,7 @@ from src.lenet_fusion import LeNet5 as LeNet5Fusion parser = argparse.ArgumentParser(description='MindSpore MNIST Example') parser.add_argument('--device_target', type=str, default="Ascend", - choices=['Ascend', 'GPU', 'CPU'], + choices=['Ascend', 'GPU'], help='device where the code will be implemented (default: Ascend)') parser.add_argument('--data_path', type=str, default="./MNIST_Data", help='path where the dataset is saved') @@ -48,16 +47,17 @@ if __name__ == "__main__": ds_eval = create_dataset(os.path.join(args.data_path, "test"), cfg.batch_size, 1) step_size = ds_eval.get_dataset_size() - # define funsion network + # define fusion network network = LeNet5Fusion(cfg.num_classes) - # convert funsion netwrok to quantization aware network + # convert fusion network to quantization aware network network = quant.convert_quant_network(network, quant_delay=0, bn_fold=False, freeze_bn=10000) + # define loss net_loss = nn.SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True, reduction="mean") + # define network optimization net_opt = nn.Momentum(network.trainable_params(), cfg.lr, cfg.momentum) - config_ck = CheckpointConfig(save_checkpoint_steps=cfg.epoch_size * step_size, - keep_checkpoint_max=cfg.keep_checkpoint_max) - ckpoint_cb = ModelCheckpoint(prefix="checkpoint_lenet", config=config_ck) + + # call back and monitor model = Model(network, net_loss, net_opt, metrics={"Accuracy": Accuracy()}) # load quantization aware network checkpoint diff --git a/model_zoo/lenet_quant/export.py b/model_zoo/lenet_quant/export.py new file mode 100644 index 0000000000..78c94f9286 --- /dev/null +++ b/model_zoo/lenet_quant/export.py @@ -0,0 +1,56 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +export quantization aware training network to infer `GEIR` backend. +""" + +import argparse +import numpy as np + +import mindspore +from mindspore import Tensor +from mindspore import context +from mindspore.train.quant import quant +from mindspore.train.serialization import load_checkpoint, load_param_into_net + +from src.config import mnist_cfg as cfg +from src.lenet_fusion import LeNet5 as LeNet5Fusion + +parser = argparse.ArgumentParser(description='MindSpore MNIST Example') +parser.add_argument('--device_target', type=str, default="Ascend", + choices=['Ascend', 'GPU'], + help='device where the code will be implemented (default: Ascend)') +parser.add_argument('--data_path', type=str, default="./MNIST_Data", + help='path where the dataset is saved') +parser.add_argument('--ckpt_path', type=str, default="", + help='if mode is test, must provide path where the trained ckpt file') +parser.add_argument('--dataset_sink_mode', type=bool, default=True, + help='dataset_sink_mode is False or True') +args = parser.parse_args() + +if __name__ == "__main__": + context.set_context(mode=context.GRAPH_MODE, device_target=args.device_target) + + # define fusion network + network = LeNet5Fusion(cfg.num_classes) + # convert fusion network to quantization aware network + network = quant.convert_quant_network(network, quant_delay=0, bn_fold=False, freeze_bn=10000) + # load quantization aware network checkpoint + param_dict = load_checkpoint(args.ckpt_path) + load_param_into_net(network, param_dict) + + # export network + inputs = Tensor(np.ones([1, 1, cfg.image_height, cfg.image_width]), mindspore.float32) + quant.export(network, inputs, file_name="lenet_quant", file_format='GEIR') diff --git a/model_zoo/lenet_quant/src/lenet.py b/model_zoo/lenet_quant/src/lenet.py index 026f1e8df5..1efcf9e7d7 100644 --- a/model_zoo/lenet_quant/src/lenet.py +++ b/model_zoo/lenet_quant/src/lenet.py @@ -34,8 +34,8 @@ class LeNet5(nn.Cell): super(LeNet5, self).__init__() self.num_class = num_class - self.conv1 = nn.Conv2d(channel, 6, 5) - self.conv2 = nn.Conv2d(6, 16, 5) + self.conv1 = nn.Conv2d(channel, 6, 5, pad_mode='valid') + self.conv2 = nn.Conv2d(6, 16, 5, pad_mode='valid') self.fc1 = nn.Dense(16 * 5 * 5, 120) self.fc2 = nn.Dense(120, 84) self.fc3 = nn.Dense(84, self.num_class) diff --git a/model_zoo/lenet_quant/src/lenet_fusion.py b/model_zoo/lenet_quant/src/lenet_fusion.py index 809276a482..88b3593502 100644 --- a/model_zoo/lenet_quant/src/lenet_fusion.py +++ b/model_zoo/lenet_quant/src/lenet_fusion.py @@ -32,11 +32,12 @@ class LeNet5(nn.Cell): def __init__(self, num_class=10, channel=1): super(LeNet5, self).__init__() + self.type = "fusion" self.num_class = num_class # change `nn.Conv2d` to `nn.Conv2dBnAct` - self.conv1 = nn.Conv2dBnAct(channel, 6, 5, activation='relu') - self.conv2 = nn.Conv2dBnAct(6, 16, 5, activation='relu') + self.conv1 = nn.Conv2dBnAct(channel, 6, 5, pad_mode='valid', activation='relu') + self.conv2 = nn.Conv2dBnAct(6, 16, 5, pad_mode='valid', activation='relu') # change `nn.Dense` to `nn.DenseBnAct` self.fc1 = nn.DenseBnAct(16 * 5 * 5, 120, activation='relu') self.fc2 = nn.DenseBnAct(120, 84, activation='relu') diff --git a/model_zoo/lenet_quant/train.py b/model_zoo/lenet_quant/train.py index 6e7a46fb38..2cff465832 100644 --- a/model_zoo/lenet_quant/train.py +++ b/model_zoo/lenet_quant/train.py @@ -22,7 +22,7 @@ import os import argparse import mindspore.nn as nn from mindspore import context -from mindspore.train.callback import ModelCheckpoint, CheckpointConfig, LossMonitor, TimeMonitor +from mindspore.train.callback import ModelCheckpoint, CheckpointConfig, LossMonitor from mindspore.train import Model from mindspore.nn.metrics import Accuracy from src.dataset import create_dataset @@ -31,7 +31,7 @@ from src.lenet_fusion import LeNet5 as LeNet5Fusion parser = argparse.ArgumentParser(description='MindSpore MNIST Example') parser.add_argument('--device_target', type=str, default="Ascend", - choices=['Ascend', 'GPU', 'CPU'], + choices=['Ascend', 'GPU'], help='device where the code will be implemented (default: Ascend)') parser.add_argument('--data_path', type=str, default="./MNIST_Data", help='path where the dataset is saved') @@ -46,16 +46,22 @@ if __name__ == "__main__": ds_train = create_dataset(os.path.join(args.data_path, "train"), cfg.batch_size, cfg.epoch_size) step_size = ds_train.get_dataset_size() + # define fusion network network = LeNet5Fusion(cfg.num_classes) + # define network loss net_loss = nn.SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True, reduction="mean") + # define network optimization net_opt = nn.Momentum(network.trainable_params(), cfg.lr, cfg.momentum) - time_cb = TimeMonitor(data_size=ds_train.get_dataset_size()) - config_ck = CheckpointConfig(save_checkpoint_steps=cfg.epoch_size * step_size, - keep_checkpoint_max=cfg.keep_checkpoint_max) - ckpoint_cb = ModelCheckpoint(prefix="checkpoint_lenet", config=config_ck) + + # call back and monitor + config_ckpt = CheckpointConfig(save_checkpoint_steps=cfg.epoch_size * step_size, + keep_checkpoint_max=cfg.keep_checkpoint_max) + ckpt_callback = ModelCheckpoint(prefix="checkpoint_lenet", config=config_ckpt) + + # define model model = Model(network, net_loss, net_opt, metrics={"Accuracy": Accuracy()}) print("============== Starting Training ==============") - model.train(cfg['epoch_size'], ds_train, callbacks=[time_cb, ckpoint_cb, LossMonitor()], + model.train(cfg['epoch_size'], ds_train, callbacks=[ckpt_callback, LossMonitor()], dataset_sink_mode=args.dataset_sink_mode) print("============== End Training ==============") diff --git a/model_zoo/lenet_quant/train_quant.py b/model_zoo/lenet_quant/train_quant.py index 04f595f322..6f27cec1e3 100644 --- a/model_zoo/lenet_quant/train_quant.py +++ b/model_zoo/lenet_quant/train_quant.py @@ -23,7 +23,7 @@ import argparse import mindspore.nn as nn from mindspore import context from mindspore.train.serialization import load_checkpoint, load_param_into_net -from mindspore.train.callback import ModelCheckpoint, CheckpointConfig, LossMonitor, TimeMonitor +from mindspore.train.callback import ModelCheckpoint, CheckpointConfig, LossMonitor from mindspore.train import Model from mindspore.nn.metrics import Accuracy from mindspore.train.quant import quant @@ -33,7 +33,7 @@ from src.lenet_fusion import LeNet5 as LeNet5Fusion parser = argparse.ArgumentParser(description='MindSpore MNIST Example') parser.add_argument('--device_target', type=str, default="Ascend", - choices=['Ascend', 'GPU', 'CPU'], + choices=['Ascend', 'GPU'], help='device where the code will be implemented (default: Ascend)') parser.add_argument('--data_path', type=str, default="./MNIST_Data", help='path where the dataset is saved') @@ -48,23 +48,30 @@ if __name__ == "__main__": ds_train = create_dataset(os.path.join(args.data_path, "train"), cfg.batch_size, cfg.epoch_size) step_size = ds_train.get_dataset_size() - # define funsion network + # define fusion network network = LeNet5Fusion(cfg.num_classes) + # load quantization aware network checkpoint param_dict = load_checkpoint(args.ckpt_path) load_param_into_net(network, param_dict) - # convert funsion netwrok to quantization aware network + + # convert fusion network to quantization aware network network = quant.convert_quant_network(network, quant_delay=0, bn_fold=False, freeze_bn=10000) + # define network loss net_loss = nn.SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True, reduction="mean") + # define network optimization net_opt = nn.Momentum(network.trainable_params(), cfg.lr, cfg.momentum) - time_cb = TimeMonitor(data_size=ds_train.get_dataset_size()) - config_ck = CheckpointConfig(save_checkpoint_steps=cfg.epoch_size * step_size, - keep_checkpoint_max=cfg.keep_checkpoint_max) - ckpoint_cb = ModelCheckpoint(prefix="checkpoint_lenet", config=config_ck) + + # call back and monitor + config_ckpt = CheckpointConfig(save_checkpoint_steps=cfg.epoch_size * step_size, + keep_checkpoint_max=cfg.keep_checkpoint_max) + ckpt_callback = ModelCheckpoint(prefix="checkpoint_lenet", config=config_ckpt) + + # define model model = Model(network, net_loss, net_opt, metrics={"Accuracy": Accuracy()}) print("============== Starting Training ==============") - model.train(cfg['epoch_size'], ds_train, callbacks=[time_cb, ckpoint_cb, LossMonitor()], + model.train(cfg['epoch_size'], ds_train, callbacks=[ckpt_callback, LossMonitor()], dataset_sink_mode=args.dataset_sink_mode) print("============== End Training ==============") diff --git a/model_zoo/lstm/eval.py b/model_zoo/lstm/eval.py index a9b81199c1..6d731fbd0d 100644 --- a/model_zoo/lstm/eval.py +++ b/model_zoo/lstm/eval.py @@ -23,8 +23,8 @@ import numpy as np from src.config import lstm_cfg as cfg from src.dataset import lstm_create_dataset, convert_to_mindrecord +from src.lstm import SentimentNet from mindspore import Tensor, nn, Model, context -from mindspore.model_zoo.lstm import SentimentNet from mindspore.nn import Accuracy from mindspore.train.callback import LossMonitor from mindspore.train.serialization import load_checkpoint, load_param_into_net diff --git a/model_zoo/lstm/src/lstm.py b/model_zoo/lstm/src/lstm.py index f014eef8df..c3ca0bbf7c 100644 --- a/model_zoo/lstm/src/lstm.py +++ b/model_zoo/lstm/src/lstm.py @@ -88,6 +88,6 @@ class SentimentNet(nn.Cell): embeddings = self.trans(embeddings, self.perm) output, _ = self.encoder(embeddings, (self.h, self.c)) # states[i] size(64,200) -> encoding.size(64,400) - encoding = self.concat((output[0], output[-1])) + encoding = self.concat((output[0], output[499])) outputs = self.decoder(encoding) return outputs diff --git a/model_zoo/lstm/train.py b/model_zoo/lstm/train.py index 732655f1de..51ae12c685 100644 --- a/model_zoo/lstm/train.py +++ b/model_zoo/lstm/train.py @@ -24,8 +24,8 @@ import numpy as np from src.config import lstm_cfg as cfg from src.dataset import convert_to_mindrecord from src.dataset import lstm_create_dataset +from src.lstm import SentimentNet from mindspore import Tensor, nn, Model, context -from mindspore.model_zoo.lstm import SentimentNet from mindspore.nn import Accuracy from mindspore.train.callback import LossMonitor, CheckpointConfig, ModelCheckpoint, TimeMonitor from mindspore.train.serialization import load_param_into_net, load_checkpoint diff --git a/model_zoo/mass/scripts/run.sh b/model_zoo/mass/scripts/run.sh index fc9606fcbd..91bed510ea 100644 --- a/model_zoo/mass/scripts/run.sh +++ b/model_zoo/mass/scripts/run.sh @@ -18,7 +18,7 @@ export DEVICE_ID=0 export RANK_ID=0 export RANK_SIZE=1 -options=`getopt -u -o ht:n:i:j:c:o:v: -l help,task:,device_num:,device_id:,hccl_json:,config:,output:,vocab -- "$@"` +options=`getopt -u -o ht:n:i:j:c:o:v: -l help,task:,device_num:,device_id:,hccl_json:,config:,output:,vocab: -- "$@"` eval set -- "$options" echo $options @@ -129,6 +129,7 @@ do esac done +file_path=$(cd "$(dirname $0)" || exit; pwd) for((i=0; i < $RANK_SIZE; i++)) do if [ $RANK_SIZE -gt 1 ] @@ -139,7 +140,6 @@ do fi echo "Working on device $i" - file_path=$(cd "$(dirname $0)" || exit; pwd) cd $file_path || exit cd ../ || exit diff --git a/model_zoo/mass/src/dataset/load_dataset.py b/model_zoo/mass/src/dataset/load_dataset.py index 9d9d558cb6..53ad5c7491 100644 --- a/model_zoo/mass/src/dataset/load_dataset.py +++ b/model_zoo/mass/src/dataset/load_dataset.py @@ -13,7 +13,6 @@ # limitations under the License. # ============================================================================ """Dataset loader to feed into model.""" -import os import mindspore.common.dtype as mstype import mindspore.dataset.engine as de import mindspore.dataset.transforms.c_transforms as deC @@ -40,12 +39,6 @@ def _load_dataset(input_files, batch_size, epoch_count=1, if not input_files: raise FileNotFoundError("Require at least one dataset.") - if not (schema_file and - os.path.exists(schema_file) - and os.path.isfile(schema_file) - and os.path.basename(schema_file).endswith(".json")): - raise FileNotFoundError("`dataset_schema` must be a existed json file.") - if not isinstance(sink_mode, bool): raise ValueError("`sink` must be type of bool.") diff --git a/model_zoo/mass/src/utils/byte_pair_encoding.py b/model_zoo/mass/src/utils/byte_pair_encoding.py index fb0e34a30d..d18b09e37f 100644 --- a/model_zoo/mass/src/utils/byte_pair_encoding.py +++ b/model_zoo/mass/src/utils/byte_pair_encoding.py @@ -16,8 +16,8 @@ import os import subprocess -ENCODER = "subword-nmt apply-bpe -c {codes} -i {input} -o {output}" -LEARN_DICT = "subword-nmt get-vocab -i {input} -o {dict_path}" +ENCODER = "subword-nmt apply-bpe -c" +LEARN_DICT = "subword-nmt get-vocab -i" def bpe_encode(codes_path, src_path, output_path, dict_path): @@ -43,10 +43,10 @@ def bpe_encode(codes_path, src_path, output_path, dict_path): raise FileNotFoundError("Dir not found.") # Encoding. - print(f" | Applying BPE encoding.") - subprocess.call(ENCODER.format(codes=codes_path, input=src_path, output=output_path), - shell=True) - print(f" | Fetching vocabulary from single file.") + print(" | Applying BPE encoding.") + commands = ENCODER.split() + [codes_path] + ["-i"] + [src_path] + ["-o"] + [output_path] + subprocess.call(commands) + print(" | Fetching vocabulary from single file.") # Learn vocab. - subprocess.call(LEARN_DICT.format(input=output_path, dict_path=dict_path), - shell=True) + commands = LEARN_DICT.split() + [output_path] + ["-o"] + [dict_path] + subprocess.call(commands) diff --git a/model_zoo/mass/src/utils/rouge_score.py b/model_zoo/mass/src/utils/rouge_score.py index f453b5d2e1..665cda3433 100644 --- a/model_zoo/mass/src/utils/rouge_score.py +++ b/model_zoo/mass/src/utils/rouge_score.py @@ -47,7 +47,7 @@ def rouge(hypothesis: List[str], target: List[str]): edited_ref.append(r + "\n") _rouge = Rouge() - scores = _rouge.get_scores(edited_hyp, target, avg=True) + scores = _rouge.get_scores(edited_hyp, edited_ref, avg=True) print(" | ROUGE Score:") print(f" | RG-1(F): {scores['rouge-1']['f'] * 100:8.2f}") print(f" | RG-2(F): {scores['rouge-2']['f'] * 100:8.2f}") diff --git a/model_zoo/mass/train.py b/model_zoo/mass/train.py index 05b96ddae3..b58075ba4e 100644 --- a/model_zoo/mass/train.py +++ b/model_zoo/mass/train.py @@ -120,6 +120,7 @@ def _build_training_pipeline(config: TransformerConfig, test_dataset (Dataset): Test dataset. """ net_with_loss = TransformerNetworkWithLoss(config, is_training=True) + net_with_loss.init_parameters_data() if config.existed_ckpt: if config.existed_ckpt.endswith(".npz"): diff --git a/model_zoo/mobilenetv2/scripts/run_infer.sh b/model_zoo/mobilenetv2/scripts/run_infer.sh index e200e600bf..7385a221d4 100644 --- a/model_zoo/mobilenetv2/scripts/run_infer.sh +++ b/model_zoo/mobilenetv2/scripts/run_infer.sh @@ -40,7 +40,7 @@ export PYTHONPATH=${BASEPATH}:$PYTHONPATH export DEVICE_ID=0 export RANK_ID=0 export RANK_SIZE=1 -if [ -d "eval" ]; +if [ -d "../eval" ]; then rm -rf ../eval fi diff --git a/model_zoo/mobilenetv2/scripts/run_train.sh b/model_zoo/mobilenetv2/scripts/run_train.sh index fc013d474c..f1d80aeac6 100644 --- a/model_zoo/mobilenetv2/scripts/run_train.sh +++ b/model_zoo/mobilenetv2/scripts/run_train.sh @@ -30,7 +30,7 @@ run_ascend() BASEPATH=$(cd "`dirname $0`" || exit; pwd) export PYTHONPATH=${BASEPATH}:$PYTHONPATH - if [ -d "train" ]; + if [ -d "../train" ]; then rm -rf ../train fi @@ -62,7 +62,7 @@ run_gpu() BASEPATH=$(cd "`dirname $0`" || exit; pwd) export PYTHONPATH=${BASEPATH}:$PYTHONPATH - if [ -d "train" ]; + if [ -d "../train" ]; then rm -rf ../train fi diff --git a/model_zoo/mobilenetv2_quant/Readme.md b/model_zoo/mobilenetv2_quant/Readme.md new file mode 100644 index 0000000000..81be5d519c --- /dev/null +++ b/model_zoo/mobilenetv2_quant/Readme.md @@ -0,0 +1,142 @@ +# MobileNetV2 Quantization Aware Training + +MobileNetV2 is a significant improvement over MobileNetV1 and pushes the state of the art for mobile visual recognition including classification, object detection and semantic segmentation. + +MobileNetV2 builds upon the ideas from MobileNetV1, using depthwise separable convolution as efficient building blocks. However, V2 introduces two new features to the architecture: 1) linear bottlenecks between the layers, and 2) shortcut connections between the bottlenecks1. + +Training MobileNetV2 with ImageNet dataset in MindSpore with quantization aware training. + +This is the simple and basic tutorial for constructing a network in MindSpore with quantization aware. + +In this readme tutorial, you will: + +1. Train a MindSpore fusion MobileNetV2 model for ImageNet from scratch using `nn.Conv2dBnAct` and `nn.DenseBnAct`. +2. Fine tune the fusion model by applying the quantization aware training auto network converter API `convert_quant_network`, after the network convergence then export a quantization aware model checkpoint file. + +[Paper](https://arxiv.org/pdf/1801.04381) Sandler, Mark, et al. "Mobilenetv2: Inverted residuals and linear bottlenecks." Proceedings of the IEEE conference on computer vision and pattern recognition. 2018. + +# Dataset + +Dataset use: ImageNet + +- Dataset size: about 125G + - Train: 120G, 1281167 images: 1000 directories + - Test: 5G, 50000 images: images should be classified into 1000 directories firstly, just like train images +- Data format: RGB images. + - Note: Data will be processed in src/dataset.py + +# Environment Requirements + +- Hardware（Ascend) + - Prepare hardware environment with Ascend processor. If you want to try Ascend, please send the [application form](https://obs-9be7.obs.cn-east-2.myhuaweicloud.com/file/other/Ascend%20Model%20Zoo%E4%BD%93%E9%AA%8C%E8%B5%84%E6%BA%90%E7%94%B3%E8%AF%B7%E8%A1%A8.docx) to ascend@huawei.com. Once approved, you can get the resources. +- Framework + - [MindSpore](http://10.90.67.50/mindspore/archive/20200506/OpenSource/me_vm_x86/) +- For more information, please check the resources below： + - [MindSpore tutorials](https://www.mindspore.cn/tutorial/zh-CN/master/index.html) + - [MindSpore API](https://www.mindspore.cn/api/zh-CN/master/index.html) + + +# Script description + +## Script and sample code + +```python +├── mobilenetv2_quant + ├── Readme.md + ├── scripts + │ ├──run_train.sh + │ ├──run_infer.sh + │ ├──run_train_quant.sh + │ ├──run_infer_quant.sh + ├── src + │ ├──config.py + │ ├──dataset.py + │ ├──luanch.py + │ ├──lr_generator.py + │ ├──mobilenetV2.py + ├── train.py + ├── eval.py +``` + +## Training process + +### Train MobileNetV2 model + +Train a MindSpore fusion MobileNetV2 model for ImageNet, like: + +- sh run_train.sh Ascend [DEVICE_NUM] [SERVER_IP(x.x.x.x)] [VISIABLE_DEVICES(0,1,2,3,4,5,6,7)] [DATASET_PATH] [CKPT_PATH] + +You can just run this command instead. + +``` bash +>>> sh run_train.sh Ascend 4 192.168.0.1 0,1,2,3 ~/imagenet/train/ ~/mobilenet.ckpt +``` + +Training result will be stored in the example path. Checkpoints will be stored at `. /checkpoint` by default, and training log will be redirected to `./train/train.log` like followings. + +``` +>>> epoch: [ 0/200], step:[ 624/ 625], loss:[5.258/5.258], time:[140412.236], lr:[0.100] +>>> epoch time: 140522.500, per step time: 224.836, avg loss: 5.258 +>>> epoch: [ 1/200], step:[ 624/ 625], loss:[3.917/3.917], time:[138221.250], lr:[0.200] +>>> epoch time: 138331.250, per step time: 221.330, avg loss: 3.917 +``` + +### Evaluate MobileNetV2 model + +Evaluate a MindSpore fusion MobileNetV2 model for ImageNet, like: + +- sh run_infer.sh Ascend [DATASET_PATH] [CHECKPOINT_PATH] + +You can just run this command instead. + +``` bash +>>> sh run_infer.sh Ascend ~/imagenet/val/ ~/train/mobilenet-200_625.ckpt +``` + +Inference result will be stored in the example path, you can find result like the followings in `val.log`. + +``` +>>> result: {'acc': 0.71976314102564111} ckpt=/path/to/checkpoint/mobilenet-200_625.ckpt +``` + +### Fine-tune for quantization aware training + +Fine tune the fusion model by applying the quantization aware training auto network converter API `convert_quant_network`, after the network convergence then export a quantization aware model checkpoint file. + +- sh run_train_quant.sh Ascend [DEVICE_NUM] [SERVER_IP(x.x.x.x)] [VISIABLE_DEVICES(0,1,2,3,4,5,6,7)] [DATASET_PATH] [CKPT_PATH] + +You can just run this command instead. + +``` bash +>>> sh run_train_quant.sh Ascend 4 192.168.0.1 0,1,2,3 ~/imagenet/train/ ~/mobilenet.ckpt +``` + +Training result will be stored in the example path. Checkpoints will be stored at `. /checkpoint` by default, and training log will be redirected to `./train/train.log` like followings. + +``` +>>> epoch: [ 0/60], step:[ 624/ 625], loss:[5.258/5.258], time:[140412.236], lr:[0.100] +>>> epoch time: 140522.500, per step time: 224.836, avg loss: 5.258 +>>> epoch: [ 1/60], step:[ 624/ 625], loss:[3.917/3.917], time:[138221.250], lr:[0.200] +>>> epoch time: 138331.250, per step time: 221.330, avg loss: 3.917 +``` + +### Evaluate quantization aware training model + +Evaluate a MindSpore fusion MobileNetV2 model for ImageNet by applying the quantization aware training, like: + +- sh run_infer_quant.sh Ascend [DATASET_PATH] [CHECKPOINT_PATH] + +You can just run this command instead. + +``` bash +>>> sh run_infer_quant.sh Ascend ~/imagenet/val/ ~/train/mobilenet-60_625.ckpt +``` + +Inference result will be stored in the example path, you can find result like the followings in `val.log`. + +``` +>>> result: {'acc': 0.71976314102564111} ckpt=/path/to/checkpoint/mobilenet-60_625.ckpt +``` + +# ModelZoo Homepage + [Link](https://gitee.com/mindspore/mindspore/tree/master/mindspore/model_zoo) diff --git a/model_zoo/mobilenetv2_quant/eval.py b/model_zoo/mobilenetv2_quant/eval.py new file mode 100644 index 0000000000..0976abbe99 --- /dev/null +++ b/model_zoo/mobilenetv2_quant/eval.py @@ -0,0 +1,76 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Evaluate MobilenetV2 on ImageNet""" + +import os +import argparse + +from mindspore import context +from mindspore import nn +from mindspore.train.model import Model +from mindspore.train.serialization import load_checkpoint, load_param_into_net +from mindspore.train.quant import quant + +from src.mobilenetV2 import mobilenetV2 +from src.dataset import create_dataset +from src.config import config_ascend + +parser = argparse.ArgumentParser(description='Image classification') +parser.add_argument('--checkpoint_path', type=str, default=None, help='Checkpoint file path') +parser.add_argument('--dataset_path', type=str, default=None, help='Dataset path') +parser.add_argument('--device_target', type=str, default=None, help='Run device target') +parser.add_argument('--quantization_aware', type=bool, default=False, help='Use quantization aware training') +args_opt = parser.parse_args() + +if __name__ == '__main__': + config_device_target = None + if args_opt.device_target == "Ascend": + config_device_target = config_ascend + device_id = int(os.getenv('DEVICE_ID')) + context.set_context(mode=context.GRAPH_MODE, device_target="Ascend", + device_id=device_id, save_graphs=False) + + else: + raise ValueError("Unsupported device target: {}.".format(args_opt.device_target)) + + # define fusion network + network = mobilenetV2(num_classes=config_device_target.num_classes) + if args_opt.quantization_aware: + # convert fusion network to quantization aware network + network = quant.convert_quant_network(network, bn_fold=True, per_channel=[True, False], symmetric=[True, False]) + # define network loss + loss = nn.SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True, reduction='mean') + + # define dataset + dataset = create_dataset(dataset_path=args_opt.dataset_path, + do_train=False, + config=config_device_target, + device_target=args_opt.device_target, + batch_size=config_device_target.batch_size) + step_size = dataset.get_dataset_size() + + # load checkpoint + if args_opt.checkpoint_path: + param_dict = load_checkpoint(args_opt.checkpoint_path) + load_param_into_net(network, param_dict) + network.set_train(False) + + # define model + model = Model(network, loss_fn=loss, metrics={'acc'}) + + print("============== Starting Validation ==============") + res = model.eval(dataset) + print("result:", res, "ckpt=", args_opt.checkpoint_path) + print("============== End Validation ==============") diff --git a/model_zoo/mobilenetv2_quant/scripts/run_infer.sh b/model_zoo/mobilenetv2_quant/scripts/run_infer.sh new file mode 100644 index 0000000000..308723af2a --- /dev/null +++ b/model_zoo/mobilenetv2_quant/scripts/run_infer.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +if [ $# != 3 ] +then + echo "Ascend: sh run_infer.sh [PLATFORM] [DATASET_PATH] [CHECKPOINT_PATH]" +exit 1 +fi + +# check dataset path +if [ ! -d $2 ] && [ ! -f $2 ] +then + echo "error: DATASET_PATH=$2 is not a directory or file" +exit 1 +fi + +# check checkpoint file +if [ ! -f $3 ] +then + echo "error: CHECKPOINT_PATH=$3 is not a file" +exit 1 +fi + +# set environment +BASEPATH=$(cd "`dirname $0`" || exit; pwd) +export DEVICE_ID=0 +export RANK_ID=0 +export RANK_SIZE=1 +if [ -d "../eval" ]; +then + rm -rf ../eval +fi +mkdir ../eval +cd ../eval || exit + +# launch +python ${BASEPATH}/../eval.py \ + --device_target=$1 \ + --dataset_path=$2 \ + --checkpoint_path=$3 \ + &> infer.log & # dataset val folder path diff --git a/model_zoo/mobilenetv2_quant/scripts/run_infer_quant.sh b/model_zoo/mobilenetv2_quant/scripts/run_infer_quant.sh new file mode 100644 index 0000000000..f8f3c10619 --- /dev/null +++ b/model_zoo/mobilenetv2_quant/scripts/run_infer_quant.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +if [ $# != 3 ] +then + echo "Ascend: sh run_infer.sh [PLATFORM] [DATASET_PATH] [CHECKPOINT_PATH]" +exit 1 +fi + +# check dataset path +if [ ! -d $2 ] && [ ! -f $2 ] +then + echo "error: DATASET_PATH=$2 is not a directory or file" +exit 1 +fi + +# check checkpoint file +if [ ! -f $3 ] +then + echo "error: CHECKPOINT_PATH=$3 is not a file" +exit 1 +fi + +# set environment +BASEPATH=$(cd "`dirname $0`" || exit; pwd) +export DEVICE_ID=0 +export RANK_ID=0 +export RANK_SIZE=1 +if [ -d "../eval" ]; +then + rm -rf ../eval +fi +mkdir ../eval +cd ../eval || exit + +# launch +python ${BASEPATH}/../eval.py \ + --device_target=$1 \ + --dataset_path=$2 \ + --checkpoint_path=$3 \ + --quantization_aware=True \ + &> infer.log & # dataset val folder path diff --git a/model_zoo/mobilenetv2_quant/scripts/run_train.sh b/model_zoo/mobilenetv2_quant/scripts/run_train.sh new file mode 100644 index 0000000000..59b105f92e --- /dev/null +++ b/model_zoo/mobilenetv2_quant/scripts/run_train.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +run_ascend() +{ + if [ $2 -lt 1 ] && [ $2 -gt 8 ] + then + echo "error: DEVICE_NUM=$2 is not in (1-9)" + exit 1 + fi + + if [ ! -d $5 ] && [ ! -f $5 ] + then + echo "error: DATASET_PATH=$5 is not a directory or file" + exit 1 + fi + + BASEPATH=$(cd "`dirname $0`" || exit; pwd) + export PYTHONPATH=${BASEPATH}:$PYTHONPATH + if [ -d "../train" ]; + then + rm -rf ../train + fi + mkdir ../train + cd ../train || exit + python ${BASEPATH}/../src/launch.py \ + --nproc_per_node=$2 \ + --visible_devices=$4 \ + --server_id=$3 \ + --training_script=${BASEPATH}/../train.py \ + --dataset_path=$5 \ + --pre_trained=$6 \ + --device_target=$1 &> train.log & # dataset train folder +} + +if [ $# -gt 6 ] || [ $# -lt 4 ] +then + echo "Usage:\n \ + Ascend: sh run_train.sh Ascend [DEVICE_NUM] [SERVER_IP(x.x.x.x)] [VISIABLE_DEVICES(0,1,2,3,4,5,6,7)] [DATASET_PATH] [CKPT_PATH]\n \ + " +exit 1 +fi + +if [ $1 = "Ascend" ] ; then + run_ascend "$@" +else + echo "Unsupported device target." +fi; + diff --git a/model_zoo/mobilenetv2_quant/scripts/run_train_quant.sh b/model_zoo/mobilenetv2_quant/scripts/run_train_quant.sh new file mode 100644 index 0000000000..c82d1b0da5 --- /dev/null +++ b/model_zoo/mobilenetv2_quant/scripts/run_train_quant.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +run_ascend() +{ + if [ $2 -lt 1 ] && [ $2 -gt 8 ] + then + echo "error: DEVICE_NUM=$2 is not in (1-9)" + exit 1 + fi + + if [ ! -d $5 ] && [ ! -f $5 ] + then + echo "error: DATASET_PATH=$5 is not a directory or file" + exit 1 + fi + + BASEPATH=$(cd "`dirname $0`" || exit; pwd) + export PYTHONPATH=${BASEPATH}:$PYTHONPATH + if [ -d "../train" ]; + then + rm -rf ../train + fi + mkdir ../train + cd ../train || exit + python ${BASEPATH}/../src/launch.py \ + --nproc_per_node=$2 \ + --visible_devices=$4 \ + --server_id=$3 \ + --training_script=${BASEPATH}/../train.py \ + --dataset_path=$5 \ + --pre_trained=$6 \ + --quantization_aware=True \ + --device_target=$1 &> train.log & # dataset train folder +} + +if [ $# -gt 6 ] || [ $# -lt 4 ] +then + echo "Usage:\n \ + Ascend: sh run_train.sh Ascend [DEVICE_NUM] [SERVER_IP(x.x.x.x)] [VISIABLE_DEVICES(0,1,2,3,4,5,6,7)] [DATASET_PATH] [CKPT_PATH]\n \ + " +exit 1 +fi + +if [ $1 = "Ascend" ] ; then + run_ascend "$@" +else + echo "Unsupported device target." +fi; + diff --git a/model_zoo/mobilenetv2_quant/src/config.py b/model_zoo/mobilenetv2_quant/src/config.py new file mode 100644 index 0000000000..97fbc52e12 --- /dev/null +++ b/model_zoo/mobilenetv2_quant/src/config.py @@ -0,0 +1,60 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +network config setting, will be used in train.py and eval.py +""" +from easydict import EasyDict as ed + +config_ascend = ed({ + "num_classes": 1000, + "image_height": 224, + "image_width": 224, + "batch_size": 256, + "data_load_mode": "mindrecord", + "epoch_size": 200, + "start_epoch": 0, + "warmup_epochs": 4, + "lr": 0.4, + "momentum": 0.9, + "weight_decay": 4e-5, + "label_smooth": 0.1, + "loss_scale": 1024, + "save_checkpoint": True, + "save_checkpoint_epochs": 1, + "keep_checkpoint_max": 200, + "save_checkpoint_path": "./checkpoint", + "quantization_aware": False, +}) + +config_ascend_quant = ed({ + "num_classes": 1000, + "image_height": 224, + "image_width": 224, + "batch_size": 192, + "data_load_mode": "mindrecord", + "epoch_size": 60, + "start_epoch": 200, + "warmup_epochs": 1, + "lr": 0.3, + "momentum": 0.9, + "weight_decay": 4e-5, + "label_smooth": 0.1, + "loss_scale": 1024, + "save_checkpoint": True, + "save_checkpoint_epochs": 1, + "keep_checkpoint_max": 200, + "save_checkpoint_path": "./checkpoint", + "quantization_aware": True, +}) diff --git a/model_zoo/mobilenetv2_quant/src/dataset.py b/model_zoo/mobilenetv2_quant/src/dataset.py new file mode 100644 index 0000000000..105a5e1396 --- /dev/null +++ b/model_zoo/mobilenetv2_quant/src/dataset.py @@ -0,0 +1,156 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +create train or eval dataset. +""" +import os +from functools import partial +import mindspore.common.dtype as mstype +import mindspore.dataset.engine as de +import mindspore.dataset.transforms.vision.c_transforms as C +import mindspore.dataset.transforms.c_transforms as C2 +import mindspore.dataset.transforms.vision.py_transforms as P +from src.config import config_ascend + + +def create_dataset(dataset_path, do_train, config, device_target, repeat_num=1, batch_size=32): + """ + create a train or eval dataset + + Args: + dataset_path(string): the path of dataset. + do_train(bool): whether dataset is used for train or eval. + repeat_num(int): the repeat times of dataset. Default: 1. + batch_size(int): the batch size of dataset. Default: 32. + + Returns: + dataset + """ + if device_target == "Ascend": + rank_size = int(os.getenv("RANK_SIZE")) + rank_id = int(os.getenv("RANK_ID")) + columns_list = ['image', 'label'] + if config_ascend.data_load_mode == "mindrecord": + load_func = partial(de.MindDataset, dataset_path, columns_list) + else: + load_func = partial(de.ImageFolderDatasetV2, dataset_path) + if do_train: + if rank_size == 1: + ds = load_func(num_parallel_workers=8, shuffle=True) + else: + ds = load_func(num_parallel_workers=8, shuffle=True, + num_shards=rank_size, shard_id=rank_id) + else: + ds = load_func(num_parallel_workers=8, shuffle=False) + else: + raise ValueError("Unsupport device_target.") + + resize_height = config.image_height + + if do_train: + buffer_size = 20480 + # apply shuffle operations + ds = ds.shuffle(buffer_size=buffer_size) + + # define map operations + decode_op = C.Decode() + resize_crop_decode_op = C.RandomCropDecodeResize(resize_height, scale=(0.08, 1.0), ratio=(0.75, 1.333)) + horizontal_flip_op = C.RandomHorizontalFlip(prob=0.5) + + resize_op = C.Resize(256) + center_crop = C.CenterCrop(resize_height) + normalize_op = C.Normalize(mean=[0.485 * 255, 0.456 * 255, 0.406 * 255], + std=[0.229 * 255, 0.224 * 255, 0.225 * 255]) + change_swap_op = C.HWC2CHW() + + if do_train: + trans = [resize_crop_decode_op, horizontal_flip_op, normalize_op, change_swap_op] + else: + trans = [decode_op, resize_op, center_crop, normalize_op, change_swap_op] + + type_cast_op = C2.TypeCast(mstype.int32) + + ds = ds.map(input_columns="image", operations=trans, num_parallel_workers=16) + ds = ds.map(input_columns="label", operations=type_cast_op, num_parallel_workers=8) + + # apply batch operations + ds = ds.batch(batch_size, drop_remainder=True) + + # apply dataset repeat operation + ds = ds.repeat(repeat_num) + + return ds + + +def create_dataset_py(dataset_path, do_train, config, device_target, repeat_num=1, batch_size=32): + """ + create a train or eval dataset + + Args: + dataset_path(string): the path of dataset. + do_train(bool): whether dataset is used for train or eval. + repeat_num(int): the repeat times of dataset. Default: 1. + batch_size(int): the batch size of dataset. Default: 32. + + Returns: + dataset + """ + if device_target == "Ascend": + rank_size = int(os.getenv("RANK_SIZE")) + rank_id = int(os.getenv("RANK_ID")) + if do_train: + if rank_size == 1: + ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True) + else: + ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True, + num_shards=rank_size, shard_id=rank_id) + else: + ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=False) + else: + raise ValueError("Unsupported device target.") + + resize_height = config.image_height + + if do_train: + buffer_size = 20480 + # apply shuffle operations + ds = ds.shuffle(buffer_size=buffer_size) + + # define map operations + decode_op = P.Decode() + resize_crop_op = P.RandomResizedCrop(resize_height, scale=(0.08, 1.0), ratio=(0.75, 1.333)) + horizontal_flip_op = P.RandomHorizontalFlip(prob=0.5) + + resize_op = P.Resize(256) + center_crop = P.CenterCrop(resize_height) + to_tensor = P.ToTensor() + normalize_op = P.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) + + if do_train: + trans = [decode_op, resize_crop_op, horizontal_flip_op, to_tensor, normalize_op] + else: + trans = [decode_op, resize_op, center_crop, to_tensor, normalize_op] + + compose = P.ComposeOp(trans) + + ds = ds.map(input_columns="image", operations=compose(), num_parallel_workers=8, python_multiprocessing=True) + + # apply batch operations + ds = ds.batch(batch_size, drop_remainder=True) + + # apply dataset repeat operation + ds = ds.repeat(repeat_num) + + return ds diff --git a/model_zoo/mobilenetv2_quant/src/launch.py b/model_zoo/mobilenetv2_quant/src/launch.py new file mode 100644 index 0000000000..08477a363a --- /dev/null +++ b/model_zoo/mobilenetv2_quant/src/launch.py @@ -0,0 +1,166 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""launch train script""" +import os +import sys +import json +import subprocess +import shutil +import platform +from argparse import ArgumentParser + + +def parse_args(): + """ + parse args . + + Args: + + Returns: + args. + + Examples: + >>> parse_args() + """ + parser = ArgumentParser(description="mindspore distributed training launch " + "helper utilty that will spawn up " + "multiple distributed processes") + parser.add_argument("--nproc_per_node", type=int, default=1, + help="The number of processes to launch on each node, " + "for D training, this is recommended to be set " + "to the number of D in your system so that " + "each process can be bound to a single D.") + parser.add_argument("--visible_devices", type=str, default="0,1,2,3,4,5,6,7", + help="will use the visible devices sequentially") + parser.add_argument("--server_id", type=str, default="", + help="server ip") + parser.add_argument("--training_script", type=str, + help="The full path to the single D training " + "program/script to be launched in parallel, " + "followed by all the arguments for the " + "training script") + # rest from the training program + args, unknown = parser.parse_known_args() + args.training_script_args = unknown + return args + + +def main(): + print("start", __file__) + args = parse_args() + print(args) + visible_devices = args.visible_devices.split(',') + assert os.path.isfile(args.training_script) + assert len(visible_devices) >= args.nproc_per_node + print('visible_devices:{}'.format(visible_devices)) + if not args.server_id: + print('pleaser input server ip!!!') + exit(0) + print('server_id:{}'.format(args.server_id)) + + # construct hccn_table + hccn_configs = open('/etc/hccn.conf', 'r').readlines() + device_ips = {} + for hccn_item in hccn_configs: + hccn_item = hccn_item.strip() + if hccn_item.startswith('address_'): + device_id, device_ip = hccn_item.split('=') + device_id = device_id.split('_')[1] + device_ips[device_id] = device_ip + print('device_id:{}, device_ip:{}'.format(device_id, device_ip)) + hccn_table = {} + arch = platform.processor() + hccn_table['board_id'] = {'aarch64': '0x002f', 'x86_64': '0x0000'}[arch] + hccn_table['chip_info'] = '910' + hccn_table['deploy_mode'] = 'lab' + hccn_table['group_count'] = '1' + hccn_table['group_list'] = [] + instance_list = [] + usable_dev = '' + for instance_id in range(args.nproc_per_node): + instance = {} + instance['devices'] = [] + device_id = visible_devices[instance_id] + device_ip = device_ips[device_id] + usable_dev += str(device_id) + instance['devices'].append({ + 'device_id': device_id, + 'device_ip': device_ip, + }) + instance['rank_id'] = str(instance_id) + instance['server_id'] = args.server_id + instance_list.append(instance) + hccn_table['group_list'].append({ + 'device_num': str(args.nproc_per_node), + 'server_num': '1', + 'group_name': '', + 'instance_count': str(args.nproc_per_node), + 'instance_list': instance_list, + }) + hccn_table['para_plane_nic_location'] = 'device' + hccn_table['para_plane_nic_name'] = [] + for instance_id in range(args.nproc_per_node): + eth_id = visible_devices[instance_id] + hccn_table['para_plane_nic_name'].append('eth{}'.format(eth_id)) + hccn_table['para_plane_nic_num'] = str(args.nproc_per_node) + hccn_table['status'] = 'completed' + + # save hccn_table to file + table_path = os.getcwd() + if not os.path.exists(table_path): + os.mkdir(table_path) + table_fn = os.path.join(table_path, + 'rank_table_{}p_{}_{}.json'.format(args.nproc_per_node, usable_dev, args.server_id)) + with open(table_fn, 'w') as table_fp: + json.dump(hccn_table, table_fp, indent=4) + sys.stdout.flush() + + # spawn the processes + processes = [] + cmds = [] + log_files = [] + env = os.environ.copy() + env['RANK_SIZE'] = str(args.nproc_per_node) + cur_path = os.getcwd() + for rank_id in range(0, args.nproc_per_node): + os.chdir(cur_path) + device_id = visible_devices[rank_id] + device_dir = os.path.join(cur_path, 'device{}'.format(rank_id)) + env['RANK_ID'] = str(rank_id) + env['DEVICE_ID'] = str(device_id) + if args.nproc_per_node > 1: + env['MINDSPORE_HCCL_CONFIG_PATH'] = table_fn + env['RANK_TABLE_FILE'] = table_fn + if os.path.exists(device_dir): + shutil.rmtree(device_dir) + os.mkdir(device_dir) + os.chdir(device_dir) + cmd = [sys.executable, '-u'] + cmd.append(args.training_script) + cmd.extend(args.training_script_args) + log_file = open('{dir}/log{id}.log'.format(dir=device_dir, id=rank_id), 'w') + process = subprocess.Popen(cmd, stdout=log_file, stderr=log_file, env=env) + processes.append(process) + cmds.append(cmd) + log_files.append(log_file) + for process, cmd, log_file in zip(processes, cmds, log_files): + process.wait() + if process.returncode != 0: + raise subprocess.CalledProcessError(returncode=process, cmd=cmd) + log_file.close() + + +if __name__ == "__main__": + main() diff --git a/model_zoo/mobilenetv2_quant/src/lr_generator.py b/model_zoo/mobilenetv2_quant/src/lr_generator.py new file mode 100644 index 0000000000..68bbfe3158 --- /dev/null +++ b/model_zoo/mobilenetv2_quant/src/lr_generator.py @@ -0,0 +1,54 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""learning rate generator""" +import math +import numpy as np + + +def get_lr(global_step, lr_init, lr_end, lr_max, warmup_epochs, total_epochs, steps_per_epoch): + """ + generate learning rate array + + Args: + global_step(int): total steps of the training + lr_init(float): init learning rate + lr_end(float): end learning rate + lr_max(float): max learning rate + warmup_epochs(int): number of warmup epochs + total_epochs(int): total epoch of training + steps_per_epoch(int): steps of one epoch + + Returns: + np.array, learning rate array + """ + lr_each_step = [] + total_steps = steps_per_epoch * total_epochs + warmup_steps = steps_per_epoch * warmup_epochs + for i in range(total_steps): + if i < warmup_steps: + lr = lr_init + (lr_max - lr_init) * i / warmup_steps + else: + lr = lr_end + \ + (lr_max - lr_end) * \ + (1. + math.cos(math.pi * (i - warmup_steps) / (total_steps - warmup_steps))) / 2. + if lr < 0.0: + lr = 0.0 + lr_each_step.append(lr) + + current_step = global_step + lr_each_step = np.array(lr_each_step).astype(np.float32) + learning_rate = lr_each_step[current_step:] + + return learning_rate diff --git a/model_zoo/mobilenetv2_quant/src/mobilenetV2.py b/model_zoo/mobilenetv2_quant/src/mobilenetV2.py new file mode 100644 index 0000000000..25dccfed10 --- /dev/null +++ b/model_zoo/mobilenetv2_quant/src/mobilenetV2.py @@ -0,0 +1,231 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""MobileNetV2 Quant model define""" + +import numpy as np + +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore import Tensor + +__all__ = ['mobilenetV2'] + + +def _make_divisible(v, divisor, min_value=None): + if min_value is None: + min_value = divisor + new_v = max(min_value, int(v + divisor / 2) // divisor * divisor) + # Make sure that round down does not go down by more than 10%. + if new_v < 0.9 * v: + new_v += divisor + return new_v + + +class GlobalAvgPooling(nn.Cell): + """ + Global avg pooling definition. + + Args: + + Returns: + Tensor, output tensor. + + Examples: + >>> GlobalAvgPooling() + """ + + def __init__(self): + super(GlobalAvgPooling, self).__init__() + self.mean = P.ReduceMean(keep_dims=False) + + def construct(self, x): + x = self.mean(x, (2, 3)) + return x + + +class ConvBNReLU(nn.Cell): + """ + Convolution/Depthwise fused with Batchnorm and ReLU block definition. + + Args: + in_planes (int): Input channel. + out_planes (int): Output channel. + kernel_size (int): Input kernel size. + stride (int): Stride size for the first convolutional layer. Default: 1. + groups (int): channel group. Convolution is 1 while Depthiwse is input channel. Default: 1. + + Returns: + Tensor, output tensor. + + Examples: + >>> ConvBNReLU(16, 256, kernel_size=1, stride=1, groups=1) + """ + + def __init__(self, in_planes, out_planes, kernel_size=3, stride=1, groups=1): + super(ConvBNReLU, self).__init__() + padding = (kernel_size - 1) // 2 + self.conv = nn.Conv2dBnAct(in_planes, out_planes, kernel_size, + stride=stride, + pad_mode='pad', + padding=padding, + group=groups, + has_bn=True, + activation='relu') + + def construct(self, x): + x = self.conv(x) + return x + + +class InvertedResidual(nn.Cell): + """ + Mobilenetv2 residual block definition. + + Args: + inp (int): Input channel. + oup (int): Output channel. + stride (int): Stride size for the first convolutional layer. Default: 1. + expand_ratio (int): expand ration of input channel + + Returns: + Tensor, output tensor. + + Examples: + >>> ResidualBlock(3, 256, 1, 1) + """ + + def __init__(self, inp, oup, stride, expand_ratio): + super(InvertedResidual, self).__init__() + assert stride in [1, 2] + + hidden_dim = int(round(inp * expand_ratio)) + self.use_res_connect = stride == 1 and inp == oup + + layers = [] + if expand_ratio != 1: + layers.append(ConvBNReLU(inp, hidden_dim, kernel_size=1)) + layers.extend([ + # dw + ConvBNReLU(hidden_dim, hidden_dim, stride=stride, groups=hidden_dim), + # pw-linear + nn.Conv2dBnAct(hidden_dim, oup, kernel_size=1, stride=1, pad_mode='pad', padding=0, group=1, has_bn=True) + ]) + self.conv = nn.SequentialCell(layers) + self.add = P.TensorAdd() + + def construct(self, x): + out = self.conv(x) + if self.use_res_connect: + out = self.add(out, x) + return out + + +class mobilenetV2(nn.Cell): + """ + mobilenetV2 fusion architecture. + + Args: + class_num (Cell): number of classes. + width_mult (int): Channels multiplier for round to 8/16 and others. Default is 1. + has_dropout (bool): Is dropout used. Default is false + inverted_residual_setting (list): Inverted residual settings. Default is None + round_nearest (list): Channel round to . Default is 8 + Returns: + Tensor, output tensor. + + Examples: + >>> mobilenetV2(num_classes=1000) + """ + + def __init__(self, num_classes=1000, width_mult=1., + has_dropout=False, inverted_residual_setting=None, round_nearest=8): + super(mobilenetV2, self).__init__() + block = InvertedResidual + input_channel = 32 + last_channel = 1280 + # setting of inverted residual blocks + self.cfgs = inverted_residual_setting + if inverted_residual_setting is None: + self.cfgs = [ + # t, c, n, s + [1, 16, 1, 1], + [6, 24, 2, 2], + [6, 32, 3, 2], + [6, 64, 4, 2], + [6, 96, 3, 1], + [6, 160, 3, 2], + [6, 320, 1, 1], + ] + + # building first layer + input_channel = _make_divisible(input_channel * width_mult, round_nearest) + self.out_channels = _make_divisible(last_channel * max(1.0, width_mult), round_nearest) + + features = [ConvBNReLU(3, input_channel, stride=2)] + # building inverted residual blocks + for t, c, n, s in self.cfgs: + output_channel = _make_divisible(c * width_mult, round_nearest) + for i in range(n): + stride = s if i == 0 else 1 + features.append(block(input_channel, output_channel, stride, expand_ratio=t)) + input_channel = output_channel + # building last several layers + features.append(ConvBNReLU(input_channel, self.out_channels, kernel_size=1)) + # make it nn.CellList + self.features = nn.SequentialCell(features) + # mobilenet head + head = ([GlobalAvgPooling(), + nn.DenseBnAct(self.out_channels, num_classes, has_bias=True, has_bn=False) + ] if not has_dropout else + [GlobalAvgPooling(), + nn.Dropout(0.2), + nn.DenseBnAct(self.out_channels, num_classes, has_bias=True, has_bn=False) + ]) + self.head = nn.SequentialCell(head) + + # init weights + self._initialize_weights() + + def construct(self, x): + x = self.features(x) + x = self.head(x) + return x + + def _initialize_weights(self): + """ + Initialize weights. + + Args: + + Returns: + None. + + Examples: + >>> _initialize_weights() + """ + for _, m in self.cells_and_names(): + if isinstance(m, nn.Conv2d): + n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels + w = Tensor(np.random.normal(0, np.sqrt(2. / n), m.weight.data.shape).astype("float32")) + m.weight.set_parameter_data(w) + if m.bias is not None: + m.bias.set_parameter_data(Tensor(np.zeros(m.bias.data.shape, dtype="float32"))) + elif isinstance(m, nn.BatchNorm2d): + m.gamma.set_parameter_data(Tensor(np.ones(m.gamma.data.shape, dtype="float32"))) + m.beta.set_parameter_data(Tensor(np.zeros(m.beta.data.shape, dtype="float32"))) + elif isinstance(m, nn.Dense): + m.weight.set_parameter_data(Tensor(np.random.normal(0, 0.01, m.weight.data.shape).astype("float32"))) + if m.bias is not None: + m.bias.set_parameter_data(Tensor(np.zeros(m.bias.data.shape, dtype="float32"))) diff --git a/model_zoo/mobilenetv2_quant/src/utils.py b/model_zoo/mobilenetv2_quant/src/utils.py new file mode 100644 index 0000000000..8690d9c380 --- /dev/null +++ b/model_zoo/mobilenetv2_quant/src/utils.py @@ -0,0 +1,113 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""MobileNetV2 utils""" + +import time +import numpy as np + +from mindspore.train.callback import Callback +from mindspore import Tensor +from mindspore import nn +from mindspore.nn.loss.loss import _Loss +from mindspore.ops import operations as P +from mindspore.ops import functional as F +from mindspore.common import dtype as mstype + + +class Monitor(Callback): + """ + Monitor loss and time. + + Args: + lr_init (numpy array): train lr + + Returns: + None + + Examples: + >>> Monitor(100,lr_init=Tensor([0.05]*100).asnumpy()) + """ + + def __init__(self, lr_init=None): + super(Monitor, self).__init__() + self.lr_init = lr_init + self.lr_init_len = len(lr_init) + + def epoch_begin(self, run_context): + self.losses = [] + self.epoch_time = time.time() + + def epoch_end(self, run_context): + cb_params = run_context.original_args() + + epoch_mseconds = (time.time() - self.epoch_time) * 1000 + per_step_mseconds = epoch_mseconds / cb_params.batch_num + print("epoch time: {:5.3f}, per step time: {:5.3f}, avg loss: {:5.3f}".format(epoch_mseconds, + per_step_mseconds, + np.mean(self.losses))) + + def step_begin(self, run_context): + self.step_time = time.time() + + def step_end(self, run_context): + cb_params = run_context.original_args() + step_mseconds = (time.time() - self.step_time) * 1000 + step_loss = cb_params.net_outputs + + if isinstance(step_loss, (tuple, list)) and isinstance(step_loss[0], Tensor): + step_loss = step_loss[0] + if isinstance(step_loss, Tensor): + step_loss = np.mean(step_loss.asnumpy()) + + self.losses.append(step_loss) + cur_step_in_epoch = (cb_params.cur_step_num - 1) % cb_params.batch_num + + print("epoch: [{:3d}/{:3d}], step:[{:5d}/{:5d}], loss:[{:5.3f}/{:5.3f}], time:[{:5.3f}], lr:[{:5.5f}]".format( + cb_params.cur_epoch_num - + 1, cb_params.epoch_num, cur_step_in_epoch, cb_params.batch_num, step_loss, + np.mean(self.losses), step_mseconds, self.lr_init[cb_params.cur_step_num - 1])) + + +class CrossEntropyWithLabelSmooth(_Loss): + """ + CrossEntropyWith LabelSmooth. + + Args: + smooth_factor (float): smooth factor, default=0. + num_classes (int): num classes + + Returns: + None. + + Examples: + >>> CrossEntropyWithLabelSmooth(smooth_factor=0., num_classes=1000) + """ + + def __init__(self, smooth_factor=0., num_classes=1000): + super(CrossEntropyWithLabelSmooth, self).__init__() + self.onehot = P.OneHot() + self.on_value = Tensor(1.0 - smooth_factor, mstype.float32) + self.off_value = Tensor(1.0 * smooth_factor / + (num_classes - 1), mstype.float32) + self.ce = nn.SoftmaxCrossEntropyWithLogits() + self.mean = P.ReduceMean(False) + self.cast = P.Cast() + + def construct(self, logit, label): + one_hot_label = self.onehot(self.cast(label, mstype.int32), F.shape(logit)[1], + self.on_value, self.off_value) + out_loss = self.ce(logit, one_hot_label) + out_loss = self.mean(out_loss, 0) + return out_loss diff --git a/model_zoo/mobilenetv2_quant/train.py b/model_zoo/mobilenetv2_quant/train.py new file mode 100644 index 0000000000..1302c3cf27 --- /dev/null +++ b/model_zoo/mobilenetv2_quant/train.py @@ -0,0 +1,131 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Train mobilenetV2 on ImageNet""" + +import os +import argparse +import random +import numpy as np + +from mindspore import context +from mindspore import Tensor +from mindspore import nn +from mindspore.train.model import Model, ParallelMode +from mindspore.train.callback import ModelCheckpoint, CheckpointConfig +from mindspore.train.serialization import load_checkpoint, load_param_into_net +from mindspore.communication.management import init +from mindspore.train.quant import quant +import mindspore.dataset.engine as de + +from src.dataset import create_dataset +from src.lr_generator import get_lr +from src.utils import Monitor, CrossEntropyWithLabelSmooth +from src.config import config_ascend, config_ascend_quant +from src.mobilenetV2 import mobilenetV2 + +random.seed(1) +np.random.seed(1) +de.config.set_seed(1) + +parser = argparse.ArgumentParser(description='Image classification') +parser.add_argument('--dataset_path', type=str, default=None, help='Dataset path') +parser.add_argument('--pre_trained', type=str, default=None, help='Pertained checkpoint path') +parser.add_argument('--device_target', type=str, default=None, help='Run device target') +parser.add_argument('--quantization_aware', type=bool, default=False, help='Use quantization aware training') +args_opt = parser.parse_args() + +if args_opt.device_target == "Ascend": + device_id = int(os.getenv('DEVICE_ID')) + rank_id = int(os.getenv('RANK_ID')) + rank_size = int(os.getenv('RANK_SIZE')) + run_distribute = rank_size > 1 + device_id = int(os.getenv('DEVICE_ID')) + context.set_context(mode=context.GRAPH_MODE, + device_target="Ascend", + device_id=device_id, save_graphs=False) +else: + raise ValueError("Unsupported device target.") + +if __name__ == '__main__': + # train on ascend + config = config_ascend_quant if args_opt.quantization_aware else config_ascend + print("training args: {}".format(args_opt)) + print("training configure: {}".format(config)) + print("parallel args: rank_id {}, device_id {}, rank_size {}".format(rank_id, device_id, rank_size)) + epoch_size = config.epoch_size + + # distribute init + if run_distribute: + context.set_auto_parallel_context(device_num=rank_size, + parallel_mode=ParallelMode.DATA_PARALLEL, + parameter_broadcast=True, + mirror_mean=True) + init() + + # define network + network = mobilenetV2(num_classes=config.num_classes) + # define loss + if config.label_smooth > 0: + loss = CrossEntropyWithLabelSmooth(smooth_factor=config.label_smooth, num_classes=config.num_classes) + else: + loss = nn.SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True, reduction='mean') + # define dataset + dataset = create_dataset(dataset_path=args_opt.dataset_path, + do_train=True, + config=config, + device_target=args_opt.device_target, + repeat_num=epoch_size, + batch_size=config.batch_size) + step_size = dataset.get_dataset_size() + # load pre trained ckpt + if args_opt.pre_trained: + param_dict = load_checkpoint(args_opt.pre_trained) + load_param_into_net(network, param_dict) + + # convert fusion network to quantization aware network + if config.quantization_aware: + network = quant.convert_quant_network(network, + bn_fold=True, + per_channel=[True, False], + symmetric=[True, False]) + + # get learning rate + lr = Tensor(get_lr(global_step=config.start_epoch * step_size, + lr_init=0, + lr_end=0, + lr_max=config.lr, + warmup_epochs=config.warmup_epochs, + total_epochs=epoch_size + config.start_epoch, + steps_per_epoch=step_size)) + + # define optimization + opt = nn.Momentum(filter(lambda x: x.requires_grad, network.get_parameters()), lr, config.momentum, + config.weight_decay) + # define model + model = Model(network, loss_fn=loss, optimizer=opt) + + print("============== Starting Training ==============") + callback = None + if rank_id == 0: + callback = [Monitor(lr_init=lr.asnumpy())] + if config.save_checkpoint: + config_ck = CheckpointConfig(save_checkpoint_steps=config.save_checkpoint_epochs * step_size, + keep_checkpoint_max=config.keep_checkpoint_max) + ckpt_cb = ModelCheckpoint(prefix="mobilenetV2", + directory=config.save_checkpoint_path, + config=config_ck) + callback += [ckpt_cb] + model.train(epoch_size, dataset, callbacks=callback) + print("============== End Training ==============") diff --git a/model_zoo/mobilenetv3/scripts/run_infer.sh b/model_zoo/mobilenetv3/scripts/run_infer.sh index e200e600bf..7385a221d4 100644 --- a/model_zoo/mobilenetv3/scripts/run_infer.sh +++ b/model_zoo/mobilenetv3/scripts/run_infer.sh @@ -40,7 +40,7 @@ export PYTHONPATH=${BASEPATH}:$PYTHONPATH export DEVICE_ID=0 export RANK_ID=0 export RANK_SIZE=1 -if [ -d "eval" ]; +if [ -d "../eval" ]; then rm -rf ../eval fi diff --git a/model_zoo/mobilenetv3/scripts/run_train.sh b/model_zoo/mobilenetv3/scripts/run_train.sh index 78b79b235f..7ac12c226d 100644 --- a/model_zoo/mobilenetv3/scripts/run_train.sh +++ b/model_zoo/mobilenetv3/scripts/run_train.sh @@ -29,7 +29,7 @@ run_ascend() BASEPATH=$(cd "`dirname $0`" || exit; pwd) export PYTHONPATH=${BASEPATH}:$PYTHONPATH - if [ -d "train" ]; + if [ -d "../train" ]; then rm -rf ../train fi @@ -60,7 +60,7 @@ run_gpu() BASEPATH=$(cd "`dirname $0`" || exit; pwd) export PYTHONPATH=${BASEPATH}:$PYTHONPATH - if [ -d "train" ]; + if [ -d "../train" ]; then rm -rf ../train fi diff --git a/model_zoo/mobilenetv3/src/dataset.py b/model_zoo/mobilenetv3/src/dataset.py index ae5070e359..d2fa12f793 100644 --- a/model_zoo/mobilenetv3/src/dataset.py +++ b/model_zoo/mobilenetv3/src/dataset.py @@ -62,7 +62,7 @@ def create_dataset(dataset_path, do_train, config, platform, repeat_num=1, batch resize_crop_op = C.RandomCropDecodeResize(resize_height, scale=(0.08, 1.0), ratio=(0.75, 1.333)) horizontal_flip_op = C.RandomHorizontalFlip(prob=0.5) - resize_op = C.Resize((256, 256)) + resize_op = C.Resize(256) center_crop = C.CenterCrop(resize_width) rescale_op = C.RandomColorAdjust(brightness=0.4, contrast=0.4, saturation=0.4) normalize_op = C.Normalize(mean=[0.485*255, 0.456*255, 0.406*255], std=[0.229*255, 0.224*255, 0.225*255]) diff --git a/model_zoo/resnet/README.md b/model_zoo/resnet/README.md new file mode 100644 index 0000000000..ad93453602 --- /dev/null +++ b/model_zoo/resnet/README.md @@ -0,0 +1,251 @@ +# ResNet Example + +## Description + +These are examples of training ResNet-50/ResNet-101 with CIFAR-10/ImageNet2012 dataset in MindSpore. +(Training ResNet-101 with dataset CIFAR-10 is unsupported now.) + +## Requirements + +- Install [MindSpore](https://www.mindspore.cn/install/en). + +- Download the dataset CIFAR-10 or ImageNet2012 + +CIFAR-10 + +> Unzip the CIFAR-10 dataset to any path you want and the folder structure should include train and eval dataset as follows: +> ``` +> . +> └─dataset +> ├─ cifar-10-batches-bin # train dataset +> └─ cifar-10-verify-bin # evaluate dataset +> ``` + +ImageNet2012 + +> Unzip the ImageNet2012 dataset to any path you want and the folder should include train and eval dataset as follows: +> +> ``` +> . +> └─dataset +> ├─ilsvrc # train dataset +> └─validation_preprocess # evaluate dataset +> ``` + + + +## Structure + +```shell +. +└──resnet + ├── README.md + ├── script + ├── run_distribute_train.sh # launch distributed training(8 pcs) + ├── run_eval.sh # launch evaluation + └── run_standalone_train.sh # launch standalone training(1 pcs) + ├── src + ├── config.py # parameter configuration + ├── dataset.py # data preprocessing + ├── crossentropy.py # loss definition for ImageNet2012 dataset + ├── lr_generator.py # generate learning rate for each step + └── resnet.py # resnet backbone, including resnet50 and resnet101 + ├── eval.py # eval net + └── train.py # train net +``` + + +## Parameter configuration + +Parameters for both training and evaluation can be set in config.py. + +- config for ResNet-50, CIFAR-10 dataset + +``` +"class_num": 10, # dataset class num +"batch_size": 32, # batch size of input tensor +"loss_scale": 1024, # loss scale +"momentum": 0.9, # momentum +"weight_decay": 1e-4, # weight decay +"epoch_size": 90, # only valid for taining, which is always 1 for inference +"save_checkpoint": True, # whether save checkpoint or not +"save_checkpoint_steps": 195, # the step interval between two checkpoints. By default, the last checkpoint will be saved after the last step +"keep_checkpoint_max": 10, # only keep the last keep_checkpoint_max checkpoint +"save_checkpoint_path": "./", # path to save checkpoint +"warmup_epochs": 5, # number of warmup epoch +"lr_decay_mode": "poly" # decay mode can be selected in steps, ploy and default +"lr_init": 0.01, # initial learning rate +"lr_end": 0.00001, # final learning rate +"lr_max": 0.1, # maximum learning rate +``` + +- config for ResNet-50, ImageNet2012 dataset + +``` +"class_num": 1001, # dataset class number +"batch_size": 32, # batch size of input tensor +"loss_scale": 1024, # loss scale +"momentum": 0.9, # momentum optimizer +"weight_decay": 1e-4, # weight decay +"epoch_size": 90, # only valid for taining, which is always 1 for inference +"pretrained_epoch_size": 1, # epoch size that model has been trained before load pretrained checkpoint +"save_checkpoint": True, # whether save checkpoint or not +"save_checkpoint_epochs": 1, # the epoch interval between two checkpoints. By default, the last checkpoint will be saved after the last epoch +"keep_checkpoint_max": 10, # only keep the last keep_checkpoint_max checkpoint +"save_checkpoint_path": "./", # path to save checkpoint relative to the executed path +"warmup_epochs": 0, # number of warmup epoch +"lr_decay_mode": "cosine", # decay mode for generating learning rate +"label_smooth": True, # label smooth +"label_smooth_factor": 0.1, # label smooth factor +"lr_init": 0, # initial learning rate +"lr_max": 0.1, # maximum learning rate +``` + +- config for ResNet-101, ImageNet2012 dataset + +``` +"class_num": 1001, # dataset class number +"batch_size": 32, # batch size of input tensor +"loss_scale": 1024, # loss scale +"momentum": 0.9, # momentum optimizer +"weight_decay": 1e-4, # weight decay +"epoch_size": 120, # epoch sizes for training +"pretrain_epoch_size": 0, # epoch size of pretrain checkpoint +"save_checkpoint": True, # whether save checkpoint or not +"save_checkpoint_epochs": 1, # the epoch interval between two checkpoints. By default, the last checkpoint will be saved after the last epoch +"keep_checkpoint_max": 10, # only keep the last keep_checkpoint_max checkpoint +"save_checkpoint_path": "./", # path to save checkpoint relative to the executed path +"warmup_epochs": 0, # number of warmup epoch +"lr_decay_mode": "cosine" # decay mode for generating learning rate +"label_smooth": 1, # label_smooth +"label_smooth_factor": 0.1, # label_smooth_factor +"lr": 0.1 # base learning rate +``` + + + +## Running the example + +### Train + +#### Usage + +``` +# distributed training +Usage: sh run_distribute_train.sh [resnet50|resnet101] [cifar10|imagenet2012] [MINDSPORE_HCCL_CONFIG_PATH] [DATASET_PATH] + [PRETRAINED_CKPT_PATH](optional) + +# standalone training +Usage: sh run_standalone_train.sh [resnet50|resnet101] [cifar10|imagenet2012] [DATASET_PATH] + [PRETRAINED_CKPT_PATH](optional) +``` + + +#### Launch + +``` +# distribute training example +sh run_distribute_train.sh resnet50 cifar10 rank_table.json ~/cifar-10-batches-bin + +# standalone training example +sh run_standalone_train.sh resnet50 cifar10 ~/cifar-10-batches-bin +``` + +> About rank_table.json, you can refer to the [distributed training tutorial](https://www.mindspore.cn/tutorial/en/master/advanced_use/distributed_training.html). + +#### Result + +Training result will be stored in the example path, whose folder name begins with "train" or "train_parallel". Under this, you can find checkpoint file together with result like the followings in log. + +- training ResNet-50 with CIFAR-10 dataset + +``` +# distribute training result(8 pcs) +epoch: 1 step: 195, loss is 1.9601055 +epoch: 2 step: 195, loss is 1.8555021 +epoch: 3 step: 195, loss is 1.6707983 +epoch: 4 step: 195, loss is 1.8162166 +epoch: 5 step: 195, loss is 1.393667 +... +``` + +- training ResNet-50 with ImageNet2012 dataset + +``` +# distribute training result(8 pcs) +epoch: 1 step: 5004, loss is 4.8995576 +epoch: 2 step: 5004, loss is 3.9235563 +epoch: 3 step: 5004, loss is 3.833077 +epoch: 4 step: 5004, loss is 3.2795618 +epoch: 5 step: 5004, loss is 3.1978393 +... +``` + +- training ResNet-101 with ImageNet2012 dataset + +``` +# distribute training result(8p) +epoch: 1 step: 5004, loss is 4.805483 +epoch: 2 step: 5004, loss is 3.2121816 +epoch: 3 step: 5004, loss is 3.429647 +epoch: 4 step: 5004, loss is 3.3667371 +epoch: 5 step: 5004, loss is 3.1718972 +... +epoch: 67 step: 5004, loss is 2.2768745 +epoch: 68 step: 5004, loss is 1.7223864 +epoch: 69 step: 5004, loss is 2.0665488 +epoch: 70 step: 5004, loss is 1.8717369 +... +``` + +### Evaluation + +#### Usage + +``` +# evaluation +Usage: sh run_eval.sh [resnet50|resnet101] [cifar10|imagenet2012] [DATASET_PATH] [CHECKPOINT_PATH] +``` + +#### Launch + +``` +# evaluation example +sh run_eval.sh resnet50 cifar10 ~/cifar10-10-verify-bin ~/resnet50_cifar10/train_parallel0/resnet-90_195.ckpt +``` + +> checkpoint can be produced in training process. + +#### Result + +Evaluation result will be stored in the example path, whose folder name is "eval". Under this, you can find result like the followings in log. + +- evaluating ResNet-50 with CIFAR-10 dataset + +``` +result: {'acc': 0.91446314102564111} ckpt=~/resnet50_cifar10/train_parallel0/resnet-90_195.ckpt +``` + +- evaluating ResNet-50 with ImageNet2012 dataset + +``` +result: {'acc': 0.7671054737516005} ckpt=train_parallel0/resnet-90_5004.ckpt +``` + +- evaluating ResNet-101 with ImageNet2012 dataset + +``` +result: {'top_5_accuracy': 0.9429417413572343, 'top_1_accuracy': 0.7853513124199744} ckpt=train_parallel0/resnet-120_5004.ckpt +``` + +### Running on GPU +``` +# distributed training example +mpirun -n 8 python train.py ---net=resnet50 --dataset=cifar10 -dataset_path=~/cifar-10-batches-bin --device_target="GPU" --run_distribute=True + +# standalone training example +python train.py --net=resnet50 --dataset=cifar10 --dataset_path=~/cifar-10-batches-bin --device_target="GPU" + +# infer example +python eval.py --net=resnet50 --dataset=cifar10 --dataset_path=~/cifar10-10-verify-bin --device_target="GPU" --checkpoint_path=resnet-90_195.ckpt +``` diff --git a/model_zoo/resnet/eval.py b/model_zoo/resnet/eval.py new file mode 100755 index 0000000000..426b8c9f3d --- /dev/null +++ b/model_zoo/resnet/eval.py @@ -0,0 +1,90 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""train resnet.""" +import os +import random +import argparse +import numpy as np +from mindspore import context +from mindspore import dataset as de +from mindspore.nn.loss import SoftmaxCrossEntropyWithLogits +from mindspore.train.model import Model +from mindspore.train.serialization import load_checkpoint, load_param_into_net +from src.crossentropy import CrossEntropy + +parser = argparse.ArgumentParser(description='Image classification') +parser.add_argument('--net', type=str, default=None, help='Resnet Model, either resnet50 or resnet101') +parser.add_argument('--dataset', type=str, default=None, help='Dataset, either cifar10 or imagenet2012') + +parser.add_argument('--checkpoint_path', type=str, default=None, help='Checkpoint file path') +parser.add_argument('--dataset_path', type=str, default=None, help='Dataset path') +parser.add_argument('--device_target', type=str, default='Ascend', help='Device target') +args_opt = parser.parse_args() + +random.seed(1) +np.random.seed(1) +de.config.set_seed(1) + +if args_opt.net == "resnet50": + from src.resnet import resnet50 as resnet + + if args_opt.dataset == "cifar10": + from src.config import config1 as config + from src.dataset import create_dataset1 as create_dataset + else: + from src.config import config2 as config + from src.dataset import create_dataset2 as create_dataset +else: + from src.resnet import resnet101 as resnet + from src.config import config3 as config + from src.dataset import create_dataset3 as create_dataset + +if __name__ == '__main__': + target = args_opt.device_target + + # init context + device_id = int(os.getenv('DEVICE_ID')) + context.set_context(mode=context.GRAPH_MODE, device_target=target, save_graphs=False, device_id=device_id) + + # create dataset + if args_opt.net == "resnet50": + dataset = create_dataset(dataset_path=args_opt.dataset_path, do_train=False, batch_size=config.batch_size, + target=target) + else: + dataset = create_dataset(dataset_path=args_opt.dataset_path, do_train=False, batch_size=config.batch_size) + step_size = dataset.get_dataset_size() + + # define net + net = resnet(class_num=config.class_num) + + # load checkpoint + param_dict = load_checkpoint(args_opt.checkpoint_path) + load_param_into_net(net, param_dict) + net.set_train(False) + + # define loss, model + if args_opt.dataset == "imagenet2012": + if not config.use_label_smooth: + config.label_smooth_factor = 0.0 + loss = CrossEntropy(smooth_factor=config.label_smooth_factor, num_classes=config.class_num) + else: + loss = SoftmaxCrossEntropyWithLogits(sparse=True, reduction='mean') + + # define model + model = Model(net, loss_fn=loss, metrics={'top_1_accuracy', 'top_5_accuracy'}) + + # eval model + res = model.eval(dataset) + print("result:", res, "ckpt=", args_opt.checkpoint_path) diff --git a/model_zoo/resnet101/scripts/run_distribute_train.sh b/model_zoo/resnet/scripts/run_distribute_train.sh similarity index 58% rename from model_zoo/resnet101/scripts/run_distribute_train.sh rename to model_zoo/resnet/scripts/run_distribute_train.sh index 65790b88c1..efcb620cd8 100755 --- a/model_zoo/resnet101/scripts/run_distribute_train.sh +++ b/model_zoo/resnet/scripts/run_distribute_train.sh @@ -14,12 +14,31 @@ # limitations under the License. # ============================================================================ -if [ $# != 2 ] && [ $# != 3 ] +if [ $# != 4 ] && [ $# != 5 ] then - echo "Usage: sh run_distribute_train.sh [MINDSPORE_HCCL_CONFIG_PATH] [DATASET_PATH] [PRETRAINED_PATH](optional)" + echo "Usage: sh run_distribute_train.sh [resnet50|resnet101] [cifar10|imagenet2012] [MINDSPORE_HCCL_CONFIG_PATH] [DATASET_PATH] [PRETRAINED_CKPT_PATH](optional)" exit 1 fi +if [ $1 != "resnet50" ] && [ $1 != "resnet101" ] +then + echo "error: the selected net is neither resnet50 nor resnet101" +exit 1 +fi + +if [ $2 != "cifar10" ] && [ $2 != "imagenet2012" ] +then + echo "error: the selected dataset is neither cifar10 nor imagenet2012" +exit 1 +fi + +if [ $1 == "resnet101" ] && [ $2 == "cifar10" ] +then + echo "error: training resnet101 with cifar10 dataset is unsupported now!" +exit 1 +fi + + get_real_path(){ if [ "${1:0:1}" == "/" ]; then echo "$1" @@ -27,14 +46,13 @@ get_real_path(){ echo "$(realpath -m $PWD/$1)" fi } -PATH1=$(get_real_path $1) -PATH2=$(get_real_path $2) -echo $PATH1 -echo $PATH2 -if [ $# == 3 ] + +PATH1=$(get_real_path $3) +PATH2=$(get_real_path $4) + +if [ $# == 5 ] then - PATH3=$(get_real_path $3) - echo $PATH3 + PATH3=$(get_real_path $5) fi if [ ! -f $PATH1 ] @@ -49,9 +67,9 @@ then exit 1 fi -if [ $# == 3 ] && [ ! -f $PATH3 ] +if [ $# == 5 ] && [ ! -f $PATH3 ] then - echo "error: PRETRAINED_PATH=$PATH3 is not a file" + echo "error: PRETRAINED_CKPT_PATH=$PATH3 is not a file" exit 1 fi @@ -73,14 +91,14 @@ do cd ./train_parallel$i || exit echo "start training for rank $RANK_ID, device $DEVICE_ID" env > env.log - if [ $# == 2 ] + if [ $# == 4 ] then - python train.py --do_train=True --run_distribute=True --device_num=$DEVICE_NUM --dataset_path=$PATH2 &> log & + python train.py --net=$1 --dataset=$2 --run_distribute=True --device_num=$DEVICE_NUM --dataset_path=$PATH2 &> log & fi - if [ $# == 3 ] + if [ $# == 5 ] then - python train.py --do_train=True --run_distribute=True --device_num=$DEVICE_NUM --dataset_path=$PATH2 --pre_trained=$PATH3 &> log & + python train.py --net=$1 --dataset=$2 --run_distribute=True --device_num=$DEVICE_NUM --dataset_path=$PATH2 --pre_trained=$PATH3 &> log & fi cd .. diff --git a/model_zoo/resnet/scripts/run_eval.sh b/model_zoo/resnet/scripts/run_eval.sh new file mode 100755 index 0000000000..496b3c1e2b --- /dev/null +++ b/model_zoo/resnet/scripts/run_eval.sh @@ -0,0 +1,84 @@ +#!/bin/bash +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +if [ $# != 4 ] +then + echo "Usage: sh run_eval.sh [resnet50|resnet101] [cifar10|imagenet2012] [DATASET_PATH] [CHECKPOINT_PATH]" +exit 1 +fi + +if [ $1 != "resnet50" ] && [ $1 != "resnet101" ] +then + echo "error: the selected net is neither resnet50 nor resnet101" +exit 1 +fi + +if [ $2 != "cifar10" ] && [ $2 != "imagenet2012" ] +then + echo "error: the selected dataset is neither cifar10 nor imagenet2012" +exit 1 +fi + +if [ $1 == "resnet101" ] && [ $2 == "cifar10" ] +then + echo "error: evaluating resnet101 with cifar10 dataset is unsupported now!" +exit 1 +fi + + +get_real_path(){ + if [ "${1:0:1}" == "/" ]; then + echo "$1" + else + echo "$(realpath -m $PWD/$1)" + fi +} + +PATH1=$(get_real_path $3) +PATH2=$(get_real_path $4) + + +if [ ! -d $PATH1 ] +then + echo "error: DATASET_PATH=$PATH1 is not a directory" +exit 1 +fi + +if [ ! -f $PATH2 ] +then + echo "error: CHECKPOINT_PATH=$PATH2 is not a file" +exit 1 +fi + +ulimit -u unlimited +export DEVICE_NUM=1 +export DEVICE_ID=0 +export RANK_SIZE=$DEVICE_NUM +export RANK_ID=0 + +if [ -d "eval" ]; +then + rm -rf ./eval +fi +mkdir ./eval +cp ../*.py ./eval +cp *.sh ./eval +cp -r ../src ./eval +cd ./eval || exit +env > env.log +echo "start evaluation for device $DEVICE_ID" +python eval.py --net=$1 --dataset=$2 --dataset_path=$PATH1 --checkpoint_path=$PATH2 &> log & +cd .. diff --git a/example/resnet50_imagenet2012/run_standalone_train.sh b/model_zoo/resnet/scripts/run_standalone_train.sh similarity index 55% rename from example/resnet50_imagenet2012/run_standalone_train.sh rename to model_zoo/resnet/scripts/run_standalone_train.sh index e0eb5efaf0..2272dbd88b 100755 --- a/example/resnet50_imagenet2012/run_standalone_train.sh +++ b/model_zoo/resnet/scripts/run_standalone_train.sh @@ -14,12 +14,31 @@ # limitations under the License. # ============================================================================ -if [ $# != 1 ] && [ $# != 2 ] +if [ $# != 3 ] && [ $# != 4 ] then - echo "Usage: sh run_standalone_train.sh [DATASET_PATH] [PRETRAINED_CKPT_PATH](optional)" + echo "Usage: sh run_standalone_train.sh [resnet50|resnet101] [cifar10|imagenet2012] [DATASET_PATH] [PRETRAINED_CKPT_PATH](optional)" exit 1 fi +if [ $1 != "resnet50" ] && [ $1 != "resnet101" ] +then + echo "error: the selected net is neither resnet50 nor resnet101" +exit 1 +fi + +if [ $2 != "cifar10" ] && [ $2 != "imagenet2012" ] +then + echo "error: the selected dataset is neither cifar10 nor imagenet2012" +exit 1 +fi + +if [ $1 == "resnet101" ] && [ $2 == "cifar10" ] +then + echo "error: training resnet101 with cifar10 dataset is unsupported now!" +exit 1 +fi + + get_real_path(){ if [ "${1:0:1}" == "/" ]; then echo "$1" @@ -28,19 +47,20 @@ get_real_path(){ fi } -PATH1=$(get_real_path $1) -if [ $# == 2 ] +PATH1=$(get_real_path $3) + +if [ $# == 4 ] then - PATH2=$(get_real_path $2) + PATH2=$(get_real_path $4) fi -if [ ! -d "$PATH1" ] +if [ ! -d $PATH1 ] then echo "error: DATASET_PATH=$PATH1 is not a directory" exit 1 fi -if [ $# == 2 ] && [ ! -f "$PATH2" ] +if [ $# == 4 ] && [ ! -f $PATH2 ] then echo "error: PRETRAINED_CKPT_PATH=$PATH2 is not a file" exit 1 @@ -50,21 +70,26 @@ ulimit -u unlimited export DEVICE_NUM=1 export DEVICE_ID=0 export RANK_ID=0 +export RANK_SIZE=1 if [ -d "train" ]; then rm -rf ./train fi mkdir ./train -cp *.py ./train +cp ../*.py ./train cp *.sh ./train +cp -r ../src ./train cd ./train || exit echo "start training for device $DEVICE_ID" env > env.log -if [ $# == 1 ] +if [ $# == 3 ] +then + python train.py --net=$1 --dataset=$2 --dataset_path=$PATH1 &> log & +fi + +if [ $# == 4 ] then - python train.py --do_train=True --dataset_path=$PATH1 &> log & -else - python train.py --do_train=True --dataset_path=$PATH1 --pre_trained=$PATH2 &> log & + python train.py --net=$1 --dataset=$2 --dataset_path=$PATH1 --pre_trained=$PATH2 &> log & fi cd .. diff --git a/example/resnet50_cifar10/config.py b/model_zoo/resnet/src/config.py similarity index 52% rename from example/resnet50_cifar10/config.py rename to model_zoo/resnet/src/config.py index 3c50a6aaed..7b1759fde0 100755 --- a/example/resnet50_cifar10/config.py +++ b/model_zoo/resnet/src/config.py @@ -17,16 +17,14 @@ network config setting, will be used in train.py and eval.py """ from easydict import EasyDict as ed -config = ed({ +# config for resent50, cifar10 +config1 = ed({ "class_num": 10, "batch_size": 32, "loss_scale": 1024, "momentum": 0.9, "weight_decay": 1e-4, "epoch_size": 90, - "buffer_size": 100, - "image_height": 224, - "image_width": 224, "save_checkpoint": True, "save_checkpoint_epochs": 5, "keep_checkpoint_max": 10, @@ -37,3 +35,45 @@ config = ed({ "lr_end": 0.00001, "lr_max": 0.1 }) + +# config for resnet50, imagenet2012 +config2 = ed({ + "class_num": 1001, + "batch_size": 32, + "loss_scale": 1024, + "momentum": 0.9, + "weight_decay": 1e-4, + "epoch_size": 90, + "pretrain_epoch_size": 1, + "save_checkpoint": True, + "save_checkpoint_epochs": 5, + "keep_checkpoint_max": 10, + "save_checkpoint_path": "./", + "warmup_epochs": 0, + "lr_decay_mode": "cosine", + "use_label_smooth": True, + "label_smooth_factor": 0.1, + "lr_init": 0, + "lr_max": 0.1 + +}) + +# config for resent101, imagenet2012 +config3 = ed({ + "class_num": 1001, + "batch_size": 32, + "loss_scale": 1024, + "momentum": 0.9, + "weight_decay": 1e-4, + "epoch_size": 120, + "pretrain_epoch_size": 0, + "save_checkpoint": True, + "save_checkpoint_epochs": 5, + "keep_checkpoint_max": 10, + "save_checkpoint_path": "./", + "warmup_epochs": 0, + "lr_decay_mode": "cosine", + "use_label_smooth": True, + "label_smooth_factor": 0.1, + "lr": 0.1 +}) diff --git a/model_zoo/resnet101/src/crossentropy.py b/model_zoo/resnet/src/crossentropy.py similarity index 98% rename from model_zoo/resnet101/src/crossentropy.py rename to model_zoo/resnet/src/crossentropy.py index 1145a41804..5118cb5161 100755 --- a/model_zoo/resnet101/src/crossentropy.py +++ b/model_zoo/resnet/src/crossentropy.py @@ -20,15 +20,18 @@ from mindspore import Tensor from mindspore.common import dtype as mstype import mindspore.nn as nn + class CrossEntropy(_Loss): """the redefined loss function with SoftmaxCrossEntropyWithLogits""" + def __init__(self, smooth_factor=0., num_classes=1001): super(CrossEntropy, self).__init__() self.onehot = P.OneHot() self.on_value = Tensor(1.0 - smooth_factor, mstype.float32) - self.off_value = Tensor(1.0 * smooth_factor / (num_classes -1), mstype.float32) + self.off_value = Tensor(1.0 * smooth_factor / (num_classes - 1), mstype.float32) self.ce = nn.SoftmaxCrossEntropyWithLogits() self.mean = P.ReduceMean(False) + def construct(self, logit, label): one_hot_label = self.onehot(label, F.shape(logit)[1], self.on_value, self.off_value) loss = self.ce(logit, one_hot_label) diff --git a/model_zoo/resnet/src/dataset.py b/model_zoo/resnet/src/dataset.py new file mode 100755 index 0000000000..ac0adc4bc9 --- /dev/null +++ b/model_zoo/resnet/src/dataset.py @@ -0,0 +1,205 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +create train or eval dataset. +""" +import os +import mindspore.common.dtype as mstype +import mindspore.dataset.engine as de +import mindspore.dataset.transforms.vision.c_transforms as C +import mindspore.dataset.transforms.c_transforms as C2 +from mindspore.communication.management import init, get_rank, get_group_size + + +def create_dataset1(dataset_path, do_train, repeat_num=1, batch_size=32, target="Ascend"): + """ + create a train or evaluate cifar10 dataset for resnet50 + Args: + dataset_path(string): the path of dataset. + do_train(bool): whether dataset is used for train or eval. + repeat_num(int): the repeat times of dataset. Default: 1 + batch_size(int): the batch size of dataset. Default: 32 + target(str): the device target. Default: Ascend + + Returns: + dataset + """ + if target == "Ascend": + device_num = int(os.getenv("DEVICE_NUM")) + rank_id = int(os.getenv("RANK_ID")) + else: + init("nccl") + rank_id = get_rank() + device_num = get_group_size() + + if device_num == 1: + ds = de.Cifar10Dataset(dataset_path, num_parallel_workers=8, shuffle=True) + else: + ds = de.Cifar10Dataset(dataset_path, num_parallel_workers=8, shuffle=True, + num_shards=device_num, shard_id=rank_id) + + # define map operations + trans = [] + if do_train: + trans += [ + C.RandomCrop((32, 32), (4, 4, 4, 4)), + C.RandomHorizontalFlip(prob=0.5) + ] + + trans += [ + C.Resize((224, 224)), + C.Rescale(1.0 / 255.0, 0.0), + C.Normalize([0.4914, 0.4822, 0.4465], [0.2023, 0.1994, 0.2010]), + C.HWC2CHW() + ] + + type_cast_op = C2.TypeCast(mstype.int32) + + ds = ds.map(input_columns="label", num_parallel_workers=8, operations=type_cast_op) + ds = ds.map(input_columns="image", num_parallel_workers=8, operations=trans) + + # apply batch operations + ds = ds.batch(batch_size, drop_remainder=True) + # apply dataset repeat operation + ds = ds.repeat(repeat_num) + + return ds + + +def create_dataset2(dataset_path, do_train, repeat_num=1, batch_size=32, target="Ascend"): + """ + create a train or eval imagenet2012 dataset for resnet50 + + Args: + dataset_path(string): the path of dataset. + do_train(bool): whether dataset is used for train or eval. + repeat_num(int): the repeat times of dataset. Default: 1 + batch_size(int): the batch size of dataset. Default: 32 + target(str): the device target. Default: Ascend + + Returns: + dataset + """ + if target == "Ascend": + device_num = int(os.getenv("DEVICE_NUM")) + rank_id = int(os.getenv("RANK_ID")) + else: + init("nccl") + rank_id = get_rank() + device_num = get_group_size() + + if device_num == 1: + ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True) + else: + ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True, + num_shards=device_num, shard_id=rank_id) + + image_size = 224 + mean = [0.485 * 255, 0.456 * 255, 0.406 * 255] + std = [0.229 * 255, 0.224 * 255, 0.225 * 255] + + # define map operations + if do_train: + trans = [ + C.RandomCropDecodeResize(image_size, scale=(0.08, 1.0), ratio=(0.75, 1.333)), + C.RandomHorizontalFlip(prob=0.5), + C.Normalize(mean=mean, std=std), + C.HWC2CHW() + ] + else: + trans = [ + C.Decode(), + C.Resize((256, 256)), + C.CenterCrop(image_size), + C.Normalize(mean=mean, std=std), + C.HWC2CHW() + ] + + type_cast_op = C2.TypeCast(mstype.int32) + + ds = ds.map(input_columns="image", num_parallel_workers=8, operations=trans) + ds = ds.map(input_columns="label", num_parallel_workers=8, operations=type_cast_op) + + # apply batch operations + ds = ds.batch(batch_size, drop_remainder=True) + + # apply dataset repeat operation + ds = ds.repeat(repeat_num) + + return ds + + +def create_dataset3(dataset_path, do_train, repeat_num=1, batch_size=32): + """ + create a train or eval imagenet2012 dataset for resnet101 + Args: + dataset_path(string): the path of dataset. + do_train(bool): whether dataset is used for train or eval. + repeat_num(int): the repeat times of dataset. Default: 1 + batch_size(int): the batch size of dataset. Default: 32 + + Returns: + dataset + """ + device_num = int(os.getenv("RANK_SIZE")) + rank_id = int(os.getenv("RANK_ID")) + + if device_num == 1: + ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True) + else: + ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True, + num_shards=device_num, shard_id=rank_id) + resize_height = 224 + rescale = 1.0 / 255.0 + shift = 0.0 + + # define map operations + decode_op = C.Decode() + + random_resize_crop_op = C.RandomResizedCrop(resize_height, (0.08, 1.0), (0.75, 1.33), max_attempts=100) + horizontal_flip_op = C.RandomHorizontalFlip(rank_id / (rank_id + 1)) + resize_op_256 = C.Resize((256, 256)) + center_crop = C.CenterCrop(224) + rescale_op = C.Rescale(rescale, shift) + normalize_op = C.Normalize((0.475, 0.451, 0.392), (0.275, 0.267, 0.278)) + changeswap_op = C.HWC2CHW() + + if do_train: + trans = [decode_op, + random_resize_crop_op, + horizontal_flip_op, + rescale_op, + normalize_op, + changeswap_op] + + else: + trans = [decode_op, + resize_op_256, + center_crop, + rescale_op, + normalize_op, + changeswap_op] + + type_cast_op = C2.TypeCast(mstype.int32) + + ds = ds.map(input_columns="image", operations=trans, num_parallel_workers=8) + ds = ds.map(input_columns="label", operations=type_cast_op, num_parallel_workers=8) + + # apply batch operations + ds = ds.batch(batch_size, drop_remainder=True) + # apply dataset repeat operation + ds = ds.repeat(repeat_num) + + return ds diff --git a/example/resnet50_cifar10/lr_generator.py b/model_zoo/resnet/src/lr_generator.py similarity index 63% rename from example/resnet50_cifar10/lr_generator.py rename to model_zoo/resnet/src/lr_generator.py index 37c8e907d7..2af8971715 100755 --- a/example/resnet50_cifar10/lr_generator.py +++ b/model_zoo/resnet/src/lr_generator.py @@ -13,15 +13,15 @@ # limitations under the License. # ============================================================================ """learning rate generator""" +import math import numpy as np -def get_lr(global_step, lr_init, lr_end, lr_max, warmup_epochs, total_epochs, steps_per_epoch, lr_decay_mode): +def get_lr(lr_init, lr_end, lr_max, warmup_epochs, total_epochs, steps_per_epoch, lr_decay_mode): """ generate learning rate array Args: - global_step(int): total steps of the training lr_init(float): init learning rate lr_end(float): end learning rate lr_max(float): max learning rate @@ -70,8 +70,47 @@ def get_lr(global_step, lr_init, lr_end, lr_max, warmup_epochs, total_epochs, st lr = lr_max - (lr_max - lr_end) * (i - warmup_steps) / (total_steps - warmup_steps) lr_each_step.append(lr) - current_step = global_step lr_each_step = np.array(lr_each_step).astype(np.float32) - learning_rate = lr_each_step[current_step:] + return lr_each_step + + +def linear_warmup_lr(current_step, warmup_steps, base_lr, init_lr): + lr_inc = (float(base_lr) - float(init_lr)) / float(warmup_steps) + lr = float(init_lr) + lr_inc * current_step + return lr + + +def warmup_cosine_annealing_lr(lr, steps_per_epoch, warmup_epochs, max_epoch=120, global_step=0): + """ + generate learning rate array with cosine + + Args: + lr(float): base learning rate + steps_per_epoch(int): steps size of one epoch + warmup_epochs(int): number of warmup epochs + max_epoch(int): total epochs of training + global_step(int): the current start index of lr array + Returns: + np.array, learning rate array + """ + base_lr = lr + warmup_init_lr = 0 + total_steps = int(max_epoch * steps_per_epoch) + warmup_steps = int(warmup_epochs * steps_per_epoch) + decay_steps = total_steps - warmup_steps + + lr_each_step = [] + for i in range(total_steps): + if i < warmup_steps: + lr = linear_warmup_lr(i + 1, warmup_steps, base_lr, warmup_init_lr) + else: + linear_decay = (total_steps - i) / decay_steps + cosine_decay = 0.5 * (1 + math.cos(math.pi * 2 * 0.47 * i / decay_steps)) + decayed = linear_decay * cosine_decay + 0.00001 + lr = base_lr * decayed + lr_each_step.append(lr) + + lr_each_step = np.array(lr_each_step).astype(np.float32) + learning_rate = lr_each_step[global_step:] return learning_rate diff --git a/model_zoo/resnet/src/resnet.py b/model_zoo/resnet/src/resnet.py new file mode 100755 index 0000000000..0e21222d21 --- /dev/null +++ b/model_zoo/resnet/src/resnet.py @@ -0,0 +1,283 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""ResNet.""" +import numpy as np +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore.common.tensor import Tensor + + +def _weight_variable(shape, factor=0.01): + init_value = np.random.randn(*shape).astype(np.float32) * factor + return Tensor(init_value) + + +def _conv3x3(in_channel, out_channel, stride=1): + weight_shape = (out_channel, in_channel, 3, 3) + weight = _weight_variable(weight_shape) + return nn.Conv2d(in_channel, out_channel, + kernel_size=3, stride=stride, padding=0, pad_mode='same', weight_init=weight) + + +def _conv1x1(in_channel, out_channel, stride=1): + weight_shape = (out_channel, in_channel, 1, 1) + weight = _weight_variable(weight_shape) + return nn.Conv2d(in_channel, out_channel, + kernel_size=1, stride=stride, padding=0, pad_mode='same', weight_init=weight) + + +def _conv7x7(in_channel, out_channel, stride=1): + weight_shape = (out_channel, in_channel, 7, 7) + weight = _weight_variable(weight_shape) + return nn.Conv2d(in_channel, out_channel, + kernel_size=7, stride=stride, padding=0, pad_mode='same', weight_init=weight) + + +def _bn(channel): + return nn.BatchNorm2d(channel, eps=1e-4, momentum=0.9, + gamma_init=1, beta_init=0, moving_mean_init=0, moving_var_init=1) + + +def _bn_last(channel): + return nn.BatchNorm2d(channel, eps=1e-4, momentum=0.9, + gamma_init=0, beta_init=0, moving_mean_init=0, moving_var_init=1) + + +def _fc(in_channel, out_channel): + weight_shape = (out_channel, in_channel) + weight = _weight_variable(weight_shape) + return nn.Dense(in_channel, out_channel, has_bias=True, weight_init=weight, bias_init=0) + + +class ResidualBlock(nn.Cell): + """ + ResNet V1 residual block definition. + + Args: + in_channel (int): Input channel. + out_channel (int): Output channel. + stride (int): Stride size for the first convolutional layer. Default: 1. + + Returns: + Tensor, output tensor. + + Examples: + >>> ResidualBlock(3, 256, stride=2) + """ + expansion = 4 + + def __init__(self, + in_channel, + out_channel, + stride=1): + super(ResidualBlock, self).__init__() + + channel = out_channel // self.expansion + self.conv1 = _conv1x1(in_channel, channel, stride=1) + self.bn1 = _bn(channel) + + self.conv2 = _conv3x3(channel, channel, stride=stride) + self.bn2 = _bn(channel) + + self.conv3 = _conv1x1(channel, out_channel, stride=1) + self.bn3 = _bn_last(out_channel) + + self.relu = nn.ReLU() + + self.down_sample = False + + if stride != 1 or in_channel != out_channel: + self.down_sample = True + self.down_sample_layer = None + + if self.down_sample: + self.down_sample_layer = nn.SequentialCell([_conv1x1(in_channel, out_channel, stride), + _bn(out_channel)]) + self.add = P.TensorAdd() + + def construct(self, x): + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + if self.down_sample: + identity = self.down_sample_layer(identity) + + out = self.add(out, identity) + out = self.relu(out) + + return out + + +class ResNet(nn.Cell): + """ + ResNet architecture. + + Args: + block (Cell): Block for network. + layer_nums (list): Numbers of block in different layers. + in_channels (list): Input channel in each layer. + out_channels (list): Output channel in each layer. + strides (list): Stride size in each layer. + num_classes (int): The number of classes that the training images are belonging to. + Returns: + Tensor, output tensor. + + Examples: + >>> ResNet(ResidualBlock, + >>> [3, 4, 6, 3], + >>> [64, 256, 512, 1024], + >>> [256, 512, 1024, 2048], + >>> [1, 2, 2, 2], + >>> 10) + """ + + def __init__(self, + block, + layer_nums, + in_channels, + out_channels, + strides, + num_classes): + super(ResNet, self).__init__() + + if not len(layer_nums) == len(in_channels) == len(out_channels) == 4: + raise ValueError("the length of layer_num, in_channels, out_channels list must be 4!") + + self.conv1 = _conv7x7(3, 64, stride=2) + self.bn1 = _bn(64) + self.relu = P.ReLU() + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, pad_mode="same") + + self.layer1 = self._make_layer(block, + layer_nums[0], + in_channel=in_channels[0], + out_channel=out_channels[0], + stride=strides[0]) + self.layer2 = self._make_layer(block, + layer_nums[1], + in_channel=in_channels[1], + out_channel=out_channels[1], + stride=strides[1]) + self.layer3 = self._make_layer(block, + layer_nums[2], + in_channel=in_channels[2], + out_channel=out_channels[2], + stride=strides[2]) + self.layer4 = self._make_layer(block, + layer_nums[3], + in_channel=in_channels[3], + out_channel=out_channels[3], + stride=strides[3]) + + self.mean = P.ReduceMean(keep_dims=True) + self.flatten = nn.Flatten() + self.end_point = _fc(out_channels[3], num_classes) + + def _make_layer(self, block, layer_num, in_channel, out_channel, stride): + """ + Make stage network of ResNet. + + Args: + block (Cell): Resnet block. + layer_num (int): Layer number. + in_channel (int): Input channel. + out_channel (int): Output channel. + stride (int): Stride size for the first convolutional layer. + + Returns: + SequentialCell, the output layer. + + Examples: + >>> _make_layer(ResidualBlock, 3, 128, 256, 2) + """ + layers = [] + + resnet_block = block(in_channel, out_channel, stride=stride) + layers.append(resnet_block) + + for _ in range(1, layer_num): + resnet_block = block(out_channel, out_channel, stride=1) + layers.append(resnet_block) + + return nn.SequentialCell(layers) + + def construct(self, x): + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + c1 = self.maxpool(x) + + c2 = self.layer1(c1) + c3 = self.layer2(c2) + c4 = self.layer3(c3) + c5 = self.layer4(c4) + + out = self.mean(c5, (2, 3)) + out = self.flatten(out) + out = self.end_point(out) + + return out + + +def resnet50(class_num=10): + """ + Get ResNet50 neural network. + + Args: + class_num (int): Class number. + + Returns: + Cell, cell instance of ResNet50 neural network. + + Examples: + >>> net = resnet50(10) + """ + return ResNet(ResidualBlock, + [3, 4, 6, 3], + [64, 256, 512, 1024], + [256, 512, 1024, 2048], + [1, 2, 2, 2], + class_num) + + +def resnet101(class_num=1001): + """ + Get ResNet101 neural network. + + Args: + class_num (int): Class number. + + Returns: + Cell, cell instance of ResNet101 neural network. + + Examples: + >>> net = resnet101(1001) + """ + return ResNet(ResidualBlock, + [3, 4, 23, 3], + [64, 256, 512, 1024], + [256, 512, 1024, 2048], + [1, 2, 2, 2], + class_num) diff --git a/model_zoo/resnet/train.py b/model_zoo/resnet/train.py new file mode 100755 index 0000000000..89ce62d733 --- /dev/null +++ b/model_zoo/resnet/train.py @@ -0,0 +1,162 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""train resnet.""" +import os +import random +import argparse +import numpy as np +from mindspore import context +from mindspore import Tensor +from mindspore import dataset as de +from mindspore.parallel._auto_parallel_context import auto_parallel_context +from mindspore.nn.optim.momentum import Momentum +from mindspore.train.model import Model, ParallelMode +from mindspore.train.callback import ModelCheckpoint, CheckpointConfig, LossMonitor, TimeMonitor +from mindspore.nn.loss import SoftmaxCrossEntropyWithLogits +from mindspore.train.loss_scale_manager import FixedLossScaleManager +from mindspore.train.serialization import load_checkpoint, load_param_into_net +from mindspore.communication.management import init, get_rank, get_group_size +import mindspore.nn as nn +import mindspore.common.initializer as weight_init +from src.lr_generator import get_lr, warmup_cosine_annealing_lr +from src.crossentropy import CrossEntropy + +parser = argparse.ArgumentParser(description='Image classification') +parser.add_argument('--net', type=str, default=None, help='Resnet Model, either resnet50 or resnet101') +parser.add_argument('--dataset', type=str, default=None, help='Dataset, either cifar10 or imagenet2012') +parser.add_argument('--run_distribute', type=bool, default=False, help='Run distribute') +parser.add_argument('--device_num', type=int, default=1, help='Device num.') + +parser.add_argument('--dataset_path', type=str, default=None, help='Dataset path') +parser.add_argument('--device_target', type=str, default='Ascend', help='Device target') +parser.add_argument('--pre_trained', type=str, default=None, help='Pretrained checkpoint path') +args_opt = parser.parse_args() + +random.seed(1) +np.random.seed(1) +de.config.set_seed(1) + +if args_opt.net == "resnet50": + from src.resnet import resnet50 as resnet + + if args_opt.dataset == "cifar10": + from src.config import config1 as config + from src.dataset import create_dataset1 as create_dataset + else: + from src.config import config2 as config + from src.dataset import create_dataset2 as create_dataset +else: + from src.resnet import resnet101 as resnet + from src.config import config3 as config + from src.dataset import create_dataset3 as create_dataset + +if __name__ == '__main__': + target = args_opt.device_target + ckpt_save_dir = config.save_checkpoint_path + + # init context + context.set_context(mode=context.GRAPH_MODE, device_target=target, save_graphs=False) + if args_opt.run_distribute: + if target == "Ascend": + device_id = int(os.getenv('DEVICE_ID')) + context.set_context(device_id=device_id, enable_auto_mixed_precision=True) + context.set_auto_parallel_context(device_num=args_opt.device_num, parallel_mode=ParallelMode.DATA_PARALLEL, + mirror_mean=True) + if args_opt.net == "resnet50": + auto_parallel_context().set_all_reduce_fusion_split_indices([107, 160]) + else: + auto_parallel_context().set_all_reduce_fusion_split_indices([180, 313]) + init() + # GPU target + else: + init("nccl") + context.set_auto_parallel_context(device_num=get_group_size(), parallel_mode=ParallelMode.DATA_PARALLEL, + mirror_mean=True) + ckpt_save_dir = config.save_checkpoint_path + "ckpt_" + str(get_rank()) + "/" + + # create dataset + if args_opt.net == "resnet50": + dataset = create_dataset(dataset_path=args_opt.dataset_path, do_train=True, repeat_num=config.epoch_size, + batch_size=config.batch_size, target=target) + else: + dataset = create_dataset(dataset_path=args_opt.dataset_path, do_train=True, repeat_num=config.epoch_size, + batch_size=config.batch_size) + step_size = dataset.get_dataset_size() + + # define net + net = resnet(class_num=config.class_num) + + # init weight + if args_opt.pre_trained: + param_dict = load_checkpoint(args_opt.pre_trained) + load_param_into_net(net, param_dict) + else: + for _, cell in net.cells_and_names(): + if isinstance(cell, nn.Conv2d): + cell.weight.default_input = weight_init.initializer(weight_init.XavierUniform(), + cell.weight.default_input.shape, + cell.weight.default_input.dtype).to_tensor() + if isinstance(cell, nn.Dense): + cell.weight.default_input = weight_init.initializer(weight_init.TruncatedNormal(), + cell.weight.default_input.shape, + cell.weight.default_input.dtype).to_tensor() + + # init lr + if args_opt.net == "resnet50": + if args_opt.dataset == "cifar10": + lr = get_lr(lr_init=config.lr_init, lr_end=config.lr_end, lr_max=config.lr_max, + warmup_epochs=config.warmup_epochs, total_epochs=config.epoch_size, steps_per_epoch=step_size, + lr_decay_mode='poly') + else: + lr = get_lr(lr_init=config.lr_init, lr_end=0.0, lr_max=config.lr_max, warmup_epochs=config.warmup_epochs, + total_epochs=config.epoch_size, steps_per_epoch=step_size, lr_decay_mode='cosine') + else: + lr = warmup_cosine_annealing_lr(config.lr, step_size, config.warmup_epochs, 120, + config.pretrain_epoch_size * step_size) + lr = Tensor(lr) + + # define opt + opt = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), lr, config.momentum, + config.weight_decay, config.loss_scale) + + # define loss, model + if target == "Ascend": + if args_opt.dataset == "imagenet2012": + if not config.use_label_smooth: + config.label_smooth_factor = 0.0 + loss = CrossEntropy(smooth_factor=config.label_smooth_factor, num_classes=config.class_num) + else: + loss = SoftmaxCrossEntropyWithLogits(sparse=True, reduction='mean') + loss_scale = FixedLossScaleManager(config.loss_scale, drop_overflow_update=False) + model = Model(net, loss_fn=loss, optimizer=opt, loss_scale_manager=loss_scale, metrics={'acc'}, + amp_level="O2", keep_batchnorm_fp32=False) + else: + # GPU target + loss = SoftmaxCrossEntropyWithLogits(sparse=True, is_grad=False, reduction='mean') + opt = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), lr, config.momentum) + model = Model(net, loss_fn=loss, optimizer=opt, metrics={'acc'}) + + # define callbacks + time_cb = TimeMonitor(data_size=step_size) + loss_cb = LossMonitor() + cb = [time_cb, loss_cb] + if config.save_checkpoint: + config_ck = CheckpointConfig(save_checkpoint_steps=config.save_checkpoint_epochs * step_size, + keep_checkpoint_max=config.keep_checkpoint_max) + ckpt_cb = ModelCheckpoint(prefix="resnet", directory=ckpt_save_dir, config=config_ck) + cb += [ckpt_cb] + + # train model + model.train(config.epoch_size, dataset, callbacks=cb) diff --git a/model_zoo/resnet101/README.md b/model_zoo/resnet101/README.md deleted file mode 100644 index 86744be372..0000000000 --- a/model_zoo/resnet101/README.md +++ /dev/null @@ -1,147 +0,0 @@ -# ResNet101 Example - -## Description - -This is an example of training ResNet101 with ImageNet dataset in MindSpore. - -## Requirements - -- Install [MindSpore](https://www.mindspore.cn/install/en). - -- Download the dataset ImageNet2012. - -> Unzip the ImageNet2012 dataset to any path you want, the folder should include train and eval dataset as follows: - -``` -. -└─dataset - ├─ilsvrc - │ - └─validation_preprocess -``` - -## Structure - -```shell -. -└─resnet101 - ├─README.md - ├─scripts - ├─run_standalone_train.sh # launch standalone training(1p) - ├─run_distribute_train.sh # launch distributed training(8p) - └─run_eval.sh # launch evaluating - ├─src - ├─config.py # parameter configuration - ├─crossentropy.py # CrossEntropy loss function - ├─dataset.py # data preprocessin - ├─lr_generator.py # generate learning rate - ├─resnet101.py # resnet101 backbone - ├─eval.py # eval net - └─train.py # train net -``` - -## Parameter configuration - -Parameters for both training and evaluating can be set in config.py. - -``` -"class_num": 1001, # dataset class number -"batch_size": 32, # batch size of input tensor -"loss_scale": 1024, # loss scale -"momentum": 0.9, # momentum optimizer -"weight_decay": 1e-4, # weight decay -"epoch_size": 120, # epoch sizes for training -"pretrain_epoch_size": 0, # epoch size of pretrain checkpoint -"buffer_size": 1000, # number of queue size in data preprocessing -"image_height": 224, # image height -"image_width": 224, # image width -"save_checkpoint": True, # whether save checkpoint or not -"save_checkpoint_epochs": 1, # the epoch interval between two checkpoints. By default, the last checkpoint will be saved after the last epoch -"keep_checkpoint_max": 10, # only keep the last keep_checkpoint_max checkpoint -"save_checkpoint_path": "./", # path to save checkpoint relative to the executed path -"warmup_epochs": 0, # number of warmup epoch -"lr_decay_mode": "cosine" # decay mode for generating learning rate -"label_smooth": 1, # label_smooth -"label_smooth_factor": 0.1, # label_smooth_factor -"lr": 0.1 # base learning rate -``` - -## Running the example - -### Train - -#### Usage - -``` -# distributed training -sh run_distribute_train.sh [MINDSPORE_HCCL_CONFIG_PATH] [DATASET_PATH] [PRETRAINED_PATH](optional) - -# standalone training -sh run_standalone_train.sh [DATASET_PATH] [PRETRAINED_PATH](optional) -``` - -#### Launch - -```bash -# distributed training example(8p) -sh run_distribute_train.sh rank_table_8p.json dataset/ilsvrc - -If you want to load pretrained ckpt file, -sh run_distribute_train.sh rank_table_8p.json dataset/ilsvrc ./ckpt/pretrained.ckpt - -# standalone training example（1p） -sh run_standalone_train.sh dataset/ilsvrc - -If you want to load pretrained ckpt file, -sh run_standalone_train.sh dataset/ilsvrc ./ckpt/pretrained.ckpt -``` - -> About rank_table.json, you can refer to the [distributed training tutorial](https://www.mindspore.cn/tutorial/en/master/advanced_use/distributed_training.html). - -#### Result - -Training result will be stored in the scripts path, whose folder name begins with "train" or "train_parallel". You can find checkpoint file together with result like the followings in log. - - -``` -# distribute training result(8p) -epoch: 1 step: 5004, loss is 4.805483 -epoch: 2 step: 5004, loss is 3.2121816 -epoch: 3 step: 5004, loss is 3.429647 -epoch: 4 step: 5004, loss is 3.3667371 -epoch: 5 step: 5004, loss is 3.1718972 -... -epoch: 67 step: 5004, loss is 2.2768745 -epoch: 68 step: 5004, loss is 1.7223864 -epoch: 69 step: 5004, loss is 2.0665488 -epoch: 70 step: 5004, loss is 1.8717369 -... -``` - -### Infer - -#### Usage - -``` -# infer -sh run_eval.sh [VALIDATION_DATASET_PATH] [CHECKPOINT_PATH] -``` - -#### Launch - -```bash -# infer with checkpoint -sh run_eval.sh dataset/validation_preprocess/ train_parallel0/resnet-120_5004.ckpt - -``` - -> checkpoint can be produced in training process. - - -#### Result - -Inference result will be stored in the scripts path, whose folder name is "eval". Under this, you can find result like the followings in log. - -``` -result: {'top_5_accuracy': 0.9429417413572343, 'top_1_accuracy': 0.7853513124199744} ckpt=train_parallel0/resnet-120_5004.ckpt -``` diff --git a/model_zoo/resnet101/eval.py b/model_zoo/resnet101/eval.py deleted file mode 100755 index 73c0289ebd..0000000000 --- a/model_zoo/resnet101/eval.py +++ /dev/null @@ -1,75 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -""" -eval. -""" -import os -import argparse -import random -import numpy as np -from mindspore import context -from mindspore.parallel._auto_parallel_context import auto_parallel_context -from mindspore.train.model import Model, ParallelMode -from mindspore.train.serialization import load_checkpoint, load_param_into_net -import mindspore.dataset.engine as de -from mindspore.communication.management import init -from src.resnet101 import resnet101 -from src.dataset import create_dataset -from src.config import config -from src.crossentropy import CrossEntropy - -random.seed(1) -np.random.seed(1) -de.config.set_seed(1) - -parser = argparse.ArgumentParser(description='Image classification') -parser.add_argument('--run_distribute', type=bool, default=False, help='Run distribute') -parser.add_argument('--device_num', type=int, default=1, help='Device num.') -parser.add_argument('--do_train', type=bool, default=False, help='Do train or not.') -parser.add_argument('--do_eval', type=bool, default=True, help='Do eval or not.') -parser.add_argument('--checkpoint_path', type=str, default=None, help='Checkpoint file path') -parser.add_argument('--dataset_path', type=str, default=None, help='Dataset path') -args_opt = parser.parse_args() - -device_id = int(os.getenv('DEVICE_ID')) - -context.set_context(mode=context.GRAPH_MODE, device_target="Ascend", save_graphs=False, device_id=device_id) - -if __name__ == '__main__': - if not args_opt.do_eval and args_opt.run_distribute: - context.set_auto_parallel_context(device_num=args_opt.device_num, parallel_mode=ParallelMode.DATA_PARALLEL, - mirror_mean=True, parameter_broadcast=True) - auto_parallel_context().set_all_reduce_fusion_split_indices([180, 313]) - init() - - epoch_size = config.epoch_size - net = resnet101(class_num=config.class_num) - - if not config.label_smooth: - config.label_smooth_factor = 0.0 - loss = CrossEntropy(smooth_factor=config.label_smooth_factor, num_classes=config.class_num) - - if args_opt.do_eval: - dataset = create_dataset(dataset_path=args_opt.dataset_path, do_train=False, batch_size=config.batch_size) - step_size = dataset.get_dataset_size() - - if args_opt.checkpoint_path: - param_dict = load_checkpoint(args_opt.checkpoint_path) - load_param_into_net(net, param_dict) - net.set_train(False) - - model = Model(net, loss_fn=loss, metrics={'top_1_accuracy', 'top_5_accuracy'}) - res = model.eval(dataset) - print("result:", res, "ckpt=", args_opt.checkpoint_path) diff --git a/model_zoo/resnet101/src/dataset.py b/model_zoo/resnet101/src/dataset.py deleted file mode 100755 index b2a074a535..0000000000 --- a/model_zoo/resnet101/src/dataset.py +++ /dev/null @@ -1,89 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -""" -create train or eval dataset. -""" -import os -import mindspore.common.dtype as mstype -import mindspore.dataset.engine as de -import mindspore.dataset.transforms.vision.c_transforms as C -import mindspore.dataset.transforms.c_transforms as C2 -from src.config import config - -def create_dataset(dataset_path, do_train, repeat_num=1, batch_size=32): - """ - create a train or evaluate dataset - Args: - dataset_path(string): the path of dataset. - do_train(bool): whether dataset is used for train or eval. - repeat_num(int): the repeat times of dataset. Default: 1 - batch_size(int): the batch size of dataset. Default: 32 - - Returns: - dataset - """ - device_num = int(os.getenv("RANK_SIZE")) - rank_id = int(os.getenv("RANK_ID")) - - if device_num == 1: - ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True) - else: - ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True, - num_shards=device_num, shard_id=rank_id) - resize_height = 224 - rescale = 1.0 / 255.0 - shift = 0.0 - - # define map operations - decode_op = C.Decode() - - random_resize_crop_op = C.RandomResizedCrop(resize_height, (0.08, 1.0), (0.75, 1.33), max_attempts=100) - horizontal_flip_op = C.RandomHorizontalFlip(rank_id / (rank_id + 1)) - resize_op_256 = C.Resize((256, 256)) - center_crop = C.CenterCrop(224) - rescale_op = C.Rescale(rescale, shift) - normalize_op = C.Normalize((0.475, 0.451, 0.392), (0.275, 0.267, 0.278)) - changeswap_op = C.HWC2CHW() - - trans = [] - if do_train: - trans = [decode_op, - random_resize_crop_op, - horizontal_flip_op, - rescale_op, - normalize_op, - changeswap_op] - - else: - trans = [decode_op, - resize_op_256, - center_crop, - rescale_op, - normalize_op, - changeswap_op] - - type_cast_op = C2.TypeCast(mstype.int32) - - ds = ds.map(input_columns="image", operations=trans, num_parallel_workers=8) - ds = ds.map(input_columns="label", operations=type_cast_op, num_parallel_workers=8) - - # apply shuffle operations - ds = ds.shuffle(buffer_size=config.buffer_size) - # apply batch operations - ds = ds.batch(batch_size, drop_remainder=True) - # apply dataset repeat operation - ds = ds.repeat(repeat_num) - - return ds diff --git a/model_zoo/resnet101/train.py b/model_zoo/resnet101/train.py deleted file mode 100755 index 1cd3627a11..0000000000 --- a/model_zoo/resnet101/train.py +++ /dev/null @@ -1,102 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -"""train_imagenet.""" -import os -import argparse -import random -import numpy as np -from mindspore import context -from mindspore import Tensor -from mindspore.parallel._auto_parallel_context import auto_parallel_context -from mindspore.nn.optim.momentum import Momentum -from mindspore.train.model import Model, ParallelMode -from mindspore.train.callback import ModelCheckpoint, CheckpointConfig, LossMonitor, TimeMonitor -from mindspore.train.loss_scale_manager import FixedLossScaleManager -from mindspore.train.serialization import load_checkpoint, load_param_into_net -import mindspore.dataset.engine as de -from mindspore.communication.management import init -import mindspore.nn as nn -import mindspore.common.initializer as weight_init -from src.resnet101 import resnet101 -from src.dataset import create_dataset -from src.lr_generator import warmup_cosine_annealing_lr -from src.config import config -from src.crossentropy import CrossEntropy - -random.seed(1) -np.random.seed(1) -de.config.set_seed(1) - -parser = argparse.ArgumentParser(description='Image classification') -parser.add_argument('--run_distribute', type=bool, default=False, help='Run distribute') -parser.add_argument('--device_num', type=int, default=1, help='Device num.') -parser.add_argument('--do_train', type=bool, default=True, help='Do train or not.') -parser.add_argument('--do_eval', type=bool, default=False, help='Do eval or not.') -parser.add_argument('--dataset_path', type=str, default=None, help='Dataset path') -parser.add_argument('--pre_trained', type=str, default=None, help='Pretrained checkpoint path') -args_opt = parser.parse_args() - -device_id = int(os.getenv('DEVICE_ID')) - -context.set_context(mode=context.GRAPH_MODE, device_target="Ascend", save_graphs=False, device_id=device_id, - enable_auto_mixed_precision=True) - -if __name__ == '__main__': - if not args_opt.do_eval and args_opt.run_distribute: - context.set_auto_parallel_context(device_num=args_opt.device_num, parallel_mode=ParallelMode.DATA_PARALLEL, - mirror_mean=True, parameter_broadcast=True) - auto_parallel_context().set_all_reduce_fusion_split_indices([180, 313]) - init() - - epoch_size = config.epoch_size - net = resnet101(class_num=config.class_num) - # weight init - for _, cell in net.cells_and_names(): - if isinstance(cell, nn.Conv2d): - cell.weight.default_input = weight_init.initializer(weight_init.XavierUniform(), - cell.weight.default_input.shape, - cell.weight.default_input.dtype).to_tensor() - if isinstance(cell, nn.Dense): - cell.weight.default_input = weight_init.initializer(weight_init.TruncatedNormal(), - cell.weight.default_input.shape, - cell.weight.default_input.dtype).to_tensor() - if not config.label_smooth: - config.label_smooth_factor = 0.0 - loss = CrossEntropy(smooth_factor=config.label_smooth_factor, num_classes=config.class_num) - if args_opt.do_train: - dataset = create_dataset(dataset_path=args_opt.dataset_path, do_train=True, - repeat_num=epoch_size, batch_size=config.batch_size) - step_size = dataset.get_dataset_size() - loss_scale = FixedLossScaleManager(config.loss_scale, drop_overflow_update=False) - if args_opt.pre_trained: - param_dict = load_checkpoint(args_opt.pre_trained) - load_param_into_net(net, param_dict) - - # learning rate strategy with cosine - lr = Tensor(warmup_cosine_annealing_lr(config.lr, step_size, config.warmup_epochs, 120, - config.pretrain_epoch_size*step_size)) - opt = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), lr, config.momentum, - config.weight_decay, config.loss_scale) - model = Model(net, loss_fn=loss, optimizer=opt, amp_level='O2', keep_batchnorm_fp32=False, - loss_scale_manager=loss_scale, metrics={'acc'}) - time_cb = TimeMonitor(data_size=step_size) - loss_cb = LossMonitor() - cb = [time_cb, loss_cb] - if config.save_checkpoint: - config_ck = CheckpointConfig(save_checkpoint_steps=config.save_checkpoint_epochs*step_size, - keep_checkpoint_max=config.keep_checkpoint_max) - ckpt_cb = ModelCheckpoint(prefix="resnet", directory=config.save_checkpoint_path, config=config_ck) - cb += [ckpt_cb] - model.train(epoch_size, dataset, callbacks=cb) diff --git a/model_zoo/resnet50_quant/Readme.md b/model_zoo/resnet50_quant/Readme.md new file mode 100644 index 0000000000..9e843b2223 --- /dev/null +++ b/model_zoo/resnet50_quant/Readme.md @@ -0,0 +1,122 @@ +# ResNet-50_quant Example + +## Description + +This is an example of training ResNet-50_quant with ImageNet2012 dataset in MindSpore. + +## Requirements + +- Install [MindSpore](https://www.mindspore.cn/install/en). + +- Download the dataset ImageNet2012 + +> Unzip the ImageNet2012 dataset to any path you want and the folder structure should include train and eval dataset as follows: +> ``` +> . +> ├── ilsvrc # train dataset +> └── ilsvrc_eval # infer dataset: images should be classified into 1000 directories firstly, just like train images +> ``` + + +## Example structure + +```shell +. +├── Resnet50_quant + ├── Readme.md + ├── scripts + │ ├──run_train.sh + │ ├──run_eval.sh + ├── src + │ ├──config.py + │ ├──crossentropy.py + │ ├──dataset.py + │ ├──luanch.py + │ ├──lr_generator.py + │ ├──utils.py + ├── models + │ ├──resnet_quant.py + ├── train.py + ├── eval.py +``` + + +## Parameter configuration + +Parameters for both training and inference can be set in config.py. + +``` +"class_num": 1001, # dataset class number +"batch_size": 32, # batch size of input tensor +"loss_scale": 1024, # loss scale +"momentum": 0.9, # momentum optimizer +"weight_decay": 1e-4, # weight decay +"epoch_size": 120, # only valid for taining, which is always 1 for inference +"pretrained_epoch_size": 90, # epoch size that model has been trained before load pretrained checkpoint +"buffer_size": 1000, # number of queue size in data preprocessing +"image_height": 224, # image height +"image_width": 224, # image width +"save_checkpoint": True, # whether save checkpoint or not +"save_checkpoint_epochs": 1, # the epoch interval between two checkpoints. By default, the last checkpoint will be saved after the last epoch +"keep_checkpoint_max": 50, # only keep the last keep_checkpoint_max checkpoint +"save_checkpoint_path": "./", # path to save checkpoint relative to the executed path +"warmup_epochs": 0, # number of warmup epoch +"lr_decay_mode": "cosine", # decay mode for generating learning rate +"label_smooth": True, # label smooth +"label_smooth_factor": 0.1, # label smooth factor +"lr_init": 0, # initial learning rate +"lr_max": 0.005, # maximum learning rate +``` + +## Running the example + +### Train + +### Usage + +- Ascend: sh run_train.sh Ascend [DEVICE_NUM] [SERVER_IP(x.x.x.x)] [VISIABLE_DEVICES(0,1,2,3,4,5,6,7)] [DATASET_PATH] [CKPT_PATH] + + +### Launch + +``` +# training example + Ascend: sh run_train.sh Ascend 8 192.168.0.1 0,1,2,3,4,5,6,7 ~/imagenet/train/ +``` + +### Result + +Training result will be stored in the example path. Checkpoints will be stored at `. /checkpoint` by default, and training log will be redirected to `./train/train.log` like followings. + +``` +epoch: 1 step: 5004, loss is 4.8995576 +epoch: 2 step: 5004, loss is 3.9235563 +epoch: 3 step: 5004, loss is 3.833077 +epoch: 4 step: 5004, loss is 3.2795618 +epoch: 5 step: 5004, loss is 3.1978393 +``` + +## Eval process + +### Usage + +- Ascend: sh run_infer.sh Ascend [DATASET_PATH] [CHECKPOINT_PATH] + +### Launch + +``` +# infer example + Ascend: sh run_infer.sh Ascend ~/imagenet/val/ ~/checkpoint/resnet50-110_5004.ckpt +``` + + +> checkpoint can be produced in training process. + +#### Result + +Inference result will be stored in the example path, whose folder name is "infer". Under this, you can find result like the followings in log. + +``` +result: {'acc': 0.75.252054737516005} ckpt=train_parallel0/resnet-110_5004.ckpt +``` + diff --git a/model_zoo/resnet50_quant/eval.py b/model_zoo/resnet50_quant/eval.py new file mode 100755 index 0000000000..481e4bb853 --- /dev/null +++ b/model_zoo/resnet50_quant/eval.py @@ -0,0 +1,78 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Evaluate Resnet50 on ImageNet""" + +import os +import argparse + +from src.config import quant_set, config_quant, config_noquant +from src.dataset import create_dataset +from src.crossentropy import CrossEntropy +from src.utils import _load_param_into_net +from models.resnet_quant import resnet50_quant + +from mindspore import context +from mindspore.train.model import Model +from mindspore.train.serialization import load_checkpoint +from mindspore.train.quant import quant + +parser = argparse.ArgumentParser(description='Image classification') +parser.add_argument('--checkpoint_path', type=str, default=None, help='Checkpoint file path') +parser.add_argument('--dataset_path', type=str, default=None, help='Dataset path') +parser.add_argument('--device_target', type=str, default='Ascend', help='Device target') +args_opt = parser.parse_args() + +context.set_context(mode=context.GRAPH_MODE, device_target=args_opt.device_target, save_graphs=False) +config = config_quant if quant_set.quantization_aware else config_noquant + +if args_opt.device_target == "Ascend": + device_id = int(os.getenv('DEVICE_ID')) + context.set_context(device_id=device_id) + +if __name__ == '__main__': + # define fusion network + net = resnet50_quant(class_num=config.class_num) + if quant_set.quantization_aware: + # convert fusion network to quantization aware network + net = quant.convert_quant_network(net, + bn_fold=True, + per_channel=[True, False], + symmetric=[True, False]) + # define network loss + if not config.use_label_smooth: + config.label_smooth_factor = 0.0 + loss = CrossEntropy(smooth_factor=config.label_smooth_factor, + num_classes=config.class_num) + + # define dataset + dataset = create_dataset(dataset_path=args_opt.dataset_path, + do_train=False, + batch_size=config.batch_size, + target=args_opt.device_target) + step_size = dataset.get_dataset_size() + + # load checkpoint + if args_opt.checkpoint_path: + param_dict = load_checkpoint(args_opt.checkpoint_path) + _load_param_into_net(net, param_dict) + net.set_train(False) + + # define model + model = Model(net, loss_fn=loss, metrics={'acc'}) + + print("============== Starting Validation ==============") + res = model.eval(dataset) + print("result:", res, "ckpt=", args_opt.checkpoint_path) + print("============== End Validation ==============") diff --git a/model_zoo/resnet50_quant/models/resnet_quant.py b/model_zoo/resnet50_quant/models/resnet_quant.py new file mode 100755 index 0000000000..63fa32222d --- /dev/null +++ b/model_zoo/resnet50_quant/models/resnet_quant.py @@ -0,0 +1,251 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""ResNet.""" +import mindspore.nn as nn +from mindspore.ops import operations as P + + +class ConvBNReLU(nn.Cell): + """ + Convolution/Depthwise fused with Batchnorm and ReLU block definition. + + Args: + in_planes (int): Input channel. + out_planes (int): Output channel. + kernel_size (int): Input kernel size. + stride (int): Stride size for the first convolutional layer. Default: 1. + groups (int): channel group. Convolution is 1 while Depthiwse is input channel. Default: 1. + + Returns: + Tensor, output tensor. + + Examples: + >>> ConvBNReLU(16, 256, kernel_size=1, stride=1, groups=1) + """ + + def __init__(self, in_planes, out_planes, kernel_size=3, stride=1, groups=1): + super(ConvBNReLU, self).__init__() + padding = (kernel_size - 1) // 2 + conv = nn.Conv2dBnAct(in_planes, out_planes, kernel_size, stride, pad_mode='pad', padding=padding, + group=groups, has_bn=True, activation='relu') + self.features = conv + + def construct(self, x): + output = self.features(x) + return output + + +class ResidualBlock(nn.Cell): + """ + ResNet V1 residual block definition. + + Args: + in_channel (int): Input channel. + out_channel (int): Output channel. + stride (int): Stride size for the first convolutional layer. Default: 1. + + Returns: + Tensor, output tensor. + + Examples: + >>> ResidualBlock(3, 256, stride=2) + """ + expansion = 4 + + def __init__(self, + in_channel, + out_channel, + stride=1): + super(ResidualBlock, self).__init__() + + channel = out_channel // self.expansion + self.conv1 = ConvBNReLU(in_channel, channel, kernel_size=1, stride=1) + self.conv2 = ConvBNReLU(channel, channel, kernel_size=3, stride=stride) + self.conv3 = nn.Conv2dBnAct(channel, out_channel, kernel_size=1, stride=1, pad_mode='same', padding=0, + has_bn=True, activation='relu') + + self.down_sample = False + if stride != 1 or in_channel != out_channel: + self.down_sample = True + self.down_sample_layer = None + + if self.down_sample: + self.down_sample_layer = nn.Conv2dBnAct(in_channel, out_channel, + kernel_size=1, stride=stride, + pad_mode='same', padding=0, has_bn=True, activation='relu') + self.add = P.TensorAdd() + self.relu = P.ReLU() + + def construct(self, x): + identity = x + out = self.conv1(x) + out = self.conv2(out) + out = self.conv3(out) + + if self.down_sample: + identity = self.down_sample_layer(identity) + + out = self.add(out, identity) + out = self.relu(out) + + return out + + +class ResNet(nn.Cell): + """ + ResNet architecture. + + Args: + block (Cell): Block for network. + layer_nums (list): Numbers of block in different layers. + in_channels (list): Input channel in each layer. + out_channels (list): Output channel in each layer. + strides (list): Stride size in each layer. + num_classes (int): The number of classes that the training images are belonging to. + Returns: + Tensor, output tensor. + + Examples: + >>> ResNet(ResidualBlock, + >>> [3, 4, 6, 3], + >>> [64, 256, 512, 1024], + >>> [256, 512, 1024, 2048], + >>> [1, 2, 2, 2], + >>> 10) + """ + + def __init__(self, + block, + layer_nums, + in_channels, + out_channels, + strides, + num_classes): + super(ResNet, self).__init__() + + if not len(layer_nums) == len(in_channels) == len(out_channels) == 4: + raise ValueError("the length of layer_num, in_channels, out_channels list must be 4!") + + self.conv1 = ConvBNReLU(3, 64, kernel_size=7, stride=2) + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, pad_mode="same") + + self.layer1 = self._make_layer(block, + layer_nums[0], + in_channel=in_channels[0], + out_channel=out_channels[0], + stride=strides[0]) + self.layer2 = self._make_layer(block, + layer_nums[1], + in_channel=in_channels[1], + out_channel=out_channels[1], + stride=strides[1]) + self.layer3 = self._make_layer(block, + layer_nums[2], + in_channel=in_channels[2], + out_channel=out_channels[2], + stride=strides[2]) + self.layer4 = self._make_layer(block, + layer_nums[3], + in_channel=in_channels[3], + out_channel=out_channels[3], + stride=strides[3]) + + self.mean = P.ReduceMean(keep_dims=True) + self.flatten = nn.Flatten() + self.end_point = nn.DenseBnAct(out_channels[3], num_classes, has_bias=True, has_bn=False) + + def _make_layer(self, block, layer_num, in_channel, out_channel, stride): + """ + Make stage network of ResNet. + + Args: + block (Cell): Resnet block. + layer_num (int): Layer number. + in_channel (int): Input channel. + out_channel (int): Output channel. + stride (int): Stride size for the first convolutional layer. + + Returns: + SequentialCell, the output layer. + + Examples: + >>> _make_layer(ResidualBlock, 3, 128, 256, 2) + """ + layers = [] + + resnet_block = block(in_channel, out_channel, stride=stride) + layers.append(resnet_block) + + for _ in range(1, layer_num): + resnet_block = block(out_channel, out_channel, stride=1) + layers.append(resnet_block) + + return nn.SequentialCell(layers) + + def construct(self, x): + x = self.conv1(x) + c1 = self.maxpool(x) + + c2 = self.layer1(c1) + c3 = self.layer2(c2) + c4 = self.layer3(c3) + c5 = self.layer4(c4) + + out = self.mean(c5, (2, 3)) + out = self.flatten(out) + out = self.end_point(out) + return out + + +def resnet50_quant(class_num=10001): + """ + Get ResNet50 neural network. + + Args: + class_num (int): Class number. + + Returns: + Cell, cell instance of ResNet50 neural network. + + Examples: + >>> net = resnet50_quant(10) + """ + return ResNet(ResidualBlock, + [3, 4, 6, 3], + [64, 256, 512, 1024], + [256, 512, 1024, 2048], + [1, 2, 2, 2], + class_num) + + +def resnet101_quant(class_num=1001): + """ + Get ResNet101 neural network. + + Args: + class_num (int): Class number. + + Returns: + Cell, cell instance of ResNet101 neural network. + + Examples: + >>> net = resnet101(1001) + """ + return ResNet(ResidualBlock, + [3, 4, 23, 3], + [64, 256, 512, 1024], + [256, 512, 1024, 2048], + [1, 2, 2, 2], + class_num) diff --git a/model_zoo/resnet50_quant/scripts/run_infer.sh b/model_zoo/resnet50_quant/scripts/run_infer.sh new file mode 100644 index 0000000000..1d74f63737 --- /dev/null +++ b/model_zoo/resnet50_quant/scripts/run_infer.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +if [ $# != 3 ] +then + echo "Ascend: sh run_infer.sh [PLATFORM] [DATASET_PATH] [CHECKPOINT_PATH]" +exit 1 +fi + +# check dataset path +if [ ! -d $2 ] && [ ! -f $2 ] +then + echo "error: DATASET_PATH=$2 is not a directory or file" +exit 1 +fi + +# check checkpoint file +if [ ! -f $3 ] +then + echo "error: CHECKPOINT_PATH=$3 is not a file" +exit 1 +fi + +# set environment +BASEPATH=$(cd "`dirname $0`" || exit; pwd) +export PYTHONPATH=${BASEPATH}:$PYTHONPATH +export DEVICE_ID=0 +export RANK_ID=0 +export RANK_SIZE=1 +if [ -d "../eval" ]; +then + rm -rf ../eval +fi +mkdir ../eval +cd ../eval || exit + +# luanch +python ${BASEPATH}/../eval.py \ + --device_target=$1 \ + --dataset_path=$2 \ + --checkpoint_path=$3 \ + &> infer.log & # dataset val folder path diff --git a/model_zoo/resnet50_quant/scripts/run_train.sh b/model_zoo/resnet50_quant/scripts/run_train.sh new file mode 100644 index 0000000000..a427201587 --- /dev/null +++ b/model_zoo/resnet50_quant/scripts/run_train.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +run_ascend() +{ + if [ $2 -lt 1 ] && [ $2 -gt 8 ] + then + echo "error: DEVICE_NUM=$2 is not in (1-8)" + exit 1 + fi + + if [ ! -d $5 ] && [ ! -f $5 ] + then + echo "error: DATASET_PATH=$5 is not a directory or file" + exit 1 + fi + + BASEPATH=$(cd "`dirname $0`" || exit; pwd) + export PYTHONPATH=${BASEPATH}:$PYTHONPATH + if [ -d "../train" ]; + then + rm -rf ../train + fi + mkdir ../train + cd ../train || exit + python ${BASEPATH}/../src/launch.py \ + --nproc_per_node=$2 \ + --visible_devices=$4 \ + --server_id=$3 \ + --training_script=${BASEPATH}/../train.py \ + --dataset_path=$5 \ + --pre_trained=$6 \ + --device_target=$1 &> train.log & # dataset train folder +} + +if [ $# -gt 6 ] || [ $# -lt 4 ] +then + echo "Usage:\n \ + Ascend: sh run_train.sh Ascend [DEVICE_NUM] [SERVER_IP(x.x.x.x)] [VISIABLE_DEVICES(0,1,2,3,4,5,6,7)] [DATASET_PATH] [CKPT_PATH]\n \ + " +exit 1 +fi + +if [ $1 = "Ascend" ] ; then + run_ascend "$@" +else + echo "not support platform" +fi; + diff --git a/example/resnet50_imagenet2012/config.py b/model_zoo/resnet50_quant/src/config.py similarity index 60% rename from example/resnet50_imagenet2012/config.py rename to model_zoo/resnet50_quant/src/config.py index cf5093d245..523702dc43 100755 --- a/example/resnet50_imagenet2012/config.py +++ b/model_zoo/resnet50_quant/src/config.py @@ -17,7 +17,10 @@ network config setting, will be used in train.py and eval.py """ from easydict import EasyDict as ed -config = ed({ +quant_set = ed({ + "quantization_aware": True, +}) +config_noquant = ed({ "class_num": 1001, "batch_size": 32, "loss_scale": 1024, @@ -28,15 +31,38 @@ config = ed({ "buffer_size": 1000, "image_height": 224, "image_width": 224, + "data_load_mode": "mindrecord", "save_checkpoint": True, - "save_checkpoint_epochs": 5, - "keep_checkpoint_max": 10, + "save_checkpoint_epochs": 1, + "keep_checkpoint_max": 50, "save_checkpoint_path": "./", "warmup_epochs": 0, "lr_decay_mode": "cosine", "use_label_smooth": True, "label_smooth_factor": 0.1, "lr_init": 0, - "lr_max": 0.1 - + "lr_max": 0.1, +}) +config_quant = ed({ + "class_num": 1001, + "batch_size": 32, + "loss_scale": 1024, + "momentum": 0.9, + "weight_decay": 1e-4, + "epoch_size": 120, + "pretrained_epoch_size": 90, + "buffer_size": 1000, + "image_height": 224, + "image_width": 224, + "data_load_mode": "mindrecord", + "save_checkpoint": True, + "save_checkpoint_epochs": 1, + "keep_checkpoint_max": 50, + "save_checkpoint_path": "./", + "warmup_epochs": 0, + "lr_decay_mode": "cosine", + "use_label_smooth": True, + "label_smooth_factor": 0.1, + "lr_init": 0, + "lr_max": 0.005, }) diff --git a/example/resnet50_imagenet2012/crossentropy.py b/model_zoo/resnet50_quant/src/crossentropy.py similarity index 100% rename from example/resnet50_imagenet2012/crossentropy.py rename to model_zoo/resnet50_quant/src/crossentropy.py diff --git a/model_zoo/resnet50_quant/src/dataset.py b/model_zoo/resnet50_quant/src/dataset.py new file mode 100755 index 0000000000..73c0780090 --- /dev/null +++ b/model_zoo/resnet50_quant/src/dataset.py @@ -0,0 +1,157 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +create train or eval dataset. +""" +import os +from functools import partial +import mindspore.common.dtype as mstype +import mindspore.dataset.engine as de +import mindspore.dataset.transforms.vision.c_transforms as C +import mindspore.dataset.transforms.c_transforms as C2 +import mindspore.dataset.transforms.vision.py_transforms as P +from mindspore.communication.management import init, get_rank, get_group_size +from src.config import quant_set, config_quant, config_noquant + +config = config_quant if quant_set.quantization_aware else config_noquant + + +def create_dataset(dataset_path, do_train, repeat_num=1, batch_size=32, target="Ascend"): + """ + create a train or eval dataset + + Args: + dataset_path(string): the path of dataset. + do_train(bool): whether dataset is used for train or eval. + repeat_num(int): the repeat times of dataset. Default: 1 + batch_size(int): the batch size of dataset. Default: 32 + target(str): the device target. Default: Ascend + + Returns: + dataset + """ + if target == "Ascend": + device_num = int(os.getenv("RANK_SIZE")) + rank_id = int(os.getenv("RANK_ID")) + else: + init("nccl") + rank_id = get_rank() + device_num = get_group_size() + + columns_list = ['image', 'label'] + if config.data_load_mode == "mindrecord": + load_func = partial(de.MindDataset, dataset_path, columns_list) + else: + load_func = partial(de.ImageFolderDatasetV2, dataset_path) + if device_num == 1: + ds = load_func(num_parallel_workers=8, shuffle=True) + else: + ds = load_func(num_parallel_workers=8, shuffle=True, + num_shards=device_num, shard_id=rank_id) + + image_size = config.image_height + mean = [0.485 * 255, 0.456 * 255, 0.406 * 255] + std = [0.229 * 255, 0.224 * 255, 0.225 * 255] + + # define map operations + if do_train: + trans = [ + C.RandomCropDecodeResize(image_size, scale=(0.08, 1.0), ratio=(0.75, 1.333)), + C.RandomHorizontalFlip(prob=0.5), + C.Normalize(mean=mean, std=std), + C.HWC2CHW() + ] + else: + trans = [ + C.Decode(), + C.Resize(256), + C.CenterCrop(image_size), + C.Normalize(mean=mean, std=std), + C.HWC2CHW() + ] + + type_cast_op = C2.TypeCast(mstype.int32) + + ds = ds.map(input_columns="image", num_parallel_workers=8, operations=trans) + ds = ds.map(input_columns="label", num_parallel_workers=8, operations=type_cast_op) + + # apply batch operations + ds = ds.batch(batch_size, drop_remainder=True) + + # apply dataset repeat operation + ds = ds.repeat(repeat_num) + + return ds + + +def create_dataset_py(dataset_path, do_train, repeat_num=1, batch_size=32, target="Ascend"): + """ + create a train or eval dataset + + Args: + dataset_path(string): the path of dataset. + do_train(bool): whether dataset is used for train or eval. + repeat_num(int): the repeat times of dataset. Default: 1 + batch_size(int): the batch size of dataset. Default: 32 + target(str): the device target. Default: Ascend + + Returns: + dataset + """ + if target == "Ascend": + device_num = int(os.getenv("RANK_SIZE")) + rank_id = int(os.getenv("RANK_ID")) + else: + init("nccl") + rank_id = get_rank() + device_num = get_group_size() + + if do_train: + if device_num == 1: + ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True) + else: + ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True, + num_shards=device_num, shard_id=rank_id) + else: + ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=False) + + image_size = 224 + + # define map operations + decode_op = P.Decode() + resize_crop_op = P.RandomResizedCrop(image_size, scale=(0.08, 1.0), ratio=(0.75, 1.333)) + horizontal_flip_op = P.RandomHorizontalFlip(prob=0.5) + + resize_op = P.Resize(256) + center_crop = P.CenterCrop(image_size) + to_tensor = P.ToTensor() + normalize_op = P.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) + + # define map operations + if do_train: + trans = [decode_op, resize_crop_op, horizontal_flip_op, to_tensor, normalize_op] + else: + trans = [decode_op, resize_op, center_crop, to_tensor, normalize_op] + + compose = P.ComposeOp(trans) + ds = ds.map(input_columns="image", operations=compose(), num_parallel_workers=8, python_multiprocessing=True) + + # apply batch operations + ds = ds.batch(batch_size, drop_remainder=True) + + # apply dataset repeat operation + ds = ds.repeat(repeat_num) + + return ds diff --git a/model_zoo/resnet50_quant/src/launch.py b/model_zoo/resnet50_quant/src/launch.py new file mode 100644 index 0000000000..abba92a540 --- /dev/null +++ b/model_zoo/resnet50_quant/src/launch.py @@ -0,0 +1,165 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""launch train script""" +import os +import sys +import json +import subprocess +import shutil +import platform +from argparse import ArgumentParser + +def parse_args(): + """ + parse args . + + Args: + + Returns: + args. + + Examples: + >>> parse_args() + """ + parser = ArgumentParser(description="mindspore distributed training launch " + "helper utilty that will spawn up " + "multiple distributed processes") + parser.add_argument("--nproc_per_node", type=int, default=1, + help="The number of processes to launch on each node, " + "for D training, this is recommended to be set " + "to the number of D in your system so that " + "each process can be bound to a single D.") + parser.add_argument("--visible_devices", type=str, default="0,1,2,3,4,5,6,7", + help="will use the visible devices sequentially") + parser.add_argument("--server_id", type=str, default="", + help="server ip") + parser.add_argument("--training_script", type=str, + help="The full path to the single D training " + "program/script to be launched in parallel, " + "followed by all the arguments for the " + "training script") + # rest from the training program + args, unknown = parser.parse_known_args() + args.training_script_args = unknown + return args + + +def main(): + print("start", __file__) + args = parse_args() + print(args) + visible_devices = args.visible_devices.split(',') + assert os.path.isfile(args.training_script) + assert len(visible_devices) >= args.nproc_per_node + print('visible_devices:{}'.format(visible_devices)) + if not args.server_id: + print('pleaser input server ip!!!') + exit(0) + print('server_id:{}'.format(args.server_id)) + + # construct hccn_table + hccn_configs = open('/etc/hccn.conf', 'r').readlines() + device_ips = {} + for hccn_item in hccn_configs: + hccn_item = hccn_item.strip() + if hccn_item.startswith('address_'): + device_id, device_ip = hccn_item.split('=') + device_id = device_id.split('_')[1] + device_ips[device_id] = device_ip + print('device_id:{}, device_ip:{}'.format(device_id, device_ip)) + hccn_table = {} + arch = platform.processor() + hccn_table['board_id'] = {'aarch64': '0x002f', 'x86_64': '0x0000'}[arch] + hccn_table['chip_info'] = '910' + hccn_table['deploy_mode'] = 'lab' + hccn_table['group_count'] = '1' + hccn_table['group_list'] = [] + instance_list = [] + usable_dev = '' + for instance_id in range(args.nproc_per_node): + instance = {} + instance['devices'] = [] + device_id = visible_devices[instance_id] + device_ip = device_ips[device_id] + usable_dev += str(device_id) + instance['devices'].append({ + 'device_id': device_id, + 'device_ip': device_ip, + }) + instance['rank_id'] = str(instance_id) + instance['server_id'] = args.server_id + instance_list.append(instance) + hccn_table['group_list'].append({ + 'device_num': str(args.nproc_per_node), + 'server_num': '1', + 'group_name': '', + 'instance_count': str(args.nproc_per_node), + 'instance_list': instance_list, + }) + hccn_table['para_plane_nic_location'] = 'device' + hccn_table['para_plane_nic_name'] = [] + for instance_id in range(args.nproc_per_node): + eth_id = visible_devices[instance_id] + hccn_table['para_plane_nic_name'].append('eth{}'.format(eth_id)) + hccn_table['para_plane_nic_num'] = str(args.nproc_per_node) + hccn_table['status'] = 'completed' + + # save hccn_table to file + table_path = os.getcwd() + if not os.path.exists(table_path): + os.mkdir(table_path) + table_fn = os.path.join(table_path, + 'rank_table_{}p_{}_{}.json'.format(args.nproc_per_node, usable_dev, args.server_id)) + with open(table_fn, 'w') as table_fp: + json.dump(hccn_table, table_fp, indent=4) + sys.stdout.flush() + + # spawn the processes + processes = [] + cmds = [] + log_files = [] + env = os.environ.copy() + env['RANK_SIZE'] = str(args.nproc_per_node) + cur_path = os.getcwd() + for rank_id in range(0, args.nproc_per_node): + os.chdir(cur_path) + device_id = visible_devices[rank_id] + device_dir = os.path.join(cur_path, 'device{}'.format(rank_id)) + env['RANK_ID'] = str(rank_id) + env['DEVICE_ID'] = str(device_id) + if args.nproc_per_node > 1: + env['MINDSPORE_HCCL_CONFIG_PATH'] = table_fn + env['RANK_TABLE_FILE'] = table_fn + if os.path.exists(device_dir): + shutil.rmtree(device_dir) + os.mkdir(device_dir) + os.chdir(device_dir) + cmd = [sys.executable, '-u'] + cmd.append(args.training_script) + cmd.extend(args.training_script_args) + log_file = open('{dir}/log{id}.log'.format(dir=device_dir, id=rank_id), 'w') + process = subprocess.Popen(cmd, stdout=log_file, stderr=log_file, env=env) + processes.append(process) + cmds.append(cmd) + log_files.append(log_file) + for process, cmd, log_file in zip(processes, cmds, log_files): + process.wait() + if process.returncode != 0: + raise subprocess.CalledProcessError(returncode=process, cmd=cmd) + log_file.close() + + +if __name__ == "__main__": + main() diff --git a/example/resnet50_imagenet2012/lr_generator.py b/model_zoo/resnet50_quant/src/lr_generator.py similarity index 100% rename from example/resnet50_imagenet2012/lr_generator.py rename to model_zoo/resnet50_quant/src/lr_generator.py diff --git a/model_zoo/resnet50_quant/src/utils.py b/model_zoo/resnet50_quant/src/utils.py new file mode 100644 index 0000000000..846fd7b894 --- /dev/null +++ b/model_zoo/resnet50_quant/src/utils.py @@ -0,0 +1,46 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""utils script""" + +def _load_param_into_net(model, params_dict): + """ + load fp32 model parameters to quantization model. + + Args: + model: quantization model + params_dict: f32 param + + Returns: + None + """ + iterable_dict = { + 'weight': iter([item for item in params_dict.items() if item[0].endswith('weight')]), + 'bias': iter([item for item in params_dict.items() if item[0].endswith('bias')]), + 'gamma': iter([item for item in params_dict.items() if item[0].endswith('gamma')]), + 'beta': iter([item for item in params_dict.items() if item[0].endswith('beta')]), + 'moving_mean': iter([item for item in params_dict.items() if item[0].endswith('moving_mean')]), + 'moving_variance': iter( + [item for item in params_dict.items() if item[0].endswith('moving_variance')]), + 'minq': iter([item for item in params_dict.items() if item[0].endswith('minq')]), + 'maxq': iter([item for item in params_dict.items() if item[0].endswith('maxq')]) + } + for name, param in model.parameters_and_names(): + key_name = name.split(".")[-1] + if key_name not in iterable_dict.keys(): + continue + value_param = next(iterable_dict[key_name], None) + if value_param is not None: + param.set_parameter_data(value_param[1].data) + print(f'init model param {name} with checkpoint param {value_param[0]}') diff --git a/model_zoo/resnet50_quant/train.py b/model_zoo/resnet50_quant/train.py new file mode 100755 index 0000000000..b026f97278 --- /dev/null +++ b/model_zoo/resnet50_quant/train.py @@ -0,0 +1,153 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Train Resnet50 on ImageNet""" + +import os +import argparse + +from mindspore import context +from mindspore import Tensor +from mindspore.parallel._auto_parallel_context import auto_parallel_context +from mindspore.nn.optim.momentum import Momentum +from mindspore.train.model import Model, ParallelMode +from mindspore.train.callback import ModelCheckpoint, CheckpointConfig, LossMonitor, TimeMonitor +from mindspore.train.loss_scale_manager import FixedLossScaleManager +from mindspore.train.serialization import load_checkpoint +from mindspore.train.quant import quant +from mindspore.communication.management import init +import mindspore.nn as nn +import mindspore.common.initializer as weight_init + +from models.resnet_quant import resnet50_quant +from src.dataset import create_dataset +from src.lr_generator import get_lr +from src.config import quant_set, config_quant, config_noquant +from src.crossentropy import CrossEntropy +from src.utils import _load_param_into_net + +parser = argparse.ArgumentParser(description='Image classification') +parser.add_argument('--run_distribute', type=bool, default=False, help='Run distribute') +parser.add_argument('--device_num', type=int, default=1, help='Device num.') +parser.add_argument('--dataset_path', type=str, default=None, help='Dataset path') +parser.add_argument('--device_target', type=str, default='Ascend', help='Device target') +parser.add_argument('--pre_trained', type=str, default=None, help='Pertained checkpoint path') +args_opt = parser.parse_args() +config = config_quant if quant_set.quantization_aware else config_noquant + +if args_opt.device_target == "Ascend": + device_id = int(os.getenv('DEVICE_ID')) + rank_id = int(os.getenv('RANK_ID')) + rank_size = int(os.getenv('RANK_SIZE')) + run_distribute = rank_size > 1 + context.set_context(mode=context.GRAPH_MODE, + device_target="Ascend", + save_graphs=False, + device_id=device_id, + enable_auto_mixed_precision=True) +else: + raise ValueError("Unsupported device target.") + +if __name__ == '__main__': + # train on ascend + print("training args: {}".format(args_opt)) + print("training configure: {}".format(config)) + print("parallel args: rank_id {}, device_id {}, rank_size {}".format(rank_id, device_id, rank_size)) + epoch_size = config.epoch_size + + # distribute init + if run_distribute: + context.set_auto_parallel_context(device_num=rank_size, + parallel_mode=ParallelMode.DATA_PARALLEL, + parameter_broadcast=True, + mirror_mean=True) + init() + context.set_auto_parallel_context(device_num=args_opt.device_num, + parallel_mode=ParallelMode.DATA_PARALLEL, + mirror_mean=True) + auto_parallel_context().set_all_reduce_fusion_split_indices([107, 160]) + + # define network + net = resnet50_quant(class_num=config.class_num) + net.set_train(True) + + # weight init and load checkpoint file + if args_opt.pre_trained: + param_dict = load_checkpoint(args_opt.pre_trained) + _load_param_into_net(net, param_dict) + epoch_size = config.epoch_size - config.pretrained_epoch_size + else: + for _, cell in net.cells_and_names(): + if isinstance(cell, nn.Conv2d): + cell.weight.default_input = weight_init.initializer(weight_init.XavierUniform(), + cell.weight.default_input.shape, + cell.weight.default_input.dtype).to_tensor() + if isinstance(cell, nn.Dense): + cell.weight.default_input = weight_init.initializer(weight_init.TruncatedNormal(), + cell.weight.default_input.shape, + cell.weight.default_input.dtype).to_tensor() + if not config.use_label_smooth: + config.label_smooth_factor = 0.0 + loss = CrossEntropy(smooth_factor=config.label_smooth_factor, num_classes=config.class_num) + loss_scale = FixedLossScaleManager(config.loss_scale, drop_overflow_update=False) + + # define dataset + dataset = create_dataset(dataset_path=args_opt.dataset_path, + do_train=True, + repeat_num=epoch_size, + batch_size=config.batch_size, + target=args_opt.device_target) + step_size = dataset.get_dataset_size() + + if quant_set.quantization_aware: + # convert fusion network to quantization aware network + net = quant.convert_quant_network(net, bn_fold=True, per_channel=[True, False], symmetric=[True, False]) + + # get learning rate + lr = get_lr(lr_init=config.lr_init, + lr_end=0.0, + lr_max=config.lr_max, + warmup_epochs=config.warmup_epochs, + total_epochs=config.epoch_size, + steps_per_epoch=step_size, + lr_decay_mode='cosine') + if args_opt.pre_trained: + lr = lr[config.pretrained_epoch_size * step_size:] + lr = Tensor(lr) + + # define optimization + opt = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), lr, config.momentum, + config.weight_decay, config.loss_scale) + + # define model + if quant_set.quantization_aware: + model = Model(net, loss_fn=loss, optimizer=opt, loss_scale_manager=loss_scale, metrics={'acc'}) + else: + model = Model(net, loss_fn=loss, optimizer=opt, loss_scale_manager=loss_scale, metrics={'acc'}, + amp_level="O2") + + print("============== Starting Training ==============") + time_callback = TimeMonitor(data_size=step_size) + loss_callback = LossMonitor() + callbacks = [time_callback, loss_callback] + if rank_id == 0: + if config.save_checkpoint: + config_ckpt = CheckpointConfig(save_checkpoint_steps=config.save_checkpoint_epochs * step_size, + keep_checkpoint_max=config.keep_checkpoint_max) + ckpt_callback = ModelCheckpoint(prefix="ResNet50", + directory=config.save_checkpoint_path, + config=config_ckpt) + callbacks += [ckpt_callback] + model.train(epoch_size, dataset, callbacks=callbacks) + print("============== End Training ==============") diff --git a/example/resnet50_imagenet2012_THOR/README.md b/model_zoo/resnet_thor/README.md similarity index 73% rename from example/resnet50_imagenet2012_THOR/README.md rename to model_zoo/resnet_thor/README.md index 6003d8d7b7..5fb17007ae 100644 --- a/example/resnet50_imagenet2012_THOR/README.md +++ b/model_zoo/resnet_thor/README.md @@ -22,14 +22,24 @@ This is an example of training ResNet-50 V1.5 with ImageNet2012 dataset by secon ```shell . -├── crossentropy.py # CrossEntropy loss function -├── config.py # parameter configuration -├── dataset_imagenet.py # data preprocessing -├── eval.py # infer script -├── model # include model file of the optimizer -├── run_distribute_train.sh # launch distributed training(8 pcs) -├── run_infer.sh # launch infering -└── train.py # train script +├── resnet_thor + ├── README.md + ├── src + ├── crossentropy.py # CrossEntropy loss function + ├── config.py # parameter configuration + ├── resnet50.py # resnet50 backbone + ├── dataset_helper.py # dataset help for minddata dataset + ├── grad_reducer_thor.py # grad reducer for thor + ├── model_thor.py # model + ├── resnet_thor.py # resnet50_thor backone + ├── thor.py # thor + ├── thor_layer.py # thor layer + └── dataset_imagenet.py # data preprocessing + ├── scripts + ├── run_distribute_train.sh # launch distributed training(8 pcs) + └── run_eval.sh # launch infering + ├── eval.py # infer script + └── train.py # train script ``` @@ -97,14 +107,14 @@ epoch: 5 step: 5004, loss is 3.3744206 ``` # infer -Usage: sh run_infer.sh [DATASET_PATH] [CHECKPOINT_PATH] +Usage: sh run_eval.sh [DATASET_PATH] [CHECKPOINT_PATH] ``` #### Launch ```bash # infer with checkpoint -sh run_infer.sh dataset/ilsvrc_eval train_parallel0/resnet-42_5004.ckpt +sh run_eval.sh dataset/ilsvrc_eval train_parallel0/resnet-42_5004.ckpt ``` > checkpoint can be produced in training process. diff --git a/example/resnet50_imagenet2012_THOR/eval.py b/model_zoo/resnet_thor/eval.py old mode 100755 new mode 100644 similarity index 93% rename from example/resnet50_imagenet2012_THOR/eval.py rename to model_zoo/resnet_thor/eval.py index db82b9fcac..fb911787ba --- a/example/resnet50_imagenet2012_THOR/eval.py +++ b/model_zoo/resnet_thor/eval.py @@ -17,13 +17,15 @@ eval. """ import os import argparse -from dataset_imagenet import create_dataset -from config import config + from mindspore import context -from mindspore.model_zoo.resnet import resnet50 from mindspore.train.model import Model from mindspore.train.serialization import load_checkpoint, load_param_into_net -from crossentropy import CrossEntropy + +from src.dataset_imagenet import create_dataset +from src.config import config +from src.crossentropy import CrossEntropy +from src.resnet50 import resnet50 parser = argparse.ArgumentParser(description='Image classification') parser.add_argument('--run_distribute', type=bool, default=False, help='Run distribute') diff --git a/example/resnet50_imagenet2012_THOR/run_distribute_train.sh b/model_zoo/resnet_thor/scripts/run_distribute_train.sh similarity index 93% rename from example/resnet50_imagenet2012_THOR/run_distribute_train.sh rename to model_zoo/resnet_thor/scripts/run_distribute_train.sh index e39034a912..6fa7457227 100644 --- a/example/resnet50_imagenet2012_THOR/run_distribute_train.sh +++ b/model_zoo/resnet_thor/scripts/run_distribute_train.sh @@ -32,6 +32,9 @@ then exit 1 fi +BASE_PATH=$(cd "`dirname $0`" || exit; pwd) +cd $BASE_PATH/../ || exit + ulimit -u unlimited export DEVICE_NUM=$3 export RANK_SIZE=$3 @@ -44,8 +47,7 @@ do rm -rf ./train_parallel$i mkdir ./train_parallel$i cp *.py ./train_parallel$i - cp *.sh ./train_parallel$i - cp -r model ./train_parallel$i + cp -r ./src ./train_parallel$i cd ./train_parallel$i || exit echo "start training for rank $RANK_ID, device $DEVICE_ID" diff --git a/model_zoo/resnet101/scripts/run_eval.sh b/model_zoo/resnet_thor/scripts/run_eval.sh old mode 100755 new mode 100644 similarity index 93% rename from model_zoo/resnet101/scripts/run_eval.sh rename to model_zoo/resnet_thor/scripts/run_eval.sh index 88f5d364ce..eafba5fbea --- a/model_zoo/resnet101/scripts/run_eval.sh +++ b/model_zoo/resnet_thor/scripts/run_eval.sh @@ -27,10 +27,10 @@ get_real_path(){ echo "$(realpath -m $PWD/$1)" fi } + PATH1=$(get_real_path $1) PATH2=$(get_real_path $2) -echo $PATH1 -echo $PATH2 + if [ ! -d $PATH1 ] then @@ -44,6 +44,9 @@ then exit 1 fi +BASE_PATH=$(cd "`dirname $0`" || exit; pwd) +cd $BASE_PATH/../ || exit + ulimit -u unlimited export DEVICE_NUM=1 export DEVICE_ID=0 @@ -55,9 +58,8 @@ then rm -rf ./eval fi mkdir ./eval -cp ../*.py ./eval -cp *.sh ./eval -cp -r ../src ./eval +cp *.py ./eval +cp -r ./src ./eval cd ./eval || exit env > env.log echo "start infering for device $DEVICE_ID" diff --git a/example/resnet50_imagenet2012_THOR/config.py b/model_zoo/resnet_thor/src/config.py similarity index 100% rename from example/resnet50_imagenet2012_THOR/config.py rename to model_zoo/resnet_thor/src/config.py diff --git a/example/resnet50_imagenet2012_THOR/crossentropy.py b/model_zoo/resnet_thor/src/crossentropy.py similarity index 100% rename from example/resnet50_imagenet2012_THOR/crossentropy.py rename to model_zoo/resnet_thor/src/crossentropy.py diff --git a/example/resnet50_imagenet2012_THOR/model/dataset_helper.py b/model_zoo/resnet_thor/src/dataset_helper.py similarity index 100% rename from example/resnet50_imagenet2012_THOR/model/dataset_helper.py rename to model_zoo/resnet_thor/src/dataset_helper.py diff --git a/example/resnet50_imagenet2012_THOR/dataset_imagenet.py b/model_zoo/resnet_thor/src/dataset_imagenet.py similarity index 100% rename from example/resnet50_imagenet2012_THOR/dataset_imagenet.py rename to model_zoo/resnet_thor/src/dataset_imagenet.py diff --git a/example/resnet50_imagenet2012_THOR/model/grad_reducer_thor.py b/model_zoo/resnet_thor/src/grad_reducer_thor.py similarity index 100% rename from example/resnet50_imagenet2012_THOR/model/grad_reducer_thor.py rename to model_zoo/resnet_thor/src/grad_reducer_thor.py diff --git a/example/resnet50_imagenet2012_THOR/model/model_thor.py b/model_zoo/resnet_thor/src/model_thor.py similarity index 99% rename from example/resnet50_imagenet2012_THOR/model/model_thor.py rename to model_zoo/resnet_thor/src/model_thor.py index 25e3dd7f82..13b8c49996 100644 --- a/example/resnet50_imagenet2012_THOR/model/model_thor.py +++ b/model_zoo/resnet_thor/src/model_thor.py @@ -32,7 +32,7 @@ from mindspore.train import amp from mindspore.train.callback import _InternalCallbackParam, RunContext, _CallbackManager from mindspore.train.parallel_utils import ParallelMode -from model.dataset_helper import DatasetHelper +from src.dataset_helper import DatasetHelper def _convert_type(types): diff --git a/model_zoo/resnet101/src/resnet101.py b/model_zoo/resnet_thor/src/resnet50.py old mode 100755 new mode 100644 similarity index 97% rename from model_zoo/resnet101/src/resnet101.py rename to model_zoo/resnet_thor/src/resnet50.py index 33f10fd6cb..00beac2fdf --- a/model_zoo/resnet101/src/resnet101.py +++ b/model_zoo/resnet_thor/src/resnet50.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ -"""ResNet101.""" +"""ResNet.""" import numpy as np import mindspore.nn as nn from mindspore.ops import operations as P @@ -240,21 +240,22 @@ class ResNet(nn.Cell): return out -def resnet101(class_num=1001): + +def resnet50(class_num=10): """ - Get ResNet101 neural network. + Get ResNet50 neural network. Args: class_num (int): Class number. Returns: - Cell, cell instance of ResNet101 neural network. + Cell, cell instance of ResNet50 neural network. Examples: - >>> net = resnet101(1001) + >>> net = resnet50(10) """ return ResNet(ResidualBlock, - [3, 4, 23, 3], + [3, 4, 6, 3], [64, 256, 512, 1024], [256, 512, 1024, 2048], [1, 2, 2, 2], diff --git a/example/resnet50_imagenet2012_THOR/model/resnet.py b/model_zoo/resnet_thor/src/resnet_thor.py similarity index 99% rename from example/resnet50_imagenet2012_THOR/model/resnet.py rename to model_zoo/resnet_thor/src/resnet_thor.py index f3305022e8..50e694854f 100644 --- a/example/resnet50_imagenet2012_THOR/model/resnet.py +++ b/model_zoo/resnet_thor/src/resnet_thor.py @@ -19,7 +19,7 @@ import mindspore.nn as nn from mindspore.common.tensor import Tensor from mindspore.ops import operations as P -from model.thor_layer import Conv2d_Thor, Dense_Thor +from src.thor_layer import Conv2d_Thor, Dense_Thor def calculate_gain(nonlinearity, param=None): diff --git a/example/resnet50_imagenet2012_THOR/model/thor.py b/model_zoo/resnet_thor/src/thor.py similarity index 99% rename from example/resnet50_imagenet2012_THOR/model/thor.py rename to model_zoo/resnet_thor/src/thor.py index 6786cb7485..cc82e6e369 100644 --- a/example/resnet50_imagenet2012_THOR/model/thor.py +++ b/model_zoo/resnet_thor/src/thor.py @@ -21,7 +21,7 @@ from mindspore.common.tensor import Tensor from mindspore.nn.optim.optimizer import Optimizer from mindspore.ops import functional as F, composite as C, operations as P from mindspore.parallel._utils import _get_device_num, _get_mirror_mean -from model.grad_reducer_thor import DistributedGradReducerThor +from src.grad_reducer_thor import DistributedGradReducerThor momentum_opt = C.MultitypeFuncGraph("momentum_opt") diff --git a/example/resnet50_imagenet2012_THOR/model/thor_layer.py b/model_zoo/resnet_thor/src/thor_layer.py similarity index 100% rename from example/resnet50_imagenet2012_THOR/model/thor_layer.py rename to model_zoo/resnet_thor/src/thor_layer.py diff --git a/example/resnet50_imagenet2012_THOR/train.py b/model_zoo/resnet_thor/train.py similarity index 96% rename from example/resnet50_imagenet2012_THOR/train.py rename to model_zoo/resnet_thor/train.py index 309018da57..47f56a0676 100644 --- a/example/resnet50_imagenet2012_THOR/train.py +++ b/model_zoo/resnet_thor/train.py @@ -26,13 +26,12 @@ from mindspore.parallel._auto_parallel_context import auto_parallel_context from mindspore.train.callback import ModelCheckpoint, CheckpointConfig, LossMonitor, TimeMonitor from mindspore.train.loss_scale_manager import FixedLossScaleManager from mindspore.train.model import ParallelMode -from model.model_thor import Model -from model.resnet import resnet50 -from model.thor import THOR - -from config import config -from crossentropy import CrossEntropy -from dataset_imagenet import create_dataset +from src.model_thor import Model +from src.resnet_thor import resnet50 +from src.thor import THOR +from src.config import config +from src.crossentropy import CrossEntropy +from src.dataset_imagenet import create_dataset random.seed(1) np.random.seed(1) diff --git a/model_zoo/resnext50/README.md b/model_zoo/resnext50/README.md new file mode 100644 index 0000000000..c44844eecc --- /dev/null +++ b/model_zoo/resnext50/README.md @@ -0,0 +1,128 @@ +# ResNext50 Example + +## Description + +This is an example of training ResNext50 with ImageNet dataset in Mindspore. + +## Requirements + +- Install [Mindspore](http://www.mindspore.cn/install/en). +- Downlaod the dataset ImageNet2012. + +## Structure + +```shell +. +└─resnext50 + ├─README.md + ├─scripts + ├─run_standalone_train.sh # launch standalone training(1p) + ├─run_distribute_train.sh # launch distributed training(8p) + └─run_eval.sh # launch evaluating + ├─src + ├─backbone + ├─_init_.py # initalize + ├─resnet.py # resnext50 backbone + ├─utils + ├─_init_.py # initalize + ├─cunstom_op.py # network operation + ├─logging.py # print log + ├─optimizers_init_.py # get parameters + ├─sampler.py # distributed sampler + ├─var_init_.py # calculate gain value + ├─_init_.py # initalize + ├─config.py # parameter configuration + ├─crossentropy.py # CrossEntropy loss function + ├─dataset.py # data preprocessing + ├─head.py # commom head + ├─image_classification.py # get resnet + ├─linear_warmup.py # linear warmup learning rate + ├─warmup_cosine_annealing.py # learning rate each step + ├─warmup_step_lr.py # warmup step learning rate + ├─eval.py # eval net + └─train.py # train net + +``` + +## Parameter Configuration + +Parameters for both training and evaluating can be set in config.py + +``` +"image_height": '224,224' # image size +"num_classes": 1000, # dataset class number +"per_batch_size": 128, # batch size of input tensor +"lr": 0.05, # base learning rate +"lr_scheduler": 'cosine_annealing', # learning rate mode +"lr_epochs": '30,60,90,120', # epoch of lr changing +"lr_gamma": 0.1, # decrease lr by a factor of exponential lr_scheduler +"eta_min": 0, # eta_min in cosine_annealing scheduler +"T_max": 150, # T-max in cosine_annealing scheduler +"max_epoch": 150, # max epoch num to train the model +"backbone": 'resnext50', # backbone metwork +"warmup_epochs" : 1, # warmup epoch +"weight_decay": 0.0001, # weight decay +"momentum": 0.9, # momentum +"is_dynamic_loss_scale": 0, # dynamic loss scale +"loss_scale": 1024, # loss scale +"label_smooth": 1, # label_smooth +"label_smooth_factor": 0.1, # label_smooth_factor +"ckpt_interval": 2000, # ckpt_interval +"ckpt_path": 'outputs/', # checkpoint save location +"is_save_on_master": 1, +"rank": 0, # local rank of distributed +"group_size": 1 # world size of distributed +``` + +## Running the example + +### Train + +#### Usage + +``` +# distribute training example(8p) +sh run_distribute_train.sh MINDSPORE_HCCL_CONFIG_PATH DATA_PATH +# standalone training +sh run_standalone_train.sh DEVICE_ID DATA_PATH +``` + +#### Launch + +```bash +# distributed training example(8p) +sh scripts/run_distribute_train.sh MINDSPORE_HCCL_CONFIG_PATH /ImageNet/train +# standalone training example +sh scripts/run_standalone_train.sh 0 /ImageNet_Original/train +``` + +#### Result + +You can find checkpoint file together with result in log. + +### Evaluation + +#### Usage + +``` +# Evaluation +sh run_eval.sh DEVICE_ID DATA_PATH PRETRAINED_CKPT_PATH +``` + +#### Launch + +```bash +# Evaluation with checkpoint +sh scripts/run_eval.sh 0 /opt/npu/datasets/classification/val /resnext50_100.ckpt +``` + +> checkpoint can be produced in training process. + +#### Result + +Evaluation result will be stored in the scripts path. Under this, you can find result like the followings in log. + +``` +acc=78,16%(TOP1) +acc=93.88%(TOP5) +``` \ No newline at end of file diff --git a/model_zoo/resnext50/eval.py b/model_zoo/resnext50/eval.py new file mode 100644 index 0000000000..ff5c83843e --- /dev/null +++ b/model_zoo/resnext50/eval.py @@ -0,0 +1,243 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Eval""" +import os +import time +import argparse +import datetime +import glob +import numpy as np +import mindspore.nn as nn + +from mindspore import Tensor, context +from mindspore.communication.management import init, get_rank, get_group_size, release +from mindspore.train.serialization import load_checkpoint, load_param_into_net +from mindspore.ops import operations as P +from mindspore.ops import functional as F +from mindspore.common import dtype as mstype + +from src.utils.logging import get_logger +from src.image_classification import get_network +from src.dataset import classification_dataset +from src.config import config + +devid = int(os.getenv('DEVICE_ID')) +context.set_context(mode=context.GRAPH_MODE, enable_auto_mixed_precision=True, + device_target="Ascend", save_graphs=False, device_id=devid) + + + +class ParameterReduce(nn.Cell): + """ParameterReduce""" + def __init__(self): + super(ParameterReduce, self).__init__() + self.cast = P.Cast() + self.reduce = P.AllReduce() + + def construct(self, x): + one = self.cast(F.scalar_to_array(1.0), mstype.float32) + out = x * one + ret = self.reduce(out) + return ret + + +def parse_args(cloud_args=None): + """parse_args""" + parser = argparse.ArgumentParser('mindspore classification test') + + # dataset related + parser.add_argument('--data_dir', type=str, default='/opt/npu/datasets/classification/val', help='eval data dir') + parser.add_argument('--per_batch_size', default=32, type=int, help='batch size for per npu') + # network related + parser.add_argument('--graph_ckpt', type=int, default=1, help='graph ckpt or feed ckpt') + parser.add_argument('--pretrained', default='', type=str, help='fully path of pretrained model to load. ' + 'If it is a direction, it will test all ckpt') + + # logging related + parser.add_argument('--log_path', type=str, default='outputs/', help='path to save log') + parser.add_argument('--is_distributed', type=int, default=0, help='if multi device') + + # roma obs + parser.add_argument('--train_url', type=str, default="", help='train url') + + args, _ = parser.parse_known_args() + args = merge_args(args, cloud_args) + args.image_size = config.image_size + args.num_classes = config.num_classes + args.backbone = config.backbone + args.rank = config.rank + args.group_size = config.group_size + + args.image_size = list(map(int, args.image_size.split(','))) + + return args + + +def get_top5_acc(top5_arg, gt_class): + sub_count = 0 + for top5, gt in zip(top5_arg, gt_class): + if gt in top5: + sub_count += 1 + return sub_count + +def merge_args(args, cloud_args): + """merge_args""" + args_dict = vars(args) + if isinstance(cloud_args, dict): + for key in cloud_args.keys(): + val = cloud_args[key] + if key in args_dict and val: + arg_type = type(args_dict[key]) + if arg_type is not type(None): + val = arg_type(val) + args_dict[key] = val + return args + +def test(cloud_args=None): + """test""" + args = parse_args(cloud_args) + + # init distributed + if args.is_distributed: + init() + args.rank = get_rank() + args.group_size = get_group_size() + + args.outputs_dir = os.path.join(args.log_path, + datetime.datetime.now().strftime('%Y-%m-%d_time_%H_%M_%S')) + + args.logger = get_logger(args.outputs_dir, args.rank) + args.logger.save_args(args) + + # network + args.logger.important_info('start create network') + if os.path.isdir(args.pretrained): + models = list(glob.glob(os.path.join(args.pretrained, '*.ckpt'))) + print(models) + if args.graph_ckpt: + f = lambda x: -1 * int(os.path.splitext(os.path.split(x)[-1])[0].split('-')[-1].split('_')[0]) + else: + f = lambda x: -1 * int(os.path.splitext(os.path.split(x)[-1])[0].split('_')[-1]) + args.models = sorted(models, key=f) + else: + args.models = [args.pretrained,] + + for model in args.models: + de_dataset = classification_dataset(args.data_dir, image_size=args.image_size, + per_batch_size=args.per_batch_size, + max_epoch=1, rank=args.rank, group_size=args.group_size, + mode='eval') + eval_dataloader = de_dataset.create_tuple_iterator() + network = get_network(args.backbone, args.num_classes) + if network is None: + raise NotImplementedError('not implement {}'.format(args.backbone)) + + param_dict = load_checkpoint(model) + param_dict_new = {} + for key, values in param_dict.items(): + if key.startswith('moments.'): + continue + elif key.startswith('network.'): + param_dict_new[key[8:]] = values + else: + param_dict_new[key] = values + + load_param_into_net(network, param_dict_new) + args.logger.info('load model {} success'.format(model)) + + # must add + network.add_flags_recursive(fp16=True) + + img_tot = 0 + top1_correct = 0 + top5_correct = 0 + network.set_train(False) + t_end = time.time() + it = 0 + for data, gt_classes in eval_dataloader: + output = network(Tensor(data, mstype.float32)) + output = output.asnumpy() + + top1_output = np.argmax(output, (-1)) + top5_output = np.argsort(output)[:, -5:] + + t1_correct = np.equal(top1_output, gt_classes).sum() + top1_correct += t1_correct + top5_correct += get_top5_acc(top5_output, gt_classes) + img_tot += args.per_batch_size + + if args.rank == 0 and it == 0: + t_end = time.time() + it = 1 + if args.rank == 0: + time_used = time.time() - t_end + fps = (img_tot - args.per_batch_size) * args.group_size / time_used + args.logger.info('Inference Performance: {:.2f} img/sec'.format(fps)) + results = [[top1_correct], [top5_correct], [img_tot]] + args.logger.info('before results={}'.format(results)) + if args.is_distributed: + model_md5 = model.replace('/', '') + tmp_dir = '/cache' + if not os.path.exists(tmp_dir): + os.mkdir(tmp_dir) + top1_correct_npy = '/cache/top1_rank_{}_{}.npy'.format(args.rank, model_md5) + top5_correct_npy = '/cache/top5_rank_{}_{}.npy'.format(args.rank, model_md5) + img_tot_npy = '/cache/img_tot_rank_{}_{}.npy'.format(args.rank, model_md5) + np.save(top1_correct_npy, top1_correct) + np.save(top5_correct_npy, top5_correct) + np.save(img_tot_npy, img_tot) + while True: + rank_ok = True + for other_rank in range(args.group_size): + top1_correct_npy = '/cache/top1_rank_{}_{}.npy'.format(other_rank, model_md5) + top5_correct_npy = '/cache/top5_rank_{}_{}.npy'.format(other_rank, model_md5) + img_tot_npy = '/cache/img_tot_rank_{}_{}.npy'.format(other_rank, model_md5) + if not os.path.exists(top1_correct_npy) or not os.path.exists(top5_correct_npy) or \ + not os.path.exists(img_tot_npy): + rank_ok = False + if rank_ok: + break + + top1_correct_all = 0 + top5_correct_all = 0 + img_tot_all = 0 + for other_rank in range(args.group_size): + top1_correct_npy = '/cache/top1_rank_{}_{}.npy'.format(other_rank, model_md5) + top5_correct_npy = '/cache/top5_rank_{}_{}.npy'.format(other_rank, model_md5) + img_tot_npy = '/cache/img_tot_rank_{}_{}.npy'.format(other_rank, model_md5) + top1_correct_all += np.load(top1_correct_npy) + top5_correct_all += np.load(top5_correct_npy) + img_tot_all += np.load(img_tot_npy) + results = [[top1_correct_all], [top5_correct_all], [img_tot_all]] + results = np.array(results) + else: + results = np.array(results) + + args.logger.info('after results={}'.format(results)) + top1_correct = results[0, 0] + top5_correct = results[1, 0] + img_tot = results[2, 0] + acc1 = 100.0 * top1_correct / img_tot + acc5 = 100.0 * top5_correct / img_tot + args.logger.info('after allreduce eval: top1_correct={}, tot={},' + 'acc={:.2f}%(TOP1)'.format(top1_correct, img_tot, acc1)) + args.logger.info('after allreduce eval: top5_correct={}, tot={},' + 'acc={:.2f}%(TOP5)'.format(top5_correct, img_tot, acc5)) + if args.is_distributed: + release() + + +if __name__ == "__main__": + test() diff --git a/model_zoo/resnext50/scripts/run_distribute_train.sh b/model_zoo/resnext50/scripts/run_distribute_train.sh new file mode 100644 index 0000000000..226cfe3cb6 --- /dev/null +++ b/model_zoo/resnext50/scripts/run_distribute_train.sh @@ -0,0 +1,55 @@ +#!/bin/bash +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +DATA_DIR=$2 +export RANK_TABLE_FILE=$1 +export RANK_SIZE=8 +PATH_CHECKPOINT="" +if [ $# == 3 ] +then + PATH_CHECKPOINT=$3 +fi + +cores=`cat /proc/cpuinfo|grep "processor" |wc -l` +echo "the number of logical core" $cores +avg_core_per_rank=`expr $cores \/ $RANK_SIZE` +core_gap=`expr $avg_core_per_rank \- 1` +echo "avg_core_per_rank" $avg_core_per_rank +echo "core_gap" $core_gap +for((i=0;i env.log + taskset -c $cmdopt python ../train.py \ + --is_distribute=1 \ + --device_id=$DEVICE_ID \ + --pretrained=$PATH_CHECKPOINT \ + --data_dir=$DATA_DIR > log.txt 2>&1 & + cd ../ +done diff --git a/model_zoo/resnext50/scripts/run_eval.sh b/model_zoo/resnext50/scripts/run_eval.sh new file mode 100644 index 0000000000..610faa874e --- /dev/null +++ b/model_zoo/resnext50/scripts/run_eval.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +DEVICE_ID=$1 +DATA_DIR=$2 +PATH_CHECKPOINT=$3 + +python eval.py \ + --device_id=$DEVICE_ID \ + --pretrained=$PATH_CHECKPOINT \ + --data_dir=$DATA_DIR > log.txt 2>&1 & diff --git a/model_zoo/resnext50/scripts/run_standalone_train.sh b/model_zoo/resnext50/scripts/run_standalone_train.sh new file mode 100644 index 0000000000..ca5d8206f3 --- /dev/null +++ b/model_zoo/resnext50/scripts/run_standalone_train.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +DEVICE_ID=$1 +DATA_DIR=$2 +PATH_CHECKPOINT="" +if [ $# == 3 ] +then + PATH_CHECKPOINT=$3 +fi + +python train.py \ + --is_distribute=0 \ + --device_id=$DEVICE_ID \ + --pretrained=$PATH_CHECKPOINT \ + --data_dir=$DATA_DIR > log.txt 2>&1 & + diff --git a/example/cv_to_mindrecord/ImageNet_Similar_Perf/imagenet/__init__.py b/model_zoo/resnext50/src/__init__.py similarity index 100% rename from example/cv_to_mindrecord/ImageNet_Similar_Perf/imagenet/__init__.py rename to model_zoo/resnext50/src/__init__.py diff --git a/example/nlp_to_mindrecord/aclImdb_preprocess/run_read.sh b/model_zoo/resnext50/src/backbone/__init__.py similarity index 94% rename from example/nlp_to_mindrecord/aclImdb_preprocess/run_read.sh rename to model_zoo/resnext50/src/backbone/__init__.py index 1ffe4de1cf..b757d07410 100644 --- a/example/nlp_to_mindrecord/aclImdb_preprocess/run_read.sh +++ b/model_zoo/resnext50/src/backbone/__init__.py @@ -1,4 +1,3 @@ -#!/bin/bash # Copyright 2020 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,5 +12,5 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ - -python create_dataset.py +"""resnet""" +from .resnet import * diff --git a/model_zoo/resnext50/src/backbone/resnet.py b/model_zoo/resnext50/src/backbone/resnet.py new file mode 100644 index 0000000000..5b69f9e1f5 --- /dev/null +++ b/model_zoo/resnext50/src/backbone/resnet.py @@ -0,0 +1,273 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +ResNet based ResNext +""" +import mindspore.nn as nn +from mindspore.ops.operations import TensorAdd, Split, Concat +from mindspore.ops import operations as P +from mindspore.common.initializer import TruncatedNormal + +from src.utils.cunstom_op import SEBlock, GroupConv + + +__all__ = ['ResNet', 'resnext50'] + + +def weight_variable(shape, factor=0.1): + return TruncatedNormal(0.02) + + +def conv7x7(in_channels, out_channels, stride=1, padding=3, has_bias=False, groups=1): + return nn.Conv2d(in_channels, out_channels, kernel_size=7, stride=stride, has_bias=has_bias, + padding=padding, pad_mode="pad", group=groups) + + +def conv3x3(in_channels, out_channels, stride=1, padding=1, has_bias=False, groups=1): + return nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=stride, has_bias=has_bias, + padding=padding, pad_mode="pad", group=groups) + + +def conv1x1(in_channels, out_channels, stride=1, padding=0, has_bias=False, groups=1): + return nn.Conv2d(in_channels, out_channels, kernel_size=1, stride=stride, has_bias=has_bias, + padding=padding, pad_mode="pad", group=groups) + + +class _DownSample(nn.Cell): + """ + Downsample for ResNext-ResNet. + + Args: + in_channels (int): Input channels. + out_channels (int): Output channels. + stride (int): Stride size for the 1*1 convolutional layer. + + Returns: + Tensor, output tensor. + + Examples: + >>>DownSample(32, 64, 2) + """ + def __init__(self, in_channels, out_channels, stride): + super(_DownSample, self).__init__() + self.conv = conv1x1(in_channels, out_channels, stride=stride, padding=0) + self.bn = nn.BatchNorm2d(out_channels) + + def construct(self, x): + out = self.conv(x) + out = self.bn(out) + return out + +class BasicBlock(nn.Cell): + """ + ResNet basic block definition. + + Args: + in_channels (int): Input channels. + out_channels (int): Output channels. + stride (int): Stride size for the first convolutional layer. Default: 1. + + Returns: + Tensor, output tensor. + + Examples: + >>>BasicBlock(32, 256, stride=2) + """ + expansion = 1 + + def __init__(self, in_channels, out_channels, stride=1, down_sample=None, use_se=False, **kwargs): + super(BasicBlock, self).__init__() + self.conv1 = conv3x3(in_channels, out_channels, stride=stride) + self.bn1 = nn.BatchNorm2d(out_channels) + self.relu = P.ReLU() + self.conv2 = conv3x3(out_channels, out_channels, stride=1) + self.bn2 = nn.BatchNorm2d(out_channels) + + self.use_se = use_se + if self.use_se: + self.se = SEBlock(out_channels) + + self.down_sample_flag = False + if down_sample is not None: + self.down_sample = down_sample + self.down_sample_flag = True + + self.add = TensorAdd() + + def construct(self, x): + identity = x + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + out = self.conv2(out) + out = self.bn2(out) + + if self.use_se: + out = self.se(out) + + if self.down_sample_flag: + identity = self.down_sample(x) + + out = self.add(out, identity) + out = self.relu(out) + return out + +class Bottleneck(nn.Cell): + """ + ResNet Bottleneck block definition. + + Args: + in_channels (int): Input channels. + out_channels (int): Output channels. + stride (int): Stride size for the initial convolutional layer. Default: 1. + + Returns: + Tensor, the ResNet unit's output. + + Examples: + >>>Bottleneck(3, 256, stride=2) + """ + expansion = 4 + + def __init__(self, in_channels, out_channels, stride=1, down_sample=None, + base_width=64, groups=1, use_se=False, **kwargs): + super(Bottleneck, self).__init__() + + width = int(out_channels * (base_width / 64.0)) * groups + self.groups = groups + self.conv1 = conv1x1(in_channels, width, stride=1) + self.bn1 = nn.BatchNorm2d(width) + self.relu = P.ReLU() + + self.conv3x3s = nn.CellList() + + self.conv2 = GroupConv(width, width, 3, stride, pad=1, groups=groups) + self.op_split = Split(axis=1, output_num=self.groups) + self.op_concat = Concat(axis=1) + + self.bn2 = nn.BatchNorm2d(width) + self.conv3 = conv1x1(width, out_channels * self.expansion, stride=1) + self.bn3 = nn.BatchNorm2d(out_channels * self.expansion) + + self.use_se = use_se + if self.use_se: + self.se = SEBlock(out_channels * self.expansion) + + self.down_sample_flag = False + if down_sample is not None: + self.down_sample = down_sample + self.down_sample_flag = True + + self.cast = P.Cast() + self.add = TensorAdd() + + def construct(self, x): + identity = x + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + out = self.conv3(out) + out = self.bn3(out) + + if self.use_se: + out = self.se(out) + + if self.down_sample_flag: + identity = self.down_sample(x) + + out = self.add(out, identity) + out = self.relu(out) + return out + +class ResNet(nn.Cell): + """ + ResNet architecture. + + Args: + block (cell): Block for network. + layers (list): Numbers of block in different layers. + width_per_group (int): Width of every group. + groups (int): Groups number. + + Returns: + Tuple, output tensor tuple. + + Examples: + >>>ResNet() + """ + def __init__(self, block, layers, width_per_group=64, groups=1, use_se=False): + super(ResNet, self).__init__() + self.in_channels = 64 + self.groups = groups + self.base_width = width_per_group + + self.conv = conv7x7(3, self.in_channels, stride=2, padding=3) + self.bn = nn.BatchNorm2d(self.in_channels) + self.relu = P.ReLU() + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, pad_mode='same') + + self.layer1 = self._make_layer(block, 64, layers[0], use_se=use_se) + self.layer2 = self._make_layer(block, 128, layers[1], stride=2, use_se=use_se) + self.layer3 = self._make_layer(block, 256, layers[2], stride=2, use_se=use_se) + self.layer4 = self._make_layer(block, 512, layers[3], stride=2, use_se=use_se) + + self.out_channels = 512 * block.expansion + self.cast = P.Cast() + + def construct(self, x): + x = self.conv(x) + x = self.bn(x) + x = self.relu(x) + x = self.maxpool(x) + x = self.layer1(x) + x = self.layer2(x) + x = self.layer3(x) + x = self.layer4(x) + + return x + + def _make_layer(self, block, out_channels, blocks_num, stride=1, use_se=False): + """_make_layer""" + down_sample = None + if stride != 1 or self.in_channels != out_channels * block.expansion: + down_sample = _DownSample(self.in_channels, + out_channels * block.expansion, + stride=stride) + + layers = [] + layers.append(block(self.in_channels, + out_channels, + stride=stride, + down_sample=down_sample, + base_width=self.base_width, + groups=self.groups, + use_se=use_se)) + self.in_channels = out_channels * block.expansion + for _ in range(1, blocks_num): + layers.append(block(self.in_channels, out_channels, + base_width=self.base_width, groups=self.groups, use_se=use_se)) + + return nn.SequentialCell(layers) + + def get_out_channels(self): + return self.out_channels + + +def resnext50(): + return ResNet(Bottleneck, [3, 4, 6, 3], width_per_group=4, groups=32) diff --git a/model_zoo/resnet101/src/config.py b/model_zoo/resnext50/src/config.py old mode 100755 new mode 100644 similarity index 63% rename from model_zoo/resnet101/src/config.py rename to model_zoo/resnext50/src/config.py index 594b28522a..c1a12aa14e --- a/model_zoo/resnet101/src/config.py +++ b/model_zoo/resnext50/src/config.py @@ -12,29 +12,34 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ -""" -network config setting, will be used in train.py and eval.py -""" +"""config""" from easydict import EasyDict as ed config = ed({ - "class_num": 1001, - "batch_size": 32, - "loss_scale": 1024, + "image_size": '224,224', + "num_classes": 1000, + + "lr": 0.4, + "lr_scheduler": 'cosine_annealing', + "lr_epochs": '30,60,90,120', + "lr_gamma": 0.1, + "eta_min": 0, + "T_max": 150, + "max_epoch": 150, + "backbone": 'resnext50', + "warmup_epochs": 1, + + "weight_decay": 0.0001, "momentum": 0.9, - "weight_decay": 1e-4, - "epoch_size": 120, - "pretrain_epoch_size": 0, - "buffer_size": 1000, - "image_height": 224, - "image_width": 224, - "save_checkpoint": True, - "save_checkpoint_epochs": 5, - "keep_checkpoint_max": 10, - "save_checkpoint_path": "./", - "warmup_epochs": 0, - "lr_decay_mode": "cosine", + "is_dynamic_loss_scale": 0, + "loss_scale": 1024, "label_smooth": 1, "label_smooth_factor": 0.1, - "lr": 0.1 + + "ckpt_interval": 1250, + "ckpt_path": 'outputs/', + "is_save_on_master": 1, + + "rank": 0, + "group_size": 1 }) diff --git a/model_zoo/resnext50/src/crossentropy.py b/model_zoo/resnext50/src/crossentropy.py new file mode 100644 index 0000000000..a0e509a51e --- /dev/null +++ b/model_zoo/resnext50/src/crossentropy.py @@ -0,0 +1,41 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +define loss function for network. +""" +from mindspore.nn.loss.loss import _Loss +from mindspore.ops import operations as P +from mindspore.ops import functional as F +from mindspore import Tensor +from mindspore.common import dtype as mstype +import mindspore.nn as nn + +class CrossEntropy(_Loss): + """ + the redefined loss function with SoftmaxCrossEntropyWithLogits. + """ + def __init__(self, smooth_factor=0., num_classes=1000): + super(CrossEntropy, self).__init__() + self.onehot = P.OneHot() + self.on_value = Tensor(1.0 - smooth_factor, mstype.float32) + self.off_value = Tensor(1.0 * smooth_factor / (num_classes -1), mstype.float32) + self.ce = nn.SoftmaxCrossEntropyWithLogits() + self.mean = P.ReduceMean(False) + + def construct(self, logit, label): + one_hot_label = self.onehot(label, F.shape(logit)[1], self.on_value, self.off_value) + loss = self.ce(logit, one_hot_label) + loss = self.mean(loss, 0) + return loss diff --git a/model_zoo/resnext50/src/dataset.py b/model_zoo/resnext50/src/dataset.py new file mode 100644 index 0000000000..9608e3c790 --- /dev/null +++ b/model_zoo/resnext50/src/dataset.py @@ -0,0 +1,155 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +dataset processing. +""" +import os +from mindspore.common import dtype as mstype +import mindspore.dataset as de +import mindspore.dataset.transforms.c_transforms as C +import mindspore.dataset.transforms.vision.c_transforms as V_C +from PIL import Image, ImageFile +from src.utils.sampler import DistributedSampler + +ImageFile.LOAD_TRUNCATED_IMAGES = True + +class TxtDataset(): + """ + create txt dataset. + + Args: + Returns: + de_dataset. + """ + def __init__(self, root, txt_name): + super(TxtDataset, self).__init__() + self.imgs = [] + self.labels = [] + fin = open(txt_name, "r") + for line in fin: + img_name, label = line.strip().split(' ') + self.imgs.append(os.path.join(root, img_name)) + self.labels.append(int(label)) + fin.close() + + def __getitem__(self, index): + img = Image.open(self.imgs[index]).convert('RGB') + return img, self.labels[index] + + def __len__(self): + return len(self.imgs) + + +def classification_dataset(data_dir, image_size, per_batch_size, max_epoch, rank, group_size, + mode='train', + input_mode='folder', + root='', + num_parallel_workers=None, + shuffle=None, + sampler=None, + class_indexing=None, + drop_remainder=True, + transform=None, + target_transform=None): + """ + A function that returns a dataset for classification. The mode of input dataset could be "folder" or "txt". + If it is "folder", all images within one folder have the same label. If it is "txt", all paths of images + are written into a textfile. + + Args: + data_dir (str): Path to the root directory that contains the dataset for "input_mode="folder"". + Or path of the textfile that contains every image's path of the dataset. + image_size (str): Size of the input images. + per_batch_size (int): the batch size of evey step during training. + max_epoch (int): the number of epochs. + rank (int): The shard ID within num_shards (default=None). + group_size (int): Number of shards that the dataset should be divided + into (default=None). + mode (str): "train" or others. Default: " train". + input_mode (str): The form of the input dataset. "folder" or "txt". Default: "folder". + root (str): the images path for "input_mode="txt"". Default: " ". + num_parallel_workers (int): Number of workers to read the data. Default: None. + shuffle (bool): Whether or not to perform shuffle on the dataset + (default=None, performs shuffle). + sampler (Sampler): Object used to choose samples from the dataset. Default: None. + class_indexing (dict): A str-to-int mapping from folder name to index + (default=None, the folder names will be sorted + alphabetically and each class will be given a + unique index starting from 0). + + Examples: + >>> from mindvision.common.datasets.classification import classification_dataset + >>> # path to imagefolder directory. This directory needs to contain sub-directories which contain the images + >>> dataset_dir = "/path/to/imagefolder_directory" + >>> de_dataset = classification_dataset(train_data_dir, image_size=[224, 244], + >>> per_batch_size=64, max_epoch=100, + >>> rank=0, group_size=4) + >>> # Path of the textfile that contains every image's path of the dataset. + >>> dataset_dir = "/path/to/dataset/images/train.txt" + >>> images_dir = "/path/to/dataset/images" + >>> de_dataset = classification_dataset(train_data_dir, image_size=[224, 244], + >>> per_batch_size=64, max_epoch=100, + >>> rank=0, group_size=4, + >>> input_mode="txt", root=images_dir) + """ + + mean = [0.485 * 255, 0.456 * 255, 0.406 * 255] + std = [0.229 * 255, 0.224 * 255, 0.225 * 255] + + if transform is None: + if mode == 'train': + transform_img = [ + V_C.RandomCropDecodeResize(image_size, scale=(0.08, 1.0), ratio=(0.75, 1.333)), + V_C.RandomHorizontalFlip(prob=0.5), + V_C.RandomColorAdjust(brightness=0.4, contrast=0.4, saturation=0.4), + V_C.Normalize(mean=mean, std=std), + V_C.HWC2CHW() + ] + else: + transform_img = [ + V_C.Decode(), + V_C.Resize((256, 256)), + V_C.CenterCrop(image_size), + V_C.Normalize(mean=mean, std=std), + V_C.HWC2CHW() + ] + else: + transform_img = transform + + if target_transform is None: + transform_label = [C.TypeCast(mstype.int32)] + else: + transform_label = target_transform + + if input_mode == 'folder': + de_dataset = de.ImageFolderDatasetV2(data_dir, num_parallel_workers=num_parallel_workers, + shuffle=shuffle, sampler=sampler, class_indexing=class_indexing, + num_shards=group_size, shard_id=rank) + else: + dataset = TxtDataset(root, data_dir) + sampler = DistributedSampler(dataset, rank, group_size, shuffle=shuffle) + de_dataset = de.GeneratorDataset(dataset, ["image", "label"], sampler=sampler) + de_dataset.set_dataset_size(len(sampler)) + + de_dataset = de_dataset.map(input_columns="image", num_parallel_workers=8, operations=transform_img) + de_dataset = de_dataset.map(input_columns="label", num_parallel_workers=8, operations=transform_label) + + columns_to_project = ["image", "label"] + de_dataset = de_dataset.project(columns=columns_to_project) + + de_dataset = de_dataset.batch(per_batch_size, drop_remainder=drop_remainder) + de_dataset = de_dataset.repeat(max_epoch) + + return de_dataset diff --git a/model_zoo/resnext50/src/head.py b/model_zoo/resnext50/src/head.py new file mode 100644 index 0000000000..a7bd85c906 --- /dev/null +++ b/model_zoo/resnext50/src/head.py @@ -0,0 +1,42 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +common architecture. +""" +import mindspore.nn as nn +from src.utils.cunstom_op import GlobalAvgPooling + +__all__ = ['CommonHead'] + +class CommonHead(nn.Cell): + """ + commom architecture definition. + + Args: + num_classes (int): Number of classes. + out_channels (int): Output channels. + + Returns: + Tensor, output tensor. + """ + def __init__(self, num_classes, out_channels): + super(CommonHead, self).__init__() + self.avgpool = GlobalAvgPooling() + self.fc = nn.Dense(out_channels, num_classes, has_bias=True).add_flags_recursive(fp16=True) + + def construct(self, x): + x = self.avgpool(x) + x = self.fc(x) + return x diff --git a/model_zoo/resnext50/src/image_classification.py b/model_zoo/resnext50/src/image_classification.py new file mode 100644 index 0000000000..d8003ad200 --- /dev/null +++ b/model_zoo/resnext50/src/image_classification.py @@ -0,0 +1,85 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +Image classifiation. +""" +import math +import mindspore.nn as nn +from mindspore.common import initializer as init +import src.backbone as backbones +import src.head as heads +from src.utils.var_init import default_recurisive_init, KaimingNormal + + +class ImageClassificationNetwork(nn.Cell): + """ + architecture of image classification network. + + Args: + Returns: + Tensor, output tensor. + """ + def __init__(self, backbone, head): + super(ImageClassificationNetwork, self).__init__() + self.backbone = backbone + self.head = head + + def construct(self, x): + x = self.backbone(x) + x = self.head(x) + return x + +class Resnet(ImageClassificationNetwork): + """ + Resnet architecture. + Args: + backbone_name (string): backbone. + num_classes (int): number of classes. + Returns: + Resnet. + """ + def __init__(self, backbone_name, num_classes): + self.backbone_name = backbone_name + backbone = backbones.__dict__[self.backbone_name]() + out_channels = backbone.get_out_channels() + head = heads.CommonHead(num_classes=num_classes, out_channels=out_channels) + super(Resnet, self).__init__(backbone, head) + + default_recurisive_init(self) + + for cell in self.cells_and_names(): + if isinstance(cell, nn.Conv2d): + cell.weight.default_input = init.initializer( + KaimingNormal(a=math.sqrt(5), mode='fan_out', nonlinearity='relu'), + cell.weight.default_input.shape, cell.weight.default_input.dtype).to_tensor() + elif isinstance(cell, nn.BatchNorm2d): + cell.gamma.default_input = init.initializer('ones', cell.gamma.default_input.shape).to_tensor() + cell.beta.default_input = init.initializer('zeros', cell.beta.default_input.shape).to_tensor() + + # Zero-initialize the last BN in each residual branch, + # so that the residual branch starts with zeros, and each residual block behaves like an identity. + # This improves the model by 0.2~0.3% according to https://arxiv.org/abs/1706.02677 + for cell in self.cells_and_names(): + if isinstance(cell, backbones.resnet.Bottleneck): + cell.bn3.gamma.default_input = init.initializer('zeros', cell.bn3.gamma.default_input.shape).to_tensor() + elif isinstance(cell, backbones.resnet.BasicBlock): + cell.bn2.gamma.default_input = init.initializer('zeros', cell.bn2.gamma.default_input.shape).to_tensor() + + + +def get_network(backbone_name, num_classes): + if backbone_name in ['resnext50']: + return Resnet(backbone_name, num_classes) + return None diff --git a/model_zoo/resnext50/src/linear_warmup.py b/model_zoo/resnext50/src/linear_warmup.py new file mode 100644 index 0000000000..af0bac631a --- /dev/null +++ b/model_zoo/resnext50/src/linear_warmup.py @@ -0,0 +1,21 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +linear warm up learning rate. +""" +def linear_warmup_lr(current_step, warmup_steps, base_lr, init_lr): + lr_inc = (float(base_lr) - float(init_lr)) / float(warmup_steps) + lr = float(init_lr) + lr_inc * current_step + return lr diff --git a/example/cv_to_mindrecord/ImageNet_Similar_Perf/template/__init__.py b/model_zoo/resnext50/src/utils/__init__.py similarity index 100% rename from example/cv_to_mindrecord/ImageNet_Similar_Perf/template/__init__.py rename to model_zoo/resnext50/src/utils/__init__.py diff --git a/model_zoo/resnext50/src/utils/cunstom_op.py b/model_zoo/resnext50/src/utils/cunstom_op.py new file mode 100644 index 0000000000..cbe89a1610 --- /dev/null +++ b/model_zoo/resnext50/src/utils/cunstom_op.py @@ -0,0 +1,108 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +network operations +""" +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore.common import dtype as mstype + + +class GlobalAvgPooling(nn.Cell): + """ + global average pooling feature map. + + Args: + mean (tuple): means for each channel. + """ + def __init__(self): + super(GlobalAvgPooling, self).__init__() + self.mean = P.ReduceMean(True) + self.shape = P.Shape() + self.reshape = P.Reshape() + + def construct(self, x): + x = self.mean(x, (2, 3)) + b, c, _, _ = self.shape(x) + x = self.reshape(x, (b, c)) + return x + + +class SEBlock(nn.Cell): + """ + squeeze and excitation block. + + Args: + channel (int): number of feature maps. + reduction (int): weight. + """ + def __init__(self, channel, reduction=16): + super(SEBlock, self).__init__() + + self.avg_pool = GlobalAvgPooling() + self.fc1 = nn.Dense(channel, channel // reduction) + self.relu = P.ReLU() + self.fc2 = nn.Dense(channel // reduction, channel) + self.sigmoid = P.Sigmoid() + self.reshape = P.Reshape() + self.shape = P.Shape() + self.sum = P.Sum() + self.cast = P.Cast() + + def construct(self, x): + b, c = self.shape(x) + y = self.avg_pool(x) + + y = self.reshape(y, (b, c)) + y = self.fc1(y) + y = self.relu(y) + y = self.fc2(y) + y = self.sigmoid(y) + y = self.reshape(y, (b, c, 1, 1)) + return x * y + +class GroupConv(nn.Cell): + """ + group convolution operation. + + Args: + in_channels (int): Input channels of feature map. + out_channels (int): Output channels of feature map. + kernel_size (int): Size of convolution kernel. + stride (int): Stride size for the group convolution layer. + + Returns: + tensor, output tensor. + """ + def __init__(self, in_channels, out_channels, kernel_size, stride, pad_mode="pad", pad=0, groups=1, has_bias=False): + super(GroupConv, self).__init__() + assert in_channels % groups == 0 and out_channels % groups == 0 + self.groups = groups + self.convs = nn.CellList() + self.op_split = P.Split(axis=1, output_num=self.groups) + self.op_concat = P.Concat(axis=1) + self.cast = P.Cast() + for _ in range(groups): + self.convs.append(nn.Conv2d(in_channels//groups, out_channels//groups, + kernel_size=kernel_size, stride=stride, has_bias=has_bias, + padding=pad, pad_mode=pad_mode, group=1)) + + def construct(self, x): + features = self.op_split(x) + outputs = () + for i in range(self.groups): + outputs = outputs + (self.convs[i](self.cast(features[i], mstype.float32)),) + out = self.op_concat(outputs) + return out diff --git a/model_zoo/resnext50/src/utils/logging.py b/model_zoo/resnext50/src/utils/logging.py new file mode 100644 index 0000000000..ac37bec4ec --- /dev/null +++ b/model_zoo/resnext50/src/utils/logging.py @@ -0,0 +1,82 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +get logger. +""" +import logging +import os +import sys +from datetime import datetime + +class LOGGER(logging.Logger): + """ + set up logging file. + + Args: + logger_name (string): logger name. + log_dir (string): path of logger. + + Returns: + string, logger path + """ + def __init__(self, logger_name, rank=0): + super(LOGGER, self).__init__(logger_name) + if rank % 8 == 0: + console = logging.StreamHandler(sys.stdout) + console.setLevel(logging.INFO) + formatter = logging.Formatter('%(asctime)s:%(levelname)s:%(message)s') + console.setFormatter(formatter) + self.addHandler(console) + + def setup_logging_file(self, log_dir, rank=0): + """set up log file""" + self.rank = rank + if not os.path.exists(log_dir): + os.makedirs(log_dir, exist_ok=True) + log_name = datetime.now().strftime('%Y-%m-%d_time_%H_%M_%S') + '_rank_{}.log'.format(rank) + self.log_fn = os.path.join(log_dir, log_name) + fh = logging.FileHandler(self.log_fn) + fh.setLevel(logging.INFO) + formatter = logging.Formatter('%(asctime)s:%(levelname)s:%(message)s') + fh.setFormatter(formatter) + self.addHandler(fh) + + def info(self, msg, *args, **kwargs): + if self.isEnabledFor(logging.INFO): + self._log(logging.INFO, msg, args, **kwargs) + + def save_args(self, args): + self.info('Args:') + args_dict = vars(args) + for key in args_dict.keys(): + self.info('--> %s: %s', key, args_dict[key]) + self.info('') + + def important_info(self, msg, *args, **kwargs): + if self.isEnabledFor(logging.INFO) and self.rank == 0: + line_width = 2 + important_msg = '\n' + important_msg += ('*'*70 + '\n')*line_width + important_msg += ('*'*line_width + '\n')*2 + important_msg += '*'*line_width + ' '*8 + msg + '\n' + important_msg += ('*'*line_width + '\n')*2 + important_msg += ('*'*70 + '\n')*line_width + self.info(important_msg, *args, **kwargs) + + +def get_logger(path, rank): + logger = LOGGER("mindversion", rank) + logger.setup_logging_file(path, rank) + return logger diff --git a/model_zoo/resnext50/src/utils/optimizers__init__.py b/model_zoo/resnext50/src/utils/optimizers__init__.py new file mode 100644 index 0000000000..d4683959b5 --- /dev/null +++ b/model_zoo/resnext50/src/utils/optimizers__init__.py @@ -0,0 +1,39 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +optimizer parameters. +""" +def get_param_groups(network): + """get param groups""" + decay_params = [] + no_decay_params = [] + for x in network.trainable_params(): + parameter_name = x.name + if parameter_name.endswith('.bias'): + # all bias not using weight decay + # print('no decay:{}'.format(parameter_name)) + no_decay_params.append(x) + elif parameter_name.endswith('.gamma'): + # bn weight bias not using weight decay, be carefully for now x not include BN + # print('no decay:{}'.format(parameter_name)) + no_decay_params.append(x) + elif parameter_name.endswith('.beta'): + # bn weight bias not using weight decay, be carefully for now x not include BN + # print('no decay:{}'.format(parameter_name)) + no_decay_params.append(x) + else: + decay_params.append(x) + + return [{'params': no_decay_params, 'weight_decay': 0.0}, {'params': decay_params}] diff --git a/model_zoo/resnext50/src/utils/sampler.py b/model_zoo/resnext50/src/utils/sampler.py new file mode 100644 index 0000000000..5b68f8325e --- /dev/null +++ b/model_zoo/resnext50/src/utils/sampler.py @@ -0,0 +1,53 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +choose samples from the dataset +""" +import math +import numpy as np + +class DistributedSampler(): + """ + sampling the dataset. + + Args: + Returns: + num_samples, number of samples. + """ + def __init__(self, dataset, rank, group_size, shuffle=True, seed=0): + self.dataset = dataset + self.rank = rank + self.group_size = group_size + self.dataset_length = len(self.dataset) + self.num_samples = int(math.ceil(self.dataset_length * 1.0 / self.group_size)) + self.total_size = self.num_samples * self.group_size + self.shuffle = shuffle + self.seed = seed + + def __iter__(self): + if self.shuffle: + self.seed = (self.seed + 1) & 0xffffffff + np.random.seed(self.seed) + indices = np.random.permutation(self.dataset_length).tolist() + else: + indices = list(range(len(self.dataset_length))) + + indices += indices[:(self.total_size - len(indices))] + indices = indices[self.rank::self.group_size] + return iter(indices) + + def __len__(self): + return self.num_samples + \ No newline at end of file diff --git a/model_zoo/resnext50/src/utils/var_init.py b/model_zoo/resnext50/src/utils/var_init.py new file mode 100644 index 0000000000..51fc109990 --- /dev/null +++ b/model_zoo/resnext50/src/utils/var_init.py @@ -0,0 +1,213 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +Initialize. +""" +import math +from functools import reduce +import numpy as np +import mindspore.nn as nn +from mindspore import Tensor +from mindspore.common import initializer as init + +def _calculate_gain(nonlinearity, param=None): + r""" + Return the recommended gain value for the given nonlinearity function. + + The values are as follows: + ================= ==================================================== + nonlinearity gain + ================= ==================================================== + Linear / Identity :math:`1` + Conv{1,2,3}D :math:`1` + Sigmoid :math:`1` + Tanh :math:`\frac{5}{3}` + ReLU :math:`\sqrt{2}` + Leaky Relu :math:`\sqrt{\frac{2}{1 + \text{negative\_slope}^2}}` + ================= ==================================================== + + Args: + nonlinearity: the non-linear function + param: optional parameter for the non-linear function + + Examples: + >>> gain = calculate_gain('leaky_relu', 0.2) # leaky_relu with negative_slope=0.2 + """ + linear_fns = ['linear', 'conv1d', 'conv2d', 'conv3d', 'conv_transpose1d', 'conv_transpose2d', 'conv_transpose3d'] + if nonlinearity in linear_fns or nonlinearity == 'sigmoid': + return 1 + if nonlinearity == 'tanh': + return 5.0 / 3 + if nonlinearity == 'relu': + return math.sqrt(2.0) + if nonlinearity == 'leaky_relu': + if param is None: + negative_slope = 0.01 + elif not isinstance(param, bool) and isinstance(param, int) or isinstance(param, float): + negative_slope = param + else: + raise ValueError("negative_slope {} not a valid number".format(param)) + return math.sqrt(2.0 / (1 + negative_slope ** 2)) + + raise ValueError("Unsupported nonlinearity {}".format(nonlinearity)) + +def _assignment(arr, num): + """Assign the value of `num` to `arr`.""" + if arr.shape == (): + arr = arr.reshape((1)) + arr[:] = num + arr = arr.reshape(()) + else: + if isinstance(num, np.ndarray): + arr[:] = num[:] + else: + arr[:] = num + return arr + +def _calculate_in_and_out(arr): + """ + Calculate n_in and n_out. + + Args: + arr (Array): Input array. + + Returns: + Tuple, a tuple with two elements, the first element is `n_in` and the second element is `n_out`. + """ + dim = len(arr.shape) + if dim < 2: + raise ValueError("If initialize data with xavier uniform, the dimension of data must greater than 1.") + + n_in = arr.shape[1] + n_out = arr.shape[0] + + if dim > 2: + counter = reduce(lambda x, y: x * y, arr.shape[2:]) + n_in *= counter + n_out *= counter + return n_in, n_out + +def _select_fan(array, mode): + mode = mode.lower() + valid_modes = ['fan_in', 'fan_out'] + if mode not in valid_modes: + raise ValueError("Mode {} not supported, please use one of {}".format(mode, valid_modes)) + + fan_in, fan_out = _calculate_in_and_out(array) + return fan_in if mode == 'fan_in' else fan_out + +class KaimingInit(init.Initializer): + r""" + Base Class. Initialize the array with He kaiming algorithm. + + Args: + a: the negative slope of the rectifier used after this layer (only + used with ``'leaky_relu'``) + mode: either ``'fan_in'`` (default) or ``'fan_out'``. Choosing ``'fan_in'`` + preserves the magnitude of the variance of the weights in the + forward pass. Choosing ``'fan_out'`` preserves the magnitudes in the + backwards pass. + nonlinearity: the non-linear function, recommended to use only with + ``'relu'`` or ``'leaky_relu'`` (default). + """ + def __init__(self, a=0, mode='fan_in', nonlinearity='leaky_relu'): + super(KaimingInit, self).__init__() + self.mode = mode + self.gain = _calculate_gain(nonlinearity, a) + def _initialize(self, arr): + pass + + +class KaimingUniform(KaimingInit): + r""" + Initialize the array with He kaiming uniform algorithm. The resulting tensor will + have values sampled from :math:`\mathcal{U}(-\text{bound}, \text{bound})` where + + .. math:: + \text{bound} = \text{gain} \times \sqrt{\frac{3}{\text{fan\_mode}}} + + Input: + arr (Array): The array to be assigned. + + Returns: + Array, assigned array. + + Examples: + >>> w = np.empty(3, 5) + >>> KaimingUniform(w, mode='fan_in', nonlinearity='relu') + """ + + def _initialize(self, arr): + fan = _select_fan(arr, self.mode) + bound = math.sqrt(3.0) * self.gain / math.sqrt(fan) + np.random.seed(0) + data = np.random.uniform(-bound, bound, arr.shape) + + _assignment(arr, data) + + +class KaimingNormal(KaimingInit): + r""" + Initialize the array with He kaiming normal algorithm. The resulting tensor will + have values sampled from :math:`\mathcal{N}(0, \text{std}^2)` where + + .. math:: + \text{std} = \frac{\text{gain}}{\sqrt{\text{fan\_mode}}} + + Input: + arr (Array): The array to be assigned. + + Returns: + Array, assigned array. + + Examples: + >>> w = np.empty(3, 5) + >>> KaimingNormal(w, mode='fan_out', nonlinearity='relu') + """ + + def _initialize(self, arr): + fan = _select_fan(arr, self.mode) + std = self.gain / math.sqrt(fan) + np.random.seed(0) + data = np.random.normal(0, std, arr.shape) + + _assignment(arr, data) + + +def default_recurisive_init(custom_cell): + """default_recurisive_init""" + for _, cell in custom_cell.cells_and_names(): + if isinstance(cell, nn.Conv2d): + cell.weight.default_input = init.initializer(KaimingUniform(a=math.sqrt(5)), + cell.weight.default_input.shape, + cell.weight.default_input.dtype).to_tensor() + if cell.bias is not None: + fan_in, _ = _calculate_in_and_out(cell.weight.default_input.asnumpy()) + bound = 1 / math.sqrt(fan_in) + np.random.seed(0) + cell.bias.default_input = Tensor(np.random.uniform(-bound, bound, cell.bias.default_input.shape), + cell.bias.default_input.dtype) + elif isinstance(cell, nn.Dense): + cell.weight.default_input = init.initializer(KaimingUniform(a=math.sqrt(5)), + cell.weight.default_input.shape, + cell.weight.default_input.dtype).to_tensor() + if cell.bias is not None: + fan_in, _ = _calculate_in_and_out(cell.weight.default_input.asnumpy()) + bound = 1 / math.sqrt(fan_in) + np.random.seed(0) + cell.bias.default_input = Tensor(np.random.uniform(-bound, bound, cell.bias.default_input.shape), + cell.bias.default_input.dtype) + elif isinstance(cell, (nn.BatchNorm2d, nn.BatchNorm1d)): + pass diff --git a/model_zoo/resnet101/src/lr_generator.py b/model_zoo/resnext50/src/warmup_cosine_annealing_lr.py old mode 100755 new mode 100644 similarity index 51% rename from model_zoo/resnet101/src/lr_generator.py rename to model_zoo/resnext50/src/warmup_cosine_annealing_lr.py index 2392e7a7bf..5d9fce9af4 --- a/model_zoo/resnet101/src/lr_generator.py +++ b/model_zoo/resnext50/src/warmup_cosine_annealing_lr.py @@ -12,45 +12,29 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ -"""learning rate generator""" +""" +warm up cosine annealing learning rate. +""" import math import numpy as np -def linear_warmup_lr(current_step, warmup_steps, base_lr, init_lr): - lr_inc = (float(base_lr) - float(init_lr)) / float(warmup_steps) - lr = float(init_lr) + lr_inc * current_step - return lr +from .linear_warmup import linear_warmup_lr -def warmup_cosine_annealing_lr(lr, steps_per_epoch, warmup_epochs, max_epoch=120, global_step=0): - """ - generate learning rate array with cosine - Args: - lr(float): base learning rate - steps_per_epoch(int): steps size of one epoch - warmup_epochs(int): number of warmup epochs - max_epoch(int): total epochs of training - global_step(int): the current start index of lr array - Returns: - np.array, learning rate array - """ +def warmup_cosine_annealing_lr(lr, steps_per_epoch, warmup_epochs, max_epoch, T_max, eta_min=0): + """warm up cosine annealing learning rate.""" base_lr = lr warmup_init_lr = 0 total_steps = int(max_epoch * steps_per_epoch) warmup_steps = int(warmup_epochs * steps_per_epoch) - decay_steps = total_steps - warmup_steps lr_each_step = [] for i in range(total_steps): + last_epoch = i // steps_per_epoch if i < warmup_steps: lr = linear_warmup_lr(i + 1, warmup_steps, base_lr, warmup_init_lr) else: - linear_decay = (total_steps - i) / decay_steps - cosine_decay = 0.5 * (1 + math.cos(math.pi * 2 * 0.47 * i / decay_steps)) - decayed = linear_decay * cosine_decay + 0.00001 - lr = base_lr * decayed + lr = eta_min + (base_lr - eta_min) * (1. + math.cos(math.pi*last_epoch / T_max)) / 2 lr_each_step.append(lr) - lr_each_step = np.array(lr_each_step).astype(np.float32) - learning_rate = lr_each_step[global_step:] - return learning_rate + return np.array(lr_each_step).astype(np.float32) diff --git a/model_zoo/resnext50/src/warmup_step_lr.py b/model_zoo/resnext50/src/warmup_step_lr.py new file mode 100644 index 0000000000..d8e85ab610 --- /dev/null +++ b/model_zoo/resnext50/src/warmup_step_lr.py @@ -0,0 +1,56 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +warm up step learning rate. +""" +from collections import Counter +import numpy as np + +from .linear_warmup import linear_warmup_lr + + +def warmup_step_lr(lr, lr_epochs, steps_per_epoch, warmup_epochs, max_epoch, gamma=0.1): + """warmup_step_lr""" + base_lr = lr + warmup_init_lr = 0 + total_steps = int(max_epoch * steps_per_epoch) + warmup_steps = int(warmup_epochs * steps_per_epoch) + milestones = lr_epochs + milestones_steps = [] + for milestone in milestones: + milestones_step = milestone * steps_per_epoch + milestones_steps.append(milestones_step) + + lr_each_step = [] + lr = base_lr + milestones_steps_counter = Counter(milestones_steps) + for i in range(total_steps): + if i < warmup_steps: + lr = linear_warmup_lr(i + 1, warmup_steps, base_lr, warmup_init_lr) + else: + lr = lr * gamma**milestones_steps_counter[i] + lr_each_step.append(lr) + + return np.array(lr_each_step).astype(np.float32) + +def multi_step_lr(lr, milestones, steps_per_epoch, max_epoch, gamma=0.1): + return warmup_step_lr(lr, milestones, steps_per_epoch, 0, max_epoch, gamma=gamma) + +def step_lr(lr, epoch_size, steps_per_epoch, max_epoch, gamma=0.1): + lr_epochs = [] + for i in range(1, max_epoch): + if i % epoch_size == 0: + lr_epochs.append(i) + return multi_step_lr(lr, lr_epochs, steps_per_epoch, max_epoch, gamma=gamma) diff --git a/model_zoo/resnext50/train.py b/model_zoo/resnext50/train.py new file mode 100644 index 0000000000..29ccd9b00c --- /dev/null +++ b/model_zoo/resnext50/train.py @@ -0,0 +1,289 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""train ImageNet.""" +import os +import time +import argparse +import datetime + +import mindspore.nn as nn +from mindspore import Tensor, context +from mindspore import ParallelMode +from mindspore.nn.optim import Momentum +from mindspore.communication.management import init, get_rank, get_group_size +from mindspore.train.callback import ModelCheckpoint +from mindspore.train.callback import CheckpointConfig, Callback +from mindspore.train.serialization import load_checkpoint, load_param_into_net +from mindspore.train.model import Model +from mindspore.train.loss_scale_manager import DynamicLossScaleManager, FixedLossScaleManager + +from src.dataset import classification_dataset +from src.crossentropy import CrossEntropy +from src.warmup_step_lr import warmup_step_lr +from src.warmup_cosine_annealing_lr import warmup_cosine_annealing_lr +from src.utils.logging import get_logger +from src.utils.optimizers__init__ import get_param_groups +from src.image_classification import get_network +from src.config import config + +devid = int(os.getenv('DEVICE_ID')) +context.set_context(mode=context.GRAPH_MODE, enable_auto_mixed_precision=True, + device_target="Ascend", save_graphs=False, device_id=devid) + +class BuildTrainNetwork(nn.Cell): + """build training network""" + def __init__(self, network, criterion): + super(BuildTrainNetwork, self).__init__() + self.network = network + self.criterion = criterion + + def construct(self, input_data, label): + output = self.network(input_data) + loss = self.criterion(output, label) + return loss + +class ProgressMonitor(Callback): + """monitor loss and time""" + def __init__(self, args): + super(ProgressMonitor, self).__init__() + self.me_epoch_start_time = 0 + self.me_epoch_start_step_num = 0 + self.args = args + self.ckpt_history = [] + + def begin(self, run_context): + self.args.logger.info('start network train...') + + def epoch_begin(self, run_context): + pass + + def epoch_end(self, run_context, *me_args): + cb_params = run_context.original_args() + me_step = cb_params.cur_step_num - 1 + + real_epoch = me_step // self.args.steps_per_epoch + time_used = time.time() - self.me_epoch_start_time + fps_mean = self.args.per_batch_size * (me_step-self.me_epoch_start_step_num) * self.args.group_size / time_used + self.args.logger.info('epoch[{}], iter[{}], loss:{}, mean_fps:{:.2f}' + 'imgs/sec'.format(real_epoch, me_step, cb_params.net_outputs, fps_mean)) + + if self.args.rank_save_ckpt_flag: + import glob + ckpts = glob.glob(os.path.join(self.args.outputs_dir, '*.ckpt')) + for ckpt in ckpts: + ckpt_fn = os.path.basename(ckpt) + if not ckpt_fn.startswith('{}-'.format(self.args.rank)): + continue + if ckpt in self.ckpt_history: + continue + self.ckpt_history.append(ckpt) + self.args.logger.info('epoch[{}], iter[{}], loss:{}, ckpt:{},' + 'ckpt_fn:{}'.format(real_epoch, me_step, cb_params.net_outputs, ckpt, ckpt_fn)) + + + self.me_epoch_start_step_num = me_step + self.me_epoch_start_time = time.time() + + def step_begin(self, run_context): + pass + + def step_end(self, run_context, *me_args): + pass + + def end(self, run_context): + self.args.logger.info('end network train...') + + +def parse_args(cloud_args=None): + """parameters""" + parser = argparse.ArgumentParser('mindspore classification training') + + # dataset related + parser.add_argument('--data_dir', type=str, default='', help='train data dir') + parser.add_argument('--per_batch_size', default=128, type=int, help='batch size for per gpu') + # network related + parser.add_argument('--pretrained', default='', type=str, help='model_path, local pretrained model to load') + + # distributed related + parser.add_argument('--is_distributed', type=int, default=1, help='if multi device') + # roma obs + parser.add_argument('--train_url', type=str, default="", help='train url') + + args, _ = parser.parse_known_args() + args = merge_args(args, cloud_args) + args.image_size = config.image_size + args.num_classes = config.num_classes + args.lr = config.lr + args.lr_scheduler = config.lr_scheduler + args.lr_epochs = config.lr_epochs + args.lr_gamma = config.lr_gamma + args.eta_min = config.eta_min + args.T_max = config.T_max + args.max_epoch = config.max_epoch + args.backbone = config.backbone + args.warmup_epochs = config.warmup_epochs + args.weight_decay = config.weight_decay + args.momentum = config.momentum + args.is_dynamic_loss_scale = config.is_dynamic_loss_scale + args.loss_scale = config.loss_scale + args.label_smooth = config.label_smooth + args.label_smooth_factor = config.label_smooth_factor + args.ckpt_interval = config.ckpt_interval + args.ckpt_path = config.ckpt_path + args.is_save_on_master = config.is_save_on_master + args.rank = config.rank + args.group_size = config.group_size + args.lr_epochs = list(map(int, args.lr_epochs.split(','))) + args.image_size = list(map(int, args.image_size.split(','))) + + return args + +def merge_args(args, cloud_args): + """dictionary""" + args_dict = vars(args) + if isinstance(cloud_args, dict): + for key in cloud_args.keys(): + val = cloud_args[key] + if key in args_dict and val: + arg_type = type(args_dict[key]) + if arg_type is not type(None): + val = arg_type(val) + args_dict[key] = val + return args + +def train(cloud_args=None): + """training process""" + args = parse_args(cloud_args) + + # init distributed + if args.is_distributed: + init() + args.rank = get_rank() + args.group_size = get_group_size() + + if args.is_dynamic_loss_scale == 1: + args.loss_scale = 1 # for dynamic loss scale can not set loss scale in momentum opt + + # select for master rank save ckpt or all rank save, compatiable for model parallel + args.rank_save_ckpt_flag = 0 + if args.is_save_on_master: + if args.rank == 0: + args.rank_save_ckpt_flag = 1 + else: + args.rank_save_ckpt_flag = 1 + + # logger + args.outputs_dir = os.path.join(args.ckpt_path, + datetime.datetime.now().strftime('%Y-%m-%d_time_%H_%M_%S')) + args.logger = get_logger(args.outputs_dir, args.rank) + + # dataloader + de_dataset = classification_dataset(args.data_dir, args.image_size, + args.per_batch_size, args.max_epoch, + args.rank, args.group_size) + de_dataset.map_model = 4 # !!!important + args.steps_per_epoch = de_dataset.get_dataset_size() + + args.logger.save_args(args) + + # network + args.logger.important_info('start create network') + # get network and init + network = get_network(args.backbone, args.num_classes) + if network is None: + raise NotImplementedError('not implement {}'.format(args.backbone)) + network.add_flags_recursive(fp16=True) + # loss + if not args.label_smooth: + args.label_smooth_factor = 0.0 + criterion = CrossEntropy(smooth_factor=args.label_smooth_factor, + num_classes=args.num_classes) + + # load pretrain model + if os.path.isfile(args.pretrained): + param_dict = load_checkpoint(args.pretrained) + param_dict_new = {} + for key, values in param_dict.items(): + if key.startswith('moments.'): + continue + elif key.startswith('network.'): + param_dict_new[key[8:]] = values + else: + param_dict_new[key] = values + load_param_into_net(network, param_dict_new) + args.logger.info('load model {} success'.format(args.pretrained)) + + # lr scheduler + if args.lr_scheduler == 'exponential': + lr = warmup_step_lr(args.lr, + args.lr_epochs, + args.steps_per_epoch, + args.warmup_epochs, + args.max_epoch, + gamma=args.lr_gamma, + ) + elif args.lr_scheduler == 'cosine_annealing': + lr = warmup_cosine_annealing_lr(args.lr, + args.steps_per_epoch, + args.warmup_epochs, + args.max_epoch, + args.T_max, + args.eta_min) + else: + raise NotImplementedError(args.lr_scheduler) + + # optimizer + opt = Momentum(params=get_param_groups(network), + learning_rate=Tensor(lr), + momentum=args.momentum, + weight_decay=args.weight_decay, + loss_scale=args.loss_scale) + + + criterion.add_flags_recursive(fp32=True) + + # package training process, adjust lr + forward + backward + optimizer + train_net = BuildTrainNetwork(network, criterion) + if args.is_distributed: + parallel_mode = ParallelMode.DATA_PARALLEL + else: + parallel_mode = ParallelMode.STAND_ALONE + if args.is_dynamic_loss_scale == 1: + loss_scale_manager = DynamicLossScaleManager(init_loss_scale=65536, scale_factor=2, scale_window=2000) + else: + loss_scale_manager = FixedLossScaleManager(args.loss_scale, drop_overflow_update=False) + + # Model api changed since TR5_branch 2020/03/09 + context.set_auto_parallel_context(parallel_mode=parallel_mode, device_num=args.group_size, + parameter_broadcast=True, mirror_mean=True) + model = Model(train_net, optimizer=opt, metrics=None, loss_scale_manager=loss_scale_manager) + + # checkpoint save + progress_cb = ProgressMonitor(args) + callbacks = [progress_cb,] + if args.rank_save_ckpt_flag: + ckpt_max_num = args.max_epoch * args.steps_per_epoch // args.ckpt_interval + ckpt_config = CheckpointConfig(save_checkpoint_steps=args.ckpt_interval, + keep_checkpoint_max=ckpt_max_num) + ckpt_cb = ModelCheckpoint(config=ckpt_config, + directory=args.outputs_dir, + prefix='{}'.format(args.rank)) + callbacks.append(ckpt_cb) + + model.train(args.max_epoch, de_dataset, callbacks=callbacks, dataset_sink_mode=True) + + +if __name__ == "__main__": + train() diff --git a/example/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/README.md b/model_zoo/utils/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/README.md similarity index 100% rename from example/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/README.md rename to model_zoo/utils/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/README.md diff --git a/example/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/create_dataset.py b/model_zoo/utils/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/create_dataset.py similarity index 100% rename from example/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/create_dataset.py rename to model_zoo/utils/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/create_dataset.py diff --git a/example/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/data/README.md b/model_zoo/utils/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/data/README.md similarity index 100% rename from example/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/data/README.md rename to model_zoo/utils/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/data/README.md diff --git a/example/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/gen_mindrecord.py b/model_zoo/utils/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/gen_mindrecord.py similarity index 100% rename from example/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/gen_mindrecord.py rename to model_zoo/utils/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/gen_mindrecord.py diff --git a/example/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/output/README.md b/model_zoo/utils/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/output/README.md similarity index 100% rename from example/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/output/README.md rename to model_zoo/utils/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/output/README.md diff --git a/example/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/run.sh b/model_zoo/utils/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/run.sh similarity index 100% rename from example/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/run.sh rename to model_zoo/utils/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/run.sh diff --git a/example/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/run_read.sh b/model_zoo/utils/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/run_read.sh similarity index 100% rename from example/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/run_read.sh rename to model_zoo/utils/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/run_read.sh diff --git a/example/cv_to_mindrecord/ImageNet_Similar_Perf/README.md b/model_zoo/utils/cv_to_mindrecord/ImageNet_Similar_Perf/README.md similarity index 100% rename from example/cv_to_mindrecord/ImageNet_Similar_Perf/README.md rename to model_zoo/utils/cv_to_mindrecord/ImageNet_Similar_Perf/README.md diff --git a/example/graph_to_mindrecord/citeseer/__init__.py b/model_zoo/utils/cv_to_mindrecord/ImageNet_Similar_Perf/imagenet/__init__.py similarity index 100% rename from example/graph_to_mindrecord/citeseer/__init__.py rename to model_zoo/utils/cv_to_mindrecord/ImageNet_Similar_Perf/imagenet/__init__.py diff --git a/example/cv_to_mindrecord/ImageNet_Similar_Perf/imagenet/mr_api.py b/model_zoo/utils/cv_to_mindrecord/ImageNet_Similar_Perf/imagenet/mr_api.py similarity index 96% rename from example/cv_to_mindrecord/ImageNet_Similar_Perf/imagenet/mr_api.py rename to model_zoo/utils/cv_to_mindrecord/ImageNet_Similar_Perf/imagenet/mr_api.py index c8129ec9ff..12272b4faf 100644 --- a/example/cv_to_mindrecord/ImageNet_Similar_Perf/imagenet/mr_api.py +++ b/model_zoo/utils/cv_to_mindrecord/ImageNet_Similar_Perf/imagenet/mr_api.py @@ -26,8 +26,8 @@ import os import pickle ######## mindrecord_schema begin ########## -mindrecord_schema = {"label": {"type": "int64"}, - "data": {"type": "bytes"}, +mindrecord_schema = {"label": {"type": "int32"}, + "image": {"type": "bytes"}, "file_name": {"type": "string"}} ######## mindrecord_schema end ########## @@ -121,5 +121,5 @@ def mindrecord_dict_data(task_id): if not image_bytes: print("The image file: {} is invalid.".format(file_name)) continue - data["data"] = image_bytes + data["image"] = image_bytes yield data diff --git a/example/cv_to_mindrecord/ImageNet_Similar_Perf/run_imagenet.sh b/model_zoo/utils/cv_to_mindrecord/ImageNet_Similar_Perf/run_imagenet.sh similarity index 100% rename from example/cv_to_mindrecord/ImageNet_Similar_Perf/run_imagenet.sh rename to model_zoo/utils/cv_to_mindrecord/ImageNet_Similar_Perf/run_imagenet.sh diff --git a/example/cv_to_mindrecord/ImageNet_Similar_Perf/run_template.sh b/model_zoo/utils/cv_to_mindrecord/ImageNet_Similar_Perf/run_template.sh similarity index 100% rename from example/cv_to_mindrecord/ImageNet_Similar_Perf/run_template.sh rename to model_zoo/utils/cv_to_mindrecord/ImageNet_Similar_Perf/run_template.sh diff --git a/example/graph_to_mindrecord/cora/__init__.py b/model_zoo/utils/cv_to_mindrecord/ImageNet_Similar_Perf/template/__init__.py similarity index 100% rename from example/graph_to_mindrecord/cora/__init__.py rename to model_zoo/utils/cv_to_mindrecord/ImageNet_Similar_Perf/template/__init__.py diff --git a/example/cv_to_mindrecord/ImageNet_Similar_Perf/template/mr_api.py b/model_zoo/utils/cv_to_mindrecord/ImageNet_Similar_Perf/template/mr_api.py similarity index 100% rename from example/cv_to_mindrecord/ImageNet_Similar_Perf/template/mr_api.py rename to model_zoo/utils/cv_to_mindrecord/ImageNet_Similar_Perf/template/mr_api.py diff --git a/example/cv_to_mindrecord/ImageNet_Similar_Perf/writer.py b/model_zoo/utils/cv_to_mindrecord/ImageNet_Similar_Perf/writer.py similarity index 100% rename from example/cv_to_mindrecord/ImageNet_Similar_Perf/writer.py rename to model_zoo/utils/cv_to_mindrecord/ImageNet_Similar_Perf/writer.py diff --git a/example/cv_to_mindrecord/README.md b/model_zoo/utils/cv_to_mindrecord/README.md similarity index 100% rename from example/cv_to_mindrecord/README.md rename to model_zoo/utils/cv_to_mindrecord/README.md diff --git a/example/graph_to_mindrecord/README.md b/model_zoo/utils/graph_to_mindrecord/README.md similarity index 100% rename from example/graph_to_mindrecord/README.md rename to model_zoo/utils/graph_to_mindrecord/README.md diff --git a/example/graph_to_mindrecord/sns/__init__.py b/model_zoo/utils/graph_to_mindrecord/citeseer/__init__.py similarity index 100% rename from example/graph_to_mindrecord/sns/__init__.py rename to model_zoo/utils/graph_to_mindrecord/citeseer/__init__.py diff --git a/example/graph_to_mindrecord/citeseer/mr_api.py b/model_zoo/utils/graph_to_mindrecord/citeseer/mr_api.py similarity index 100% rename from example/graph_to_mindrecord/citeseer/mr_api.py rename to model_zoo/utils/graph_to_mindrecord/citeseer/mr_api.py diff --git a/tests/ut/python/pynative_mode/ge/model/__init__.py b/model_zoo/utils/graph_to_mindrecord/cora/__init__.py similarity index 100% rename from tests/ut/python/pynative_mode/ge/model/__init__.py rename to model_zoo/utils/graph_to_mindrecord/cora/__init__.py diff --git a/example/graph_to_mindrecord/cora/mr_api.py b/model_zoo/utils/graph_to_mindrecord/cora/mr_api.py similarity index 100% rename from example/graph_to_mindrecord/cora/mr_api.py rename to model_zoo/utils/graph_to_mindrecord/cora/mr_api.py diff --git a/example/graph_to_mindrecord/graph_map_schema.py b/model_zoo/utils/graph_to_mindrecord/graph_map_schema.py similarity index 100% rename from example/graph_to_mindrecord/graph_map_schema.py rename to model_zoo/utils/graph_to_mindrecord/graph_map_schema.py diff --git a/model_zoo/utils/graph_to_mindrecord/pubmed/__init__.py b/model_zoo/utils/graph_to_mindrecord/pubmed/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/model_zoo/utils/graph_to_mindrecord/pubmed/mr_api.py b/model_zoo/utils/graph_to_mindrecord/pubmed/mr_api.py new file mode 100644 index 0000000000..4e04e90cdd --- /dev/null +++ b/model_zoo/utils/graph_to_mindrecord/pubmed/mr_api.py @@ -0,0 +1,105 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +User-defined API for MindRecord GNN writer. +""" +import os + +import pickle as pkl +import numpy as np +import scipy.sparse as sp + +# parse args from command line parameter 'graph_api_args' +# args delimiter is ':' +args = os.environ['graph_api_args'].split(':') +PUBMED_PATH = args[0] +dataset_str = 'pubmed' + +# profile: (num_features, feature_data_types, feature_shapes) +node_profile = (2, ["float32", "int32"], [[-1], [-1]]) +edge_profile = (0, [], []) + + +def _normalize_cora_features(features): + row_sum = np.array(features.sum(1)) + r_inv = np.power(row_sum * 1.0, -1).flatten() + r_inv[np.isinf(r_inv)] = 0. + r_mat_inv = sp.diags(r_inv) + features = r_mat_inv.dot(features) + return features + + +def _parse_index_file(filename): + """Parse index file.""" + index = [] + for line in open(filename): + index.append(int(line.strip())) + return index + + +def yield_nodes(task_id=0): + """ + Generate node data + + Yields: + data (dict): data row which is dict. + """ + print("Node task is {}".format(task_id)) + + names = ['tx', 'ty', 'allx', 'ally'] + objects = [] + for name in names: + with open("{}/ind.{}.{}".format(PUBMED_PATH, dataset_str, name), 'rb') as f: + objects.append(pkl.load(f, encoding='latin1')) + tx, ty, allx, ally = tuple(objects) + test_idx_reorder = _parse_index_file( + "{}/ind.{}.test.index".format(PUBMED_PATH, dataset_str)) + test_idx_range = np.sort(test_idx_reorder) + + features = sp.vstack((allx, tx)).tolil() + features[test_idx_reorder, :] = features[test_idx_range, :] + features = _normalize_cora_features(features) + features = features.A + + labels = np.vstack((ally, ty)) + labels[test_idx_reorder, :] = labels[test_idx_range, :] + + line_count = 0 + for i, label in enumerate(labels): + node = {'id': i, 'type': 0, 'feature_1': features[i].tolist(), + 'feature_2': label.tolist().index(1)} + line_count += 1 + yield node + print('Processed {} lines for nodes.'.format(line_count)) + + +def yield_edges(task_id=0): + """ + Generate edge data + + Yields: + data (dict): data row which is dict. + """ + print("Edge task is {}".format(task_id)) + with open("{}/ind.{}.graph".format(PUBMED_PATH, dataset_str), 'rb') as f: + graph = pkl.load(f, encoding='latin1') + line_count = 0 + for i in graph: + for dst_id in graph[i]: + edge = {'id': line_count, + 'src_id': i, 'dst_id': dst_id, 'type': 0} + line_count += 1 + yield edge + print('Processed {} lines for edges.'.format(line_count)) diff --git a/example/graph_to_mindrecord/read_citeseer.sh b/model_zoo/utils/graph_to_mindrecord/read_citeseer.sh similarity index 100% rename from example/graph_to_mindrecord/read_citeseer.sh rename to model_zoo/utils/graph_to_mindrecord/read_citeseer.sh diff --git a/example/graph_to_mindrecord/read_cora.sh b/model_zoo/utils/graph_to_mindrecord/read_cora.sh similarity index 100% rename from example/graph_to_mindrecord/read_cora.sh rename to model_zoo/utils/graph_to_mindrecord/read_cora.sh diff --git a/example/graph_to_mindrecord/reader.py b/model_zoo/utils/graph_to_mindrecord/reader.py similarity index 100% rename from example/graph_to_mindrecord/reader.py rename to model_zoo/utils/graph_to_mindrecord/reader.py diff --git a/model_zoo/utils/graph_to_mindrecord/sns/__init__.py b/model_zoo/utils/graph_to_mindrecord/sns/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/example/graph_to_mindrecord/sns/mr_api.py b/model_zoo/utils/graph_to_mindrecord/sns/mr_api.py similarity index 100% rename from example/graph_to_mindrecord/sns/mr_api.py rename to model_zoo/utils/graph_to_mindrecord/sns/mr_api.py diff --git a/example/graph_to_mindrecord/write_citeseer.sh b/model_zoo/utils/graph_to_mindrecord/write_citeseer.sh similarity index 100% rename from example/graph_to_mindrecord/write_citeseer.sh rename to model_zoo/utils/graph_to_mindrecord/write_citeseer.sh diff --git a/example/graph_to_mindrecord/write_cora.sh b/model_zoo/utils/graph_to_mindrecord/write_cora.sh similarity index 100% rename from example/graph_to_mindrecord/write_cora.sh rename to model_zoo/utils/graph_to_mindrecord/write_cora.sh diff --git a/model_zoo/utils/graph_to_mindrecord/write_pubmed.sh b/model_zoo/utils/graph_to_mindrecord/write_pubmed.sh new file mode 100644 index 0000000000..309505e27d --- /dev/null +++ b/model_zoo/utils/graph_to_mindrecord/write_pubmed.sh @@ -0,0 +1,12 @@ +#!/bin/bash +SRC_PATH=/tmp/pubmed/dataset +MINDRECORD_PATH=/tmp/pubmed/mindrecord + +rm -f $MINDRECORD_PATH/* + +python writer.py --mindrecord_script pubmed \ +--mindrecord_file "$MINDRECORD_PATH/pubmed_mr" \ +--mindrecord_partitions 1 \ +--mindrecord_header_size_by_bit 18 \ +--mindrecord_page_size_by_bit 20 \ +--graph_api_args "$SRC_PATH" diff --git a/example/graph_to_mindrecord/write_sns.sh b/model_zoo/utils/graph_to_mindrecord/write_sns.sh similarity index 100% rename from example/graph_to_mindrecord/write_sns.sh rename to model_zoo/utils/graph_to_mindrecord/write_sns.sh diff --git a/example/graph_to_mindrecord/writer.py b/model_zoo/utils/graph_to_mindrecord/writer.py similarity index 100% rename from example/graph_to_mindrecord/writer.py rename to model_zoo/utils/graph_to_mindrecord/writer.py diff --git a/example/nlp_to_mindrecord/README.md b/model_zoo/utils/nlp_to_mindrecord/README.md similarity index 100% rename from example/nlp_to_mindrecord/README.md rename to model_zoo/utils/nlp_to_mindrecord/README.md diff --git a/example/nlp_to_mindrecord/aclImdb/README.md b/model_zoo/utils/nlp_to_mindrecord/aclImdb/README.md similarity index 100% rename from example/nlp_to_mindrecord/aclImdb/README.md rename to model_zoo/utils/nlp_to_mindrecord/aclImdb/README.md diff --git a/example/nlp_to_mindrecord/aclImdb/create_dataset.py b/model_zoo/utils/nlp_to_mindrecord/aclImdb/create_dataset.py similarity index 100% rename from example/nlp_to_mindrecord/aclImdb/create_dataset.py rename to model_zoo/utils/nlp_to_mindrecord/aclImdb/create_dataset.py diff --git a/example/nlp_to_mindrecord/CLUERNER2020/data/README.md b/model_zoo/utils/nlp_to_mindrecord/aclImdb/data/README.md similarity index 100% rename from example/nlp_to_mindrecord/CLUERNER2020/data/README.md rename to model_zoo/utils/nlp_to_mindrecord/aclImdb/data/README.md diff --git a/example/nlp_to_mindrecord/aclImdb/gen_mindrecord.py b/model_zoo/utils/nlp_to_mindrecord/aclImdb/gen_mindrecord.py similarity index 100% rename from example/nlp_to_mindrecord/aclImdb/gen_mindrecord.py rename to model_zoo/utils/nlp_to_mindrecord/aclImdb/gen_mindrecord.py diff --git a/example/nlp_to_mindrecord/aclImdb/output/README.md b/model_zoo/utils/nlp_to_mindrecord/aclImdb/output/README.md similarity index 100% rename from example/nlp_to_mindrecord/aclImdb/output/README.md rename to model_zoo/utils/nlp_to_mindrecord/aclImdb/output/README.md diff --git a/example/nlp_to_mindrecord/aclImdb/run.sh b/model_zoo/utils/nlp_to_mindrecord/aclImdb/run.sh similarity index 100% rename from example/nlp_to_mindrecord/aclImdb/run.sh rename to model_zoo/utils/nlp_to_mindrecord/aclImdb/run.sh diff --git a/example/nlp_to_mindrecord/CLUERNER2020/run_read.sh b/model_zoo/utils/nlp_to_mindrecord/aclImdb/run_read.sh similarity index 100% rename from example/nlp_to_mindrecord/CLUERNER2020/run_read.sh rename to model_zoo/utils/nlp_to_mindrecord/aclImdb/run_read.sh diff --git a/example/nlp_to_mindrecord/aclImdb_preprocess/README.md b/model_zoo/utils/nlp_to_mindrecord/aclImdb_preprocess/README.md similarity index 100% rename from example/nlp_to_mindrecord/aclImdb_preprocess/README.md rename to model_zoo/utils/nlp_to_mindrecord/aclImdb_preprocess/README.md diff --git a/example/nlp_to_mindrecord/aclImdb_preprocess/create_dataset.py b/model_zoo/utils/nlp_to_mindrecord/aclImdb_preprocess/create_dataset.py similarity index 100% rename from example/nlp_to_mindrecord/aclImdb_preprocess/create_dataset.py rename to model_zoo/utils/nlp_to_mindrecord/aclImdb_preprocess/create_dataset.py diff --git a/example/nlp_to_mindrecord/aclImdb/data/README.md b/model_zoo/utils/nlp_to_mindrecord/aclImdb_preprocess/data/README.md similarity index 100% rename from example/nlp_to_mindrecord/aclImdb/data/README.md rename to model_zoo/utils/nlp_to_mindrecord/aclImdb_preprocess/data/README.md diff --git a/example/nlp_to_mindrecord/aclImdb_preprocess/gen_mindrecord.py b/model_zoo/utils/nlp_to_mindrecord/aclImdb_preprocess/gen_mindrecord.py similarity index 100% rename from example/nlp_to_mindrecord/aclImdb_preprocess/gen_mindrecord.py rename to model_zoo/utils/nlp_to_mindrecord/aclImdb_preprocess/gen_mindrecord.py diff --git a/example/nlp_to_mindrecord/aclImdb_preprocess/output/README.md b/model_zoo/utils/nlp_to_mindrecord/aclImdb_preprocess/output/README.md similarity index 100% rename from example/nlp_to_mindrecord/aclImdb_preprocess/output/README.md rename to model_zoo/utils/nlp_to_mindrecord/aclImdb_preprocess/output/README.md diff --git a/example/nlp_to_mindrecord/aclImdb_preprocess/run.sh b/model_zoo/utils/nlp_to_mindrecord/aclImdb_preprocess/run.sh similarity index 100% rename from example/nlp_to_mindrecord/aclImdb_preprocess/run.sh rename to model_zoo/utils/nlp_to_mindrecord/aclImdb_preprocess/run.sh diff --git a/example/nlp_to_mindrecord/aclImdb/run_read.sh b/model_zoo/utils/nlp_to_mindrecord/aclImdb_preprocess/run_read.sh similarity index 100% rename from example/nlp_to_mindrecord/aclImdb/run_read.sh rename to model_zoo/utils/nlp_to_mindrecord/aclImdb_preprocess/run_read.sh diff --git a/model_zoo/vgg16/src/config.py b/model_zoo/vgg16/src/config.py index 8c6ffee98b..a34cf7a1d3 100644 --- a/model_zoo/vgg16/src/config.py +++ b/model_zoo/vgg16/src/config.py @@ -19,7 +19,9 @@ from easydict import EasyDict as edict cifar_cfg = edict({ 'num_classes': 10, - 'lr_init': 0.05, + 'lr_init': 0.01, + 'lr_max': 0.1, + 'warmup_epochs': 5, 'batch_size': 64, 'epoch_size': 70, 'momentum': 0.9, diff --git a/model_zoo/vgg16/train.py b/model_zoo/vgg16/train.py index c582cdd679..33a4f0310c 100644 --- a/model_zoo/vgg16/train.py +++ b/model_zoo/vgg16/train.py @@ -38,20 +38,25 @@ random.seed(1) np.random.seed(1) -def lr_steps(global_step, lr_max=None, total_epochs=None, steps_per_epoch=None): +def lr_steps(global_step, lr_init, lr_max, warmup_epochs, total_epochs, steps_per_epoch): """Set learning rate.""" lr_each_step = [] total_steps = steps_per_epoch * total_epochs - decay_epoch_index = [0.3 * total_steps, 0.6 * total_steps, 0.8 * total_steps] + warmup_steps = steps_per_epoch * warmup_epochs + if warmup_steps != 0: + inc_each_step = (float(lr_max) - float(lr_init)) / float(warmup_steps) + else: + inc_each_step = 0 for i in range(total_steps): - if i < decay_epoch_index[0]: - lr_each_step.append(lr_max) - elif i < decay_epoch_index[1]: - lr_each_step.append(lr_max * 0.1) - elif i < decay_epoch_index[2]: - lr_each_step.append(lr_max * 0.01) + if i < warmup_steps: + lr_value = float(lr_init) + inc_each_step * float(i) else: - lr_each_step.append(lr_max * 0.001) + base = (1.0 - (float(i) - float(warmup_steps)) / (float(total_steps) - float(warmup_steps))) + lr_value = float(lr_max) * base * base + if lr_value < 0.0: + lr_value = 0.0 + lr_each_step.append(lr_value) + current_step = global_step lr_each_step = np.array(lr_each_step).astype(np.float32) learning_rate = lr_each_step[current_step:] @@ -86,7 +91,8 @@ if __name__ == '__main__': if args_opt.pre_trained: load_param_into_net(net, load_checkpoint(args_opt.pre_trained)) - lr = lr_steps(0, lr_max=cfg.lr_init, total_epochs=cfg.epoch_size, steps_per_epoch=batch_num) + lr = lr_steps(0, lr_init=cfg.lr_init, lr_max=cfg.lr_max, warmup_epochs=cfg.warmup_epochs, + total_epochs=cfg.epoch_size, steps_per_epoch=batch_num) opt = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), Tensor(lr), cfg.momentum, weight_decay=cfg.weight_decay) loss = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction='mean', is_grad=False) diff --git a/model_zoo/warpctc/README.md b/model_zoo/warpctc/README.md new file mode 100644 index 0000000000..cb941255bf --- /dev/null +++ b/model_zoo/warpctc/README.md @@ -0,0 +1,137 @@ +# Warpctc Example + +## Description + +These is an example of training Warpctc with self-generated captcha image dataset in MindSpore. + +## Requirements + +- Install [MindSpore](https://www.mindspore.cn/install/en). + +- Generate captcha images. + +> The [captcha](https://github.com/lepture/captcha) library can be used to generate captcha images. You can generate the train and test dataset by yourself or just run the script `scripts/run_process_data.sh`. By default, the shell script will generate 10000 test images and 50000 train images separately. +> ``` +> $ cd scripts +> $ sh run_process_data.sh +> +> # after execution, you will find the dataset like the follows: +> . +> └─warpctc +> └─data +> ├─ train # train dataset +> └─ test # evaluate dataset +> ... + + +## Structure + +```shell +. +└──warpct + ├── README.md + ├── script + ├── run_distribute_train.sh # launch distributed training(8 pcs) + ├── run_eval.sh # launch evaluation + ├── run_process_data.sh # launch dataset generation + └── run_standalone_train.sh # launch standalone training(1 pcs) + ├── src + ├── config.py # parameter configuration + ├── dataset.py # data preprocessing + ├── loss.py # ctcloss definition + ├── lr_generator.py # generate learning rate for each step + ├── metric.py # accuracy metric for warpctc network + ├── warpctc.py # warpctc network definition + └── warpctc_for_train.py # warp network with grad, loss and gradient clip + ├── eval.py # eval net + ├── process_data.py # dataset generation script + └── train.py # train net +``` + + +## Parameter configuration + +Parameters for both training and evaluation can be set in config.py. + +``` +"max_captcha_digits": 4, # max number of digits in each +"captcha_width": 160, # width of captcha images +"captcha_height": 64, # height of capthca images +"batch_size": 64, # batch size of input tensor +"epoch_size": 30, # only valid for taining, which is always 1 for inference +"hidden_size": 512, # hidden size in LSTM layers +"learning_rate": 0.01, # initial learning rate +"momentum": 0.9 # momentum of SGD optimizer +"save_checkpoint": True, # whether save checkpoint or not +"save_checkpoint_steps": 98, # the step interval between two checkpoints. By default, the last checkpoint will be saved after the last step +"keep_checkpoint_max": 30, # only keep the last keep_checkpoint_max checkpoint +"save_checkpoint_path": "./", # path to save checkpoint +``` + +## Running the example + +### Train + +#### Usage + +``` +# distributed training +Usage: sh run_distribute_train.sh [MINDSPORE_HCCL_CONFIG_PATH] [DATASET_PATH] + +# standalone training +Usage: sh run_standalone_train.sh [DATASET_PATH] +``` + + +#### Launch + +``` +# distribute training example +sh run_distribute_train.sh rank_table.json ../data/train + +# standalone training example +sh run_standalone_train.sh ../data/train +``` + +> About rank_table.json, you can refer to the [distributed training tutorial](https://www.mindspore.cn/tutorial/en/master/advanced_use/distributed_training.html). + +#### Result + +Training result will be stored in folder `scripts`, whose name begins with "train" or "train_parallel". Under this, you can find checkpoint file together with result like the followings in log. + +``` +# distribute training result(8 pcs) +Epoch: [ 1/ 30], step: [ 98/ 98], loss: [0.5853/0.5853], time: [376813.7944] +Epoch: [ 2/ 30], step: [ 98/ 98], loss: [0.4007/0.4007], time: [75882.0951] +Epoch: [ 3/ 30], step: [ 98/ 98], loss: [0.0921/0.0921], time: [75150.9385] +Epoch: [ 4/ 30], step: [ 98/ 98], loss: [0.1472/0.1472], time: [75135.0193] +Epoch: [ 5/ 30], step: [ 98/ 98], loss: [0.0186/0.0186], time: [75199.5809] +... +``` + + +### Evaluation + +#### Usage + +``` +# evaluation +Usage: sh run_eval.sh [DATASET_PATH] [CHECKPOINT_PATH] +``` + +#### Launch + +``` +# evaluation example +sh run_eval.sh ../data/test warpctc-30-98.ckpt +``` + +> checkpoint can be produced in training process. + +#### Result + +Evaluation result will be stored in the example path, whose folder name is "eval". Under this, you can find result like the followings in log. + +``` +result: {'WarpCTCAccuracy': 0.9901472929936306} +``` diff --git a/model_zoo/warpctc/eval.py b/model_zoo/warpctc/eval.py new file mode 100755 index 0000000000..df62c7c755 --- /dev/null +++ b/model_zoo/warpctc/eval.py @@ -0,0 +1,65 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Warpctc evaluation""" +import os +import math as m +import random +import argparse +import numpy as np +from mindspore import context +from mindspore import dataset as de +from mindspore.train.model import Model +from mindspore.train.serialization import load_checkpoint, load_param_into_net + +from src.loss import CTCLoss +from src.config import config as cf +from src.dataset import create_dataset +from src.warpctc import StackedRNN +from src.metric import WarpCTCAccuracy + +random.seed(1) +np.random.seed(1) +de.config.set_seed(1) + +parser = argparse.ArgumentParser(description="Warpctc training") +parser.add_argument("--dataset_path", type=str, default=None, help="Dataset, default is None.") +parser.add_argument("--checkpoint_path", type=str, default=None, help="checkpoint file path, default is None") +args_opt = parser.parse_args() + +device_id = int(os.getenv('DEVICE_ID')) +context.set_context(mode=context.GRAPH_MODE, + device_target="Ascend", + save_graphs=False, + device_id=device_id) + +if __name__ == '__main__': + max_captcha_digits = cf.max_captcha_digits + input_size = m.ceil(cf.captcha_height / 64) * 64 * 3 + # create dataset + dataset = create_dataset(dataset_path=args_opt.dataset_path, repeat_num=1, batch_size=cf.batch_size) + step_size = dataset.get_dataset_size() + # define loss + loss = CTCLoss(max_sequence_length=cf.captcha_width, max_label_length=max_captcha_digits, batch_size=cf.batch_size) + # define net + net = StackedRNN(input_size=input_size, batch_size=cf.batch_size, hidden_size=cf.hidden_size) + # load checkpoint + param_dict = load_checkpoint(args_opt.checkpoint_path) + load_param_into_net(net, param_dict) + net.set_train(False) + # define model + model = Model(net, loss_fn=loss, metrics={'WarpCTCAccuracy': WarpCTCAccuracy()}) + # start evaluation + res = model.eval(dataset) + print("result:", res, flush=True) diff --git a/model_zoo/warpctc/process_data.py b/model_zoo/warpctc/process_data.py new file mode 100755 index 0000000000..567ad10933 --- /dev/null +++ b/model_zoo/warpctc/process_data.py @@ -0,0 +1,71 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Generate train and test dataset""" +import os +import math as m +import random +from multiprocessing import Process +from captcha.image import ImageCaptcha + + +def _generate_captcha_per_process(path, total, start, end, img_width, img_height, max_digits): + captcha = ImageCaptcha(width=img_width, height=img_height) + filename_head = '{:0>' + str(len(str(total))) + '}-' + for i in range(start, end): + digits = '' + digits_length = random.randint(1, max_digits) + for _ in range(0, digits_length): + integer = random.randint(0, 9) + digits += str(integer) + captcha.write(digits, os.path.join(path, filename_head.format(i) + digits + '.png')) + + +def generate_captcha(name, img_num, img_width, img_height, max_digits, process_num=16): + """ + generate captcha images + + Args: + name(str): name of folder, under which captcha images are saved in + img_num(int): number of generated captcha images + img_width(int): width of generated captcha images + img_height(int): height of generated captcha images + max_digits(int): max number of digits in each captcha images. For each captcha images, number of digits is in + range [1,max_digits] + process_num(int): number of process to generate captcha images, default is 16 + """ + cur_script_path = os.path.dirname(os.path.realpath(__file__)) + path = os.path.join(cur_script_path, "data", name) + print("Generating dataset [{}] under {}...".format(name, path)) + if os.path.exists(path): + os.system("rm -rf {}".format(path)) + os.system("mkdir -p {}".format(path)) + img_num_per_thread = m.ceil(img_num / process_num) + + processes = [] + for i in range(process_num): + start = i * img_num_per_thread + end = start + img_num_per_thread if i != (process_num - 1) else img_num + p = Process(target=_generate_captcha_per_process, + args=(path, img_num, start, end, img_width, img_height, max_digits)) + p.start() + processes.append(p) + for p in processes: + p.join() + print("Generating dataset [{}] finished, total number is {}!".format(name, img_num)) + + +if __name__ == '__main__': + generate_captcha("test", img_num=10000, img_width=160, img_height=64, max_digits=4) + generate_captcha("train", img_num=50000, img_width=160, img_height=64, max_digits=4) diff --git a/example/resnet50_cifar10/run_distribute_train.sh b/model_zoo/warpctc/scripts/run_distribute_train.sh similarity index 53% rename from example/resnet50_cifar10/run_distribute_train.sh rename to model_zoo/warpctc/scripts/run_distribute_train.sh index e4bdd775b3..3cebf6d195 100755 --- a/example/resnet50_cifar10/run_distribute_train.sh +++ b/model_zoo/warpctc/scripts/run_distribute_train.sh @@ -14,13 +14,12 @@ # limitations under the License. # ============================================================================ -if [ $# != 2 ] -then - echo "Usage: sh run_distribute_train.sh [MINDSPORE_HCCL_CONFIG_PATH] [DATASET_PATH]" -exit 1 +if [ $# != 2 ]; then + echo "Usage: sh run_distribute_train.sh [MINDSPORE_HCCL_CONFIG_PATH] [DATASET_PATH]" + exit 1 fi -get_real_path(){ +get_real_path() { if [ "${1:0:1}" == "/" ]; then echo "$1" else @@ -31,34 +30,33 @@ get_real_path(){ PATH1=$(get_real_path $1) PATH2=$(get_real_path $2) -if [ ! -f "$PATH1" ] -then - echo "error: MINDSPORE_HCCL_CONFIG_PATH=$PATH1 is not a file" -exit 1 -fi +if [ ! -f $PATH1 ]; then + echo "error: MINDSPORE_HCCL_CONFIG_PATH=$PATH1 is not a file" + exit 1 +fi -if [ ! -d "$PATH2" ] -then - echo "error: DATASET_PATH=$PATH2 is not a directory" -exit 1 -fi +if [ ! -d $PATH2 ]; then + echo "error: DATASET_PATH=$PATH2 is not a directory" + exit 1 +fi ulimit -u unlimited export DEVICE_NUM=8 export RANK_SIZE=8 export MINDSPORE_HCCL_CONFIG_PATH=$PATH1 +export RANK_TABLE_FILE=$PATH1 -for((i=0; i<${DEVICE_NUM}; i++)) -do - export DEVICE_ID=$i - export RANK_ID=$i - rm -rf ./train_parallel$i - mkdir ./train_parallel$i - cp *.py ./train_parallel$i - cp *.sh ./train_parallel$i - cd ./train_parallel$i || exit - echo "start training for rank $RANK_ID, device $DEVICE_ID" - env > env.log - python train.py --do_train=True --run_distribute=True --device_num=$DEVICE_NUM --dataset_path=$PATH2 &> log & - cd .. +for ((i = 0; i < ${DEVICE_NUM}; i++)); do + export DEVICE_ID=$i + export RANK_ID=$i + rm -rf ./train_parallel$i + mkdir ./train_parallel$i + cp ../*.py ./train_parallel$i + cp *.sh ./train_parallel$i + cp -r ../src ./train_parallel$i + cd ./train_parallel$i || exit + echo "start training for rank $RANK_ID, device $DEVICE_ID" + env >env.log + python train.py --run_distribute=True --device_num=$DEVICE_NUM --dataset_path=$PATH2 &>log & + cd .. done diff --git a/example/resnet50_imagenet2012_THOR/run_infer.sh b/model_zoo/warpctc/scripts/run_eval.sh similarity index 63% rename from example/resnet50_imagenet2012_THOR/run_infer.sh rename to model_zoo/warpctc/scripts/run_eval.sh index 14d7faf981..659de6d72a 100755 --- a/example/resnet50_imagenet2012_THOR/run_infer.sh +++ b/model_zoo/warpctc/scripts/run_eval.sh @@ -14,13 +14,12 @@ # limitations under the License. # ============================================================================ -if [ $# != 2 ] -then - echo "Usage: sh run_infer.sh [DATASET_PATH] [CHECKPOINT_PATH]" -exit 1 +if [ $# != 2 ]; then + echo "Usage: sh run_eval.sh [DATASET_PATH] [CHECKPOINT_PATH]" + exit 1 fi -get_real_path(){ +get_real_path() { if [ "${1:0:1}" == "/" ]; then echo "$1" else @@ -31,18 +30,15 @@ get_real_path(){ PATH1=$(get_real_path $1) PATH2=$(get_real_path $2) +if [ ! -d $PATH1 ]; then + echo "error: DATASET_PATH=$PATH1 is not a directory" + exit 1 +fi -if [ ! -d $PATH1 ] -then - echo "error: DATASET_PATH=$1 is not a directory" -exit 1 -fi - -if [ ! -f $PATH2 ] -then - echo "error: CHECKPOINT_PATH=$2 is not a file" -exit 1 -fi +if [ ! -f $PATH2 ]; then + echo "error: CHECKPOINT_PATH=$PATH2 is not a file" + exit 1 +fi ulimit -u unlimited export DEVICE_NUM=1 @@ -50,15 +46,15 @@ export DEVICE_ID=0 export RANK_SIZE=$DEVICE_NUM export RANK_ID=0 -if [ -d "infer" ]; -then - rm -rf ./infer +if [ -d "eval" ]; then + rm -rf ./eval fi -mkdir ./infer -cp *.py ./infer -cp *.sh ./infer -cd ./infer || exit -env > env.log -echo "start infering for device $DEVICE_ID" -python eval.py --do_eval=True --dataset_path=$PATH1 --checkpoint_path=$PATH2 &> log & +mkdir ./eval +cp ../*.py ./eval +cp *.sh ./eval +cp -r ../src ./eval +cd ./eval || exit +env >env.log +echo "start evaluation for device $DEVICE_ID" +python eval.py --dataset_path=$PATH1 --checkpoint_path=$PATH2 &>log & cd .. diff --git a/model_zoo/warpctc/scripts/run_process_data.sh b/model_zoo/warpctc/scripts/run_process_data.sh new file mode 100755 index 0000000000..56b89f1a72 --- /dev/null +++ b/model_zoo/warpctc/scripts/run_process_data.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +CUR_PATH=$(dirname $PWD/$0) +cd $CUR_PATH/../ && + python process_data.py && + cd - || exit \ No newline at end of file diff --git a/example/resnet50_cifar10/run_standalone_train.sh b/model_zoo/warpctc/scripts/run_standalone_train.sh similarity index 73% rename from example/resnet50_cifar10/run_standalone_train.sh rename to model_zoo/warpctc/scripts/run_standalone_train.sh index cb08cde6c9..22a16ef4c8 100755 --- a/example/resnet50_cifar10/run_standalone_train.sh +++ b/model_zoo/warpctc/scripts/run_standalone_train.sh @@ -14,13 +14,12 @@ # limitations under the License. # ============================================================================ -if [ $# != 1 ] -then - echo "Usage: sh run_standalone_train.sh [DATASET_PATH]" -exit 1 +if [ $# != 1 ]; then + echo "Usage: sh run_standalone_train.sh [DATASET_PATH]" + exit 1 fi -get_real_path(){ +get_real_path() { if [ "${1:0:1}" == "/" ]; then echo "$1" else @@ -30,26 +29,26 @@ get_real_path(){ PATH1=$(get_real_path $1) -if [ ! -d "$PATH1" ] -then - echo "error: DATASET_PATH=$PATH1 is not a directory" -exit 1 -fi +if [ ! -d $PATH1 ]; then + echo "error: DATASET_PATH=$PATH1 is not a directory" + exit 1 +fi ulimit -u unlimited export DEVICE_NUM=1 export DEVICE_ID=0 export RANK_ID=0 +export RANK_SIZE=1 -if [ -d "train" ]; -then - rm -rf ./train +if [ -d "train" ]; then + rm -rf ./train fi mkdir ./train -cp *.py ./train +cp ../*.py ./train cp *.sh ./train +cp -r ../src ./train cd ./train || exit echo "start training for device $DEVICE_ID" -env > env.log -python train.py --do_train=True --dataset_path=$PATH1 &> log & +env >env.log +python train.py --dataset=$PATH1 &>log & cd .. diff --git a/model_zoo/warpctc/src/config.py b/model_zoo/warpctc/src/config.py new file mode 100755 index 0000000000..ed9c2968de --- /dev/null +++ b/model_zoo/warpctc/src/config.py @@ -0,0 +1,31 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Network parameters.""" +from easydict import EasyDict + +config = EasyDict({ + "max_captcha_digits": 4, + "captcha_width": 160, + "captcha_height": 64, + "batch_size": 64, + "epoch_size": 30, + "hidden_size": 512, + "learning_rate": 0.01, + "momentum": 0.9, + "save_checkpoint": True, + "save_checkpoint_steps": 98, + "keep_checkpoint_max": 30, + "save_checkpoint_path": "./", +}) diff --git a/model_zoo/warpctc/src/dataset.py b/model_zoo/warpctc/src/dataset.py new file mode 100755 index 0000000000..76e592b906 --- /dev/null +++ b/model_zoo/warpctc/src/dataset.py @@ -0,0 +1,92 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Dataset preprocessing.""" +import os +import math as m +import numpy as np +import mindspore.common.dtype as mstype +import mindspore.dataset.engine as de +import mindspore.dataset.transforms.c_transforms as c +import mindspore.dataset.transforms.vision.c_transforms as vc +from PIL import Image +from src.config import config as cf + + +class _CaptchaDataset(): + """ + create train or evaluation dataset for warpctc + + Args: + img_root_dir(str): root path of images + max_captcha_digits(int): max number of digits in images. + blank(int): value reserved for blank label, default is 10. When parsing label from image file names, if label + length is less than max_captcha_digits, the remaining labels are padding with blank. + """ + + def __init__(self, img_root_dir, max_captcha_digits, blank=10): + if not os.path.exists(img_root_dir): + raise RuntimeError("the input image dir {} is invalid!".format(img_root_dir)) + self.img_root_dir = img_root_dir + self.img_names = [i for i in os.listdir(img_root_dir) if i.endswith('.png')] + self.max_captcha_digits = max_captcha_digits + self.blank = blank + + def __len__(self): + return len(self.img_names) + + def __getitem__(self, item): + img_name = self.img_names[item] + im = Image.open(os.path.join(self.img_root_dir, img_name)) + r, g, b = im.split() + im = Image.merge("RGB", (b, g, r)) + image = np.array(im) + label_str = os.path.splitext(img_name)[0] + label_str = label_str[label_str.find('-') + 1:] + label = [int(i) for i in label_str] + label.extend([int(self.blank)] * (self.max_captcha_digits - len(label))) + label = np.array(label) + return image, label + + +def create_dataset(dataset_path, repeat_num=1, batch_size=1): + """ + create train or evaluation dataset for warpctc + + Args: + dataset_path(int): dataset path + repeat_num(int): dataset repetition num, default is 1 + batch_size(int): batch size of generated dataset, default is 1 + """ + rank_size = int(os.environ.get("RANK_SIZE")) if os.environ.get("RANK_SIZE") else 1 + rank_id = int(os.environ.get("RANK_ID")) if os.environ.get("RANK_ID") else 0 + + dataset = _CaptchaDataset(dataset_path, cf.max_captcha_digits) + ds = de.GeneratorDataset(dataset, ["image", "label"], shuffle=True, num_shards=rank_size, shard_id=rank_id) + ds.set_dataset_size(m.ceil(len(dataset) / rank_size)) + image_trans = [ + vc.Rescale(1.0 / 255.0, 0.0), + vc.Normalize([0.9010, 0.9049, 0.9025], std=[0.1521, 0.1347, 0.1458]), + vc.Resize((m.ceil(cf.captcha_height / 16) * 16, cf.captcha_width)), + vc.HWC2CHW() + ] + label_trans = [ + c.TypeCast(mstype.int32) + ] + ds = ds.map(input_columns=["image"], num_parallel_workers=8, operations=image_trans) + ds = ds.map(input_columns=["label"], num_parallel_workers=8, operations=label_trans) + + ds = ds.batch(batch_size) + ds = ds.repeat(repeat_num) + return ds diff --git a/model_zoo/warpctc/src/loss.py b/model_zoo/warpctc/src/loss.py new file mode 100755 index 0000000000..8ea4c20e94 --- /dev/null +++ b/model_zoo/warpctc/src/loss.py @@ -0,0 +1,49 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""CTC Loss.""" +import numpy as np +from mindspore.nn.loss.loss import _Loss +from mindspore import Tensor, Parameter +from mindspore.common import dtype as mstype +from mindspore.ops import operations as P + + +class CTCLoss(_Loss): + """ + CTCLoss definition + + Args: + max_sequence_length(int): max number of sequence length. For captcha images, the value is equal to image + width + max_label_length(int): max number of label length for each input. + batch_size(int): batch size of input logits + """ + + def __init__(self, max_sequence_length, max_label_length, batch_size): + super(CTCLoss, self).__init__() + self.sequence_length = Parameter(Tensor(np.array([max_sequence_length] * batch_size), mstype.int32), + name="sequence_length") + labels_indices = [] + for i in range(batch_size): + for j in range(max_label_length): + labels_indices.append([i, j]) + self.labels_indices = Parameter(Tensor(np.array(labels_indices), mstype.int64), name="labels_indices") + self.reshape = P.Reshape() + self.ctc_loss = P.CTCLoss(ctc_merge_repeated=True) + + def construct(self, logit, label): + labels_values = self.reshape(label, (-1,)) + loss, _ = self.ctc_loss(logit, self.labels_indices, labels_values, self.sequence_length) + return loss diff --git a/model_zoo/warpctc/src/lr_schedule.py b/model_zoo/warpctc/src/lr_schedule.py new file mode 100755 index 0000000000..a0ae6c886a --- /dev/null +++ b/model_zoo/warpctc/src/lr_schedule.py @@ -0,0 +1,36 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Learning rate generator.""" + + +def get_lr(epoch_size, step_size, lr_init): + """ + generate learning rate for each step, which decays in every 10 epoch + + Args: + epoch_size(int): total epoch number + step_size(int): total step number in each step + lr_init(int): initial learning rate + + Returns: + List, learning rate array + """ + lr = lr_init + lrs = [] + for i in range(1, epoch_size + 1): + if i % 10 == 0: + lr *= 0.1 + lrs.extend([lr for _ in range(step_size)]) + return lrs diff --git a/model_zoo/warpctc/src/metric.py b/model_zoo/warpctc/src/metric.py new file mode 100755 index 0000000000..d1060d0781 --- /dev/null +++ b/model_zoo/warpctc/src/metric.py @@ -0,0 +1,89 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Metric for accuracy evaluation.""" +from mindspore import nn + +BLANK_LABLE = 10 + + +class WarpCTCAccuracy(nn.Metric): + """ + Define accuracy metric for warpctc network. + """ + + def __init__(self): + super(WarpCTCAccuracy).__init__() + self._correct_num = 0 + self._total_num = 0 + self._count = 0 + + def clear(self): + self._correct_num = 0 + self._total_num = 0 + + def update(self, *inputs): + if len(inputs) != 2: + raise ValueError('WarpCTCAccuracy need 2 inputs (y_pred, y), but got {}'.format(len(inputs))) + + y_pred = self._convert_data(inputs[0]) + y = self._convert_data(inputs[1]) + + self._count += 1 + + pred_lbls = self._get_prediction(y_pred) + + for b_idx, target in enumerate(y): + if self._is_eq(pred_lbls[b_idx], target): + self._correct_num += 1 + self._total_num += 1 + + def eval(self): + if self._total_num == 0: + raise RuntimeError('Accuary can not be calculated, because the number of samples is 0.') + return self._correct_num / self._total_num + + @staticmethod + def _is_eq(pred_lbl, target): + """ + check whether predict label is equal to target label + """ + target = target.tolist() + pred_diff = len(target) - len(pred_lbl) + if pred_diff > 0: + # padding by BLANK_LABLE + pred_lbl.extend([BLANK_LABLE] * pred_diff) + return pred_lbl == target + + @staticmethod + def _get_prediction(y_pred): + """ + parse predict result to labels + """ + seq_len, batch_size, _ = y_pred.shape + indices = y_pred.argmax(axis=2) + + lens = [seq_len] * batch_size + pred_lbls = [] + for i in range(batch_size): + idx = indices[:, i] + last_idx = BLANK_LABLE + pred_lbl = [] + for j in range(lens[i]): + cur_idx = idx[j] + if cur_idx not in [last_idx, BLANK_LABLE]: + pred_lbl.append(cur_idx) + last_idx = cur_idx + pred_lbls.append(pred_lbl) + return pred_lbls diff --git a/model_zoo/warpctc/src/warpctc.py b/model_zoo/warpctc/src/warpctc.py new file mode 100755 index 0000000000..9669fc4bfd --- /dev/null +++ b/model_zoo/warpctc/src/warpctc.py @@ -0,0 +1,90 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Warpctc network definition.""" + +import numpy as np +import mindspore.nn as nn +from mindspore import Tensor, Parameter +from mindspore.common import dtype as mstype +from mindspore.ops import operations as P +from mindspore.ops import functional as F + + +class StackedRNN(nn.Cell): + """ + Define a stacked RNN network which contains two LSTM layers and one full-connect layer. + + Args: + input_size(int): Size of time sequence. Usually, the input_size is equal to three times of image height for + captcha images. + batch_size(int): batch size of input data, default is 64 + hidden_size(int): the hidden size in LSTM layers, default is 512 + """ + def __init__(self, input_size, batch_size=64, hidden_size=512): + super(StackedRNN, self).__init__() + self.batch_size = batch_size + self.input_size = input_size + self.num_classes = 11 + self.reshape = P.Reshape() + self.cast = P.Cast() + k = (1 / hidden_size) ** 0.5 + self.h1 = Tensor(np.zeros(shape=(batch_size, hidden_size)).astype(np.float16)) + self.c1 = Tensor(np.zeros(shape=(batch_size, hidden_size)).astype(np.float16)) + self.w1 = Parameter(np.random.uniform(-k, k, (4 * hidden_size, input_size + hidden_size, 1, 1)) + .astype(np.float16), name="w1") + self.w2 = Parameter(np.random.uniform(-k, k, (4 * hidden_size, hidden_size + hidden_size, 1, 1)) + .astype(np.float16), name="w2") + self.b1 = Parameter(np.random.uniform(-k, k, (4 * hidden_size, 1, 1, 1)).astype(np.float16), name="b1") + self.b2 = Parameter(np.random.uniform(-k, k, (4 * hidden_size, 1, 1, 1)).astype(np.float16), name="b2") + + self.h2 = Tensor(np.zeros(shape=(batch_size, hidden_size)).astype(np.float16)) + self.c2 = Tensor(np.zeros(shape=(batch_size, hidden_size)).astype(np.float16)) + + self.basic_lstm_cell = P.BasicLSTMCell(keep_prob=1.0, forget_bias=0.0, state_is_tuple=True, activation="tanh") + + self.fc_weight = np.random.random((self.num_classes, hidden_size)).astype(np.float32) + self.fc_bias = np.random.random((self.num_classes)).astype(np.float32) + + self.fc = nn.Dense(in_channels=hidden_size, out_channels=self.num_classes, weight_init=Tensor(self.fc_weight), + bias_init=Tensor(self.fc_bias)) + + self.fc.to_float(mstype.float32) + self.expand_dims = P.ExpandDims() + self.concat = P.Concat() + self.transpose = P.Transpose() + + def construct(self, x): + x = self.cast(x, mstype.float16) + x = self.transpose(x, (3, 0, 2, 1)) + x = self.reshape(x, (-1, self.batch_size, self.input_size)) + h1 = self.h1 + c1 = self.c1 + h2 = self.h2 + c2 = self.c2 + + c1, h1, _, _, _, _, _ = self.basic_lstm_cell(x[0, :, :], h1, c1, self.w1, self.b1) + c2, h2, _, _, _, _, _ = self.basic_lstm_cell(h1, h2, c2, self.w2, self.b2) + + h2_after_fc = self.fc(h2) + output = self.expand_dims(h2_after_fc, 0) + for i in range(1, F.shape(x)[0]): + c1, h1, _, _, _, _, _ = self.basic_lstm_cell(x[i, :, :], h1, c1, self.w1, self.b1) + c2, h2, _, _, _, _, _ = self.basic_lstm_cell(h1, h2, c2, self.w2, self.b2) + + h2_after_fc = self.fc(h2) + h2_after_fc = self.expand_dims(h2_after_fc, 0) + output = self.concat((output, h2_after_fc)) + + return output diff --git a/model_zoo/warpctc/src/warpctc_for_train.py b/model_zoo/warpctc/src/warpctc_for_train.py new file mode 100755 index 0000000000..d847f47c62 --- /dev/null +++ b/model_zoo/warpctc/src/warpctc_for_train.py @@ -0,0 +1,114 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Automatic differentiation with grad clip.""" +from mindspore.parallel._utils import (_get_device_num, _get_mirror_mean, + _get_parallel_mode) +from mindspore.train.parallel_utils import ParallelMode +from mindspore.common import dtype as mstype +from mindspore.ops import composite as C +from mindspore.ops import functional as F +from mindspore.ops import operations as P +from mindspore.nn.cell import Cell +from mindspore.nn.wrap.grad_reducer import DistributedGradReducer +import mindspore.nn as nn +from mindspore.common.tensor import Tensor +import numpy as np + +compute_norm = C.MultitypeFuncGraph("compute_norm") + + +@compute_norm.register("Tensor") +def _compute_norm(grad): + norm = nn.Norm() + norm = norm(F.cast(grad, mstype.float32)) + ret = F.expand_dims(F.cast(norm, mstype.float32), 0) + return ret + + +grad_div = C.MultitypeFuncGraph("grad_div") + + +@grad_div.register("Tensor", "Tensor") +def _grad_div(val, grad): + div = P.Div() + mul = P.Mul() + grad = mul(grad, 10.0) + ret = div(grad, val) + return ret + + +class TrainOneStepCellWithGradClip(Cell): + """ + Network training package class. + + Wraps the network with an optimizer. The resulting Cell be trained with input data and label. + Backward graph with grad clip will be created in the construct function to do parameter updating. + Different parallel modes are available to run the training. + + Args: + network (Cell): The training network. + optimizer (Cell): Optimizer for updating the weights. + sens (Number): The scaling number to be filled as the input of backpropagation. Default value is 1.0. + + Inputs: + - data (Tensor) - Tensor of shape :(N, ...). + - label (Tensor) - Tensor of shape :(N, ...). + + Outputs: + Tensor, a scalar Tensor with shape :math:`()`. + """ + + def __init__(self, network, optimizer, sens=1.0): + super(TrainOneStepCellWithGradClip, self).__init__(auto_prefix=False) + self.network = network + self.network.set_grad() + self.network.add_flags(defer_inline=True) + self.weights = optimizer.parameters + self.optimizer = optimizer + self.grad = C.GradOperation('grad', get_by_list=True, sens_param=True) + self.sens = sens + self.reducer_flag = False + self.grad_reducer = None + self.hyper_map = C.HyperMap() + self.greater = P.Greater() + self.select = P.Select() + self.norm = nn.Norm(keep_dims=True) + self.dtype = P.DType() + self.cast = P.Cast() + self.concat = P.Concat(axis=0) + self.ten = Tensor(np.array([10.0]).astype(np.float32)) + parallel_mode = _get_parallel_mode() + if parallel_mode in (ParallelMode.DATA_PARALLEL, ParallelMode.HYBRID_PARALLEL): + self.reducer_flag = True + if self.reducer_flag: + mean = _get_mirror_mean() + degree = _get_device_num() + self.grad_reducer = DistributedGradReducer(optimizer.parameters, mean, degree) + + def construct(self, data, label): + weights = self.weights + loss = self.network(data, label) + sens = P.Fill()(P.DType()(loss), P.Shape()(loss), self.sens) + grads = self.grad(self.network, weights)(data, label, sens) + norm = self.hyper_map(F.partial(compute_norm), grads) + norm = self.concat(norm) + norm = self.norm(norm) + cond = self.greater(norm, self.cast(self.ten, self.dtype(norm))) + clip_val = self.select(cond, norm, self.cast(self.ten, self.dtype(norm))) + grads = self.hyper_map(F.partial(grad_div, clip_val), grads) + if self.reducer_flag: + # apply grad reducer on grads + grads = self.grad_reducer(grads) + return F.depend(loss, self.optimizer(grads)) diff --git a/model_zoo/warpctc/train.py b/model_zoo/warpctc/train.py new file mode 100755 index 0000000000..651d2a73a4 --- /dev/null +++ b/model_zoo/warpctc/train.py @@ -0,0 +1,84 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Warpctc training""" +import os +import math as m +import random +import argparse +import numpy as np +import mindspore.nn as nn +from mindspore import context +from mindspore import dataset as de +from mindspore.train.model import Model, ParallelMode +from mindspore.nn.wrap import WithLossCell +from mindspore.train.callback import TimeMonitor, LossMonitor, CheckpointConfig, ModelCheckpoint +from mindspore.communication.management import init + +from src.loss import CTCLoss +from src.config import config as cf +from src.dataset import create_dataset +from src.warpctc import StackedRNN +from src.warpctc_for_train import TrainOneStepCellWithGradClip +from src.lr_schedule import get_lr + +random.seed(1) +np.random.seed(1) +de.config.set_seed(1) + +parser = argparse.ArgumentParser(description="Warpctc training") +parser.add_argument("--run_distribute", type=bool, default=False, help="Run distribute, default is false.") +parser.add_argument('--device_num', type=int, default=1, help='Device num, default is 1.') +parser.add_argument('--dataset_path', type=str, default=None, help='Dataset path, default is None') +args_opt = parser.parse_args() + +device_id = int(os.getenv('DEVICE_ID')) +context.set_context(mode=context.GRAPH_MODE, + device_target="Ascend", + save_graphs=False, + device_id=device_id) + +if __name__ == '__main__': + if args_opt.run_distribute: + context.reset_auto_parallel_context() + context.set_auto_parallel_context(device_num=args_opt.device_num, + parallel_mode=ParallelMode.DATA_PARALLEL, + mirror_mean=True) + init() + max_captcha_digits = cf.max_captcha_digits + input_size = m.ceil(cf.captcha_height / 64) * 64 * 3 + # create dataset + dataset = create_dataset(dataset_path=args_opt.dataset_path, repeat_num=cf.epoch_size, batch_size=cf.batch_size) + step_size = dataset.get_dataset_size() + # define lr + lr_init = cf.learning_rate if not args_opt.run_distribute else cf.learning_rate * args_opt.device_num + lr = get_lr(cf.epoch_size, step_size, lr_init) + # define loss + loss = CTCLoss(max_sequence_length=cf.captcha_width, max_label_length=max_captcha_digits, batch_size=cf.batch_size) + # define net + net = StackedRNN(input_size=input_size, batch_size=cf.batch_size, hidden_size=cf.hidden_size) + # define opt + opt = nn.SGD(params=net.trainable_params(), learning_rate=lr, momentum=cf.momentum) + net = WithLossCell(net, loss) + net = TrainOneStepCellWithGradClip(net, opt).set_train() + # define model + model = Model(net) + # define callbacks + callbacks = [LossMonitor(), TimeMonitor(data_size=step_size)] + if cf.save_checkpoint: + config_ck = CheckpointConfig(save_checkpoint_steps=cf.save_checkpoint_steps, + keep_checkpoint_max=cf.keep_checkpoint_max) + ckpt_cb = ModelCheckpoint(prefix="waptctc", directory=cf.save_checkpoint_path, config=config_ck) + callbacks.append(ckpt_cb) + model.train(cf.epoch_size, dataset, callbacks=callbacks) diff --git a/model_zoo/wide_and_deep/README.md b/model_zoo/wide_and_deep/README.md index 54367ef173..000e6a5335 100644 --- a/model_zoo/wide_and_deep/README.md +++ b/model_zoo/wide_and_deep/README.md @@ -37,6 +37,7 @@ To train and evaluate the model, command as follows: python train_and_eval.py ``` Arguments: + * `--device_target`: Device where the code will be implemented (Default: Ascend). * `--data_path`: This should be set to the same directory given to the data_download's data_dir argument. * `--epochs`: Total train epochs. * `--batch_size`: Training batch size. @@ -57,6 +58,7 @@ To train the model in one device, command as follows: python train.py ``` Arguments: + * `--device_target`: Device where the code will be implemented (Default: Ascend). * `--data_path`: This should be set to the same directory given to the data_download's data_dir argument. * `--epochs`: Total train epochs. * `--batch_size`: Training batch size. @@ -87,6 +89,7 @@ To evaluate the model, command as follows: python eval.py ``` Arguments: + * `--device_target`: Device where the code will be implemented (Default: Ascend). * `--data_path`: This should be set to the same directory given to the data_download's data_dir argument. * `--epochs`: Total train epochs. * `--batch_size`: Training batch size. diff --git a/model_zoo/wide_and_deep/eval.py b/model_zoo/wide_and_deep/eval.py index 72c30b1f5c..bc3846533f 100644 --- a/model_zoo/wide_and_deep/eval.py +++ b/model_zoo/wide_and_deep/eval.py @@ -26,11 +26,11 @@ from src.datasets import create_dataset from src.metrics import AUCMetric from src.config import WideDeepConfig -context.set_context(mode=context.GRAPH_MODE, device_target="Davinci", - save_graphs=True) - def get_WideDeep_net(config): + """ + Get network of wide&deep model. + """ WideDeep_net = WideDeepModel(config) loss_net = NetWithLossClass(WideDeep_net, config) @@ -91,4 +91,5 @@ if __name__ == "__main__": widedeep_config = WideDeepConfig() widedeep_config.argparse_init() + context.set_context(mode=context.GRAPH_MODE, device_target=widedeep_config.device_target) test_eval(widedeep_config) diff --git a/model_zoo/wide_and_deep/script/run_multigpu_train.sh b/model_zoo/wide_and_deep/script/run_multigpu_train.sh index 987eeaa65e..e0e08ab80e 100644 --- a/model_zoo/wide_and_deep/script/run_multigpu_train.sh +++ b/model_zoo/wide_and_deep/script/run_multigpu_train.sh @@ -14,7 +14,7 @@ # limitations under the License. # ============================================================================ -# bash run_multigpu_train.sh +# bash run_multigpu_train.sh RANK_SIZE EPOCH_SIZE DATASET script_self=$(readlink -f "$0") self_path=$(dirname "${script_self}") RANK_SIZE=$1 @@ -25,4 +25,5 @@ mpirun --allow-run-as-root -n $RANK_SIZE \ python -s ${self_path}/../train_and_eval_distribute.py \ --device_target="GPU" \ --data_path=$DATASET \ + --batch_size=8000 \ --epochs=$EPOCH_SIZE > log.txt 2>&1 & diff --git a/example/nlp_to_mindrecord/zhwiki/run_read.sh b/model_zoo/wide_and_deep/script/run_standalone_train_for_gpu.sh similarity index 61% rename from example/nlp_to_mindrecord/zhwiki/run_read.sh rename to model_zoo/wide_and_deep/script/run_standalone_train_for_gpu.sh index 3cc368457b..693c62b847 100644 --- a/example/nlp_to_mindrecord/zhwiki/run_read.sh +++ b/model_zoo/wide_and_deep/script/run_standalone_train_for_gpu.sh @@ -14,21 +14,14 @@ # limitations under the License. # ============================================================================ -file_list=() -file_index=0 +# bash run_standalone_train_for_gpu.sh EPOCH_SIZE DATASET +script_self=$(readlink -f "$0") +self_path=$(dirname "${script_self}") +EPOCH_SIZE=$1 +DATASET=$2 -# get all the mindrecord file from output dir -function getdir() { - elements=`ls $1/[A-Z]*.mindrecord` - for element in ${elements[*]}; - do - file_list[$file_index]=$element - file_index=`expr $file_index + 1` - done -} - -getdir "./output" -echo "Get all the mindrecord files: "${file_list[*]} - -# create dataset for train -python create_dataset.py --input_file ${file_list[*]} +python -s ${self_path}/../train_and_eval.py \ + --device_target="GPU" \ + --data_path=$DATASET \ + --batch_size=16000 \ + --epochs=$EPOCH_SIZE > log.txt 2>&1 & diff --git a/model_zoo/wide_and_deep/src/preprocess_data.py b/model_zoo/wide_and_deep/src/preprocess_data.py index 35d13b841d..75562aa71b 100644 --- a/model_zoo/wide_and_deep/src/preprocess_data.py +++ b/model_zoo/wide_and_deep/src/preprocess_data.py @@ -17,6 +17,8 @@ import os import pickle import collections import argparse +import urllib.request +import tarfile import numpy as np from mindspore.mindrecord import FileWriter @@ -257,10 +259,15 @@ if __name__ == '__main__': download_data_path = data_path + "origin_data/" mkdir_path(download_data_path) - os.system( - "wget -P {} -c https://s3-eu-west-1.amazonaws.com/kaggle-display-advertising-challenge-dataset/dac.tar.gz --no-check-certificate".format( - download_data_path)) - os.system("tar -zxvf {}dac.tar.gz".format(download_data_path)) + url = "https://s3-eu-west-1.amazonaws.com/kaggle-display-advertising-challenge-dataset/dac.tar.gz" + file_name = download_data_path + '/' + url.split('/')[-1] + urllib.request.urlretrieve(url, filename=file_name) + + tar = tarfile.open(file_name) + names = tar.getnames() + for name in names: + tar.extract(name, path=download_data_path) + tar.close() criteo_stats = CriteoStatsDict() data_file_path = data_path + "origin_data/train.txt" diff --git a/model_zoo/wide_and_deep/train.py b/model_zoo/wide_and_deep/train.py index ac9750c547..a043be3dc6 100644 --- a/model_zoo/wide_and_deep/train.py +++ b/model_zoo/wide_and_deep/train.py @@ -15,16 +15,16 @@ import os from mindspore import Model, context from mindspore.train.callback import ModelCheckpoint, CheckpointConfig, TimeMonitor - from src.wide_and_deep import PredictWithSigmoid, TrainStepWrap, NetWithLossClass, WideDeepModel from src.callbacks import LossCallBack from src.datasets import create_dataset from src.config import WideDeepConfig -context.set_context(mode=context.GRAPH_MODE, device_target="Ascend", save_graphs=True) - def get_WideDeep_net(configure): + """ + Get network of wide&deep model. + """ WideDeep_net = WideDeepModel(configure) loss_net = NetWithLossClass(WideDeep_net, configure) @@ -72,7 +72,7 @@ def test_train(configure): model = Model(train_net) callback = LossCallBack(config=configure) - ckptconfig = CheckpointConfig(save_checkpoint_steps=1, + ckptconfig = CheckpointConfig(save_checkpoint_steps=ds_train.get_dataset_size(), keep_checkpoint_max=5) ckpoint_cb = ModelCheckpoint(prefix='widedeep_train', directory=configure.ckpt_path, config=ckptconfig) model.train(epochs, ds_train, callbacks=[TimeMonitor(ds_train.get_dataset_size()), callback, ckpoint_cb]) @@ -82,4 +82,5 @@ if __name__ == "__main__": config = WideDeepConfig() config.argparse_init() + context.set_context(mode=context.GRAPH_MODE, device_target=config.device_target) test_train(config) diff --git a/model_zoo/wide_and_deep/train_and_eval.py b/model_zoo/wide_and_deep/train_and_eval.py index 0b37b67a11..e0ab6b2e9e 100644 --- a/model_zoo/wide_and_deep/train_and_eval.py +++ b/model_zoo/wide_and_deep/train_and_eval.py @@ -15,7 +15,7 @@ import os from mindspore import Model, context -from mindspore.train.callback import ModelCheckpoint, CheckpointConfig +from mindspore.train.callback import ModelCheckpoint, CheckpointConfig, TimeMonitor from src.wide_and_deep import PredictWithSigmoid, TrainStepWrap, NetWithLossClass, WideDeepModel from src.callbacks import LossCallBack, EvalCallBack @@ -23,10 +23,11 @@ from src.datasets import create_dataset from src.metrics import AUCMetric from src.config import WideDeepConfig -context.set_context(mode=context.GRAPH_MODE, device_target="Davinci") - def get_WideDeep_net(config): + """ + Get network of wide&deep model. + """ WideDeep_net = WideDeepModel(config) loss_net = NetWithLossClass(WideDeep_net, config) @@ -87,11 +88,13 @@ def test_train_eval(config): out = model.eval(ds_eval) print("=====" * 5 + "model.eval() initialized: {}".format(out)) - model.train(epochs, ds_train, callbacks=[eval_callback, callback, ckpoint_cb]) + model.train(epochs, ds_train, + callbacks=[TimeMonitor(ds_train.get_dataset_size()), eval_callback, callback, ckpoint_cb]) if __name__ == "__main__": wide_deep_config = WideDeepConfig() wide_deep_config.argparse_init() + context.set_context(mode=context.GRAPH_MODE, device_target=wide_deep_config.device_target) test_train_eval(wide_deep_config) diff --git a/model_zoo/wide_and_deep/train_and_eval_auto_parallel.py b/model_zoo/wide_and_deep/train_and_eval_auto_parallel.py index 780c95540c..4c86931b2e 100644 --- a/model_zoo/wide_and_deep/train_and_eval_auto_parallel.py +++ b/model_zoo/wide_and_deep/train_and_eval_auto_parallel.py @@ -40,6 +40,9 @@ init() def get_WideDeep_net(config): + """ + Get network of wide&deep model. + """ WideDeep_net = WideDeepModel(config) loss_net = NetWithLossClass(WideDeep_net, config) loss_net = VirtualDatasetCellTriple(loss_net) diff --git a/model_zoo/wide_and_deep/train_and_eval_distribute.py b/model_zoo/wide_and_deep/train_and_eval_distribute.py index db98bacfec..71f2b11cba 100644 --- a/model_zoo/wide_and_deep/train_and_eval_distribute.py +++ b/model_zoo/wide_and_deep/train_and_eval_distribute.py @@ -33,6 +33,9 @@ sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) def get_WideDeep_net(config): + """ + Get network of wide&deep model. + """ WideDeep_net = WideDeepModel(config) loss_net = NetWithLossClass(WideDeep_net, config) train_net = TrainStepWrap(loss_net) @@ -90,8 +93,12 @@ def train_and_eval(config): callback = LossCallBack(config=config) ckptconfig = CheckpointConfig(save_checkpoint_steps=ds_train.get_dataset_size(), keep_checkpoint_max=5) - ckpoint_cb = ModelCheckpoint(prefix='widedeep_train', - directory=config.ckpt_path, config=ckptconfig) + if config.device_target == "Ascend": + ckpoint_cb = ModelCheckpoint(prefix='widedeep_train', + directory=config.ckpt_path, config=ckptconfig) + elif config.device_target == "GPU": + ckpoint_cb = ModelCheckpoint(prefix='widedeep_train_' + str(get_rank()), + directory=config.ckpt_path, config=ckptconfig) out = model.eval(ds_eval) print("=====" * 5 + "model.eval() initialized: {}".format(out)) model.train(epochs, ds_train, diff --git a/model_zoo/yolov3_darknet53/README.md b/model_zoo/yolov3_darknet53/README.md new file mode 100644 index 0000000000..9a5bfed7d3 --- /dev/null +++ b/model_zoo/yolov3_darknet53/README.md @@ -0,0 +1,132 @@ +# YOLOV3-DarkNet53 Example + +## Description + +This is an example of training YOLOV3-DarkNet53 with COCO2014 dataset in MindSpore. + +## Requirements + +- Install [MindSpore](https://www.mindspore.cn/install/en). + +- Download the dataset COCO2014. + +> Unzip the COCO2014 dataset to any path you want, the folder should include train and eval dataset as follows: + +``` +. +└─dataset + ├─train2014 + ├─val2014 + └─annotations +``` + +## Structure + +```shell +. +└─yolov3_darknet53 + ├─README.md + ├─scripts + ├─run_standalone_train.sh # launch standalone training(1p) + ├─run_distribute_train.sh # launch distributed training(8p) + └─run_eval.sh # launch evaluating + ├─src + ├─config.py # parameter configuration + ├─darknet.py # backbone of network + ├─distributed_sampler.py # iterator of dataset + ├─initializer.py # initializer of parameters + ├─logger.py # log function + ├─loss.py # loss function + ├─lr_scheduler.py # generate learning rate + ├─transforms.py # Preprocess data + ├─util.py # util function + ├─yolo.py # yolov3 network + ├─yolo_dataset.py # create dataset for YOLOV3 + ├─eval.py # eval net + └─train.py # train net +``` + +## Running the example + +### Train + +#### Usage + +``` +# distributed training +sh run_distribute_train.sh [DATASET_PATH] [PRETRAINED_BACKBONE] [MINDSPORE_HCCL_CONFIG_PATH] + +# standalone training +sh run_standalone_train.sh [DATASET_PATH] [PRETRAINED_BACKBONE] +``` + +#### Launch + +```bash +# distributed training example(8p) +sh run_distribute_train.sh dataset/coco2014 backbone/backbone.ckpt rank_table_8p.json + +# standalone training example(1p) +sh run_standalone_train.sh dataset/coco2014 backbone/backbone.ckpt +``` + +> About rank_table.json, you can refer to the [distributed training tutorial](https://www.mindspore.cn/tutorial/en/master/advanced_use/distributed_training.html). + +#### Result + +Training result will be stored in the scripts path, whose folder name begins with "train" or "train_parallel". You can find checkpoint file together with result like the followings in log.txt. + +``` +# distribute training result(8p) +epoch[0], iter[0], loss:14623.384766, 1.23 imgs/sec, lr:7.812499825377017e-05 +epoch[0], iter[100], loss:1486.253051, 15.01 imgs/sec, lr:0.007890624925494194 +epoch[0], iter[200], loss:288.579535, 490.41 imgs/sec, lr:0.015703124925494194 +epoch[0], iter[300], loss:153.136754, 531.99 imgs/sec, lr:0.023515624925494194 +epoch[1], iter[400], loss:106.429322, 405.14 imgs/sec, lr:0.03132812678813934 +... +epoch[318], iter[102000], loss:34.135306, 431.06 imgs/sec, lr:9.63797629083274e-06 +epoch[319], iter[102100], loss:35.652469, 449.52 imgs/sec, lr:2.409552052995423e-06 +epoch[319], iter[102200], loss:34.652273, 384.02 imgs/sec, lr:2.409552052995423e-06 +epoch[319], iter[102300], loss:35.430038, 423.49 imgs/sec, lr:2.409552052995423e-06 +... +``` + +### Infer + +#### Usage + +``` +# infer +sh run_eval.sh [DATASET_PATH] [CHECKPOINT_PATH] +``` + +#### Launch + +```bash +# infer with checkpoint +sh run_eval.sh dataset/coco2014/ checkpoint/0-319_102400.ckpt + +``` + +> checkpoint can be produced in training process. + + +#### Result + +Inference result will be stored in the scripts path, whose folder name is "eval". Under this, you can find result like the followings in log.txt. + +``` +=============coco eval reulst========= + Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.311 + Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.528 + Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ] = 0.322 + Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.127 + Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.323 + Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.428 + Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 1 ] = 0.259 + Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 10 ] = 0.398 + Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.423 + Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.224 + Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.442 + Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.551 +``` diff --git a/model_zoo/yolov3_darknet53/eval.py b/model_zoo/yolov3_darknet53/eval.py new file mode 100644 index 0000000000..6680b10476 --- /dev/null +++ b/model_zoo/yolov3_darknet53/eval.py @@ -0,0 +1,328 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""YoloV3 eval.""" +import os +import argparse +import datetime +import time +import sys +from collections import defaultdict + +import numpy as np +from pycocotools.coco import COCO +from pycocotools.cocoeval import COCOeval + +from mindspore import Tensor +from mindspore.train import ParallelMode +from mindspore import context +from mindspore.train.serialization import load_checkpoint, load_param_into_net +import mindspore as ms + +from src.yolo import YOLOV3DarkNet53 +from src.logger import get_logger +from src.yolo_dataset import create_yolo_dataset +from src.config import ConfigYOLOV3DarkNet53 + +devid = int(os.getenv('DEVICE_ID')) +context.set_context(mode=context.GRAPH_MODE, device_target="Ascend", save_graphs=True, device_id=devid) + + +class Redirct: + def __init__(self): + self.content = "" + + def write(self, content): + self.content += content + + def flush(self): + self.content = "" + + +class DetectionEngine: + """Detection engine.""" + def __init__(self, args): + self.ignore_threshold = args.ignore_threshold + self.labels = ['person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus', 'train', 'truck', 'boat', + 'traffic light', 'fire hydrant', 'stop sign', 'parking meter', 'bench', 'bird', 'cat', + 'dog', 'horse', 'sheep', 'cow', 'elephant', 'bear', 'zebra', 'giraffe', 'backpack', + 'umbrella', 'handbag', 'tie', 'suitcase', 'frisbee', 'skis', 'snowboard', 'sports ball', + 'kite', 'baseball bat', 'baseball glove', 'skateboard', 'surfboard', 'tennis racket', + 'bottle', 'wine glass', 'cup', 'fork', 'knife', 'spoon', 'bowl', 'banana', 'apple', + 'sandwich', 'orange', 'broccoli', 'carrot', 'hot dog', 'pizza', 'donut', 'cake', 'chair', + 'couch', 'potted plant', 'bed', 'dining table', 'toilet', 'tv', 'laptop', 'mouse', 'remote', + 'keyboard', 'cell phone', 'microwave', 'oven', 'toaster', 'sink', 'refrigerator', 'book', + 'clock', 'vase', 'scissors', 'teddy bear', 'hair drier', 'toothbrush'] + self.num_classes = len(self.labels) + self.results = {} + self.file_path = '' + self.save_prefix = args.outputs_dir + self.annFile = args.annFile + self._coco = COCO(self.annFile) + self._img_ids = list(sorted(self._coco.imgs.keys())) + self.det_boxes = [] + self.nms_thresh = args.nms_thresh + self.coco_catIds = self._coco.getCatIds() + + def do_nms_for_results(self): + """Get result boxes.""" + for img_id in self.results: + for clsi in self.results[img_id]: + dets = self.results[img_id][clsi] + dets = np.array(dets) + keep_index = self._nms(dets, self.nms_thresh) + + keep_box = [{'image_id': int(img_id), + 'category_id': int(clsi), + 'bbox': list(dets[i][:4].astype(float)), + 'score': dets[i][4].astype(float)} + for i in keep_index] + self.det_boxes.extend(keep_box) + + def _nms(self, dets, thresh): + """Calculate NMS.""" + # conver xywh -> xmin ymin xmax ymax + x1 = dets[:, 0] + y1 = dets[:, 1] + x2 = x1 + dets[:, 2] + y2 = y1 + dets[:, 3] + scores = dets[:, 4] + + areas = (x2 - x1 + 1) * (y2 - y1 + 1) + order = scores.argsort()[::-1] + + keep = [] + while order.size > 0: + i = order[0] + keep.append(i) + xx1 = np.maximum(x1[i], x1[order[1:]]) + yy1 = np.maximum(y1[i], y1[order[1:]]) + xx2 = np.minimum(x2[i], x2[order[1:]]) + yy2 = np.minimum(y2[i], y2[order[1:]]) + + w = np.maximum(0.0, xx2 - xx1 + 1) + h = np.maximum(0.0, yy2 - yy1 + 1) + inter = w * h + ovr = inter / (areas[i] + areas[order[1:]] - inter) + + inds = np.where(ovr <= thresh)[0] + order = order[inds + 1] + return keep + + def write_result(self): + """Save result to file.""" + import json + t = datetime.datetime.now().strftime('_%Y_%m_%d_%H_%M_%S') + try: + self.file_path = self.save_prefix + '/predict' + t + '.json' + f = open(self.file_path, 'w') + json.dump(self.det_boxes, f) + except IOError as e: + raise RuntimeError("Unable to open json file to dump. What(): {}".format(str(e))) + else: + f.close() + return self.file_path + + def get_eval_result(self): + """Get eval result.""" + cocoGt = COCO(self.annFile) + cocoDt = cocoGt.loadRes(self.file_path) + cocoEval = COCOeval(cocoGt, cocoDt, 'bbox') + cocoEval.evaluate() + cocoEval.accumulate() + rdct = Redirct() + stdout = sys.stdout + sys.stdout = rdct + cocoEval.summarize() + sys.stdout = stdout + return rdct.content + + def detect(self, outputs, batch, image_shape, image_id): + """Detect boxes.""" + outputs_num = len(outputs) + # output [|32, 52, 52, 3, 85| ] + for batch_id in range(batch): + for out_id in range(outputs_num): + # 32, 52, 52, 3, 85 + out_item = outputs[out_id] + # 52, 52, 3, 85 + out_item_single = out_item[batch_id, :] + # get number of items in one head, [B, gx, gy, anchors, 5+80] + dimensions = out_item_single.shape[:-1] + out_num = 1 + for d in dimensions: + out_num *= d + ori_w, ori_h = image_shape[batch_id] + img_id = int(image_id[batch_id]) + x = out_item_single[..., 0] * ori_w + y = out_item_single[..., 1] * ori_h + w = out_item_single[..., 2] * ori_w + h = out_item_single[..., 3] * ori_h + + conf = out_item_single[..., 4:5] + cls_emb = out_item_single[..., 5:] + + cls_argmax = np.expand_dims(np.argmax(cls_emb, axis=-1), axis=-1) + x = x.reshape(-1) + y = y.reshape(-1) + w = w.reshape(-1) + h = h.reshape(-1) + cls_emb = cls_emb.reshape(-1, 80) + conf = conf.reshape(-1) + cls_argmax = cls_argmax.reshape(-1) + + x_top_left = x - w / 2. + y_top_left = y - h / 2. + # creat all False + flag = np.random.random(cls_emb.shape) > sys.maxsize + for i in range(flag.shape[0]): + c = cls_argmax[i] + flag[i, c] = True + confidence = cls_emb[flag] * conf + for x_lefti, y_lefti, wi, hi, confi, clsi in zip(x_top_left, y_top_left, w, h, confidence, cls_argmax): + if confi < self.ignore_threshold: + continue + if img_id not in self.results: + self.results[img_id] = defaultdict(list) + x_lefti = max(0, x_lefti) + y_lefti = max(0, y_lefti) + wi = min(wi, ori_w) + hi = min(hi, ori_h) + # transform catId to match coco + coco_clsi = self.coco_catIds[clsi] + self.results[img_id][coco_clsi].append([x_lefti, y_lefti, wi, hi, confi]) + + +def parse_args(): + """Parse arguments.""" + parser = argparse.ArgumentParser('mindspore coco testing') + + # dataset related + parser.add_argument('--data_dir', type=str, default='', help='train data dir') + parser.add_argument('--per_batch_size', default=1, type=int, help='batch size for per gpu') + + # network related + parser.add_argument('--pretrained', default='', type=str, help='model_path, local pretrained model to load') + + # logging related + parser.add_argument('--log_path', type=str, default='outputs/', help='checkpoint save location') + + # detect_related + parser.add_argument('--nms_thresh', type=float, default=0.5, help='threshold for NMS') + parser.add_argument('--annFile', type=str, default='', help='path to annotation') + parser.add_argument('--testing_shape', type=str, default='', help='shape for test ') + parser.add_argument('--ignore_threshold', type=float, default=0.001, help='threshold to throw low quality boxes') + + args, _ = parser.parse_known_args() + + args.data_root = os.path.join(args.data_dir, 'val2014') + args.annFile = os.path.join(args.data_dir, 'annotations/instances_val2014.json') + + return args + + +def conver_testing_shape(args): + """Convert testing shape to list.""" + testing_shape = [int(args.testing_shape), int(args.testing_shape)] + return testing_shape + + +def test(): + """The function of eval.""" + start_time = time.time() + args = parse_args() + + # logger + args.outputs_dir = os.path.join(args.log_path, + datetime.datetime.now().strftime('%Y-%m-%d_time_%H_%M_%S')) + rank_id = int(os.environ.get('RANK_ID')) + args.logger = get_logger(args.outputs_dir, rank_id) + + context.reset_auto_parallel_context() + parallel_mode = ParallelMode.STAND_ALONE + context.set_auto_parallel_context(parallel_mode=parallel_mode, mirror_mean=True, device_num=1) + + args.logger.info('Creating Network....') + network = YOLOV3DarkNet53(is_training=False) + + args.logger.info(args.pretrained) + if os.path.isfile(args.pretrained): + param_dict = load_checkpoint(args.pretrained) + param_dict_new = {} + for key, values in param_dict.items(): + if key.startswith('moments.'): + continue + elif key.startswith('yolo_network.'): + param_dict_new[key[13:]] = values + else: + param_dict_new[key] = values + load_param_into_net(network, param_dict_new) + args.logger.info('load_model {} success'.format(args.pretrained)) + else: + args.logger.info('{} not exists or not a pre-trained file'.format(args.pretrained)) + assert FileNotFoundError('{} not exists or not a pre-trained file'.format(args.pretrained)) + exit(1) + + data_root = args.data_root + ann_file = args.annFile + + config = ConfigYOLOV3DarkNet53() + if args.testing_shape: + config.test_img_shape = conver_testing_shape(args) + + ds, data_size = create_yolo_dataset(data_root, ann_file, is_training=False, batch_size=args.per_batch_size, + max_epoch=1, device_num=1, rank=rank_id, shuffle=False, + config=config) + + args.logger.info('testing shape : {}'.format(config.test_img_shape)) + args.logger.info('totol {} images to eval'.format(data_size)) + + network.set_train(False) + + # init detection engine + detection = DetectionEngine(args) + + input_shape = Tensor(tuple(config.test_img_shape), ms.float32) + args.logger.info('Start inference....') + for i, data in enumerate(ds.create_dict_iterator()): + image = Tensor(data["image"]) + + image_shape = Tensor(data["image_shape"]) + image_id = Tensor(data["img_id"]) + + prediction = network(image, input_shape) + output_big, output_me, output_small = prediction + output_big = output_big.asnumpy() + output_me = output_me.asnumpy() + output_small = output_small.asnumpy() + image_id = image_id.asnumpy() + image_shape = image_shape.asnumpy() + + detection.detect([output_small, output_me, output_big], args.per_batch_size, image_shape, image_id) + if i % 1000 == 0: + args.logger.info('Processing... {:.2f}% '.format(i * args.per_batch_size / data_size * 100)) + + args.logger.info('Calculating mAP...') + detection.do_nms_for_results() + result_file_path = detection.write_result() + args.logger.info('result file path: {}'.format(result_file_path)) + eval_result = detection.get_eval_result() + + cost_time = time.time() - start_time + args.logger.info('\n=============coco eval reulst=========\n' + eval_result) + args.logger.info('testing cost time {:.2f}h'.format(cost_time / 3600.)) + + +if __name__ == "__main__": + test() diff --git a/example/resnet50_imagenet2012/run_distribute_train.sh b/model_zoo/yolov3_darknet53/scripts/run_distribute_train.sh old mode 100755 new mode 100644 similarity index 51% rename from example/resnet50_imagenet2012/run_distribute_train.sh rename to model_zoo/yolov3_darknet53/scripts/run_distribute_train.sh index 22157608e6..c6e83ae8f8 --- a/example/resnet50_imagenet2012/run_distribute_train.sh +++ b/model_zoo/yolov3_darknet53/scripts/run_distribute_train.sh @@ -14,9 +14,9 @@ # limitations under the License. # ============================================================================ -if [ $# != 2 ] && [ $# != 3 ] -then - echo "Usage: sh run_distribute_train.sh [MINDSPORE_HCCL_CONFIG_PATH] [DATASET_PATH] [PRETRAINED_CKPT_PATH](optional)" +if [ $# != 3 ] +then + echo "Usage: sh run_distribute_train.sh [DATASET_PATH] [PRETRAINED_BACKBONE] [MINDSPORE_HCCL_CONFIG_PATH]" exit 1 fi @@ -28,36 +28,34 @@ get_real_path(){ fi } -PATH1=$(get_real_path $1) -PATH2=$(get_real_path $2) -if [ $# == 3 ] -then - PATH3=$(get_real_path $3) -fi +DATASET_PATH=$(get_real_path $1) +PRETRAINED_BACKBONE=$(get_real_path $2) +MINDSPORE_HCCL_CONFIG_PATH=$(get_real_path $3) +echo $DATASET_PATH +echo $PRETRAINED_BACKBONE +echo $MINDSPORE_HCCL_CONFIG_PATH -if [ ! -f "$PATH1" ] -then - echo "error: MINDSPORE_HCCL_CONFIG_PATH=$PATH1 is not a file" +if [ ! -d $DATASET_PATH ] +then + echo "error: DATASET_PATH=$DATASET_PATH is not a directory" exit 1 -fi +fi -if [ ! -d "$PATH2" ] -then - echo "error: DATASET_PATH=$PATH2 is not a directory" +if [ ! -f $PRETRAINED_BACKBONE ] +then + echo "error: PRETRAINED_PATH=$PRETRAINED_BACKBONE is not a file" exit 1 -fi +fi -if [ $# == 3 ] && [ ! -f "$PATH3" ] +if [ ! -f $MINDSPORE_HCCL_CONFIG_PATH ] then - echo "error: PRETRAINED_CKPT_PATH=$PATH3 is not a file" + echo "error: MINDSPORE_HCCL_CONFIG_PATH=$MINDSPORE_HCCL_CONFIG_PATH is not a file" exit 1 fi -ulimit -u unlimited export DEVICE_NUM=8 export RANK_SIZE=8 -export MINDSPORE_HCCL_CONFIG_PATH=$PATH1 -export RANK_TABLE_FILE=$PATH1 +export MINDSPORE_HCCL_CONFIG_PATH=$MINDSPORE_HCCL_CONFIG_PATH for((i=0; i<${DEVICE_NUM}; i++)) do @@ -65,16 +63,19 @@ do export RANK_ID=$i rm -rf ./train_parallel$i mkdir ./train_parallel$i - cp *.py ./train_parallel$i - cp *.sh ./train_parallel$i + cp ../*.py ./train_parallel$i + cp -r ../src ./train_parallel$i cd ./train_parallel$i || exit echo "start training for rank $RANK_ID, device $DEVICE_ID" env > env.log - if [ $# == 2 ] - then - python train.py --do_train=True --run_distribute=True --device_num=$DEVICE_NUM --dataset_path=$PATH2 &> log & - else - python train.py --do_train=True --run_distribute=True --device_num=$DEVICE_NUM --dataset_path=$PATH2 --pre_trained=$PATH3 &> log & - fi + python train.py \ + --data_dir=$DATASET_PATH \ + --pretrained_backbone=$PRETRAINED_BACKBONE \ + --is_distributed=1 \ + --lr=0.1 \ + --T_max=320 \ + --max_epoch=320 \ + --warmup_epochs=4 \ + --lr_scheduler=cosine_annealing > log.txt 2>&1 & cd .. done diff --git a/example/resnet50_imagenet2012/run_infer.sh b/model_zoo/yolov3_darknet53/scripts/run_eval.sh old mode 100755 new mode 100644 similarity index 69% rename from example/resnet50_imagenet2012/run_infer.sh rename to model_zoo/yolov3_darknet53/scripts/run_eval.sh index 1482b63f5f..ff6b79e516 --- a/example/resnet50_imagenet2012/run_infer.sh +++ b/model_zoo/yolov3_darknet53/scripts/run_eval.sh @@ -15,8 +15,8 @@ # ============================================================================ if [ $# != 2 ] -then - echo "Usage: sh run_infer.sh [DATASET_PATH] [CHECKPOINT_PATH]" +then + echo "Usage: sh run_eval.sh [DATASET_PATH] [CHECKPOINT_PATH]" exit 1 fi @@ -27,38 +27,40 @@ get_real_path(){ echo "$(realpath -m $PWD/$1)" fi } +DATASET_PATH=$(get_real_path $1) +CHECKPOINT_PATH=$(get_real_path $2) +echo $DATASET_PATH +echo $CHECKPOINT_PATH -PATH1=$(get_real_path $1) -PATH2=$(get_real_path $2) - - -if [ ! -d $PATH1 ] -then +if [ ! -d $DATASET_PATH ] +then echo "error: DATASET_PATH=$PATH1 is not a directory" exit 1 -fi +fi -if [ ! -f $PATH2 ] -then +if [ ! -f $CHECKPOINT_PATH ] +then echo "error: CHECKPOINT_PATH=$PATH2 is not a file" exit 1 -fi +fi -ulimit -u unlimited export DEVICE_NUM=1 export DEVICE_ID=0 export RANK_SIZE=$DEVICE_NUM export RANK_ID=0 -if [ -d "infer" ]; +if [ -d "eval" ]; then - rm -rf ./infer + rm -rf ./eval fi -mkdir ./infer -cp *.py ./infer -cp *.sh ./infer -cd ./infer || exit +mkdir ./eval +cp ../*.py ./eval +cp -r ../src ./eval +cd ./eval || exit env > env.log echo "start infering for device $DEVICE_ID" -python eval.py --do_eval=True --dataset_path=$PATH1 --checkpoint_path=$PATH2 &> log & +python eval.py \ + --data_dir=$DATASET_PATH \ + --pretrained=$CHECKPOINT_PATH \ + --testing_shape=416 > log.txt 2>&1 & cd .. diff --git a/model_zoo/resnet101/scripts/run_standalone_train.sh b/model_zoo/yolov3_darknet53/scripts/run_standalone_train.sh old mode 100755 new mode 100644 similarity index 65% rename from model_zoo/resnet101/scripts/run_standalone_train.sh rename to model_zoo/yolov3_darknet53/scripts/run_standalone_train.sh index 7214d114d5..2597dca24e --- a/model_zoo/resnet101/scripts/run_standalone_train.sh +++ b/model_zoo/yolov3_darknet53/scripts/run_standalone_train.sh @@ -14,9 +14,9 @@ # limitations under the License. # ============================================================================ -if [ $# != 1 ] && [ $# != 2 ] -then - echo "Usage: sh run_standalone_train.sh [DATASET_PATH] [PRETRAINED_PATH](optional)" +if [ $# != 2 ] +then + echo "Usage: sh run_standalone_train.sh [DATASET_PATH] [PRETRAINED_BACKBONE]" exit 1 fi @@ -27,27 +27,24 @@ get_real_path(){ echo "$(realpath -m $PWD/$1)" fi } -PATH1=$(get_real_path $1) -echo $PATH1 -if [ $# == 2 ] -then - PATH2=$(get_real_path $2) - echo $PATH2 -fi -if [ ! -d $PATH1 ] -then - echo "error: DATASET_PATH=$PATH1 is not a directory" +DATASET_PATH=$(get_real_path $1) +echo $DATASET_PATH +PRETRAINED_BACKBONE=$(get_real_path $2) +echo $PRETRAINED_BACKBONE + +if [ ! -d $DATASET_PATH ] +then + echo "error: DATASET_PATH=$DATASET_PATH is not a directory" exit 1 fi -if [ $# == 2 ] && [ ! -f $PATH2 ] +if [ ! -f $PRETRAINED_BACKBONE ] then - echo "error: PRETRAINED_PATH=$PATH2 is not a file" + echo "error: PRETRAINED_PATH=$PRETRAINED_BACKBONE is not a file" exit 1 fi -ulimit -u unlimited export DEVICE_NUM=1 export DEVICE_ID=0 export RANK_ID=0 @@ -59,18 +56,18 @@ then fi mkdir ./train cp ../*.py ./train -cp *.sh ./train cp -r ../src ./train cd ./train || exit echo "start training for device $DEVICE_ID" env > env.log -if [ $# == 1 ] -then - python train.py --do_train=True --dataset_path=$PATH1 &> log & -fi -if [ $# == 2 ] -then - python train.py --do_train=True --dataset_path=$PATH1 --pre_trained=$PATH2 &> log & -fi -cd .. +python train.py \ + --data_dir=$DATASET_PATH \ + --pretrained_backbone=$PRETRAINED_BACKBONE \ + --is_distributed=0 \ + --lr=0.1 \ + --T_max=320 \ + --max_epoch=320 \ + --warmup_epochs=4 \ + --lr_scheduler=cosine_annealing > log.txt 2>&1 & +cd .. \ No newline at end of file diff --git a/model_zoo/yolov3_darknet53/src/config.py b/model_zoo/yolov3_darknet53/src/config.py new file mode 100644 index 0000000000..16831b048d --- /dev/null +++ b/model_zoo/yolov3_darknet53/src/config.py @@ -0,0 +1,68 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Config parameters for Darknet based yolov3_darknet53 models.""" + + +class ConfigYOLOV3DarkNet53: + """ + Config parameters for the yolov3_darknet53. + + Examples: + ConfigYOLOV3DarkNet53() + """ + # train_param + # data augmentation related + hue = 0.1 + saturation = 1.5 + value = 1.5 + jitter = 0.3 + + resize_rate = 1 + multi_scale = [[320, 320], + [352, 352], + [384, 384], + [416, 416], + [448, 448], + [480, 480], + [512, 512], + [544, 544], + [576, 576], + [608, 608] + ] + + num_classes = 80 + max_box = 50 + + backbone_input_shape = [32, 64, 128, 256, 512] + backbone_shape = [64, 128, 256, 512, 1024] + backbone_layers = [1, 2, 8, 8, 4] + + # confidence under ignore_threshold means no object when training + ignore_threshold = 0.7 + + # h->w + anchor_scales = [(10, 13), + (16, 30), + (33, 23), + (30, 61), + (62, 45), + (59, 119), + (116, 90), + (156, 198), + (373, 326)] + out_channel = 255 + + # test_param + test_img_shape = [416, 416] diff --git a/model_zoo/yolov3_darknet53/src/darknet.py b/model_zoo/yolov3_darknet53/src/darknet.py new file mode 100644 index 0000000000..4a2eb1de78 --- /dev/null +++ b/model_zoo/yolov3_darknet53/src/darknet.py @@ -0,0 +1,211 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""DarkNet model.""" +import mindspore.nn as nn +from mindspore.ops import operations as P + + +def conv_block(in_channels, + out_channels, + kernel_size, + stride, + dilation=1): + """Get a conv2d batchnorm and relu layer""" + pad_mode = 'same' + padding = 0 + + return nn.SequentialCell( + [nn.Conv2d(in_channels, + out_channels, + kernel_size=kernel_size, + stride=stride, + padding=padding, + dilation=dilation, + pad_mode=pad_mode), + nn.BatchNorm2d(out_channels, momentum=0.1), + nn.ReLU()] + ) + + +class ResidualBlock(nn.Cell): + """ + DarkNet V1 residual block definition. + + Args: + in_channels: Integer. Input channel. + out_channels: Integer. Output channel. + + Returns: + Tensor, output tensor. + Examples: + ResidualBlock(3, 208) + """ + expansion = 4 + + def __init__(self, + in_channels, + out_channels): + + super(ResidualBlock, self).__init__() + out_chls = out_channels//2 + self.conv1 = conv_block(in_channels, out_chls, kernel_size=1, stride=1) + self.conv2 = conv_block(out_chls, out_channels, kernel_size=3, stride=1) + self.add = P.TensorAdd() + + def construct(self, x): + identity = x + out = self.conv1(x) + out = self.conv2(out) + out = self.add(out, identity) + + return out + + +class DarkNet(nn.Cell): + """ + DarkNet V1 network. + + Args: + block: Cell. Block for network. + layer_nums: List. Numbers of different layers. + in_channels: Integer. Input channel. + out_channels: Integer. Output channel. + detect: Bool. Whether detect or not. Default:False. + + Returns: + Tuple, tuple of output tensor,(f1,f2,f3,f4,f5). + + Examples: + DarkNet(ResidualBlock, + [1, 2, 8, 8, 4], + [32, 64, 128, 256, 512], + [64, 128, 256, 512, 1024], + 100) + """ + def __init__(self, + block, + layer_nums, + in_channels, + out_channels, + detect=False): + super(DarkNet, self).__init__() + + self.outchannel = out_channels[-1] + self.detect = detect + + if not len(layer_nums) == len(in_channels) == len(out_channels) == 5: + raise ValueError("the length of layer_num, inchannel, outchannel list must be 5!") + self.conv0 = conv_block(3, + in_channels[0], + kernel_size=3, + stride=1) + self.conv1 = conv_block(in_channels[0], + out_channels[0], + kernel_size=3, + stride=2) + self.conv2 = conv_block(in_channels[1], + out_channels[1], + kernel_size=3, + stride=2) + self.conv3 = conv_block(in_channels[2], + out_channels[2], + kernel_size=3, + stride=2) + self.conv4 = conv_block(in_channels[3], + out_channels[3], + kernel_size=3, + stride=2) + self.conv5 = conv_block(in_channels[4], + out_channels[4], + kernel_size=3, + stride=2) + + self.layer1 = self._make_layer(block, + layer_nums[0], + in_channel=out_channels[0], + out_channel=out_channels[0]) + self.layer2 = self._make_layer(block, + layer_nums[1], + in_channel=out_channels[1], + out_channel=out_channels[1]) + self.layer3 = self._make_layer(block, + layer_nums[2], + in_channel=out_channels[2], + out_channel=out_channels[2]) + self.layer4 = self._make_layer(block, + layer_nums[3], + in_channel=out_channels[3], + out_channel=out_channels[3]) + self.layer5 = self._make_layer(block, + layer_nums[4], + in_channel=out_channels[4], + out_channel=out_channels[4]) + + def _make_layer(self, block, layer_num, in_channel, out_channel): + """ + Make Layer for DarkNet. + + :param block: Cell. DarkNet block. + :param layer_num: Integer. Layer number. + :param in_channel: Integer. Input channel. + :param out_channel: Integer. Output channel. + + Examples: + _make_layer(ConvBlock, 1, 128, 256) + """ + layers = [] + darkblk = block(in_channel, out_channel) + layers.append(darkblk) + + for _ in range(1, layer_num): + darkblk = block(out_channel, out_channel) + layers.append(darkblk) + + return nn.SequentialCell(layers) + + def construct(self, x): + c1 = self.conv0(x) + c2 = self.conv1(c1) + c3 = self.layer1(c2) + c4 = self.conv2(c3) + c5 = self.layer2(c4) + c6 = self.conv3(c5) + c7 = self.layer3(c6) + c8 = self.conv4(c7) + c9 = self.layer4(c8) + c10 = self.conv5(c9) + c11 = self.layer5(c10) + if self.detect: + return c7, c9, c11 + + return c11 + + def get_out_channels(self): + return self.outchannel + + +def darknet53(): + """ + Get DarkNet53 neural network. + + Returns: + Cell, cell instance of DarkNet53 neural network. + + Examples: + darknet53() + """ + return DarkNet(ResidualBlock, [1, 2, 8, 8, 4], + [32, 64, 128, 256, 512], + [64, 128, 256, 512, 1024]) diff --git a/model_zoo/yolov3_darknet53/src/distributed_sampler.py b/model_zoo/yolov3_darknet53/src/distributed_sampler.py new file mode 100644 index 0000000000..d31048ee9b --- /dev/null +++ b/model_zoo/yolov3_darknet53/src/distributed_sampler.py @@ -0,0 +1,60 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Yolo dataset distributed sampler.""" +from __future__ import division +import math +import numpy as np + + +class DistributedSampler: + """Distributed sampler.""" + def __init__(self, dataset_size, num_replicas=None, rank=None, shuffle=True): + if num_replicas is None: + print("***********Setting world_size to 1 since it is not passed in ******************") + num_replicas = 1 + if rank is None: + print("***********Setting rank to 0 since it is not passed in ******************") + rank = 0 + self.dataset_size = dataset_size + self.num_replicas = num_replicas + self.rank = rank + self.epoch = 0 + self.num_samples = int(math.ceil(dataset_size * 1.0 / self.num_replicas)) + self.total_size = self.num_samples * self.num_replicas + self.shuffle = shuffle + + def __iter__(self): + # deterministically shuffle based on epoch + if self.shuffle: + indices = np.random.RandomState(seed=self.epoch).permutation(self.dataset_size) + # np.array type. number from 0 to len(dataset_size)-1, used as index of dataset + indices = indices.tolist() + self.epoch += 1 + # change to list type + else: + indices = list(range(self.dataset_size)) + + # add extra samples to make it evenly divisible + indices += indices[:(self.total_size - len(indices))] + assert len(indices) == self.total_size + + # subsample + indices = indices[self.rank:self.total_size:self.num_replicas] + assert len(indices) == self.num_samples + + return iter(indices) + + def __len__(self): + return self.num_samples diff --git a/model_zoo/yolov3_darknet53/src/initializer.py b/model_zoo/yolov3_darknet53/src/initializer.py new file mode 100644 index 0000000000..f3c03a8ad1 --- /dev/null +++ b/model_zoo/yolov3_darknet53/src/initializer.py @@ -0,0 +1,179 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Parameter init.""" +import math +import numpy as np +from mindspore.common import initializer as init +from mindspore.common.initializer import Initializer as MeInitializer +import mindspore.nn as nn +from mindspore import Tensor + + +np.random.seed(5) + + +def calculate_gain(nonlinearity, param=None): + r"""Return the recommended gain value for the given nonlinearity function. + The values are as follows: + + ================= ==================================================== + nonlinearity gain + ================= ==================================================== + Linear / Identity :math:`1` + Conv{1,2,3}D :math:`1` + Sigmoid :math:`1` + Tanh :math:`\frac{5}{3}` + ReLU :math:`\sqrt{2}` + Leaky Relu :math:`\sqrt{\frac{2}{1 + \text{negative\_slope}^2}}` + ================= ==================================================== + + Args: + nonlinearity: the non-linear function (`nn.functional` name) + param: optional parameter for the non-linear function + + Examples: + >>> gain = nn.init.calculate_gain('leaky_relu', 0.2) # leaky_relu with negative_slope=0.2 + """ + linear_fns = ['linear', 'conv1d', 'conv2d', 'conv3d', 'conv_transpose1d', 'conv_transpose2d', 'conv_transpose3d'] + if nonlinearity in linear_fns or nonlinearity == 'sigmoid': + return 1 + if nonlinearity == 'tanh': + return 5.0 / 3 + if nonlinearity == 'relu': + return math.sqrt(2.0) + if nonlinearity == 'leaky_relu': + if param is None: + negative_slope = 0.01 + elif not isinstance(param, bool) and isinstance(param, int) or isinstance(param, float): + # True/False are instances of int, hence check above + negative_slope = param + else: + raise ValueError("negative_slope {} not a valid number".format(param)) + return math.sqrt(2.0 / (1 + negative_slope ** 2)) + + raise ValueError("Unsupported nonlinearity {}".format(nonlinearity)) + + +def _assignment(arr, num): + """Assign the value of 'num' and 'arr'.""" + if arr.shape == (): + arr = arr.reshape((1)) + arr[:] = num + arr = arr.reshape(()) + else: + if isinstance(num, np.ndarray): + arr[:] = num[:] + else: + arr[:] = num + return arr + + +def _calculate_correct_fan(array, mode): + mode = mode.lower() + valid_modes = ['fan_in', 'fan_out'] + if mode not in valid_modes: + raise ValueError("Mode {} not supported, please use one of {}".format(mode, valid_modes)) + + fan_in, fan_out = _calculate_fan_in_and_fan_out(array) + return fan_in if mode == 'fan_in' else fan_out + + +def kaiming_uniform_(arr, a=0, mode='fan_in', nonlinearity='leaky_relu'): + r"""Fills the input `Tensor` with values according to the method + described in `Delving deep into rectifiers: Surpassing human-level + performance on ImageNet classification` - He, K. et al. (2015), using a + uniform distribution. The resulting tensor will have values sampled from + :math:`\mathcal{U}(-\text{bound}, \text{bound})` where + + .. math:: + \text{bound} = \text{gain} \times \sqrt{\frac{3}{\text{fan\_mode}}} + + Also known as He initialization. + + Args: + tensor: an n-dimensional `Tensor` + a: the negative slope of the rectifier used after this layer (only + used with ``'leaky_relu'``) + mode: either ``'fan_in'`` (default) or ``'fan_out'``. Choosing ``'fan_in'`` + preserves the magnitude of the variance of the weights in the + forward pass. Choosing ``'fan_out'`` preserves the magnitudes in the + backwards pass. + nonlinearity: the non-linear function (`nn.functional` name), + recommended to use only with ``'relu'`` or ``'leaky_relu'`` (default). + + Examples: + >>> w = np.empty(3, 5) + >>> nn.init.kaiming_uniform_(w, mode='fan_in', nonlinearity='relu') + """ + fan = _calculate_correct_fan(arr, mode) + gain = calculate_gain(nonlinearity, a) + std = gain / math.sqrt(fan) + bound = math.sqrt(3.0) * std # Calculate uniform bounds from standard deviation + return np.random.uniform(-bound, bound, arr.shape) + + +def _calculate_fan_in_and_fan_out(arr): + """Calculate fan in and fan out.""" + dimensions = len(arr.shape) + if dimensions < 2: + raise ValueError("Fan in and fan out can not be computed for array with fewer than 2 dimensions") + + num_input_fmaps = arr.shape[1] + num_output_fmaps = arr.shape[0] + receptive_field_size = 1 + if dimensions > 2: + receptive_field_size = arr[0][0].size + fan_in = num_input_fmaps * receptive_field_size + fan_out = num_output_fmaps * receptive_field_size + + return fan_in, fan_out + + +class KaimingUniform(MeInitializer): + """Kaiming uniform initializer.""" + def __init__(self, a=0, mode='fan_in', nonlinearity='leaky_relu'): + super(KaimingUniform, self).__init__() + self.a = a + self.mode = mode + self.nonlinearity = nonlinearity + + def _initialize(self, arr): + tmp = kaiming_uniform_(arr, self.a, self.mode, self.nonlinearity) + _assignment(arr, tmp) + + +def default_recurisive_init(custom_cell): + """Initialize parameter.""" + for _, cell in custom_cell.cells_and_names(): + if isinstance(cell, nn.Conv2d): + cell.weight.default_input = init.initializer(KaimingUniform(a=math.sqrt(5)), + cell.weight.default_input.shape, + cell.weight.default_input.dtype).to_tensor() + if cell.bias is not None: + fan_in, _ = _calculate_fan_in_and_fan_out(cell.weight.default_input.asnumpy()) + bound = 1 / math.sqrt(fan_in) + cell.bias.default_input = Tensor(np.random.uniform(-bound, bound, cell.bias.default_input.shape), + cell.bias.default_input.dtype) + elif isinstance(cell, nn.Dense): + cell.weight.default_input = init.initializer(KaimingUniform(a=math.sqrt(5)), + cell.weight.default_input.shape, + cell.weight.default_input.dtype).to_tensor() + if cell.bias is not None: + fan_in, _ = _calculate_fan_in_and_fan_out(cell.weight.default_input.asnumpy()) + bound = 1 / math.sqrt(fan_in) + cell.bias.default_input = Tensor(np.random.uniform(-bound, bound, cell.bias.default_input.shape), + cell.bias.default_input.dtype) + elif isinstance(cell, (nn.BatchNorm2d, nn.BatchNorm1d)): + pass diff --git a/model_zoo/yolov3_darknet53/src/logger.py b/model_zoo/yolov3_darknet53/src/logger.py new file mode 100644 index 0000000000..b41ab405fc --- /dev/null +++ b/model_zoo/yolov3_darknet53/src/logger.py @@ -0,0 +1,80 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Custom Logger.""" +import os +import sys +import logging +from datetime import datetime + + +class LOGGER(logging.Logger): + """ + Logger. + + Args: + logger_name: String. Logger name. + rank: Integer. Rank id. + """ + def __init__(self, logger_name, rank=0): + super(LOGGER, self).__init__(logger_name) + self.rank = rank + if rank % 8 == 0: + console = logging.StreamHandler(sys.stdout) + console.setLevel(logging.INFO) + formatter = logging.Formatter('%(asctime)s:%(levelname)s:%(message)s') + console.setFormatter(formatter) + self.addHandler(console) + + def setup_logging_file(self, log_dir, rank=0): + """Setup logging file.""" + self.rank = rank + if not os.path.exists(log_dir): + os.makedirs(log_dir, exist_ok=True) + log_name = datetime.now().strftime('%Y-%m-%d_time_%H_%M_%S') + '_rank_{}.log'.format(rank) + self.log_fn = os.path.join(log_dir, log_name) + fh = logging.FileHandler(self.log_fn) + fh.setLevel(logging.INFO) + formatter = logging.Formatter('%(asctime)s:%(levelname)s:%(message)s') + fh.setFormatter(formatter) + self.addHandler(fh) + + def info(self, msg, *args, **kwargs): + if self.isEnabledFor(logging.INFO): + self._log(logging.INFO, msg, args, **kwargs) + + def save_args(self, args): + self.info('Args:') + args_dict = vars(args) + for key in args_dict.keys(): + self.info('--> %s: %s', key, args_dict[key]) + self.info('') + + def important_info(self, msg, *args, **kwargs): + if self.isEnabledFor(logging.INFO) and self.rank == 0: + line_width = 2 + important_msg = '\n' + important_msg += ('*'*70 + '\n')*line_width + important_msg += ('*'*line_width + '\n')*2 + important_msg += '*'*line_width + ' '*8 + msg + '\n' + important_msg += ('*'*line_width + '\n')*2 + important_msg += ('*'*70 + '\n')*line_width + self.info(important_msg, *args, **kwargs) + + +def get_logger(path, rank): + """Get Logger.""" + logger = LOGGER('yolov3_darknet53', rank) + logger.setup_logging_file(path, rank) + return logger diff --git a/model_zoo/yolov3_darknet53/src/loss.py b/model_zoo/yolov3_darknet53/src/loss.py new file mode 100644 index 0000000000..acdc6dba60 --- /dev/null +++ b/model_zoo/yolov3_darknet53/src/loss.py @@ -0,0 +1,70 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""YOLOV3 loss.""" +from mindspore.ops import operations as P +import mindspore.nn as nn + + +class XYLoss(nn.Cell): + """Loss for x and y.""" + def __init__(self): + super(XYLoss, self).__init__() + self.cross_entropy = P.SigmoidCrossEntropyWithLogits() + self.reduce_sum = P.ReduceSum() + + def construct(self, object_mask, box_loss_scale, predict_xy, true_xy): + xy_loss = object_mask * box_loss_scale * self.cross_entropy(predict_xy, true_xy) + xy_loss = self.reduce_sum(xy_loss, ()) + return xy_loss + + +class WHLoss(nn.Cell): + """Loss for w and h.""" + def __init__(self): + super(WHLoss, self).__init__() + self.square = P.Square() + self.reduce_sum = P.ReduceSum() + + def construct(self, object_mask, box_loss_scale, predict_wh, true_wh): + wh_loss = object_mask * box_loss_scale * 0.5 * P.Square()(true_wh - predict_wh) + wh_loss = self.reduce_sum(wh_loss, ()) + return wh_loss + + +class ConfidenceLoss(nn.Cell): + """Loss for confidence.""" + def __init__(self): + super(ConfidenceLoss, self).__init__() + self.cross_entropy = P.SigmoidCrossEntropyWithLogits() + self.reduce_sum = P.ReduceSum() + + def construct(self, object_mask, predict_confidence, ignore_mask): + confidence_loss = self.cross_entropy(predict_confidence, object_mask) + confidence_loss = object_mask * confidence_loss + (1 - object_mask) * confidence_loss * ignore_mask + confidence_loss = self.reduce_sum(confidence_loss, ()) + return confidence_loss + + +class ClassLoss(nn.Cell): + """Loss for classification.""" + def __init__(self): + super(ClassLoss, self).__init__() + self.cross_entropy = P.SigmoidCrossEntropyWithLogits() + self.reduce_sum = P.ReduceSum() + + def construct(self, object_mask, predict_class, class_probs): + class_loss = object_mask * self.cross_entropy(predict_class, class_probs) + class_loss = self.reduce_sum(class_loss, ()) + return class_loss diff --git a/model_zoo/yolov3_darknet53/src/lr_scheduler.py b/model_zoo/yolov3_darknet53/src/lr_scheduler.py new file mode 100644 index 0000000000..ae30454745 --- /dev/null +++ b/model_zoo/yolov3_darknet53/src/lr_scheduler.py @@ -0,0 +1,144 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Learning rate scheduler.""" +import math +from collections import Counter + +import numpy as np + + +def linear_warmup_lr(current_step, warmup_steps, base_lr, init_lr): + """Linear learning rate.""" + lr_inc = (float(base_lr) - float(init_lr)) / float(warmup_steps) + lr = float(init_lr) + lr_inc * current_step + return lr + + +def warmup_step_lr(lr, lr_epochs, steps_per_epoch, warmup_epochs, max_epoch, gamma=0.1): + """Warmup step learning rate.""" + base_lr = lr + warmup_init_lr = 0 + total_steps = int(max_epoch * steps_per_epoch) + warmup_steps = int(warmup_epochs * steps_per_epoch) + milestones = lr_epochs + milestones_steps = [] + for milestone in milestones: + milestones_step = milestone * steps_per_epoch + milestones_steps.append(milestones_step) + + lr_each_step = [] + lr = base_lr + milestones_steps_counter = Counter(milestones_steps) + for i in range(total_steps): + if i < warmup_steps: + lr = linear_warmup_lr(i + 1, warmup_steps, base_lr, warmup_init_lr) + else: + lr = lr * gamma**milestones_steps_counter[i] + lr_each_step.append(lr) + + return np.array(lr_each_step).astype(np.float32) + + +def multi_step_lr(lr, milestones, steps_per_epoch, max_epoch, gamma=0.1): + return warmup_step_lr(lr, milestones, steps_per_epoch, 0, max_epoch, gamma=gamma) + + +def step_lr(lr, epoch_size, steps_per_epoch, max_epoch, gamma=0.1): + lr_epochs = [] + for i in range(1, max_epoch): + if i % epoch_size == 0: + lr_epochs.append(i) + return multi_step_lr(lr, lr_epochs, steps_per_epoch, max_epoch, gamma=gamma) + + +def warmup_cosine_annealing_lr(lr, steps_per_epoch, warmup_epochs, max_epoch, T_max, eta_min=0): + """Cosine annealing learning rate.""" + base_lr = lr + warmup_init_lr = 0 + total_steps = int(max_epoch * steps_per_epoch) + warmup_steps = int(warmup_epochs * steps_per_epoch) + + lr_each_step = [] + for i in range(total_steps): + last_epoch = i // steps_per_epoch + if i < warmup_steps: + lr = linear_warmup_lr(i + 1, warmup_steps, base_lr, warmup_init_lr) + else: + lr = eta_min + (base_lr - eta_min) * (1. + math.cos(math.pi*last_epoch / T_max)) / 2 + lr_each_step.append(lr) + + return np.array(lr_each_step).astype(np.float32) + + +def warmup_cosine_annealing_lr_V2(lr, steps_per_epoch, warmup_epochs, max_epoch, T_max, eta_min=0): + """Cosine annealing learning rate V2.""" + base_lr = lr + warmup_init_lr = 0 + total_steps = int(max_epoch * steps_per_epoch) + warmup_steps = int(warmup_epochs * steps_per_epoch) + + last_lr = 0 + last_epoch_V1 = 0 + + T_max_V2 = int(max_epoch*1/3) + + lr_each_step = [] + for i in range(total_steps): + last_epoch = i // steps_per_epoch + if i < warmup_steps: + lr = linear_warmup_lr(i + 1, warmup_steps, base_lr, warmup_init_lr) + else: + if i < total_steps*2/3: + lr = eta_min + (base_lr - eta_min) * (1. + math.cos(math.pi*last_epoch / T_max)) / 2 + last_lr = lr + last_epoch_V1 = last_epoch + else: + base_lr = last_lr + last_epoch = last_epoch-last_epoch_V1 + lr = eta_min + (base_lr - eta_min) * (1. + math.cos(math.pi * last_epoch / T_max_V2)) / 2 + + lr_each_step.append(lr) + return np.array(lr_each_step).astype(np.float32) + + +def warmup_cosine_annealing_lr_sample(lr, steps_per_epoch, warmup_epochs, max_epoch, T_max, eta_min=0): + """Warmup cosine annealing learning rate.""" + start_sample_epoch = 60 + step_sample = 2 + tobe_sampled_epoch = 60 + end_sampled_epoch = start_sample_epoch + step_sample*tobe_sampled_epoch + max_sampled_epoch = max_epoch+tobe_sampled_epoch + T_max = max_sampled_epoch + + base_lr = lr + warmup_init_lr = 0 + total_steps = int(max_epoch * steps_per_epoch) + total_sampled_steps = int(max_sampled_epoch * steps_per_epoch) + warmup_steps = int(warmup_epochs * steps_per_epoch) + + lr_each_step = [] + + for i in range(total_sampled_steps): + last_epoch = i // steps_per_epoch + if last_epoch in range(start_sample_epoch, end_sampled_epoch, step_sample): + continue + if i < warmup_steps: + lr = linear_warmup_lr(i + 1, warmup_steps, base_lr, warmup_init_lr) + else: + lr = eta_min + (base_lr - eta_min) * (1. + math.cos(math.pi*last_epoch / T_max)) / 2 + lr_each_step.append(lr) + + assert total_steps == len(lr_each_step) + return np.array(lr_each_step).astype(np.float32) diff --git a/model_zoo/yolov3_darknet53/src/transforms.py b/model_zoo/yolov3_darknet53/src/transforms.py new file mode 100644 index 0000000000..837d1a25ea --- /dev/null +++ b/model_zoo/yolov3_darknet53/src/transforms.py @@ -0,0 +1,577 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Preprocess dataset.""" +import random +import threading +import copy + +import numpy as np +from PIL import Image +import cv2 + + +def _rand(a=0., b=1.): + return np.random.rand() * (b - a) + a + + +def bbox_iou(bbox_a, bbox_b, offset=0): + """Calculate Intersection-Over-Union(IOU) of two bounding boxes. + + Parameters + ---------- + bbox_a : numpy.ndarray + An ndarray with shape :math:`(N, 4)`. + bbox_b : numpy.ndarray + An ndarray with shape :math:`(M, 4)`. + offset : float or int, default is 0 + The ``offset`` is used to control the whether the width(or height) is computed as + (right - left + ``offset``). + Note that the offset must be 0 for normalized bboxes, whose ranges are in ``[0, 1]``. + + Returns + ------- + numpy.ndarray + An ndarray with shape :math:`(N, M)` indicates IOU between each pairs of + bounding boxes in `bbox_a` and `bbox_b`. + + """ + if bbox_a.shape[1] < 4 or bbox_b.shape[1] < 4: + raise IndexError("Bounding boxes axis 1 must have at least length 4") + + tl = np.maximum(bbox_a[:, None, :2], bbox_b[:, :2]) + br = np.minimum(bbox_a[:, None, 2:4], bbox_b[:, 2:4]) + + area_i = np.prod(br - tl + offset, axis=2) * (tl < br).all(axis=2) + area_a = np.prod(bbox_a[:, 2:4] - bbox_a[:, :2] + offset, axis=1) + area_b = np.prod(bbox_b[:, 2:4] - bbox_b[:, :2] + offset, axis=1) + return area_i / (area_a[:, None] + area_b - area_i) + + +def statistic_normalize_img(img, statistic_norm): + """Statistic normalize images.""" + # img: RGB + if isinstance(img, Image.Image): + img = np.array(img) + img = img/255. + mean = np.array([0.485, 0.456, 0.406]) + std = np.array([0.229, 0.224, 0.225]) + if statistic_norm: + img = (img - mean) / std + return img + + +def get_interp_method(interp, sizes=()): + """Get the interpolation method for resize functions. + The major purpose of this function is to wrap a random interp method selection + and a auto-estimation method. + + Parameters + ---------- + interp : int + interpolation method for all resizing operations + + Possible values: + 0: Nearest Neighbors Interpolation. + 1: Bilinear interpolation. + 2: Bicubic interpolation over 4x4 pixel neighborhood. + 3: Nearest Neighbors. [Originally it should be Area-based, + as we cannot find Area-based, so we use NN instead. + Area-based (resampling using pixel area relation). It may be a + preferred method for image decimation, as it gives moire-free + results. But when the image is zoomed, it is similar to the Nearest + Neighbors method. (used by default). + 4: Lanczos interpolation over 8x8 pixel neighborhood. + 9: Cubic for enlarge, area for shrink, bilinear for others + 10: Random select from interpolation method metioned above. + Note: + When shrinking an image, it will generally look best with AREA-based + interpolation, whereas, when enlarging an image, it will generally look best + with Bicubic (slow) or Bilinear (faster but still looks OK). + More details can be found in the documentation of OpenCV, please refer to + http://docs.opencv.org/master/da/d54/group__imgproc__transform.html. + sizes : tuple of int + (old_height, old_width, new_height, new_width), if None provided, auto(9) + will return Area(2) anyway. + + Returns + ------- + int + interp method from 0 to 4 + """ + if interp == 9: + if sizes: + assert len(sizes) == 4 + oh, ow, nh, nw = sizes + if nh > oh and nw > ow: + return 2 + if nh < oh and nw < ow: + return 0 + return 1 + return 2 + if interp == 10: + return random.randint(0, 4) + if interp not in (0, 1, 2, 3, 4): + raise ValueError('Unknown interp method %d' % interp) + return interp + + +def pil_image_reshape(interp): + """Reshape pil image.""" + reshape_type = { + 0: Image.NEAREST, + 1: Image.BILINEAR, + 2: Image.BICUBIC, + 3: Image.NEAREST, + 4: Image.LANCZOS, + } + return reshape_type[interp] + + +def _preprocess_true_boxes(true_boxes, anchors, in_shape, num_classes, + max_boxes, label_smooth, label_smooth_factor=0.1): + """Preprocess annotation boxes.""" + anchors = np.array(anchors) + num_layers = anchors.shape[0] // 3 + anchor_mask = [[6, 7, 8], [3, 4, 5], [0, 1, 2]] + true_boxes = np.array(true_boxes, dtype='float32') + input_shape = np.array(in_shape, dtype='int32') + boxes_xy = (true_boxes[..., 0:2] + true_boxes[..., 2:4]) // 2. + # trans to box center point + boxes_wh = true_boxes[..., 2:4] - true_boxes[..., 0:2] + # input_shape is [h, w] + true_boxes[..., 0:2] = boxes_xy / input_shape[::-1] + true_boxes[..., 2:4] = boxes_wh / input_shape[::-1] + # true_boxes = [xywh] + + grid_shapes = [input_shape // 32, input_shape // 16, input_shape // 8] + # grid_shape [h, w] + y_true = [np.zeros((grid_shapes[l][0], grid_shapes[l][1], len(anchor_mask[l]), + 5 + num_classes), dtype='float32') for l in range(num_layers)] + # y_true [gridy, gridx] + anchors = np.expand_dims(anchors, 0) + anchors_max = anchors / 2. + anchors_min = -anchors_max + valid_mask = boxes_wh[..., 0] > 0 + + wh = boxes_wh[valid_mask] + if wh.size > 0: + wh = np.expand_dims(wh, -2) + boxes_max = wh / 2. + boxes_min = -boxes_max + + intersect_min = np.maximum(boxes_min, anchors_min) + intersect_max = np.minimum(boxes_max, anchors_max) + intersect_wh = np.maximum(intersect_max - intersect_min, 0.) + intersect_area = intersect_wh[..., 0] * intersect_wh[..., 1] + box_area = wh[..., 0] * wh[..., 1] + anchor_area = anchors[..., 0] * anchors[..., 1] + iou = intersect_area / (box_area + anchor_area - intersect_area) + + best_anchor = np.argmax(iou, axis=-1) + for t, n in enumerate(best_anchor): + for l in range(num_layers): + if n in anchor_mask[l]: + i = np.floor(true_boxes[t, 0] * grid_shapes[l][1]).astype('int32') # grid_y + j = np.floor(true_boxes[t, 1] * grid_shapes[l][0]).astype('int32') # grid_x + + k = anchor_mask[l].index(n) + c = true_boxes[t, 4].astype('int32') + y_true[l][j, i, k, 0:4] = true_boxes[t, 0:4] + y_true[l][j, i, k, 4] = 1. + + # lable-smooth + if label_smooth: + sigma = label_smooth_factor/(num_classes-1) + y_true[l][j, i, k, 5:] = sigma + y_true[l][j, i, k, 5+c] = 1-label_smooth_factor + else: + y_true[l][j, i, k, 5 + c] = 1. + + # pad_gt_boxes for avoiding dynamic shape + pad_gt_box0 = np.zeros(shape=[max_boxes, 4], dtype=np.float32) + pad_gt_box1 = np.zeros(shape=[max_boxes, 4], dtype=np.float32) + pad_gt_box2 = np.zeros(shape=[max_boxes, 4], dtype=np.float32) + + mask0 = np.reshape(y_true[0][..., 4:5], [-1]) + gt_box0 = np.reshape(y_true[0][..., 0:4], [-1, 4]) + # gt_box [boxes, [x,y,w,h]] + gt_box0 = gt_box0[mask0 == 1] + # gt_box0: get all boxes which have object + pad_gt_box0[:gt_box0.shape[0]] = gt_box0 + # gt_box0.shape[0]: total number of boxes in gt_box0 + # top N of pad_gt_box0 is real box, and after are pad by zero + + mask1 = np.reshape(y_true[1][..., 4:5], [-1]) + gt_box1 = np.reshape(y_true[1][..., 0:4], [-1, 4]) + gt_box1 = gt_box1[mask1 == 1] + pad_gt_box1[:gt_box1.shape[0]] = gt_box1 + + mask2 = np.reshape(y_true[2][..., 4:5], [-1]) + gt_box2 = np.reshape(y_true[2][..., 0:4], [-1, 4]) + + gt_box2 = gt_box2[mask2 == 1] + pad_gt_box2[:gt_box2.shape[0]] = gt_box2 + return y_true[0], y_true[1], y_true[2], pad_gt_box0, pad_gt_box1, pad_gt_box2 + + +def _reshape_data(image, image_size): + """Reshape image.""" + if not isinstance(image, Image.Image): + image = Image.fromarray(image) + ori_w, ori_h = image.size + ori_image_shape = np.array([ori_w, ori_h], np.int32) + # original image shape fir:H sec:W + h, w = image_size + interp = get_interp_method(interp=9, sizes=(ori_h, ori_w, h, w)) + image = image.resize((w, h), pil_image_reshape(interp)) + image_data = statistic_normalize_img(image, statistic_norm=True) + if len(image_data.shape) == 2: + image_data = np.expand_dims(image_data, axis=-1) + image_data = np.concatenate([image_data, image_data, image_data], axis=-1) + image_data = image_data.astype(np.float32) + return image_data, ori_image_shape + + +def color_distortion(img, hue, sat, val, device_num): + """Color distortion.""" + hue = _rand(-hue, hue) + sat = _rand(1, sat) if _rand() < .5 else 1 / _rand(1, sat) + val = _rand(1, val) if _rand() < .5 else 1 / _rand(1, val) + if device_num != 1: + cv2.setNumThreads(1) + x = cv2.cvtColor(img, cv2.COLOR_RGB2HSV_FULL) + x = x / 255. + x[..., 0] += hue + x[..., 0][x[..., 0] > 1] -= 1 + x[..., 0][x[..., 0] < 0] += 1 + x[..., 1] *= sat + x[..., 2] *= val + x[x > 1] = 1 + x[x < 0] = 0 + x = x * 255. + x = x.astype(np.uint8) + image_data = cv2.cvtColor(x, cv2.COLOR_HSV2RGB_FULL) + return image_data + + +def filp_pil_image(img): + return img.transpose(Image.FLIP_LEFT_RIGHT) + + +def convert_gray_to_color(img): + if len(img.shape) == 2: + img = np.expand_dims(img, axis=-1) + img = np.concatenate([img, img, img], axis=-1) + return img + + +def _is_iou_satisfied_constraint(min_iou, max_iou, box, crop_box): + iou = bbox_iou(box, crop_box) + return min_iou <= iou.min() and max_iou >= iou.max() + + +def _choose_candidate_by_constraints(max_trial, input_w, input_h, image_w, image_h, jitter, box, use_constraints): + """Choose candidate by constraints.""" + if use_constraints: + constraints = ( + (0.1, None), + (0.3, None), + (0.5, None), + (0.7, None), + (0.9, None), + (None, 1), + ) + else: + constraints = ( + (None, None), + ) + # add default candidate + candidates = [(0, 0, input_w, input_h)] + for constraint in constraints: + min_iou, max_iou = constraint + min_iou = -np.inf if min_iou is None else min_iou + max_iou = np.inf if max_iou is None else max_iou + + for _ in range(max_trial): + # box_data should have at least one box + new_ar = float(input_w) / float(input_h) * _rand(1 - jitter, 1 + jitter) / _rand(1 - jitter, 1 + jitter) + scale = _rand(0.25, 2) + + if new_ar < 1: + nh = int(scale * input_h) + nw = int(nh * new_ar) + else: + nw = int(scale * input_w) + nh = int(nw / new_ar) + + dx = int(_rand(0, input_w - nw)) + dy = int(_rand(0, input_h - nh)) + + if box.size > 0: + t_box = copy.deepcopy(box) + t_box[:, [0, 2]] = t_box[:, [0, 2]] * float(nw) / float(image_w) + dx + t_box[:, [1, 3]] = t_box[:, [1, 3]] * float(nh) / float(image_h) + dy + + crop_box = np.array((0, 0, input_w, input_h)) + if not _is_iou_satisfied_constraint(min_iou, max_iou, t_box, crop_box[np.newaxis]): + continue + else: + candidates.append((dx, dy, nw, nh)) + else: + raise Exception("!!! annotation box is less than 1") + return candidates + + +def _correct_bbox_by_candidates(candidates, input_w, input_h, image_w, + image_h, flip, box, box_data, allow_outside_center): + """Calculate correct boxes.""" + while candidates: + if len(candidates) > 1: + # ignore default candidate which do not crop + candidate = candidates.pop(np.random.randint(1, len(candidates))) + else: + candidate = candidates.pop(np.random.randint(0, len(candidates))) + dx, dy, nw, nh = candidate + t_box = copy.deepcopy(box) + t_box[:, [0, 2]] = t_box[:, [0, 2]] * float(nw) / float(image_w) + dx + t_box[:, [1, 3]] = t_box[:, [1, 3]] * float(nh) / float(image_h) + dy + if flip: + t_box[:, [0, 2]] = input_w - t_box[:, [2, 0]] + + if allow_outside_center: + pass + else: + t_box = t_box[np.logical_and((t_box[:, 0] + t_box[:, 2])/2. >= 0., (t_box[:, 1] + t_box[:, 3])/2. >= 0.)] + t_box = t_box[np.logical_and((t_box[:, 0] + t_box[:, 2]) / 2. <= input_w, + (t_box[:, 1] + t_box[:, 3]) / 2. <= input_h)] + + # recorrect x, y for case x,y < 0 reset to zero, after dx and dy, some box can smaller than zero + t_box[:, 0:2][t_box[:, 0:2] < 0] = 0 + # recorrect w,h not higher than input size + t_box[:, 2][t_box[:, 2] > input_w] = input_w + t_box[:, 3][t_box[:, 3] > input_h] = input_h + box_w = t_box[:, 2] - t_box[:, 0] + box_h = t_box[:, 3] - t_box[:, 1] + # discard invalid box: w or h smaller than 1 pixel + t_box = t_box[np.logical_and(box_w > 1, box_h > 1)] + + if t_box.shape[0] > 0: + # break if number of find t_box + box_data[: len(t_box)] = t_box + return box_data, candidate + raise Exception('all candidates can not satisfied re-correct bbox') + + +def _data_aug(image, box, jitter, hue, sat, val, image_input_size, max_boxes, + anchors, num_classes, max_trial=10, device_num=1): + """Crop an image randomly with bounding box constraints. + + This data augmentation is used in training of + Single Shot Multibox Detector [#]_. More details can be found in + data augmentation section of the original paper. + .. [#] Wei Liu, Dragomir Anguelov, Dumitru Erhan, Christian Szegedy, + Scott Reed, Cheng-Yang Fu, Alexander C. Berg. + SSD: Single Shot MultiBox Detector. ECCV 2016.""" + + if not isinstance(image, Image.Image): + image = Image.fromarray(image) + + image_w, image_h = image.size + input_h, input_w = image_input_size + + np.random.shuffle(box) + if len(box) > max_boxes: + box = box[:max_boxes] + flip = _rand() < .5 + box_data = np.zeros((max_boxes, 5)) + + candidates = _choose_candidate_by_constraints(use_constraints=False, + max_trial=max_trial, + input_w=input_w, + input_h=input_h, + image_w=image_w, + image_h=image_h, + jitter=jitter, + box=box) + box_data, candidate = _correct_bbox_by_candidates(candidates=candidates, + input_w=input_w, + input_h=input_h, + image_w=image_w, + image_h=image_h, + flip=flip, + box=box, + box_data=box_data, + allow_outside_center=True) + dx, dy, nw, nh = candidate + interp = get_interp_method(interp=10) + image = image.resize((nw, nh), pil_image_reshape(interp)) + # place image, gray color as back graoud + new_image = Image.new('RGB', (input_w, input_h), (128, 128, 128)) + new_image.paste(image, (dx, dy)) + image = new_image + + if flip: + image = filp_pil_image(image) + + image = np.array(image) + + image = convert_gray_to_color(image) + + image_data = color_distortion(image, hue, sat, val, device_num) + image_data = statistic_normalize_img(image_data, statistic_norm=True) + + image_data = image_data.astype(np.float32) + + return image_data, box_data + + +def preprocess_fn(image, box, config, input_size, device_num): + """Preprocess data function.""" + config_anchors = config.anchor_scales + anchors = np.array([list(x) for x in config_anchors]) + max_boxes = config.max_box + num_classes = config.num_classes + jitter = config.jitter + hue = config.hue + sat = config.saturation + val = config.value + image, anno = _data_aug(image, box, jitter=jitter, hue=hue, sat=sat, val=val, + image_input_size=input_size, max_boxes=max_boxes, + num_classes=num_classes, anchors=anchors, device_num=device_num) + return image, anno + + +def reshape_fn(image, img_id, config): + input_size = config.test_img_shape + image, ori_image_shape = _reshape_data(image, image_size=input_size) + return image, ori_image_shape, img_id + + +class MultiScaleTrans: + """Multi scale transform.""" + def __init__(self, config, device_num): + self.config = config + self.seed = 0 + self.size_list = [] + self.resize_rate = config.resize_rate + self.dataset_size = config.dataset_size + self.size_dict = {} + self.seed_num = int(1e6) + self.seed_list = self.generate_seed_list(seed_num=self.seed_num) + self.resize_count_num = int(np.ceil(self.dataset_size / self.resize_rate)) + self.device_num = device_num + + def generate_seed_list(self, init_seed=1234, seed_num=int(1e6), seed_range=(1, 1000)): + seed_list = [] + random.seed(init_seed) + for _ in range(seed_num): + seed = random.randint(seed_range[0], seed_range[1]) + seed_list.append(seed) + return seed_list + + def __call__(self, imgs, annos, batchInfo): + epoch_num = batchInfo.get_epoch_num() + size_idx = int(batchInfo.get_batch_num() / self.resize_rate) + seed_key = self.seed_list[(epoch_num * self.resize_count_num + size_idx) % self.seed_num] + ret_imgs = [] + ret_annos = [] + + if self.size_dict.get(seed_key, None) is None: + random.seed(seed_key) + new_size = random.choice(self.config.multi_scale) + self.size_dict[seed_key] = new_size + seed = seed_key + + input_size = self.size_dict[seed] + for img, anno in zip(imgs, annos): + img, anno = preprocess_fn(img, anno, self.config, input_size, self.device_num) + ret_imgs.append(img.transpose(2, 0, 1).copy()) + ret_annos.append(anno) + return np.array(ret_imgs), np.array(ret_annos) + + +def thread_batch_preprocess_true_box(annos, config, input_shape, result_index, batch_bbox_true_1, batch_bbox_true_2, + batch_bbox_true_3, batch_gt_box1, batch_gt_box2, batch_gt_box3): + """Preprocess true box for multi-thread.""" + i = 0 + for anno in annos: + bbox_true_1, bbox_true_2, bbox_true_3, gt_box1, gt_box2, gt_box3 = \ + _preprocess_true_boxes(true_boxes=anno, anchors=config.anchor_scales, in_shape=input_shape, + num_classes=config.num_classes, max_boxes=config.max_box, + label_smooth=config.label_smooth, label_smooth_factor=config.label_smooth_factor) + batch_bbox_true_1[result_index + i] = bbox_true_1 + batch_bbox_true_2[result_index + i] = bbox_true_2 + batch_bbox_true_3[result_index + i] = bbox_true_3 + batch_gt_box1[result_index + i] = gt_box1 + batch_gt_box2[result_index + i] = gt_box2 + batch_gt_box3[result_index + i] = gt_box3 + i = i + 1 + + +def batch_preprocess_true_box(annos, config, input_shape): + """Preprocess true box with multi-thread.""" + batch_bbox_true_1 = [] + batch_bbox_true_2 = [] + batch_bbox_true_3 = [] + batch_gt_box1 = [] + batch_gt_box2 = [] + batch_gt_box3 = [] + threads = [] + + step = 4 + for index in range(0, len(annos), step): + for _ in range(step): + batch_bbox_true_1.append(None) + batch_bbox_true_2.append(None) + batch_bbox_true_3.append(None) + batch_gt_box1.append(None) + batch_gt_box2.append(None) + batch_gt_box3.append(None) + step_anno = annos[index: index + step] + t = threading.Thread(target=thread_batch_preprocess_true_box, + args=(step_anno, config, input_shape, index, batch_bbox_true_1, batch_bbox_true_2, + batch_bbox_true_3, batch_gt_box1, batch_gt_box2, batch_gt_box3)) + t.start() + threads.append(t) + + for t in threads: + t.join() + + return np.array(batch_bbox_true_1), np.array(batch_bbox_true_2), np.array(batch_bbox_true_3), \ + np.array(batch_gt_box1), np.array(batch_gt_box2), np.array(batch_gt_box3) + + +def batch_preprocess_true_box_single(annos, config, input_shape): + """Preprocess true boxes.""" + batch_bbox_true_1 = [] + batch_bbox_true_2 = [] + batch_bbox_true_3 = [] + batch_gt_box1 = [] + batch_gt_box2 = [] + batch_gt_box3 = [] + for anno in annos: + bbox_true_1, bbox_true_2, bbox_true_3, gt_box1, gt_box2, gt_box3 = \ + _preprocess_true_boxes(true_boxes=anno, anchors=config.anchor_scales, in_shape=input_shape, + num_classes=config.num_classes, max_boxes=config.max_box, + label_smooth=config.label_smooth, label_smooth_factor=config.label_smooth_factor) + batch_bbox_true_1.append(bbox_true_1) + batch_bbox_true_2.append(bbox_true_2) + batch_bbox_true_3.append(bbox_true_3) + batch_gt_box1.append(gt_box1) + batch_gt_box2.append(gt_box2) + batch_gt_box3.append(gt_box3) + + return np.array(batch_bbox_true_1), np.array(batch_bbox_true_2), np.array(batch_bbox_true_3), \ + np.array(batch_gt_box1), np.array(batch_gt_box2), np.array(batch_gt_box3) diff --git a/model_zoo/yolov3_darknet53/src/util.py b/model_zoo/yolov3_darknet53/src/util.py new file mode 100644 index 0000000000..1a3da99181 --- /dev/null +++ b/model_zoo/yolov3_darknet53/src/util.py @@ -0,0 +1,177 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Util class or function.""" +from mindspore.train.serialization import load_checkpoint +import mindspore.nn as nn + + +class AverageMeter: + """Computes and stores the average and current value""" + + def __init__(self, name, fmt=':f', tb_writer=None): + self.name = name + self.fmt = fmt + self.reset() + self.tb_writer = tb_writer + self.cur_step = 1 + self.val = 0 + self.avg = 0 + self.sum = 0 + self.count = 0 + + def reset(self): + self.val = 0 + self.avg = 0 + self.sum = 0 + self.count = 0 + + def update(self, val, n=1): + self.val = val + self.sum += val * n + self.count += n + self.avg = self.sum / self.count + if self.tb_writer is not None: + self.tb_writer.add_scalar(self.name, self.val, self.cur_step) + self.cur_step += 1 + + def __str__(self): + fmtstr = '{name}:{avg' + self.fmt + '}' + return fmtstr.format(**self.__dict__) + + +def load_backbone(net, ckpt_path, args): + """Load darknet53 backbone checkpoint.""" + param_dict = load_checkpoint(ckpt_path) + yolo_backbone_prefix = 'feature_map.backbone' + darknet_backbone_prefix = 'network.backbone' + find_param = [] + not_found_param = [] + + for name, cell in net.cells_and_names(): + if name.startswith(yolo_backbone_prefix): + name = name.replace(yolo_backbone_prefix, darknet_backbone_prefix) + if isinstance(cell, (nn.Conv2d, nn.Dense)): + darknet_weight = '{}.weight'.format(name) + darknet_bias = '{}.bias'.format(name) + if darknet_weight in param_dict: + cell.weight.default_input = param_dict[darknet_weight].data + find_param.append(darknet_weight) + else: + not_found_param.append(darknet_weight) + if darknet_bias in param_dict: + cell.bias.default_input = param_dict[darknet_bias].data + find_param.append(darknet_bias) + else: + not_found_param.append(darknet_bias) + elif isinstance(cell, (nn.BatchNorm2d, nn.BatchNorm1d)): + darknet_moving_mean = '{}.moving_mean'.format(name) + darknet_moving_variance = '{}.moving_variance'.format(name) + darknet_gamma = '{}.gamma'.format(name) + darknet_beta = '{}.beta'.format(name) + if darknet_moving_mean in param_dict: + cell.moving_mean.default_input = param_dict[darknet_moving_mean].data + find_param.append(darknet_moving_mean) + else: + not_found_param.append(darknet_moving_mean) + if darknet_moving_variance in param_dict: + cell.moving_variance.default_input = param_dict[darknet_moving_variance].data + find_param.append(darknet_moving_variance) + else: + not_found_param.append(darknet_moving_variance) + if darknet_gamma in param_dict: + cell.gamma.default_input = param_dict[darknet_gamma].data + find_param.append(darknet_gamma) + else: + not_found_param.append(darknet_gamma) + if darknet_beta in param_dict: + cell.beta.default_input = param_dict[darknet_beta].data + find_param.append(darknet_beta) + else: + not_found_param.append(darknet_beta) + + args.logger.info('================found_param {}========='.format(len(find_param))) + args.logger.info(find_param) + args.logger.info('================not_found_param {}========='.format(len(not_found_param))) + args.logger.info(not_found_param) + args.logger.info('=====load {} successfully ====='.format(ckpt_path)) + + return net + + +def default_wd_filter(x): + """default weight decay filter.""" + parameter_name = x.name + if parameter_name.endswith('.bias'): + # all bias not using weight decay + return False + if parameter_name.endswith('.gamma'): + # bn weight bias not using weight decay, be carefully for now x not include BN + return False + if parameter_name.endswith('.beta'): + # bn weight bias not using weight decay, be carefully for now x not include BN + return False + + return True + + +def get_param_groups(network): + """Param groups for optimizer.""" + decay_params = [] + no_decay_params = [] + for x in network.trainable_params(): + parameter_name = x.name + if parameter_name.endswith('.bias'): + # all bias not using weight decay + no_decay_params.append(x) + elif parameter_name.endswith('.gamma'): + # bn weight bias not using weight decay, be carefully for now x not include BN + no_decay_params.append(x) + elif parameter_name.endswith('.beta'): + # bn weight bias not using weight decay, be carefully for now x not include BN + no_decay_params.append(x) + else: + decay_params.append(x) + + return [{'params': no_decay_params, 'weight_decay': 0.0}, {'params': decay_params}] + + +class ShapeRecord: + """Log image shape.""" + def __init__(self): + self.shape_record = { + 320: 0, + 352: 0, + 384: 0, + 416: 0, + 448: 0, + 480: 0, + 512: 0, + 544: 0, + 576: 0, + 608: 0, + 'total': 0 + } + + def set(self, shape): + if len(shape) > 1: + shape = shape[0] + shape = int(shape) + self.shape_record[shape] += 1 + self.shape_record['total'] += 1 + + def show(self, logger): + for key in self.shape_record: + rate = self.shape_record[key] / float(self.shape_record['total']) + logger.info('shape {}: {:.2f}%'.format(key, rate*100)) diff --git a/model_zoo/yolov3_darknet53/src/yolo.py b/model_zoo/yolov3_darknet53/src/yolo.py new file mode 100644 index 0000000000..09cce0c97f --- /dev/null +++ b/model_zoo/yolov3_darknet53/src/yolo.py @@ -0,0 +1,437 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""YOLOv3 based on DarkNet.""" +import mindspore as ms +import mindspore.nn as nn +from mindspore.common.tensor import Tensor +from mindspore import context +from mindspore.parallel._auto_parallel_context import auto_parallel_context +from mindspore.communication.management import get_group_size +from mindspore.ops import operations as P +from mindspore.ops import functional as F +from mindspore.ops import composite as C + +from src.darknet import DarkNet, ResidualBlock +from src.config import ConfigYOLOV3DarkNet53 +from src.loss import XYLoss, WHLoss, ConfidenceLoss, ClassLoss + + +def _conv_bn_relu(in_channel, + out_channel, + ksize, + stride=1, + padding=0, + dilation=1, + alpha=0.1, + momentum=0.9, + eps=1e-5, + pad_mode="same"): + """Get a conv2d batchnorm and relu layer""" + return nn.SequentialCell( + [nn.Conv2d(in_channel, + out_channel, + kernel_size=ksize, + stride=stride, + padding=padding, + dilation=dilation, + pad_mode=pad_mode), + nn.BatchNorm2d(out_channel, momentum=momentum, eps=eps), + nn.LeakyReLU(alpha)] + ) + + +class YoloBlock(nn.Cell): + """ + YoloBlock for YOLOv3. + + Args: + in_channels: Integer. Input channel. + out_chls: Interger. Middle channel. + out_channels: Integer. Output channel. + + Returns: + Tuple, tuple of output tensor,(f1,f2,f3). + + Examples: + YoloBlock(1024, 512, 255) + + """ + def __init__(self, in_channels, out_chls, out_channels): + super(YoloBlock, self).__init__() + out_chls_2 = out_chls*2 + + self.conv0 = _conv_bn_relu(in_channels, out_chls, ksize=1) + self.conv1 = _conv_bn_relu(out_chls, out_chls_2, ksize=3) + + self.conv2 = _conv_bn_relu(out_chls_2, out_chls, ksize=1) + self.conv3 = _conv_bn_relu(out_chls, out_chls_2, ksize=3) + + self.conv4 = _conv_bn_relu(out_chls_2, out_chls, ksize=1) + self.conv5 = _conv_bn_relu(out_chls, out_chls_2, ksize=3) + + self.conv6 = nn.Conv2d(out_chls_2, out_channels, kernel_size=1, stride=1, has_bias=True) + + def construct(self, x): + c1 = self.conv0(x) + c2 = self.conv1(c1) + + c3 = self.conv2(c2) + c4 = self.conv3(c3) + + c5 = self.conv4(c4) + c6 = self.conv5(c5) + + out = self.conv6(c6) + return c5, out + + +class YOLOv3(nn.Cell): + """ + YOLOv3 Network. + + Note: + backbone = darknet53 + + Args: + backbone_shape: List. Darknet output channels shape. + backbone: Cell. Backbone Network. + out_channel: Interger. Output channel. + + Returns: + Tensor, output tensor. + + Examples: + YOLOv3(backbone_shape=[64, 128, 256, 512, 1024] + backbone=darknet53(), + out_channel=255) + """ + def __init__(self, backbone_shape, backbone, out_channel): + super(YOLOv3, self).__init__() + self.out_channel = out_channel + self.backbone = backbone + self.backblock0 = YoloBlock(backbone_shape[-1], out_chls=backbone_shape[-2], out_channels=out_channel) + + self.conv1 = _conv_bn_relu(in_channel=backbone_shape[-2], out_channel=backbone_shape[-2]//2, ksize=1) + self.backblock1 = YoloBlock(in_channels=backbone_shape[-2]+backbone_shape[-3], + out_chls=backbone_shape[-3], + out_channels=out_channel) + + self.conv2 = _conv_bn_relu(in_channel=backbone_shape[-3], out_channel=backbone_shape[-3]//2, ksize=1) + self.backblock2 = YoloBlock(in_channels=backbone_shape[-3]+backbone_shape[-4], + out_chls=backbone_shape[-4], + out_channels=out_channel) + self.concat = P.Concat(axis=1) + + def construct(self, x): + # input_shape of x is (batch_size, 3, h, w) + # feature_map1 is (batch_size, backbone_shape[2], h/8, w/8) + # feature_map2 is (batch_size, backbone_shape[3], h/16, w/16) + # feature_map3 is (batch_size, backbone_shape[4], h/32, w/32) + img_hight = P.Shape()(x)[2] + img_width = P.Shape()(x)[3] + feature_map1, feature_map2, feature_map3 = self.backbone(x) + con1, big_object_output = self.backblock0(feature_map3) + + con1 = self.conv1(con1) + ups1 = P.ResizeNearestNeighbor((img_hight / 16, img_width / 16))(con1) + con1 = self.concat((ups1, feature_map2)) + con2, medium_object_output = self.backblock1(con1) + + con2 = self.conv2(con2) + ups2 = P.ResizeNearestNeighbor((img_hight / 8, img_width / 8))(con2) + con3 = self.concat((ups2, feature_map1)) + _, small_object_output = self.backblock2(con3) + + return big_object_output, medium_object_output, small_object_output + + +class DetectionBlock(nn.Cell): + """ + YOLOv3 detection Network. It will finally output the detection result. + + Args: + scale: Character. + config: ConfigYOLOV3DarkNet53, Configuration instance. + is_training: Bool, Whether train or not, default True. + + Returns: + Tuple, tuple of output tensor,(f1,f2,f3). + + Examples: + DetectionBlock(scale='l',stride=32) + """ + + def __init__(self, scale, config=ConfigYOLOV3DarkNet53(), is_training=True): + super(DetectionBlock, self).__init__() + self.config = config + if scale == 's': + idx = (0, 1, 2) + elif scale == 'm': + idx = (3, 4, 5) + elif scale == 'l': + idx = (6, 7, 8) + else: + raise KeyError("Invalid scale value for DetectionBlock") + self.anchors = Tensor([self.config.anchor_scales[i] for i in idx], ms.float32) + self.num_anchors_per_scale = 3 + self.num_attrib = 4+1+self.config.num_classes + self.lambda_coord = 1 + + self.sigmoid = nn.Sigmoid() + self.reshape = P.Reshape() + self.tile = P.Tile() + self.concat = P.Concat(axis=-1) + self.conf_training = is_training + + def construct(self, x, input_shape): + num_batch = P.Shape()(x)[0] + grid_size = P.Shape()(x)[2:4] + + # Reshape and transpose the feature to [n, grid_size[0], grid_size[1], 3, num_attrib] + prediction = P.Reshape()(x, (num_batch, + self.num_anchors_per_scale, + self.num_attrib, + grid_size[0], + grid_size[1])) + prediction = P.Transpose()(prediction, (0, 3, 4, 1, 2)) + + range_x = range(grid_size[1]) + range_y = range(grid_size[0]) + grid_x = P.Cast()(F.tuple_to_array(range_x), ms.float32) + grid_y = P.Cast()(F.tuple_to_array(range_y), ms.float32) + # Tensor of shape [grid_size[0], grid_size[1], 1, 1] representing the coordinate of x/y axis for each grid + # [batch, gridx, gridy, 1, 1] + grid_x = self.tile(self.reshape(grid_x, (1, 1, -1, 1, 1)), (1, grid_size[0], 1, 1, 1)) + grid_y = self.tile(self.reshape(grid_y, (1, -1, 1, 1, 1)), (1, 1, grid_size[1], 1, 1)) + # Shape is [grid_size[0], grid_size[1], 1, 2] + grid = self.concat((grid_x, grid_y)) + + box_xy = prediction[:, :, :, :, :2] + box_wh = prediction[:, :, :, :, 2:4] + box_confidence = prediction[:, :, :, :, 4:5] + box_probs = prediction[:, :, :, :, 5:] + + # gridsize1 is x + # gridsize0 is y + box_xy = (self.sigmoid(box_xy) + grid) / P.Cast()(F.tuple_to_array((grid_size[1], grid_size[0])), ms.float32) + # box_wh is w->h + box_wh = P.Exp()(box_wh) * self.anchors / input_shape + box_confidence = self.sigmoid(box_confidence) + box_probs = self.sigmoid(box_probs) + + if self.conf_training: + return grid, prediction, box_xy, box_wh + return self.concat((box_xy, box_wh, box_confidence, box_probs)) + + +class Iou(nn.Cell): + """Calculate the iou of boxes""" + def __init__(self): + super(Iou, self).__init__() + self.min = P.Minimum() + self.max = P.Maximum() + + def construct(self, box1, box2): + # box1: pred_box [batch, gx, gy, anchors, 1, 4] ->4: [x_center, y_center, w, h] + # box2: gt_box [batch, 1, 1, 1, maxbox, 4] + # convert to topLeft and rightDown + box1_xy = box1[:, :, :, :, :, :2] + box1_wh = box1[:, :, :, :, :, 2:4] + box1_mins = box1_xy - box1_wh / F.scalar_to_array(2.0) # topLeft + box1_maxs = box1_xy + box1_wh / F.scalar_to_array(2.0) # rightDown + + box2_xy = box2[:, :, :, :, :, :2] + box2_wh = box2[:, :, :, :, :, 2:4] + box2_mins = box2_xy - box2_wh / F.scalar_to_array(2.0) + box2_maxs = box2_xy + box2_wh / F.scalar_to_array(2.0) + + intersect_mins = self.max(box1_mins, box2_mins) + intersect_maxs = self.min(box1_maxs, box2_maxs) + intersect_wh = self.max(intersect_maxs - intersect_mins, F.scalar_to_array(0.0)) + # P.squeeze: for effiecient slice + intersect_area = P.Squeeze(-1)(intersect_wh[:, :, :, :, :, 0:1]) * \ + P.Squeeze(-1)(intersect_wh[:, :, :, :, :, 1:2]) + box1_area = P.Squeeze(-1)(box1_wh[:, :, :, :, :, 0:1]) * P.Squeeze(-1)(box1_wh[:, :, :, :, :, 1:2]) + box2_area = P.Squeeze(-1)(box2_wh[:, :, :, :, :, 0:1]) * P.Squeeze(-1)(box2_wh[:, :, :, :, :, 1:2]) + iou = intersect_area / (box1_area + box2_area - intersect_area) + # iou : [batch, gx, gy, anchors, maxboxes] + return iou + + +class YoloLossBlock(nn.Cell): + """ + Loss block cell of YOLOV3 network. + """ + def __init__(self, scale, config=ConfigYOLOV3DarkNet53()): + super(YoloLossBlock, self).__init__() + self.config = config + if scale == 's': + # anchor mask + idx = (0, 1, 2) + elif scale == 'm': + idx = (3, 4, 5) + elif scale == 'l': + idx = (6, 7, 8) + else: + raise KeyError("Invalid scale value for DetectionBlock") + self.anchors = Tensor([self.config.anchor_scales[i] for i in idx], ms.float32) + self.ignore_threshold = Tensor(self.config.ignore_threshold, ms.float32) + self.concat = P.Concat(axis=-1) + self.iou = Iou() + self.reduce_max = P.ReduceMax(keep_dims=False) + self.xy_loss = XYLoss() + self.wh_loss = WHLoss() + self.confidenceLoss = ConfidenceLoss() + self.classLoss = ClassLoss() + + def construct(self, grid, prediction, pred_xy, pred_wh, y_true, gt_box, input_shape): + # prediction : origin output from yolo + # pred_xy: (sigmoid(xy)+grid)/grid_size + # pred_wh: (exp(wh)*anchors)/input_shape + # y_true : after normalize + # gt_box: [batch, maxboxes, xyhw] after normalize + + object_mask = y_true[:, :, :, :, 4:5] + class_probs = y_true[:, :, :, :, 5:] + + grid_shape = P.Shape()(prediction)[1:3] + grid_shape = P.Cast()(F.tuple_to_array(grid_shape[::-1]), ms.float32) + + pred_boxes = self.concat((pred_xy, pred_wh)) + true_xy = y_true[:, :, :, :, :2] * grid_shape - grid + true_wh = y_true[:, :, :, :, 2:4] + true_wh = P.Select()(P.Equal()(true_wh, 0.0), + P.Fill()(P.DType()(true_wh), + P.Shape()(true_wh), 1.0), + true_wh) + true_wh = P.Log()(true_wh / self.anchors * input_shape) + # 2-w*h for large picture, use small scale, since small obj need more precise + box_loss_scale = 2 - y_true[:, :, :, :, 2:3] * y_true[:, :, :, :, 3:4] + + gt_shape = P.Shape()(gt_box) + gt_box = P.Reshape()(gt_box, (gt_shape[0], 1, 1, 1, gt_shape[1], gt_shape[2])) + + # add one more dimension for broadcast + iou = self.iou(P.ExpandDims()(pred_boxes, -2), gt_box) + # gt_box is x,y,h,w after normalize + # [batch, grid[0], grid[1], num_anchor, num_gt] + best_iou = self.reduce_max(iou, -1) + # [batch, grid[0], grid[1], num_anchor] + + # ignore_mask IOU too small + ignore_mask = best_iou < self.ignore_threshold + ignore_mask = P.Cast()(ignore_mask, ms.float32) + ignore_mask = P.ExpandDims()(ignore_mask, -1) + # ignore_mask backpro will cause a lot maximunGrad and minimumGrad time consume. + # so we turn off its gradient + ignore_mask = F.stop_gradient(ignore_mask) + + xy_loss = self.xy_loss(object_mask, box_loss_scale, prediction[:, :, :, :, :2], true_xy) + wh_loss = self.wh_loss(object_mask, box_loss_scale, prediction[:, :, :, :, 2:4], true_wh) + confidence_loss = self.confidenceLoss(object_mask, prediction[:, :, :, :, 4:5], ignore_mask) + class_loss = self.classLoss(object_mask, prediction[:, :, :, :, 5:], class_probs) + loss = xy_loss + wh_loss + confidence_loss + class_loss + batch_size = P.Shape()(prediction)[0] + return loss / batch_size + + +class YOLOV3DarkNet53(nn.Cell): + """ + Darknet based YOLOV3 network. + + Args: + is_training: Bool. Whether train or not. + + Returns: + Cell, cell instance of Darknet based YOLOV3 neural network. + + Examples: + YOLOV3DarkNet53(True) + """ + + def __init__(self, is_training): + super(YOLOV3DarkNet53, self).__init__() + self.config = ConfigYOLOV3DarkNet53() + + # YOLOv3 network + self.feature_map = YOLOv3(backbone=DarkNet(ResidualBlock, self.config.backbone_layers, + self.config.backbone_input_shape, + self.config.backbone_shape, + detect=True), + backbone_shape=self.config.backbone_shape, + out_channel=self.config.out_channel) + + # prediction on the default anchor boxes + self.detect_1 = DetectionBlock('l', is_training=is_training) + self.detect_2 = DetectionBlock('m', is_training=is_training) + self.detect_3 = DetectionBlock('s', is_training=is_training) + + def construct(self, x, input_shape): + big_object_output, medium_object_output, small_object_output = self.feature_map(x) + output_big = self.detect_1(big_object_output, input_shape) + output_me = self.detect_2(medium_object_output, input_shape) + output_small = self.detect_3(small_object_output, input_shape) + # big is the final output which has smallest feature map + return output_big, output_me, output_small + + +class YoloWithLossCell(nn.Cell): + """YOLOV3 loss.""" + def __init__(self, network): + super(YoloWithLossCell, self).__init__() + self.yolo_network = network + self.config = ConfigYOLOV3DarkNet53() + self.loss_big = YoloLossBlock('l', self.config) + self.loss_me = YoloLossBlock('m', self.config) + self.loss_small = YoloLossBlock('s', self.config) + + def construct(self, x, y_true_0, y_true_1, y_true_2, gt_0, gt_1, gt_2, input_shape): + yolo_out = self.yolo_network(x, input_shape) + loss_l = self.loss_big(*yolo_out[0], y_true_0, gt_0, input_shape) + loss_m = self.loss_me(*yolo_out[1], y_true_1, gt_1, input_shape) + loss_s = self.loss_small(*yolo_out[2], y_true_2, gt_2, input_shape) + return loss_l + loss_m + loss_s + + +class TrainingWrapper(nn.Cell): + """Training wrapper.""" + def __init__(self, network, optimizer, sens=1.0): + super(TrainingWrapper, self).__init__(auto_prefix=False) + self.network = network + self.weights = optimizer.parameters + self.optimizer = optimizer + self.grad = C.GradOperation('grad', get_by_list=True, sens_param=True) + self.sens = sens + self.reducer_flag = False + self.grad_reducer = None + self.parallel_mode = context.get_auto_parallel_context("parallel_mode") + if self.parallel_mode in [ms.ParallelMode.DATA_PARALLEL, ms.ParallelMode.HYBRID_PARALLEL]: + self.reducer_flag = True + if self.reducer_flag: + mean = context.get_auto_parallel_context("mirror_mean") + if auto_parallel_context().get_device_num_is_set(): + degree = context.get_auto_parallel_context("device_num") + else: + degree = get_group_size() + self.grad_reducer = nn.DistributedGradReducer(optimizer.parameters, mean, degree) + + def construct(self, *args): + weights = self.weights + loss = self.network(*args) + sens = P.Fill()(P.DType()(loss), P.Shape()(loss), self.sens) + grads = self.grad(self.network, weights)(*args, sens) + if self.reducer_flag: + grads = self.grad_reducer(grads) + return F.depend(loss, self.optimizer(grads)) diff --git a/model_zoo/yolov3_darknet53/src/yolo_dataset.py b/model_zoo/yolov3_darknet53/src/yolo_dataset.py new file mode 100644 index 0000000000..45657db823 --- /dev/null +++ b/model_zoo/yolov3_darknet53/src/yolo_dataset.py @@ -0,0 +1,184 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""YOLOV3 dataset.""" +import os + +from PIL import Image +from pycocotools.coco import COCO +import mindspore.dataset as de +import mindspore.dataset.transforms.vision.c_transforms as CV + +from src.distributed_sampler import DistributedSampler +from src.transforms import reshape_fn, MultiScaleTrans + + +min_keypoints_per_image = 10 + + +def _has_only_empty_bbox(anno): + return all(any(o <= 1 for o in obj["bbox"][2:]) for obj in anno) + + +def _count_visible_keypoints(anno): + return sum(sum(1 for v in ann["keypoints"][2::3] if v > 0) for ann in anno) + + +def has_valid_annotation(anno): + """Check annotation file.""" + # if it's empty, there is no annotation + if not anno: + return False + # if all boxes have close to zero area, there is no annotation + if _has_only_empty_bbox(anno): + return False + # keypoints task have a slight different critera for considering + # if an annotation is valid + if "keypoints" not in anno[0]: + return True + # for keypoint detection tasks, only consider valid images those + # containing at least min_keypoints_per_image + if _count_visible_keypoints(anno) >= min_keypoints_per_image: + return True + return False + + +class COCOYoloDataset: + """YOLOV3 Dataset for COCO.""" + def __init__(self, root, ann_file, remove_images_without_annotations=True, + filter_crowd_anno=True, is_training=True): + self.coco = COCO(ann_file) + self.root = root + self.img_ids = list(sorted(self.coco.imgs.keys())) + self.filter_crowd_anno = filter_crowd_anno + self.is_training = is_training + + # filter images without any annotations + if remove_images_without_annotations: + img_ids = [] + for img_id in self.img_ids: + ann_ids = self.coco.getAnnIds(imgIds=img_id, iscrowd=None) + anno = self.coco.loadAnns(ann_ids) + if has_valid_annotation(anno): + img_ids.append(img_id) + self.img_ids = img_ids + + self.categories = {cat["id"]: cat["name"] for cat in self.coco.cats.values()} + + self.cat_ids_to_continuous_ids = { + v: i for i, v in enumerate(self.coco.getCatIds()) + } + self.continuous_ids_cat_ids = { + v: k for k, v in self.cat_ids_to_continuous_ids.items() + } + + def __getitem__(self, index): + """ + Args: + index (int): Index + + Returns: + (img, target) (tuple): target is a dictionary contains "bbox", "segmentation" or "keypoints", + generated by the image's annotation. img is a PIL image. + """ + coco = self.coco + img_id = self.img_ids[index] + img_path = coco.loadImgs(img_id)[0]["file_name"] + img = Image.open(os.path.join(self.root, img_path)).convert("RGB") + if not self.is_training: + return img, img_id + + ann_ids = coco.getAnnIds(imgIds=img_id) + target = coco.loadAnns(ann_ids) + # filter crowd annotations + if self.filter_crowd_anno: + annos = [anno for anno in target if anno["iscrowd"] == 0] + else: + annos = [anno for anno in target] + + target = {} + boxes = [anno["bbox"] for anno in annos] + target["bboxes"] = boxes + + classes = [anno["category_id"] for anno in annos] + classes = [self.cat_ids_to_continuous_ids[cl] for cl in classes] + target["labels"] = classes + + bboxes = target['bboxes'] + labels = target['labels'] + out_target = [] + for bbox, label in zip(bboxes, labels): + tmp = [] + # convert to [x_min y_min x_max y_max] + bbox = self._convetTopDown(bbox) + tmp.extend(bbox) + tmp.append(int(label)) + # tmp [x_min y_min x_max y_max, label] + out_target.append(tmp) + return img, out_target + + def __len__(self): + return len(self.img_ids) + + def _convetTopDown(self, bbox): + x_min = bbox[0] + y_min = bbox[1] + w = bbox[2] + h = bbox[3] + return [x_min, y_min, x_min+w, y_min+h] + + +def create_yolo_dataset(image_dir, anno_path, batch_size, max_epoch, device_num, rank, + config=None, is_training=True, shuffle=True): + """Create dataset for YOLOV3.""" + if is_training: + filter_crowd = True + remove_empty_anno = True + else: + filter_crowd = False + remove_empty_anno = False + + yolo_dataset = COCOYoloDataset(root=image_dir, ann_file=anno_path, filter_crowd_anno=filter_crowd, + remove_images_without_annotations=remove_empty_anno, is_training=is_training) + distributed_sampler = DistributedSampler(len(yolo_dataset), device_num, rank, shuffle=shuffle) + hwc_to_chw = CV.HWC2CHW() + + config.dataset_size = len(yolo_dataset) + num_parallel_workers1 = int(64 / device_num) + num_parallel_workers2 = int(16 / device_num) + if is_training: + multi_scale_trans = MultiScaleTrans(config, device_num) + if device_num != 8: + ds = de.GeneratorDataset(yolo_dataset, column_names=["image", "annotation"], + num_parallel_workers=num_parallel_workers1, + sampler=distributed_sampler) + ds = ds.batch(batch_size, per_batch_map=multi_scale_trans, input_columns=['image', 'annotation'], + num_parallel_workers=num_parallel_workers2, drop_remainder=True) + else: + ds = de.GeneratorDataset(yolo_dataset, column_names=["image", "annotation"], sampler=distributed_sampler) + ds = ds.batch(batch_size, per_batch_map=multi_scale_trans, input_columns=['image', 'annotation'], + num_parallel_workers=8, drop_remainder=True) + else: + ds = de.GeneratorDataset(yolo_dataset, column_names=["image", "img_id"], + sampler=distributed_sampler) + compose_map_func = (lambda image, img_id: reshape_fn(image, img_id, config)) + ds = ds.map(input_columns=["image", "img_id"], + output_columns=["image", "image_shape", "img_id"], + columns_order=["image", "image_shape", "img_id"], + operations=compose_map_func, num_parallel_workers=8) + ds = ds.map(input_columns=["image"], operations=hwc_to_chw, num_parallel_workers=8) + ds = ds.batch(batch_size, drop_remainder=True) + ds = ds.repeat(max_epoch) + + return ds, len(yolo_dataset) diff --git a/model_zoo/yolov3_darknet53/train.py b/model_zoo/yolov3_darknet53/train.py new file mode 100644 index 0000000000..0781cedccb --- /dev/null +++ b/model_zoo/yolov3_darknet53/train.py @@ -0,0 +1,338 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""YoloV3 train.""" +import os +import time +import argparse +import datetime + +from mindspore import ParallelMode +from mindspore.nn.optim.momentum import Momentum +from mindspore import Tensor +import mindspore.nn as nn +from mindspore import context +from mindspore.communication.management import init, get_rank, get_group_size +from mindspore.train.callback import ModelCheckpoint, RunContext +from mindspore.train.callback import _InternalCallbackParam, CheckpointConfig +import mindspore as ms +from mindspore.train.serialization import load_checkpoint, load_param_into_net + +from src.yolo import YOLOV3DarkNet53, YoloWithLossCell, TrainingWrapper +from src.logger import get_logger +from src.util import AverageMeter, load_backbone, get_param_groups +from src.lr_scheduler import warmup_step_lr, warmup_cosine_annealing_lr, \ + warmup_cosine_annealing_lr_V2, warmup_cosine_annealing_lr_sample +from src.yolo_dataset import create_yolo_dataset +from src.initializer import default_recurisive_init +from src.config import ConfigYOLOV3DarkNet53 +from src.transforms import batch_preprocess_true_box, batch_preprocess_true_box_single +from src.util import ShapeRecord + + +devid = int(os.getenv('DEVICE_ID')) +context.set_context(mode=context.GRAPH_MODE, enable_auto_mixed_precision=True, + device_target="Ascend", save_graphs=True, device_id=devid) + + +class BuildTrainNetwork(nn.Cell): + def __init__(self, network, criterion): + super(BuildTrainNetwork, self).__init__() + self.network = network + self.criterion = criterion + + def construct(self, input_data, label): + output = self.network(input_data) + loss = self.criterion(output, label) + return loss + + +def parse_args(): + """Parse train arguments.""" + parser = argparse.ArgumentParser('mindspore coco training') + + # dataset related + parser.add_argument('--data_dir', type=str, default='', help='train data dir') + parser.add_argument('--per_batch_size', default=32, type=int, help='batch size for per gpu') + + # network related + parser.add_argument('--pretrained_backbone', default='', type=str, help='model_path, local pretrained backbone' + ' model to load') + parser.add_argument('--resume_yolov3', default='', type=str, help='path of pretrained yolov3') + + # optimizer and lr related + parser.add_argument('--lr_scheduler', default='exponential', type=str, + help='lr-scheduler, option type: exponential, cosine_annealing') + parser.add_argument('--lr', default=0.001, type=float, help='learning rate of the training') + parser.add_argument('--lr_epochs', type=str, default='220,250', help='epoch of lr changing') + parser.add_argument('--lr_gamma', type=float, default=0.1, + help='decrease lr by a factor of exponential lr_scheduler') + parser.add_argument('--eta_min', type=float, default=0., help='eta_min in cosine_annealing scheduler') + parser.add_argument('--T_max', type=int, default=320, help='T-max in cosine_annealing scheduler') + parser.add_argument('--max_epoch', type=int, default=320, help='max epoch num to train the model') + parser.add_argument('--warmup_epochs', default=0, type=float, help='warmup epoch') + parser.add_argument('--weight_decay', type=float, default=0.0005, help='weight decay') + parser.add_argument('--momentum', type=float, default=0.9, help='momentum') + + # loss related + parser.add_argument('--loss_scale', type=int, default=1024, help='static loss scale') + parser.add_argument('--label_smooth', type=int, default=0, help='whether to use label smooth in CE') + parser.add_argument('--label_smooth_factor', type=float, default=0.1, help='smooth strength of original one-hot') + + # logging related + parser.add_argument('--log_interval', type=int, default=100, help='logging interval') + parser.add_argument('--ckpt_path', type=str, default='outputs/', help='checkpoint save location') + parser.add_argument('--ckpt_interval', type=int, default=None, help='ckpt_interval') + + parser.add_argument('--is_save_on_master', type=int, default=1, help='save ckpt on master or all rank') + + # distributed related + parser.add_argument('--is_distributed', type=int, default=1, help='if multi device') + parser.add_argument('--rank', type=int, default=0, help='local rank of distributed') + parser.add_argument('--group_size', type=int, default=1, help='world size of distributed') + + # roma obs + parser.add_argument('--train_url', type=str, default="", help='train url') + + # profiler init + parser.add_argument('--need_profiler', type=int, default=0, help='whether use profiler') + + # reset default config + parser.add_argument('--training_shape', type=str, default="", help='fix training shape') + parser.add_argument('--resize_rate', type=int, default=None, help='resize rate for multi-scale training') + + args, _ = parser.parse_known_args() + if args.lr_scheduler == 'cosine_annealing' and args.max_epoch > args.T_max: + args.T_max = args.max_epoch + + args.lr_epochs = list(map(int, args.lr_epochs.split(','))) + args.data_root = os.path.join(args.data_dir, 'train2014') + args.annFile = os.path.join(args.data_dir, 'annotations/instances_train2014.json') + + return args + + +def conver_training_shape(args): + training_shape = [int(args.training_shape), int(args.training_shape)] + return training_shape + + +def train(): + """Train function.""" + args = parse_args() + + # init distributed + if args.is_distributed: + init() + args.rank = get_rank() + args.group_size = get_group_size() + + # select for master rank save ckpt or all rank save, compatiable for model parallel + args.rank_save_ckpt_flag = 0 + if args.is_save_on_master: + if args.rank == 0: + args.rank_save_ckpt_flag = 1 + else: + args.rank_save_ckpt_flag = 1 + + # logger + args.outputs_dir = os.path.join(args.ckpt_path, + datetime.datetime.now().strftime('%Y-%m-%d_time_%H_%M_%S')) + args.logger = get_logger(args.outputs_dir, args.rank) + args.logger.save_args(args) + + if args.need_profiler: + from mindinsight.profiler.profiling import Profiler + profiler = Profiler(output_path=args.outputs_dir, is_detail=True, is_show_op_path=True) + + loss_meter = AverageMeter('loss') + + context.reset_auto_parallel_context() + if args.is_distributed: + parallel_mode = ParallelMode.DATA_PARALLEL + degree = get_group_size() + else: + parallel_mode = ParallelMode.STAND_ALONE + degree = 1 + context.set_auto_parallel_context(parallel_mode=parallel_mode, mirror_mean=True, device_num=degree) + + network = YOLOV3DarkNet53(is_training=True) + # default is kaiming-normal + default_recurisive_init(network) + + if args.pretrained_backbone: + network = load_backbone(network, args.pretrained_backbone, args) + args.logger.info('load pre-trained backbone {} into network'.format(args.pretrained_backbone)) + else: + args.logger.info('Not load pre-trained backbone, please be careful') + + if args.resume_yolov3: + param_dict = load_checkpoint(args.resume_yolov3) + param_dict_new = {} + for key, values in param_dict.items(): + if key.startswith('moments.'): + continue + elif key.startswith('yolo_network.'): + param_dict_new[key[13:]] = values + args.logger.info('in resume {}'.format(key)) + else: + param_dict_new[key] = values + args.logger.info('in resume {}'.format(key)) + + args.logger.info('resume finished') + load_param_into_net(network, param_dict_new) + args.logger.info('load_model {} success'.format(args.resume_yolov3)) + + network = YoloWithLossCell(network) + args.logger.info('finish get network') + + config = ConfigYOLOV3DarkNet53() + + config.label_smooth = args.label_smooth + config.label_smooth_factor = args.label_smooth_factor + + if args.training_shape: + config.multi_scale = [conver_training_shape(args)] + if args.resize_rate: + config.resize_rate = args.resize_rate + + ds, data_size = create_yolo_dataset(image_dir=args.data_root, anno_path=args.annFile, is_training=True, + batch_size=args.per_batch_size, max_epoch=args.max_epoch, + device_num=args.group_size, rank=args.rank, config=config) + args.logger.info('Finish loading dataset') + + args.steps_per_epoch = int(data_size / args.per_batch_size / args.group_size) + + if not args.ckpt_interval: + args.ckpt_interval = args.steps_per_epoch + + # lr scheduler + if args.lr_scheduler == 'exponential': + lr = warmup_step_lr(args.lr, + args.lr_epochs, + args.steps_per_epoch, + args.warmup_epochs, + args.max_epoch, + gamma=args.lr_gamma, + ) + elif args.lr_scheduler == 'cosine_annealing': + lr = warmup_cosine_annealing_lr(args.lr, + args.steps_per_epoch, + args.warmup_epochs, + args.max_epoch, + args.T_max, + args.eta_min) + elif args.lr_scheduler == 'cosine_annealing_V2': + lr = warmup_cosine_annealing_lr_V2(args.lr, + args.steps_per_epoch, + args.warmup_epochs, + args.max_epoch, + args.T_max, + args.eta_min) + elif args.lr_scheduler == 'cosine_annealing_sample': + lr = warmup_cosine_annealing_lr_sample(args.lr, + args.steps_per_epoch, + args.warmup_epochs, + args.max_epoch, + args.T_max, + args.eta_min) + else: + raise NotImplementedError(args.lr_scheduler) + + opt = Momentum(params=get_param_groups(network), + learning_rate=Tensor(lr), + momentum=args.momentum, + weight_decay=args.weight_decay, + loss_scale=args.loss_scale) + + network = TrainingWrapper(network, opt) + network.set_train() + + if args.rank_save_ckpt_flag: + # checkpoint save + ckpt_max_num = args.max_epoch * args.steps_per_epoch // args.ckpt_interval + ckpt_config = CheckpointConfig(save_checkpoint_steps=args.ckpt_interval, + keep_checkpoint_max=ckpt_max_num) + ckpt_cb = ModelCheckpoint(config=ckpt_config, + directory=args.outputs_dir, + prefix='{}'.format(args.rank)) + cb_params = _InternalCallbackParam() + cb_params.train_network = network + cb_params.epoch_num = ckpt_max_num + cb_params.cur_epoch_num = 1 + run_context = RunContext(cb_params) + ckpt_cb.begin(run_context) + + old_progress = -1 + t_end = time.time() + data_loader = ds.create_dict_iterator() + + shape_record = ShapeRecord() + for i, data in enumerate(data_loader): + images = data["image"] + input_shape = images.shape[2:4] + args.logger.info('iter[{}], shape{}'.format(i, input_shape[0])) + shape_record.set(input_shape) + + images = Tensor(images) + annos = data["annotation"] + if args.group_size == 1: + batch_y_true_0, batch_y_true_1, batch_y_true_2, batch_gt_box0, batch_gt_box1, batch_gt_box2 = \ + batch_preprocess_true_box(annos, config, input_shape) + else: + batch_y_true_0, batch_y_true_1, batch_y_true_2, batch_gt_box0, batch_gt_box1, batch_gt_box2 = \ + batch_preprocess_true_box_single(annos, config, input_shape) + + batch_y_true_0 = Tensor(batch_y_true_0) + batch_y_true_1 = Tensor(batch_y_true_1) + batch_y_true_2 = Tensor(batch_y_true_2) + batch_gt_box0 = Tensor(batch_gt_box0) + batch_gt_box1 = Tensor(batch_gt_box1) + batch_gt_box2 = Tensor(batch_gt_box2) + + input_shape = Tensor(tuple(input_shape[::-1]), ms.float32) + loss = network(images, batch_y_true_0, batch_y_true_1, batch_y_true_2, batch_gt_box0, batch_gt_box1, + batch_gt_box2, input_shape) + loss_meter.update(loss.asnumpy()) + + if args.rank_save_ckpt_flag: + # ckpt progress + cb_params.cur_step_num = i + 1 # current step number + cb_params.batch_num = i + 2 + ckpt_cb.step_end(run_context) + + if i % args.log_interval == 0: + time_used = time.time() - t_end + epoch = int(i / args.steps_per_epoch) + fps = args.per_batch_size * (i - old_progress) * args.group_size / time_used + if args.rank == 0: + args.logger.info( + 'epoch[{}], iter[{}], {}, {:.2f} imgs/sec, lr:{}'.format(epoch, i, loss_meter, fps, lr[i])) + t_end = time.time() + loss_meter.reset() + old_progress = i + + if (i + 1) % args.steps_per_epoch == 0 and args.rank_save_ckpt_flag: + cb_params.cur_epoch_num += 1 + + if args.need_profiler: + if i == 10: + profiler.analyse() + break + + args.logger.info('==========end training===============') + + +if __name__ == "__main__": + train() diff --git a/model_zoo/yolov3/README.md b/model_zoo/yolov3_resnet18/README.md similarity index 100% rename from model_zoo/yolov3/README.md rename to model_zoo/yolov3_resnet18/README.md diff --git a/model_zoo/yolov3/eval.py b/model_zoo/yolov3_resnet18/eval.py similarity index 100% rename from model_zoo/yolov3/eval.py rename to model_zoo/yolov3_resnet18/eval.py diff --git a/model_zoo/yolov3/scripts/run_distribute_train.sh b/model_zoo/yolov3_resnet18/scripts/run_distribute_train.sh similarity index 100% rename from model_zoo/yolov3/scripts/run_distribute_train.sh rename to model_zoo/yolov3_resnet18/scripts/run_distribute_train.sh diff --git a/model_zoo/yolov3/scripts/run_eval.sh b/model_zoo/yolov3_resnet18/scripts/run_eval.sh similarity index 100% rename from model_zoo/yolov3/scripts/run_eval.sh rename to model_zoo/yolov3_resnet18/scripts/run_eval.sh diff --git a/model_zoo/yolov3/scripts/run_standalone_train.sh b/model_zoo/yolov3_resnet18/scripts/run_standalone_train.sh similarity index 100% rename from model_zoo/yolov3/scripts/run_standalone_train.sh rename to model_zoo/yolov3_resnet18/scripts/run_standalone_train.sh diff --git a/model_zoo/yolov3/src/config.py b/model_zoo/yolov3_resnet18/src/config.py similarity index 100% rename from model_zoo/yolov3/src/config.py rename to model_zoo/yolov3_resnet18/src/config.py diff --git a/model_zoo/yolov3/src/dataset.py b/model_zoo/yolov3_resnet18/src/dataset.py similarity index 100% rename from model_zoo/yolov3/src/dataset.py rename to model_zoo/yolov3_resnet18/src/dataset.py diff --git a/model_zoo/yolov3/src/utils.py b/model_zoo/yolov3_resnet18/src/utils.py similarity index 100% rename from model_zoo/yolov3/src/utils.py rename to model_zoo/yolov3_resnet18/src/utils.py diff --git a/model_zoo/yolov3/src/yolov3.py b/model_zoo/yolov3_resnet18/src/yolov3.py similarity index 100% rename from model_zoo/yolov3/src/yolov3.py rename to model_zoo/yolov3_resnet18/src/yolov3.py diff --git a/model_zoo/yolov3/train.py b/model_zoo/yolov3_resnet18/train.py similarity index 100% rename from model_zoo/yolov3/train.py rename to model_zoo/yolov3_resnet18/train.py diff --git a/serving/core/ms_service_pb2.py b/serving/core/ms_service_pb2.py new file mode 100644 index 0000000000..9feec026f9 --- /dev/null +++ b/serving/core/ms_service_pb2.py @@ -0,0 +1,318 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: ms_service.proto + +from google.protobuf.internal import enum_type_wrapper +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='ms_service.proto', + package='ms_serving', + syntax='proto3', + serialized_options=None, + serialized_pb=b'\n\x10ms_service.proto\x12\nms_serving\"2\n\x0ePredictRequest\x12 \n\x04\x64\x61ta\x18\x01 \x03(\x0b\x32\x12.ms_serving.Tensor\"2\n\x0cPredictReply\x12\"\n\x06result\x18\x01 \x03(\x0b\x32\x12.ms_serving.Tensor\"\x1b\n\x0bTensorShape\x12\x0c\n\x04\x64ims\x18\x01 \x03(\x03\"p\n\x06Tensor\x12-\n\x0ctensor_shape\x18\x01 \x01(\x0b\x32\x17.ms_serving.TensorShape\x12)\n\x0btensor_type\x18\x02 \x01(\x0e\x32\x14.ms_serving.DataType\x12\x0c\n\x04\x64\x61ta\x18\x03 \x01(\x0c*\xc9\x01\n\x08\x44\x61taType\x12\x0e\n\nMS_UNKNOWN\x10\x00\x12\x0b\n\x07MS_BOOL\x10\x01\x12\x0b\n\x07MS_INT8\x10\x02\x12\x0c\n\x08MS_UINT8\x10\x03\x12\x0c\n\x08MS_INT16\x10\x04\x12\r\n\tMS_UINT16\x10\x05\x12\x0c\n\x08MS_INT32\x10\x06\x12\r\n\tMS_UINT32\x10\x07\x12\x0c\n\x08MS_INT64\x10\x08\x12\r\n\tMS_UINT64\x10\t\x12\x0e\n\nMS_FLOAT16\x10\n\x12\x0e\n\nMS_FLOAT32\x10\x0b\x12\x0e\n\nMS_FLOAT64\x10\x0c\x32\x8e\x01\n\tMSService\x12\x41\n\x07Predict\x12\x1a.ms_serving.PredictRequest\x1a\x18.ms_serving.PredictReply\"\x00\x12>\n\x04Test\x12\x1a.ms_serving.PredictRequest\x1a\x18.ms_serving.PredictReply\"\x00\x62\x06proto3' +) + +_DATATYPE = _descriptor.EnumDescriptor( + name='DataType', + full_name='ms_serving.DataType', + filename=None, + file=DESCRIPTOR, + values=[ + _descriptor.EnumValueDescriptor( + name='MS_UNKNOWN', index=0, number=0, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='MS_BOOL', index=1, number=1, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='MS_INT8', index=2, number=2, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='MS_UINT8', index=3, number=3, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='MS_INT16', index=4, number=4, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='MS_UINT16', index=5, number=5, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='MS_INT32', index=6, number=6, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='MS_UINT32', index=7, number=7, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='MS_INT64', index=8, number=8, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='MS_UINT64', index=9, number=9, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='MS_FLOAT16', index=10, number=10, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='MS_FLOAT32', index=11, number=11, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='MS_FLOAT64', index=12, number=12, + serialized_options=None, + type=None), + ], + containing_type=None, + serialized_options=None, + serialized_start=280, + serialized_end=481, +) +_sym_db.RegisterEnumDescriptor(_DATATYPE) + +DataType = enum_type_wrapper.EnumTypeWrapper(_DATATYPE) +MS_UNKNOWN = 0 +MS_BOOL = 1 +MS_INT8 = 2 +MS_UINT8 = 3 +MS_INT16 = 4 +MS_UINT16 = 5 +MS_INT32 = 6 +MS_UINT32 = 7 +MS_INT64 = 8 +MS_UINT64 = 9 +MS_FLOAT16 = 10 +MS_FLOAT32 = 11 +MS_FLOAT64 = 12 + + + +_PREDICTREQUEST = _descriptor.Descriptor( + name='PredictRequest', + full_name='ms_serving.PredictRequest', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='data', full_name='ms_serving.PredictRequest.data', index=0, + number=1, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=32, + serialized_end=82, +) + + +_PREDICTREPLY = _descriptor.Descriptor( + name='PredictReply', + full_name='ms_serving.PredictReply', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='result', full_name='ms_serving.PredictReply.result', index=0, + number=1, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=84, + serialized_end=134, +) + + +_TENSORSHAPE = _descriptor.Descriptor( + name='TensorShape', + full_name='ms_serving.TensorShape', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='dims', full_name='ms_serving.TensorShape.dims', index=0, + number=1, type=3, cpp_type=2, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=136, + serialized_end=163, +) + + +_TENSOR = _descriptor.Descriptor( + name='Tensor', + full_name='ms_serving.Tensor', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='tensor_shape', full_name='ms_serving.Tensor.tensor_shape', index=0, + number=1, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='tensor_type', full_name='ms_serving.Tensor.tensor_type', index=1, + number=2, type=14, cpp_type=8, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='data', full_name='ms_serving.Tensor.data', index=2, + number=3, type=12, cpp_type=9, label=1, + has_default_value=False, default_value=b"", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=165, + serialized_end=277, +) + +_PREDICTREQUEST.fields_by_name['data'].message_type = _TENSOR +_PREDICTREPLY.fields_by_name['result'].message_type = _TENSOR +_TENSOR.fields_by_name['tensor_shape'].message_type = _TENSORSHAPE +_TENSOR.fields_by_name['tensor_type'].enum_type = _DATATYPE +DESCRIPTOR.message_types_by_name['PredictRequest'] = _PREDICTREQUEST +DESCRIPTOR.message_types_by_name['PredictReply'] = _PREDICTREPLY +DESCRIPTOR.message_types_by_name['TensorShape'] = _TENSORSHAPE +DESCRIPTOR.message_types_by_name['Tensor'] = _TENSOR +DESCRIPTOR.enum_types_by_name['DataType'] = _DATATYPE +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + +PredictRequest = _reflection.GeneratedProtocolMessageType('PredictRequest', (_message.Message,), { + 'DESCRIPTOR' : _PREDICTREQUEST, + '__module__' : 'ms_service_pb2' + # @@protoc_insertion_point(class_scope:ms_serving.PredictRequest) + }) +_sym_db.RegisterMessage(PredictRequest) + +PredictReply = _reflection.GeneratedProtocolMessageType('PredictReply', (_message.Message,), { + 'DESCRIPTOR' : _PREDICTREPLY, + '__module__' : 'ms_service_pb2' + # @@protoc_insertion_point(class_scope:ms_serving.PredictReply) + }) +_sym_db.RegisterMessage(PredictReply) + +TensorShape = _reflection.GeneratedProtocolMessageType('TensorShape', (_message.Message,), { + 'DESCRIPTOR' : _TENSORSHAPE, + '__module__' : 'ms_service_pb2' + # @@protoc_insertion_point(class_scope:ms_serving.TensorShape) + }) +_sym_db.RegisterMessage(TensorShape) + +Tensor = _reflection.GeneratedProtocolMessageType('Tensor', (_message.Message,), { + 'DESCRIPTOR' : _TENSOR, + '__module__' : 'ms_service_pb2' + # @@protoc_insertion_point(class_scope:ms_serving.Tensor) + }) +_sym_db.RegisterMessage(Tensor) + + + +_MSSERVICE = _descriptor.ServiceDescriptor( + name='MSService', + full_name='ms_serving.MSService', + file=DESCRIPTOR, + index=0, + serialized_options=None, + serialized_start=484, + serialized_end=626, + methods=[ + _descriptor.MethodDescriptor( + name='Predict', + full_name='ms_serving.MSService.Predict', + index=0, + containing_service=None, + input_type=_PREDICTREQUEST, + output_type=_PREDICTREPLY, + serialized_options=None, + ), + _descriptor.MethodDescriptor( + name='Test', + full_name='ms_serving.MSService.Test', + index=1, + containing_service=None, + input_type=_PREDICTREQUEST, + output_type=_PREDICTREPLY, + serialized_options=None, + ), +]) +_sym_db.RegisterServiceDescriptor(_MSSERVICE) + +DESCRIPTOR.services_by_name['MSService'] = _MSSERVICE + +# @@protoc_insertion_point(module_scope) diff --git a/serving/python_example/ms_service_pb2_grpc.py b/serving/core/ms_service_pb2_grpc.py similarity index 100% rename from serving/python_example/ms_service_pb2_grpc.py rename to serving/core/ms_service_pb2_grpc.py diff --git a/serving/core/server.cc b/serving/core/server.cc index add9d16bee..5ba7ad36a7 100644 --- a/serving/core/server.cc +++ b/serving/core/server.cc @@ -22,6 +22,7 @@ #include #include #include +#include #include "mindspore/ccsrc/utils/log_adapter.h" #include "serving/ms_service.grpc.pb.h" @@ -40,7 +41,7 @@ namespace serving { using MSTensorPtr = std::shared_ptr; Status Session::CreatDeviceSession(const std::string &device, uint32_t device_id) { - session_ = inference::MSSession::CreateSession(device + "Inference", device_id); + session_ = inference::MSSession::CreateSession(device, device_id); if (session_ == nullptr) { MS_LOG(ERROR) << "Creat Session Failed"; return FAILED; @@ -67,6 +68,7 @@ Status Session::Predict(const std::vector &inputs, inference::Multi MS_LOG(INFO) << "run Predict"; *outputs = session_->RunGraph(graph_id_, inputs); + MS_LOG(INFO) << "run Predict finished"; return SUCCESS; } @@ -80,12 +82,16 @@ Status Session::Warmup(const MindSporeModelPtr model) { std::string file_name = model->GetModelPath() + '/' + model->GetModelName(); char *graphBuf = ReadFile(file_name.c_str(), &size); if (graphBuf == nullptr) { - MS_LOG(ERROR) << "Load graph model failed, file name is " << file_name.c_str(); + MS_LOG(ERROR) << "Read model file failed, file name is " << file_name.c_str(); return FAILED; } last_graph_ = inference::LoadModel(graphBuf, size, device_type_); + if (last_graph_ == nullptr) { + MS_LOG(ERROR) << "Load graph model failed, file name is " << file_name.c_str(); + return FAILED; + } graph_id_ = session_->CompileGraph(last_graph_); - MS_LOG(INFO) << "Session Warmup"; + MS_LOG(INFO) << "Session Warmup finished"; return SUCCESS; } @@ -95,6 +101,9 @@ Status Session::Clear() { } namespace { +static const uint32_t uint32max = 0x7FFFFFFF; +std::promise exit_requested; + const std::map type2id_map{ {ms_serving::MS_UNKNOWN, TypeId::kNumberTypeBegin}, {ms_serving::MS_BOOL, TypeId::kNumberTypeBool}, {ms_serving::MS_INT8, TypeId::kNumberTypeInt8}, {ms_serving::MS_UINT8, TypeId::kNumberTypeUInt8}, @@ -141,7 +150,7 @@ MSTensorPtr ServingTensor2MSTensor(const ms_serving::Tensor &tensor) { } TypeId type = iter->second; auto ms_tensor = std::shared_ptr(inference::MSTensor::CreateTensor(type, shape)); - memcpy_s(ms_tensor->MutableData(), tensor.data().size(), tensor.data().data(), tensor.data().size()); + memcpy_s(ms_tensor->MutableData(), ms_tensor->Size(), tensor.data().data(), tensor.data().size()); return ms_tensor; } @@ -166,10 +175,7 @@ void ClearEnv() { Session::Instance().Clear(); inference::ExitInference(); } -void HandleSignal(int sig) { - ClearEnv(); - exit(0); -} +void HandleSignal(int sig) { exit_requested.set_value(); } #ifdef ENABLE_D static rtContext_t g_ctx = nullptr; @@ -221,6 +227,7 @@ class MSServiceImpl final : public MSService::Service { Status Server::BuildAndStart() { // handle exit signal signal(SIGINT, HandleSignal); + signal(SIGTERM, HandleSignal); Status res; auto option_args = Options::Instance().GetArgs(); std::string server_address = "0.0.0.0:" + std::to_string(option_args->grpc_port); @@ -247,31 +254,36 @@ Status Server::BuildAndStart() { rtError_t rt_ret = rtCtxGetCurrent(&ctx); if (rt_ret != RT_ERROR_NONE || ctx == nullptr) { MS_LOG(ERROR) << "the ascend device context is null"; + ClearEnv(); return FAILED; } g_ctx = ctx; #endif - MSServiceImpl service; + MSServiceImpl ms_service; grpc::EnableDefaultHealthCheckService(true); grpc::reflection::InitProtoReflectionServerBuilderPlugin(); // Set the port is not reuseable auto option = grpc::MakeChannelArgumentOption(GRPC_ARG_ALLOW_REUSEPORT, 0); - grpc::ServerBuilder builder; - builder.SetOption(std::move(option)); - // Listen on the given address without any authentication mechanism. - builder.AddListeningPort(server_address, grpc::InsecureServerCredentials()); - // Register "service" as the instance through which we'll communicate with - // clients. In this case it corresponds to an *synchronous* service. - builder.RegisterService(&service); - // Finally assemble the server. - std::unique_ptr server(builder.BuildAndStart()); - MS_LOG(INFO) << "Server listening on " << server_address << std::endl; - - // Wait for the server to shutdown. Note that some other thread must be - // responsible for shutting down the server for this call to ever return. - server->Wait(); + grpc::ServerBuilder serverBuilder; + serverBuilder.SetOption(std::move(option)); + serverBuilder.SetMaxMessageSize(uint32max); + serverBuilder.AddListeningPort(server_address, grpc::InsecureServerCredentials()); + serverBuilder.RegisterService(&ms_service); + std::unique_ptr server(serverBuilder.BuildAndStart()); + if (server == nullptr) { + MS_LOG(ERROR) << "The serving server create failed"; + ClearEnv(); + return FAILED; + } + auto grpc_server_run = [&server]() { server->Wait(); }; + std::thread serving_thread(grpc_server_run); + MS_LOG(INFO) << "MS Serving listening on " << server_address; + auto exit_future = exit_requested.get_future(); + exit_future.wait(); + ClearEnv(); + server->Shutdown(); + serving_thread.join(); return SUCCESS; } - } // namespace serving } // namespace mindspore diff --git a/serving/core/util/file_system_operation.cc b/serving/core/util/file_system_operation.cc index a5143995de..1af512a54c 100644 --- a/serving/core/util/file_system_operation.cc +++ b/serving/core/util/file_system_operation.cc @@ -29,7 +29,6 @@ namespace mindspore { namespace serving { - char *ReadFile(const char *file, size_t *size) { if (file == nullptr) { MS_LOG(ERROR) << "file is nullptr"; @@ -70,8 +69,8 @@ bool DirOrFileExist(const std::string &file_path) { } std::vector GetAllSubDirs(const std::string &dir_path) { - DIR *dir; - struct dirent *ptr; + DIR *dir = nullptr; + struct dirent *ptr = nullptr; std::vector SubDirs; if ((dir = opendir(dir_path.c_str())) == NULL) { diff --git a/serving/core/util/option_parser.cc b/serving/core/util/option_parser.cc index 9cbd7eaee8..c7f00e3733 100644 --- a/serving/core/util/option_parser.cc +++ b/serving/core/util/option_parser.cc @@ -36,17 +36,16 @@ bool RemovePrefix(std::string *str, const std::string &prefix) { bool Option::ParseInt32(std::string *arg) { if (RemovePrefix(arg, "--") && RemovePrefix(arg, name_) && RemovePrefix(arg, "=")) { - char extra; int32_t parsed_value; - if (sscanf(arg->data(), "%d%c", &parsed_value, &extra) != 1) { - std::cout << "Parse " << name_ << "Error for option " << *arg << std::endl; + try { + parsed_value = std::stoi(arg->data()); + } catch (std::invalid_argument) { + std::cout << "Parse " << name_ << " Error for option " << *arg << std::endl; return false; - } else { - *int32_default_ = parsed_value; } + *int32_default_ = parsed_value; return true; } - return false; } @@ -76,17 +75,16 @@ bool Option::ParseString(std::string *arg) { bool Option::ParseFloat(std::string *arg) { if (RemovePrefix(arg, "--") && RemovePrefix(arg, name_) && RemovePrefix(arg, "=")) { - char extra; float parsed_value; - if (sscanf(arg->data(), "%f%c", &parsed_value, &extra) != 1) { - std::cout << "Parse " << name_ << "Error for option " << *arg << std::endl; + try { + parsed_value = std::stof(arg->data()); + } catch (std::invalid_argument) { + std::cout << "Parse " << name_ << " Error for option " << *arg << std::endl; return false; - } else { - *float_default_ = parsed_value; } + *float_default_ = parsed_value; return true; } - return false; } @@ -159,10 +157,11 @@ Options::Options() : args_(nullptr) { CreateOptions(); } void Options::CreateOptions() { args_ = std::make_shared(); std::vector