| @@ -23,7 +23,7 @@ usage() | |||||
| { | { | ||||
| echo "Usage:" | echo "Usage:" | ||||
| echo "bash build.sh [-d] [-r] [-v] [-c on|off] [-t ut|st] [-g on|off] [-h] [-b ge] [-m infer|train] \\" | echo "bash build.sh [-d] [-r] [-v] [-c on|off] [-t ut|st] [-g on|off] [-h] [-b ge] [-m infer|train] \\" | ||||
| echo " [-a on|off] [-p on|off] [-i] [-L] [-R] [-D on|off] [-j[n]] [-e gpu|ascend|cpu|npu] \\" | |||||
| echo " [-a on|off] [-p on|off] [-i] [-R] [-D on|off] [-j[n]] [-e gpu|ascend|cpu|npu] \\" | |||||
| echo " [-P on|off] [-z [on|off]] [-M on|off] [-V 9.2|10.1|310|910] [-I arm64|arm32|x86_64] [-K] \\" | echo " [-P on|off] [-z [on|off]] [-M on|off] [-V 9.2|10.1|310|910] [-I arm64|arm32|x86_64] [-K] \\" | ||||
| echo " [-B on|off] [-E] [-l on|off] [-n full|lite|off] [-T on|off] [-H on|off] \\" | echo " [-B on|off] [-E] [-l on|off] [-n full|lite|off] [-T on|off] [-H on|off] \\" | ||||
| echo " [-A [cpp|java|object-c] [-C on|off] [-o on|off] [-S on|off] [-k on|off] [-W sse|neon|avx|off] \\" | echo " [-A [cpp|java|object-c] [-C on|off] [-o on|off] [-S on|off] [-k on|off] [-W sse|neon|avx|off] \\" | ||||
| @@ -43,7 +43,6 @@ usage() | |||||
| echo " -p Enable pipeline profile, print to stdout, default off" | echo " -p Enable pipeline profile, print to stdout, default off" | ||||
| echo " -R Enable pipeline profile, record to json, default off" | echo " -R Enable pipeline profile, record to json, default off" | ||||
| echo " -i Enable increment building, default off" | echo " -i Enable increment building, default off" | ||||
| echo " -L Enable load ANF-IR as input of 'infer', default off" | |||||
| echo " -j[n] Set the threads when building (Default: -j8)" | echo " -j[n] Set the threads when building (Default: -j8)" | ||||
| echo " -e Use cpu, gpu, npu or ascend" | echo " -e Use cpu, gpu, npu or ascend" | ||||
| echo " -P Enable dump anf graph to file in ProtoBuffer format, default on" | echo " -P Enable dump anf graph to file in ProtoBuffer format, default on" | ||||
| @@ -93,7 +92,6 @@ checkopts() | |||||
| ENABLE_ASAN="off" | ENABLE_ASAN="off" | ||||
| ENABLE_PROFILE="off" | ENABLE_PROFILE="off" | ||||
| INC_BUILD="off" | INC_BUILD="off" | ||||
| ENABLE_LOAD_IR="off" | |||||
| ENABLE_TIMELINE="off" | ENABLE_TIMELINE="off" | ||||
| ENABLE_DUMP2PROTO="on" | ENABLE_DUMP2PROTO="on" | ||||
| ENABLE_DUMP_IR="on" | ENABLE_DUMP_IR="on" | ||||
| @@ -125,7 +123,7 @@ checkopts() | |||||
| ENABLE_HIDDEN="on" | ENABLE_HIDDEN="on" | ||||
| LITE_ENABLE_GPU="" | LITE_ENABLE_GPU="" | ||||
| # Process the options | # Process the options | ||||
| while getopts 'drvj:c:t:hsb:a:g:p:ie:m:l:I:LRP:D:zM:V:K:B:En:T:A:C:o:S:k:W:H:' opt | |||||
| while getopts 'drvj:c:t:hsb:a:g:p:ie:m:l:I:RP:D:zM:V:K:B:En:T:A:C:o:S:k:W:H:' opt | |||||
| do | do | ||||
| OPTARG=$(echo ${OPTARG} | tr '[A-Z]' '[a-z]') | OPTARG=$(echo ${OPTARG} | tr '[A-Z]' '[a-z]') | ||||
| case "${opt}" in | case "${opt}" in | ||||
| @@ -210,10 +208,6 @@ checkopts() | |||||
| fi | fi | ||||
| TRAIN_MODE=$(echo "$OPTARG" | tr '[a-z]' '[A-Z]') | TRAIN_MODE=$(echo "$OPTARG" | tr '[a-z]' '[A-Z]') | ||||
| ;; | ;; | ||||
| L) | |||||
| ENABLE_LOAD_IR="on" | |||||
| echo "build with enable load anf ir" | |||||
| ;; | |||||
| R) | R) | ||||
| ENABLE_TIMELINE="on" | ENABLE_TIMELINE="on" | ||||
| echo "enable time_line record" | echo "enable time_line record" | ||||
| @@ -424,7 +418,6 @@ build_mindspore() | |||||
| mkdir -pv "${BUILD_PATH}/mindspore" | mkdir -pv "${BUILD_PATH}/mindspore" | ||||
| cd "${BUILD_PATH}/mindspore" | cd "${BUILD_PATH}/mindspore" | ||||
| CMAKE_ARGS="-DDEBUG_MODE=$DEBUG_MODE -DBUILD_PATH=$BUILD_PATH" | CMAKE_ARGS="-DDEBUG_MODE=$DEBUG_MODE -DBUILD_PATH=$BUILD_PATH" | ||||
| CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_LOAD_ANF_IR=$ENABLE_LOAD_IR" | |||||
| if [[ "X$ENABLE_COVERAGE" = "Xon" ]]; then | if [[ "X$ENABLE_COVERAGE" = "Xon" ]]; then | ||||
| CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_COVERAGE=ON" | CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_COVERAGE=ON" | ||||
| fi | fi | ||||
| @@ -111,8 +111,6 @@ class AnfExporter { | |||||
| void ExportIR(const std::string &filename, const std::string &id, const FuncGraphPtr &func_graph); | void ExportIR(const std::string &filename, const std::string &id, const FuncGraphPtr &func_graph); | ||||
| void ExportIR(const std::string &filename, const std::vector<TaggedGraph> &graphs); | void ExportIR(const std::string &filename, const std::vector<TaggedGraph> &graphs); | ||||
| std::vector<FuncGraphPtr> ImportIR(const std::string &filename); | |||||
| } // namespace mindspore | } // namespace mindspore | ||||
| #endif // MINDSPORE_CCSRC_DEBUG_ANF_IR_UTILS_H_ | #endif // MINDSPORE_CCSRC_DEBUG_ANF_IR_UTILS_H_ | ||||
| @@ -667,61 +667,6 @@ bool ExecutorPy::Compile(const py::object &obj, const py::tuple &args, const py: | |||||
| return ret_value; | return ret_value; | ||||
| } | } | ||||
| #ifdef ENABLE_LOAD_ANF_IR | |||||
| // get MindSpore Intermediate Representation File | |||||
| std::string GetMsIrFile(void) { | |||||
| std::string file; | |||||
| const char *path = getenv("MS_IR_FILE"); | |||||
| if (path == nullptr) { | |||||
| return file; | |||||
| } | |||||
| char real_path[PATH_MAX] = {0}; | |||||
| if (realpath(path, real_path) == nullptr) { | |||||
| MS_LOG(ERROR) << "MS IR path error, " << path; | |||||
| return file; | |||||
| } | |||||
| file = real_path; | |||||
| return file; | |||||
| } | |||||
| void RunPipelineAction(const ActionItem &action, pipeline::ResourcePtr resource, bool *result) { | |||||
| MS_EXCEPTION_IF_NULL(resource); | |||||
| MS_EXCEPTION_IF_NULL(result); | |||||
| std::string ir_file = GetMsIrFile(); | |||||
| (void)parse::python_adapter::set_python_scoped(); | |||||
| if (ir_file.empty()) { | |||||
| *result = action.second(resource); | |||||
| return; | |||||
| } | |||||
| // when in loading anf ir mode, action `parse` do nothing | |||||
| if (action.first == "parse") { | |||||
| return; | |||||
| } | |||||
| // load MindSpore IR from file | |||||
| if (action.first == "symbol_resolve") { | |||||
| MS_LOG(DEBUG) << action.first << " read ir file: " << ir_file; | |||||
| std::vector<FuncGraphPtr> graphs = ImportIR(ir_file); | |||||
| if (graphs.size() == 0) { | |||||
| MS_LOG(EXCEPTION) << action.first << " read ir file " << ir_file << " failed as no graph found"; | |||||
| } | |||||
| auto manager = resource->manager(); | |||||
| MS_EXCEPTION_IF_NULL(manager); | |||||
| for (auto &graph : graphs) { | |||||
| manager->AddFuncGraph(graph); | |||||
| } | |||||
| resource->set_func_graph(graphs[0]); | |||||
| return; | |||||
| } | |||||
| // do normal action when not in `parse` and `symbol_resolve` stage | |||||
| *result = action.second(resource); | |||||
| } | |||||
| #endif | |||||
| void Pipeline::Run() { | void Pipeline::Run() { | ||||
| MS_LOG(INFO) << "Pipeline run"; | MS_LOG(INFO) << "Pipeline run"; | ||||
| MS_EXCEPTION_IF_NULL(resource_); | MS_EXCEPTION_IF_NULL(resource_); | ||||
| @@ -737,11 +682,7 @@ void Pipeline::Run() { | |||||
| bool result = true; | bool result = true; | ||||
| WITH(MsProfile::GetProfile()->Step(action.first))[&result, &action, this]() { | WITH(MsProfile::GetProfile()->Step(action.first))[&result, &action, this]() { | ||||
| MS_LOG(DEBUG) << "Action " << action.first << " start ..."; | MS_LOG(DEBUG) << "Action " << action.first << " start ..."; | ||||
| #ifdef ENABLE_LOAD_ANF_IR | |||||
| RunPipelineAction(action, resource_, &result); | |||||
| #else | |||||
| result = action.second(resource_); | result = action.second(resource_); | ||||
| #endif | |||||
| MS_LOG(DEBUG) << "Action " << action.first << " end."; | MS_LOG(DEBUG) << "Action " << action.first << " end."; | ||||
| }; | }; | ||||
| if (action.first == "task_emit") { | if (action.first == "task_emit") { | ||||
| @@ -1,124 +0,0 @@ | |||||
| #!/bin/bash | |||||
| # Copyright 2019 Huawei Technologies Co., Ltd | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||||
| # you may not use this file except in compliance with the License. | |||||
| # You may obtain a copy of the License at | |||||
| # | |||||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||||
| # | |||||
| # Unless required by applicable law or agreed to in writing, software | |||||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
| # See the License for the specific language governing permissions and | |||||
| # limitations under the License. | |||||
| # ============================================================================ | |||||
| set -x | |||||
| set -e | |||||
| export SAVE_GRAPHS=YES | |||||
| # print usage message | |||||
| function usage() | |||||
| { | |||||
| echo "Usage:" | |||||
| echo "bash $0 [-g] [-d] [-a] [-h] [-f file]" | |||||
| echo "e.g. $0 -f 3_specialize.dat" | |||||
| echo "" | |||||
| echo "Options:" | |||||
| echo " -g Generate ir file for debug" | |||||
| echo " -d Debug dumped ir" | |||||
| echo " -a Execute all steps, default" | |||||
| echo " -f File to be parse" | |||||
| echo " -h Print usage" | |||||
| } | |||||
| # check and set options | |||||
| function checkopts() | |||||
| { | |||||
| # init variable | |||||
| MODE_GEN=0 | |||||
| MODE_DBG=1 | |||||
| MODE_ALL=2 | |||||
| FILE_NAME="3_optimize.dat" | |||||
| mode="${MODE_ALL}" # default execute all steps | |||||
| # Process the options | |||||
| while getopts 'gdaf:h' opt | |||||
| do | |||||
| case "${opt}" in | |||||
| g) | |||||
| mode="${MODE_GEN}" | |||||
| ;; | |||||
| d) | |||||
| mode="${MODE_DBG}" | |||||
| ;; | |||||
| a) | |||||
| mode="${MODE_ALL}" | |||||
| ;; | |||||
| f) | |||||
| FILE_NAME="$OPTARG" | |||||
| if ! [ -f "${FILE_NAME}" ]; then | |||||
| echo "File $FILE_NAME does not exist" | |||||
| usage | |||||
| exit 1 | |||||
| fi | |||||
| ;; | |||||
| h) | |||||
| usage | |||||
| exit 0 | |||||
| ;; | |||||
| *) | |||||
| echo "Unknown option ${opt}!" | |||||
| usage | |||||
| exit 1 | |||||
| esac | |||||
| done | |||||
| } | |||||
| # init variable | |||||
| # check options | |||||
| checkopts "$@" | |||||
| CUR_PATH=$(pwd) | |||||
| cd "`dirname $0`/.." | |||||
| cd build/mindspore/ | |||||
| make -j8 | |||||
| cp -v mindspore/ccsrc/_c_expression.cpython-*.so ../../mindspore/ | |||||
| cd - | |||||
| UT_NAME="./tests/ut/python/model/test_lenet.py::test_lenet5_train_sens" | |||||
| #UT_NAME="./tests/python/ops/test_math_ops.py::test_matmul_grad" | |||||
| #UT_NAME="./tests/python/exec/resnet_example.py::test_compile" | |||||
| #UT_NAME="./tests/perf_test/test_bert_train.py::test_bert_train" | |||||
| if [[ "${mode}" == "${MODE_GEN}" || "${mode}" == "${MODE_ALL}" ]]; then | |||||
| rm -rf pkl_objs | |||||
| mkdir -p pkl_objs | |||||
| echo "MS_IR_PATH=$(pwd)/pkl_objs pytest -s ${UT_NAME}" | |||||
| MS_IR_PATH=$(pwd)/pkl_objs/ pytest -s "${UT_NAME}" | |||||
| #pytest -s $UT_NAME | |||||
| # 1_resolve.dat | |||||
| # 3_specialize.dat | |||||
| # 4_simplify_data_structures.dat | |||||
| # 5_opt.dat | |||||
| # 6_opt2.dat | |||||
| # 7_opt_ge_adaptor_special.dat | |||||
| # 8_cconv.dat | |||||
| # 9_validate.dat | |||||
| cp "${FILE_NAME}" anf_ir_file.dbg | |||||
| rm -rf pkl_objs.dbg | |||||
| cp -rf pkl_objs pkl_objs.dbg | |||||
| fi | |||||
| if [[ "${mode}" == "${MODE_DBG}" || "${mode}" == "${MODE_ALL}" ]]; then | |||||
| echo "MS_IR_FILE=$(pwd)/anf_ir_file.dbg MS_IR_PATH=$(pwd)/pkl_objs.dbg/ pytest -s ${UT_NAME}" | |||||
| MS_IR_FILE=$(pwd)/anf_ir_file.dbg MS_IR_PATH=$(pwd)/pkl_objs.dbg/ pytest -s "${UT_NAME}" | |||||
| fi | |||||
| cd $CUR_PATH | |||||