From 0f73fad662ce0e61f9606b23cc5aac227cfc8755 Mon Sep 17 00:00:00 2001 From: wjm Date: Tue, 8 Jun 2021 15:42:38 +0800 Subject: [PATCH] add mapping --- parser/common/CMakeLists.txt | 1 + parser/common/convert/message2operator.cc | 169 ++++++++++++++++++ parser/common/convert/message2operator.h | 38 ++++ parser/onnx/onnx_parser.cc | 11 +- tests/ut/parser/CMakeLists.txt | 1 + .../onnx_parser_unittest.cc | 5 + 6 files changed, 220 insertions(+), 5 deletions(-) create mode 100644 parser/common/convert/message2operator.cc create mode 100644 parser/common/convert/message2operator.h diff --git a/parser/common/CMakeLists.txt b/parser/common/CMakeLists.txt index 379d4d8..d5b87e8 100644 --- a/parser/common/CMakeLists.txt +++ b/parser/common/CMakeLists.txt @@ -15,6 +15,7 @@ set(SRC_LIST "../tensorflow/tensorflow_fusion_op_parser.cc" "../tensorflow/tensorflow_util.cc" "convert/pb2json.cc" + "convert/message2operator.cc" "op_def/ir_pb_converter.cc" "op_def/defs.cc" "op_def/op_schema.cc" diff --git a/parser/common/convert/message2operator.cc b/parser/common/convert/message2operator.cc new file mode 100644 index 0000000..9d7ab64 --- /dev/null +++ b/parser/common/convert/message2operator.cc @@ -0,0 +1,169 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "message2operator.h" + +#include + +#include "common/convert/pb2json.h" +#include "common/util.h" +#include "framework/common/debug/ge_log.h" + +namespace ge { +namespace { +const int kMaxParseDepth = 5; +const uint32_t kInteval = 2; +} // namespace + +Status Message2Operator::ParseOperatorAttrs(const google::protobuf::Message *message, int depth, ge::Operator &ops) { + if (depth > kMaxParseDepth) { + REPORT_INNER_ERROR("E19999", "Message depth:%d can not exceed %d.", depth, kMaxParseDepth); + GELOGE(FAILED, "[Check][Param]Message depth can not exceed %d.", kMaxParseDepth); + return FAILED; + } + + const google::protobuf::Reflection *reflection = message->GetReflection(); + GE_CHECK_NOTNULL(reflection); + std::vector field_desc; + reflection->ListFields(*message, &field_desc); + + for (auto &field : field_desc) { + GE_CHECK_NOTNULL(field); + if (field->is_repeated()) { + if (ParseRepeatedField(reflection, message, field, depth, ops) != SUCCESS) { + GELOGE(FAILED, "[Parse][RepeatedField] %s failed.", field->name().c_str()); + return FAILED; + } + } else { + if (ParseField(reflection, message, field, depth, ops) != SUCCESS) { + GELOGE(FAILED, "[Parse][Field] %s failed.", field->name().c_str()); + return FAILED; + } + } + } + return SUCCESS; +} + +Status Message2Operator::ParseField(const google::protobuf::Reflection *reflection, + const google::protobuf::Message *message, + const google::protobuf::FieldDescriptor *field, int depth, ge::Operator &ops) { + GELOGD("Start to parse field: %s.", field->name().c_str()); + switch (field->cpp_type()) { +#define CASE_FIELD_TYPE(cpptype, method, valuetype, logtype) \ + case google::protobuf::FieldDescriptor::CPPTYPE_##cpptype: { \ + valuetype value = reflection->Get##method(*message, field); \ + GELOGD("Parse result(%s : %" #logtype ")", field->name().c_str(), value); \ + (void)ops.SetAttr(field->name(), value); \ + break; \ + } + CASE_FIELD_TYPE(INT32, Int32, int32_t, d); + CASE_FIELD_TYPE(UINT32, UInt32, uint32_t, u); + CASE_FIELD_TYPE(INT64, Int64, int64_t, ld); + CASE_FIELD_TYPE(FLOAT, Float, float, f); + CASE_FIELD_TYPE(BOOL, Bool, bool, d); +#undef CASE_FIELD_TYPE + case google::protobuf::FieldDescriptor::CPPTYPE_ENUM: { + GE_CHECK_NOTNULL(reflection->GetEnum(*message, field)); + int value = reflection->GetEnum(*message, field)->number(); + GELOGD("Parse result(%s : %d)", field->name().c_str(), value); + (void)ops.SetAttr(field->name(), value); + break; + } + case google::protobuf::FieldDescriptor::CPPTYPE_STRING: { + string value = reflection->GetString(*message, field); + GELOGD("Parse result(%s : %s)", field->name().c_str(), value.c_str()); + (void)ops.SetAttr(field->name(), value); + break; + } + case google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE: { + const google::protobuf::Message &sub_message = reflection->GetMessage(*message, field); + if (ParseOperatorAttrs(&sub_message, depth + 1, ops) != SUCCESS) { + GELOGE(FAILED, "[Parse][OperatorAttrs] of %s failed.", field->name().c_str()); + return FAILED; + } + break; + } + default: { + ErrorManager::GetInstance().ATCReportErrMessage("E11032", {"name", "reason"}, + {field->name().c_str(), "Unsupported field type"}); + GELOGE(FAILED, "[Check][FieldType]Unsupported field type, name: %s.", field->name().c_str()); + return FAILED; + } + } + GELOGD("Parse field: %s success.", field->name().c_str()); + return SUCCESS; +} + +Status Message2Operator::ParseRepeatedField(const google::protobuf::Reflection *reflection, + const google::protobuf::Message *message, + const google::protobuf::FieldDescriptor *field, int depth, + ge::Operator &ops) { + GELOGD("Start to parse field: %s.", field->name().c_str()); + int field_size = reflection->FieldSize(*message, field); + if (field_size <= 0) { + REPORT_INNER_ERROR("E19999", "Size of repeated field %s must bigger than 0", field->name().c_str()); + GELOGE(FAILED, "[Check][Size]Size of repeated field %s must bigger than 0", field->name().c_str()); + return FAILED; + } + + switch (field->cpp_type()) { +#define CASE_FIELD_TYPE_REPEATED(cpptype, method, valuetype) \ + case google::protobuf::FieldDescriptor::CPPTYPE_##cpptype: { \ + std::vector attr_value; \ + for (int i = 0; i < field_size; i++) { \ + valuetype value = reflection->GetRepeated##method(*message, field, i); \ + attr_value.push_back(value); \ + } \ + (void)ops.SetAttr(field->name(), attr_value); \ + break; \ + } + CASE_FIELD_TYPE_REPEATED(INT32, Int32, int32_t); + CASE_FIELD_TYPE_REPEATED(UINT32, UInt32, uint32_t); + CASE_FIELD_TYPE_REPEATED(INT64, Int64, int64_t); + CASE_FIELD_TYPE_REPEATED(FLOAT, Float, float); + CASE_FIELD_TYPE_REPEATED(BOOL, Bool, bool); + CASE_FIELD_TYPE_REPEATED(STRING, String, string); +#undef CASE_FIELD_TYPE_REPEATED + case google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE: { + nlohmann::json message_json; + Pb2Json::RepeatedMessage2Json(*message, field, reflection, std::set(), message_json[field->name()], + false); + std::string repeated_message_str; + try { + repeated_message_str = message_json.dump(kInteval, ' ', false, Json::error_handler_t::ignore); + } catch (std::exception &e) { + ErrorManager::GetInstance().ATCReportErrMessage("E19007", {"exception"}, {e.what()}); + GELOGE(FAILED, "[Parse][JSON]Failed to convert JSON to string, reason: %s.", e.what()); + return FAILED; + } catch (...) { + ErrorManager::GetInstance().ATCReportErrMessage("E19008"); + GELOGE(FAILED, "[Parse][JSON]Failed to convert JSON to string."); + return FAILED; + } + (void)ops.SetAttr(field->name(), repeated_message_str); + break; + } + default: { + ErrorManager::GetInstance().ATCReportErrMessage("E11032", {"name", "reason"}, + {field->name().c_str(), "Unsupported field type"}); + GELOGE(FAILED, "[Check][FieldType]Unsupported field type, name: %s.", field->name().c_str()); + return FAILED; + } + } + GELOGD("Parse repeated field: %s success.", field->name().c_str()); + return SUCCESS; +} +} // namespace ge \ No newline at end of file diff --git a/parser/common/convert/message2operator.h b/parser/common/convert/message2operator.h new file mode 100644 index 0000000..0a7b8af --- /dev/null +++ b/parser/common/convert/message2operator.h @@ -0,0 +1,38 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PARSER_MESSAGE2OPERATOR_H +#define PARSER_MESSAGE2OPERATOR_H + +#include "external/ge/ge_api_error_codes.h" +#include "external/graph/operator.h" +#include "google/protobuf/message.h" + +namespace ge { +class Message2Operator { + public: + static Status ParseOperatorAttrs(const google::protobuf::Message *message, int depth, ge::Operator &ops); + + private: + static Status ParseField(const google::protobuf::Reflection *reflection, const google::protobuf::Message *message, + const google::protobuf::FieldDescriptor *field, int depth, ge::Operator &ops); + + static Status ParseRepeatedField(const google::protobuf::Reflection *reflection, + const google::protobuf::Message *message, + const google::protobuf::FieldDescriptor *field, int depth, ge::Operator &ops); +}; +} // namespace ge +#endif // PARSER_MESSAGE2OPERATOR_H diff --git a/parser/onnx/onnx_parser.cc b/parser/onnx/onnx_parser.cc index eef2041..218a1f4 100644 --- a/parser/onnx/onnx_parser.cc +++ b/parser/onnx/onnx_parser.cc @@ -18,6 +18,7 @@ #include #include #include +#include "common/convert/message2operator.h" #include "common/convert/pb2json.h" #include "common/util.h" #include "common/util/error_manager/error_manager.h" @@ -570,13 +571,13 @@ Status OnnxModelParser::ParseOpParam(const ge::onnx::NodeProto *node_proto, ge:: status = op_parser->ParseParams(node_proto, op); } else { ge::Operator op_src(node_proto->name(), op_type); - /*status = domi::AutoMappingFn(node_def, op_src); + status = Message2Operator::ParseOperatorAttrs(node_proto, 1, op_src); if (status != SUCCESS) { - REPORT_CALL_ERROR("E19999", "Auto mapping node_def:%s(%s) to operator failed", - node_def->name().c_str(), node_def->op().c_str()); - GELOGE(status, "Node[%s] auto mapping failed.", node_name.c_str()); + REPORT_CALL_ERROR("E19999", "Auto mapping node:%s(%s) to operator failed", + node_proto->name().c_str(), op_type.c_str()); + GELOGE(status, "Node[%s] auto mapping failed.", node_proto->name().c_str()); return status; - }*/ + } std::shared_ptr onnx_custom_op_parser = std::dynamic_pointer_cast(op_parser); status = onnx_custom_op_parser->ParseParams(op_src, op); diff --git a/tests/ut/parser/CMakeLists.txt b/tests/ut/parser/CMakeLists.txt index 7130e20..9f16487 100644 --- a/tests/ut/parser/CMakeLists.txt +++ b/tests/ut/parser/CMakeLists.txt @@ -221,6 +221,7 @@ set(PARSER_SRC_FILES "${PARSER_DIR}/parser/caffe/caffe_reshape_parser.cc" "${PARSER_DIR}/parser/common/acl_graph_parser_util.cc" "${PARSER_DIR}/parser/common/convert/pb2json.cc" + "${PARSER_DIR}/parser/common/convert/message2operator.cc" "${PARSER_DIR}/parser/common/data_op_parser.cc" "${PARSER_DIR}/parser/common/model_saver.cc" "${PARSER_DIR}/parser/common/op_def/arg_op.cc" diff --git a/tests/ut/parser/testcase/onnx_parser_testcase/onnx_parser_unittest.cc b/tests/ut/parser/testcase/onnx_parser_testcase/onnx_parser_unittest.cc index ae4cb49..ed63849 100644 --- a/tests/ut/parser/testcase/onnx_parser_testcase/onnx_parser_unittest.cc +++ b/tests/ut/parser/testcase/onnx_parser_testcase/onnx_parser_unittest.cc @@ -40,6 +40,10 @@ static Status ParseParams(const google::protobuf::Message* op_src, ge::Operator& } static Status ParseParamByOpFunc(const ge::Operator &op_src, ge::Operator& op_dest) { + string node_info; + if(op_src.GetAttr("attribute", node_info)==ge::GRAPH_SUCCESS) { + //std::cout<