!431 UT test

Merge pull request !431 from 李正龙/RE_UT_Test

tests/ut/parser/parser_ut_utils.cc

@@ -17,6 +17,7 @@
 #include "ut/parser/parser_ut_utils.h"
 #include "framework/common/debug/ge_log.h"
 #include "graph/utils/graph_utils.h"
+#include <limits.h>
 
 namespace ge {
 void ParerUTestsUtils::ClearParserInnerCtx() {
@@ -66,5 +67,68 @@ void GraphBuilder::AddDataEdge(const NodePtr &src_node, int src_idx, const NodeP
 void GraphBuilder::AddControlEdge(const NodePtr &src_node, const NodePtr &dst_node) {
   GraphUtils::AddEdge(src_node->GetOutControlAnchor(), dst_node->GetInControlAnchor());
 }
+
+ge::MemBuffer* MemBufferFromFile(const char *path) {
+    char path_temp[PATH_MAX + 1] = {0x00};
+    if(strlen(path) > PATH_MAX || nullptr == realpath(path, path_temp)) {
+        return nullptr;
+    }
+    FILE *fp = fopen(path_temp, "r+");
+    if (fp == nullptr) {
+        return nullptr;
+    }
+
+    // get model file length
+    if (0 != fseek(fp, 0, SEEK_END)) {
+        fclose(fp);
+        return nullptr;
+    }
+    long file_length = ftell(fp);
+    if (fseek(fp, 0, SEEK_SET)) {
+        fclose(fp);
+        return nullptr;
+    }
+    if (file_length <= 0) {
+        fclose(fp);
+        return nullptr;
+    }
+
+    // alloc model buffer
+    void *data = malloc((unsigned int)file_length);
+    if (!data) {
+        fclose(fp);
+        return nullptr;
+    }
+
+    // read file into memory
+    uint32_t read_size = (uint32_t)fread(data, 1, (unsigned int)file_length, fp);
+
+    // check if read success
+    if ((long)read_size != file_length) {
+        free(data);
+        data = nullptr;
+        fclose(fp);
+        return nullptr;
+    }
+
+    // close model file
+    fclose(fp);
+
+    // create an MemBuffer
+    MemBuffer* membuf = new MemBuffer();
+    if (!membuf) {
+        free(data);
+        data = nullptr;
+        return nullptr;
+    }
+    membuf->data = malloc((unsigned int)read_size);
+
+    // set size && data
+    membuf->size = (uint32_t)read_size;
+    memcpy((char*)membuf->data, (char*)data, read_size);
+    free(data);
+    return membuf;
+}
+
 }  // namespace ut
 }  // namespace ge

tests/ut/parser/parser_ut_utils.h

@@ -21,9 +21,15 @@
 #include "graph/compute_graph.h"
 
 namespace ge {
+struct MemBuffer {
+  void *data;
+  uint32_t size;
+};
+
 class ParerUTestsUtils {
  public:
   static void ClearParserInnerCtx();
+  static MemBuffer* MemBufferFromFile(const char *path);
 };
 namespace ut {
 class GraphBuilder {

tests/ut/parser/testcase/tensorflow_parser_testcase/tensorflow_model/caffe.proto

@@ -0,0 +1,3 @@
+{
+  "a": "b"
+}

tests/ut/parser/testcase/tensorflow_parser_testcase/tensorflow_model/caffe_abs.pbtxt

@@ -0,0 +1,14 @@
+name: "TestAbs"
+layer {
+  name: "data"
+  type: "Input"
+  top: "data"
+  input_param { shape: { dim: 64 dim: 1 dim: 28 dim: 28 } }
+}
+
+layer {
+  name: "abs"
+  type: "AbsVal"
+  bottom: "data"
+  top: "abs_out" 
+}

tests/ut/parser/testcase/tensorflow_parser_testcase/tensorflow_model/conv2d_depthwise_pb_gen.py

@@ -0,0 +1,26 @@
+import tensorflow as tf
+import os
+
+pb_file_path = os.getcwd()
+
+with tf.compat.v1.Session(graph=tf.Graph()) as sess:
+    # NHWC 
+    fmap_shape = [17, 101, 101, 17]
+    filter_size = [5, 5, 17, 1]
+    dy_shape = [17, 49, 49, 17]
+    strideh, stridew = [2, 2]
+    padding = 'VALID'
+    tensor_x1 = tf.compat.v1.placeholder(dtype="float16", shape=fmap_shape)
+    tensor_x2 = tf.compat.v1.placeholder(dtype="float16", shape=fmap_shape)
+    tensor_x = tf.add(tensor_x1, tensor_x2)
+    tensor_dy1 = tf.compat.v1.placeholder(dtype="float16", shape=dy_shape)
+    tensor_dy2 = tf.compat.v1.placeholder(dtype="float16", shape=dy_shape)
+    tensor_dy = tf.add(tensor_dy1, tensor_dy2)
+    
+    op = tf.nn.depthwise_conv2d_backprop_filter(tensor_x, filter_size, tensor_dy,
+                                  strides=[1, strideh, stridew, 1],
+                                  padding=padding,
+                                  data_format='NHWC',
+                                  dilations=[1,1,1,1])
+
+    tf.io.write_graph(sess.graph, logdir="./", name="test_depth_wise_conv2d.pb", as_text=False)

tests/ut/parser/testcase/tensorflow_parser_testcase/tensorflow_model/onnx_clip_v9.py

@@ -0,0 +1,28 @@
+import onnx
+from onnx import helper
+from onnx import AttributeProto, TensorProto, GraphProto
+
+
+def make_clip_V9():
+    X = helper.make_tensor_value_info("X", TensorProto.FLOAT, [3, 4, 5])
+    Y = helper.make_tensor_value_info("Y", TensorProto.FLOAT, [3, 4, 5])
+    node_def = helper.make_node('Clip',
+                                inputs=['X'],
+                                outputs=['Y'],
+                                max = 1.0,
+                                min = -1.0,
+                                )
+    graph = helper.make_graph(
+        [node_def],
+        "test_clip_case_V9",
+        [X],
+        [Y],
+    )
+
+    model = helper.make_model(graph, producer_name="onnx-mul_test")
+    model.opset_import[0].version = 9
+    onnx.save(model, "./onnx_clip_v9.onnx")
+
+
+if __name__ == '__main__':
+    make_clip_V9()

tests/ut/parser/testcase/tensorflow_parser_testcase/tensorflow_model/tensor_array_pb_gen.py

@@ -0,0 +1,14 @@
+import tensorflow as tf
+import os
+from tensorflow.python.framework import graph_util
+from tensorflow.python.ops import gen_data_flow_ops
+import numpy as np
+
+with tf.compat.v1.Session(graph=tf.Graph()) as sess:
+    size = tf.compat.v1.placeholder(dtype="int32", shape=())
+    value = tf.compat.v1.placeholder(dtype="float32", shape=(2,2))
+    index = tf.compat.v1.placeholder(dtype="int32", shape=())
+    flow = tf.compat.v1.placeholder(dtype="float32", shape=())
+    handleTensor = gen_data_flow_ops.tensor_array_v3(size= size, dtype = np.float32)
+    output = gen_data_flow_ops.tensor_array_write_v3(handle = handleTensor[0], index=index, value=value, flow_in=flow)
+    tf.io.write_graph(sess.graph, logdir="./", name="tensor_array.pb", as_text=False)

tests/ut/parser/testcase/tensorflow_parser_testcase/tensorflow_model/test_VarIsInitializedOp_pb_gen.py

@@ -0,0 +1,14 @@
+import tensorflow as tf
+import numpy as np
+
+def generate_VarIsInitializedOp_pb():
+    with tf.compat.v1.Session(graph=tf.Graph()) as sess:
+        x = tf.compat.v1.placeholder(dtype="int32", shape=())
+        y = tf.Variable(tf.compat.v1.random_normal(shape=[4,3],mean=0,stddev=1), dtype="float32", name='y')
+        init = tf.compat.v1.global_variables_initializer()
+        sess.run(init)
+        op = tf.compat.v1.raw_ops.VarIsInitializedOp(resource=y, name="VarIsInitializedOp")
+    tf.io.write_graph(sess.graph, logdir="./", name="test_VarIsInitializedOp.pb", as_text=False)
+
+if __name__=='__main__':
+    generate_VarIsInitializedOp_pb()

tests/ut/parser/testcase/tensorflow_parser_testcase/tensorflow_model/test_avgpool3dgrad_pb_gen.py

@@ -0,0 +1,41 @@
+import tensorflow as tf
+import os
+import numpy as np
+from tensorflow.python.framework import graph_util
+
+pb_file_path = os.getcwd()
+
+def generate_case_1():
+    with tf.compat.v1.Session(graph=tf.Graph()) as sess:
+        grads_1 = tf.compat.v1.placeholder(dtype="float16", shape=(1,2,2,2,1))
+        grads_2 = tf.compat.v1.placeholder(dtype="float16", shape=(1,2,2,2,1))
+        grads = tf.add(grads_1, grads_2)
+        orig_input_shape = tf.constant(np.array([1,3,3,3,1]).astype("int32"))
+        op = tf.raw_ops.AvgPool3DGrad(orig_input_shape=orig_input_shape,
+                                      grad=grads,
+                                      ksize=[1,2,2,2,1],
+                                      strides=[1,1,1,1,1],
+                                      padding="VALID",
+                                      data_format='NDHWC',
+                                      name='AvgPool3DGrad')
+
+        tf.io.write_graph(sess.graph, logdir="./", name="avgpool3dgrad_case_1.pb", as_text=False)
+
+def generate_case_2():
+    with tf.compat.v1.Session(graph=tf.Graph()) as sess:
+        grads_1 = tf.compat.v1.placeholder(dtype="float16", shape=(1,1,2,2,2))
+        grads_2 = tf.compat.v1.placeholder(dtype="float16", shape=(1,1,2,2,2))
+        grads = tf.add(grads_1, grads_2)
+        orig_input_shape = tf.constant(np.array([1,1,3,3,3]).astype("int32"), )
+        op = tf.raw_ops.AvgPool3DGrad(orig_input_shape=orig_input_shape,
+                                      grad=grads,
+                                      ksize=[1,1,2,2,2],
+                                      strides=[1,1,1,1,1],
+                                      padding="VALID",
+                                      data_format='NCDHW',
+                                      name='AvgPool3DGrad')
+
+        tf.io.write_graph(sess.graph, logdir="./", name="avgpool3dgrad.pb.txt", as_text=False)
+
+if __name__=='__main__':
+    generate_case_2()

tests/ut/parser/testcase/tensorflow_parser_testcase/tensorflow_model/test_blocklstm_pb.gen.py

@@ -0,0 +1,34 @@
+import tensorflow as tf
+import os
+
+pb_file_path = os.getcwd()
+
+def generate_case_0():
+    with tf.compat.v1.Session(graph=tf.Graph()) as sess:
+        input_dtype = tf.float32
+        input_shape0 = [1, ]
+        input_shape1 = [202, 1, 768]
+        input_shape2 = [1, 1]
+        input_shape3 = [1, 1]
+        input_shape4 = [769, 4]
+        input_shape5 = [1, ]
+        input_shape6 = [1, ]
+        input_shape7 = [1, ]
+        input_shape8 = [4, ]
+
+        d0 = tf.compat.v1.placeholder(dtype=tf.int64, shape=input_shape0)
+        d1 = tf.compat.v1.placeholder(dtype=input_dtype, shape=input_shape1)
+        d2 = tf.compat.v1.placeholder(dtype=input_dtype, shape=input_shape2)
+        d3 = tf.compat.v1.placeholder(dtype=input_dtype, shape=input_shape3)
+        d4 = tf.compat.v1.placeholder(dtype=input_dtype, shape=input_shape4)
+        d5 = tf.compat.v1.placeholder(dtype=input_dtype, shape=input_shape5)
+        d6 = tf.compat.v1.placeholder(dtype=input_dtype, shape=input_shape6)
+        d7 = tf.compat.v1.placeholder(dtype=input_dtype, shape=input_shape7)
+        d8 = tf.compat.v1.placeholder(dtype=input_dtype, shape=input_shape8)
+
+        i1, cs1, f1, o1, ci1, co1, h1 = tf.raw_ops.BlockLSTM(seq_len_max=d0, x=d1, cs_prev=d2, h_prev=d3, w=d4, wci=d5, wcf=d6, wco=d7, b=d8,
+                                                             forget_bias=1, cell_clip=3, use_peephole=False, name="blockLSTM")
+        tf.io.write_graph(sess.graph, logdir="./", name="blocklstm_case.pb", as_text=False)
+
+if __name__=='__main__':
+    generate_case_0()

tests/ut/parser/testcase/tensorflow_parser_testcase/tensorflow_model/test_constant_pb_gen.py

@@ -0,0 +1,21 @@
+import tensorflow as tf
+import numpy as np
+from tensorflow.python.framework import graph_util
+
+def generate_constant_pb():
+    with tf.compat.v1.Session(graph=tf.Graph()) as sess:
+        grads_1 = tf.compat.v1.placeholder(dtype="float16", shape=(1,1,2,2,2))
+        grads_2 = tf.compat.v1.placeholder(dtype="float16", shape=(1,1,2,2,2))
+        grads = tf.add(grads_1, grads_2)
+        orig_input_shape = tf.constant(np.array([1,1,3,3,3]).astype("int32"), )
+        op = tf.raw_ops.AvgPool3DGrad(orig_input_shape=orig_input_shape,
+                                      grad=grads,
+                                      ksize=[1,1,2,2,2],
+                                      strides=[1,1,1,1,1],
+                                      padding="VALID",
+                                      data_format='NCDHW',
+                                      name='AvgPool3DGrad')
+        tf.io.write_graph(sess.graph, logdir="./", name="test_constant.pb", as_text=False)
+
+if __name__ == "__main__":
+    generate_constant_pb()

tests/ut/parser/testcase/tensorflow_parser_testcase/tensorflow_model/test_conv2d_pb_gen.py

@@ -0,0 +1,26 @@
+import tensorflow as tf
+import os
+from tensorflow.python.framework import graph_util
+
+pb_file_path = os.getcwd()
+
+def generate_conv2d_pb():
+    with tf.compat.v1.Session(graph=tf.Graph()) as sess:
+        input_x = tf.compat.v1.placeholder(dtype="float32", shape=(1,56,56,64))
+        input_filter = tf.compat.v1.placeholder(dtype="float32", shape=(3,3,64,64))
+        op = tf.nn.conv2d(input_x, input_filter, strides=[1,1,1,1], padding=[[0,0],[1,1],[1,1],[0,0]],
+                          data_format="NHWC", dilations=[1,1,1,1], name='conv2d_res')
+        tf.io.write_graph(sess.graph, logdir="./", name="conv2d.pb", as_text=False)
+
+def generate_add_pb():
+    with tf.compat.v1.Session(graph=tf.Graph()) as sess:
+        x = tf.compat.v1.placeholder(tf.int32, name='x')
+        y = tf.compat.v1.placeholder(tf.int32, name='y')
+        b = tf.Variable(1, name='b')
+        xy = tf.multiply(x, y)
+        op = tf.add(xy, b, name='op_to_store')
+        tf.io.write_graph(sess.graph, logdir="./", name="model.pb", as_text=False)
+
+if __name__=='__main__':
+    generate_conv2d_pb()
+    generate_add_pb()

tests/ut/parser/testcase/tensorflow_parser_testcase/tensorflow_model/test_enter_pb_gen.py

@@ -0,0 +1,13 @@
+import tensorflow as tf
+from tensorflow.python.ops import control_flow_ops
+
+def generate_enter_pb():
+    with tf.compat.v1.Session(graph=tf.Graph()) as sess:
+        x = tf.compat.v1.placeholder(dtype="int32", shape=())
+        y = tf.compat.v1.placeholder(dtype="int32", shape=())
+        output1 = control_flow_ops.enter(x, frame_name = "output1")
+        output2 = control_flow_ops.enter(y, frame_name = "output2")
+    tf.io.write_graph(sess.graph, logdir="./", name="test_enter.pb", as_text=False)
+
+if __name__=='__main__':
+    generate_enter_pb()

tests/ut/parser/testcase/tensorflow_parser_testcase/tensorflow_model/test_fill_pb_gen.py

@@ -0,0 +1,12 @@
+import tensorflow as tf
+import numpy as np
+
+def generate_fill_pb():
+    with tf.compat.v1.Session(graph=tf.Graph()) as sess:
+        x = tf.compat.v1.placeholder(dtype="int32", shape=(2,2))
+        y = tf.fill([1,2], value = 5)
+        z = tf.add(x,y)
+    tf.io.write_graph(sess.graph, logdir="./", name="test_fill.pb", as_text=False)
+
+if __name__ == "__main__":
+    generate_fill_pb()

tests/ut/parser/testcase/tensorflow_parser_testcase/tensorflow_model/test_identity_pb_gen.py

@@ -0,0 +1,13 @@
+import tensorflow as tf
+
+def generate_identity_pb():
+    with tf.compat.v1.Session(graph=tf.Graph()) as sess:
+        x = tf.compat.v1.placeholder(dtype="int32", shape=())
+        x_plus_1 = tf.add(x, 1, name='x_plus')
+        with tf.control_dependencies([x_plus_1]):
+            y = x
+            z = tf.identity(x,name='identity')
+    tf.io.write_graph(sess.graph, logdir="./", name="test_identity.pb", as_text=False)
+
+if __name__=='__main__':
+    generate_identity_pb()

tests/ut/parser/testcase/tensorflow_parser_testcase/tensorflow_model/test_merge_pb_gen.py

@@ -0,0 +1,18 @@
+import tensorflow as tf
+from tensorflow.python.framework import graph_util
+import numpy as np
+
+def generate_merge_pb():
+    with tf.compat.v1.Session(graph=tf.Graph()) as sess:
+        dist = tf.compat.v1.placeholder(tf.float32, [100])
+        tf.compat.v1.summary.histogram(name="Merge", values=dist)
+        writer = tf.compat.v1.summary.FileWriter("./tf_summary_merge_pb")
+        op = tf.compat.v1.summary.merge_all()
+        for step in range(10):
+            mean_moving_normal = np.random.normal(loc=step, scale=1, size=[100])
+            summ = sess.run(op, feed_dict = {dist : mean_moving_normal})
+            writer.add_summary(summ, global_step=step)
+        tf.io.write_graph(sess.graph, logdir="./", name="merge.pb", as_text=False)
+
+if __name__=='__main__':
+    generate_merge_pb()

tests/ut/parser/testcase/tensorflow_parser_testcase/tensorflow_model/test_no_op_pb_gen.py

@@ -0,0 +1,14 @@
+import tensorflow as tf
+import numpy as np
+
+def generate_no_op_pb():
+    with tf.compat.v1.Session(graph=tf.Graph()) as sess:
+        x = tf.compat.v1.placeholder(dtype="int32", shape=())
+        y = tf.compat.v1.placeholder(dtype="int32", shape=())
+        add_op = tf.add(x, y)
+        y2 = tf.no_op(name="train")
+        z = tf.group([add_op, y2])
+    tf.io.write_graph(sess.graph, logdir="./", name="test_no_op.pb", as_text=False)
+
+if __name__=='__main__':
+    generate_no_op_pb()

tests/ut/parser/testcase/tensorflow_parser_testcase/tensorflow_model/test_reshape_pb_gen.py

@@ -0,0 +1,10 @@
+import tensorflow as tf
+
+def generate_reshape_pb():
+    with tf.compat.v1.Session(graph=tf.Graph()) as sess:
+        x = tf.compat.v1.placeholder(dtype="int32", shape=(2,2))
+        y = tf.compat.v1.reshape(x, [2,2])
+    tf.io.write_graph(sess.graph, logdir="./", name="test_reshape.pb", as_text=False)
+
+if __name__ == "__main__":
+    generate_reshape_pb()

tests/ut/parser/testcase/tensorflow_parser_testcase/tensorflow_model/test_sequeeze_pb_gen.py

@@ -0,0 +1,13 @@
+import tensorflow as tf
+import numpy as np
+
+def generate_sequeeze_pb():
+    with tf.compat.v1.Session(graph=tf.Graph()) as sess:
+        x = tf.compat.v1.placeholder(dtype="int32", shape=(2,2))
+        y = tf.constant([[1,2],[2,3]])
+        z = tf.add(x,y)
+        op = tf.squeeze(z,name = "squeeze")
+    tf.io.write_graph(sess.graph, logdir="./", name="test_sequeeze.pb", as_text=False)
+
+if __name__ == "__main__":
+    generate_sequeeze_pb()

tests/ut/parser/testcase/tensorflow_parser_testcase/tensorflow_model/test_shape_n_pb_gen.py

@@ -0,0 +1,11 @@
+import tensorflow as tf
+import numpy as np
+
+def generate_shape_n_pb():
+    with tf.compat.v1.Session(graph=tf.Graph()) as sess:
+        x = tf.compat.v1.placeholder(dtype="int32", shape=(2,2))
+        y = tf.shape_n([1,2], name= "shape_n")
+    tf.io.write_graph(sess.graph, logdir="./", name="test_shape_n.pb", as_text=False)
+
+if __name__ == "__main__":
+    generate_shape_n_pb()

tests/ut/parser/testcase/tensorflow_parser_testcase/tensorflow_model/test_switch_pb_gen.py

@@ -0,0 +1,13 @@
+import tensorflow as tf
+from tensorflow.python.ops import control_flow_ops
+
+def generate_switch_pb():
+    with tf.compat.v1.Session(graph=tf.Graph()) as sess:
+        x = tf.compat.v1.placeholder(dtype="int32", shape=())
+        y = tf.compat.v1.placeholder(dtype="int32", shape=())
+        output1 = control_flow_ops.switch(x,False)
+        output2 = control_flow_ops.switch(y,True)
+    tf.io.write_graph(sess.graph, logdir="./", name="test_switch.pb", as_text=False)
+
+if __name__=='__main__':
+    generate_switch_pb()

tests/ut/parser/testcase/tensorflow_parser_testcase/tensorflow_model/test_variableV2_pb_gen.py

@@ -0,0 +1,13 @@
+import tensorflow as tf
+
+def generate_VariableV2_pb():
+    with tf.compat.v1.Session(graph=tf.Graph()) as sess:
+        x = tf.compat.v1.placeholder(dtype="int32", shape=(2,3))
+        op = tf.raw_ops.VariableV2(shape=[2,3], dtype="int32", name="VariableV2")
+        init = tf.compat.v1.global_variables_initializer()
+        op_add = tf.add(x,op)
+        sess.run(init)
+    tf.io.write_graph(sess.graph, logdir="./", name="test_VariableV2.pb", as_text=False)
+
+if __name__=='__main__':
+    generate_VariableV2_pb()

tests/ut/parser/testcase/tensorflow_parser_testcase/tensorflow_model/tf_add.pb

@@ -0,0 +1,13 @@
+
+8
+PlaceholderPlaceholder*
+dtype0*
+shape:

+:
+
Placeholder_1Placeholder*
+dtype0*
+shape:

+6
+
+add_test_1AddPlaceholder
Placeholder_1*
+
T0"†