From e8ee209877d472846fa18ae0b23caceafdfe3e78 Mon Sep 17 00:00:00 2001 From: ougongchang Date: Mon, 8 Mar 2021 16:09:09 +0800 Subject: [PATCH] Optimize summary ut and delete duplicate test cases Change the summary test case to stage3 to avoid multiple processes that cause the summary test case to fail --- tests/ut/python/runtest.sh | 28 +++- .../train/summary/test_graph_summary.py | 125 --------------- .../train/summary/test_image_summary.py | 5 +- tests/ut/python/train/summary/test_summary.py | 140 ----------------- .../train/summary/test_summary_collector.py | 24 +-- .../train/summary/test_summary_record.py | 2 +- .../train/summary/test_tensor_summary.py | 145 ------------------ 7 files changed, 40 insertions(+), 429 deletions(-) delete mode 100644 tests/ut/python/train/summary/test_graph_summary.py delete mode 100644 tests/ut/python/train/summary/test_summary.py delete mode 100644 tests/ut/python/train/summary/test_tensor_summary.py diff --git a/tests/ut/python/runtest.sh b/tests/ut/python/runtest.sh index 18b3d7ba93..8deefbbe32 100755 --- a/tests/ut/python/runtest.sh +++ b/tests/ut/python/runtest.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2019 Huawei Technologies Co., Ltd +# Copyright 2019-2021 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ -CURRPATH=$(cd $(dirname $0); pwd) +CURRPATH=$(cd "$(dirname $0)"; pwd) IGNORE_EXEC="--ignore=$CURRPATH/exec" PROJECT_PATH=$(cd ${CURRPATH}/../../..; pwd) @@ -36,7 +36,7 @@ if [ $# -eq 1 ] && ([ "$1" == "stage1" ] || [ "$1" == "stage2" ] || [ "$1" == elif [ $1 == "stage2" ]; then echo "run python parallel\train\ops ut" - pytest -n 4 --dist=loadfile -v $CURRPATH/parallel $CURRPATH/train + pytest -n 4 --dist=loadfile -v $CURRPATH/parallel $CURRPATH/train --ignore=$CURRPATH/train/summary RET=$? if [ ${RET} -ne 0 ]; then exit ${RET} @@ -53,6 +53,16 @@ if [ $# -eq 1 ] && ([ "$1" == "stage1" ] || [ "$1" == "stage2" ] || [ "$1" == fi pytest $CURRPATH/pynative_mode + RET=$? + if [ ${RET} -ne 0 ]; then + exit ${RET} + fi + + pytest -v $CURRPATH/train/summary + RET=$? + if [ ${RET} -ne 0 ]; then + exit ${RET} + fi fi else echo "run all python ut" @@ -62,7 +72,7 @@ else exit ${RET} fi - pytest -n 4 --dist=loadfile -v $CURRPATH/parallel $CURRPATH/train + pytest -n 4 --dist=loadfile -v $CURRPATH/parallel $CURRPATH/train --ignore=$CURRPATH/train/summary RET=$? if [ ${RET} -ne 0 ]; then exit ${RET} @@ -81,6 +91,16 @@ else fi pytest $CURRPATH/pynative_mode + RET=$? + if [ ${RET} -ne 0 ]; then + exit ${RET} + fi + + pytest -v $CURRPATH/train/summary + RET=$? + if [ ${RET} -ne 0 ]; then + exit ${RET} + fi fi RET=$? diff --git a/tests/ut/python/train/summary/test_graph_summary.py b/tests/ut/python/train/summary/test_graph_summary.py deleted file mode 100644 index 643ddbdea2..0000000000 --- a/tests/ut/python/train/summary/test_graph_summary.py +++ /dev/null @@ -1,125 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -""" test_graph_summary """ -import logging -import os -import numpy as np - -import mindspore.nn as nn -from mindspore import Model, context -from mindspore.nn.optim import Momentum -from mindspore.train.summary import SummaryRecord -from mindspore.train.callback import SummaryCollector -from .....dataset_mock import MindData - -CUR_DIR = os.getcwd() -SUMMARY_DIR = CUR_DIR + "/test_temp_summary_event_file/" -GRAPH_TEMP = CUR_DIR + "/ms_output-resnet50.pb" - -log = logging.getLogger("test") -log.setLevel(level=logging.ERROR) - - -class Net(nn.Cell): - """ Net definition """ - - def __init__(self): - super(Net, self).__init__() - self.conv = nn.Conv2d(3, 64, 3, has_bias=False, weight_init='normal', pad_mode='valid') - self.bn = nn.BatchNorm2d(64) - self.relu = nn.ReLU() - self.flatten = nn.Flatten() - self.fc = nn.Dense(64 * 222 * 222, 3) # padding=0 - - def construct(self, x): - x = self.conv(x) - x = self.bn(x) - x = self.relu(x) - x = self.flatten(x) - out = self.fc(x) - return out - - -class LossNet(nn.Cell): - """ LossNet definition """ - - def __init__(self): - super(LossNet, self).__init__() - self.conv = nn.Conv2d(3, 64, 3, has_bias=False, weight_init='normal', pad_mode='valid') - self.bn = nn.BatchNorm2d(64) - self.relu = nn.ReLU() - self.flatten = nn.Flatten() - self.fc = nn.Dense(64 * 222 * 222, 3) # padding=0 - self.loss = nn.SoftmaxCrossEntropyWithLogits() - - def construct(self, x, y): - x = self.conv(x) - x = self.bn(x) - x = self.relu(x) - x = self.flatten(x) - x = self.fc(x) - out = self.loss(x, y) - return out - - -def get_model(): - """ get_model """ - net = Net() - loss = nn.SoftmaxCrossEntropyWithLogits() - optim = Momentum(net.trainable_params(), learning_rate=0.1, momentum=0.9) - model = Model(net, loss_fn=loss, optimizer=optim, metrics=None) - return model - - -def get_dataset(): - """ get_datasetdataset """ - dataset_types = (np.float32, np.float32) - dataset_shapes = ((2, 3, 224, 224), (2, 3)) - - dataset = MindData(size=2, batch_size=2, - np_types=dataset_types, - output_shapes=dataset_shapes, - input_indexs=(0, 1)) - return dataset - - -# Test 1: summary sample of graph -def test_graph_summary_sample(): - """ test_graph_summary_sample """ - log.debug("begin test_graph_summary_sample") - dataset = get_dataset() - net = Net() - loss = nn.SoftmaxCrossEntropyWithLogits() - optim = Momentum(net.trainable_params(), 0.1, 0.9) - context.set_context(mode=context.GRAPH_MODE) - model = Model(net, loss_fn=loss, optimizer=optim, metrics=None) - with SummaryRecord(SUMMARY_DIR, file_suffix="_MS_GRAPH", network=model._train_network) as test_writer: - model.train(2, dataset) - for i in range(1, 5): - test_writer.record(i) - - -def test_graph_summary_callback(): - dataset = get_dataset() - net = Net() - loss = nn.SoftmaxCrossEntropyWithLogits() - optim = Momentum(net.trainable_params(), 0.1, 0.9) - context.set_context(mode=context.GRAPH_MODE) - model = Model(net, loss_fn=loss, optimizer=optim, metrics=None) - summary_collector = SummaryCollector(SUMMARY_DIR, - collect_freq=1, - keep_default_action=False, - collect_specified_data={'collect_graph': True}) - model.train(1, dataset, callbacks=[summary_collector]) diff --git a/tests/ut/python/train/summary/test_image_summary.py b/tests/ut/python/train/summary/test_image_summary.py index 6801f3f1b4..d13a269a52 100644 --- a/tests/ut/python/train/summary/test_image_summary.py +++ b/tests/ut/python/train/summary/test_image_summary.py @@ -34,9 +34,8 @@ log.setLevel(level=logging.ERROR) def make_image_tensor(shape, dtype=float): """ make_image_tensor """ - # pylint: disable=unused-argument - numel = np.prod(shape) - x = (np.arange(numel, dtype=float)).reshape(shape) + number = np.prod(shape) + x = (np.arange(number, dtype=dtype)).reshape(shape) return x diff --git a/tests/ut/python/train/summary/test_summary.py b/tests/ut/python/train/summary/test_summary.py deleted file mode 100644 index 535bd6ffff..0000000000 --- a/tests/ut/python/train/summary/test_summary.py +++ /dev/null @@ -1,140 +0,0 @@ -# Copyright 2020-2021 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -"""Test summary.""" -import os -import random - -import numpy as np - - -import mindspore.nn as nn -from mindspore.common.tensor import Tensor -from mindspore.ops import operations as P -from mindspore.train.summary.summary_record import SummaryRecord, _cache_summary_tensor_data - -CUR_DIR = os.getcwd() -SUMMARY_DIR = CUR_DIR + "/test_temp_summary_event_file/" - - -def get_test_data(step): - """ get_test_data """ - test_data_list = [] - tag1 = "x1[:Scalar]" - tag2 = "x2[:Scalar]" - np1 = np.array(step + 1).astype(np.float32) - np2 = np.array(step + 2).astype(np.float32) - - dict1 = {} - dict1["name"] = tag1 - dict1["data"] = Tensor(np1) - - dict2 = {} - dict2["name"] = tag2 - dict2["data"] = Tensor(np2) - - test_data_list.append(dict1) - test_data_list.append(dict2) - - return test_data_list - - -def test_scalar_summary_sample(): - """ test_scalar_summary_sample """ - with SummaryRecord(SUMMARY_DIR, file_suffix="_MS_SCALAR") as test_writer: - for i in range(1, 5): - test_data = get_test_data(i) - _cache_summary_tensor_data(test_data) - test_writer.record(i) - - -def get_test_data_shape_1(step): - """ get_test_data_shape_1 """ - test_data_list = [] - tag1 = "x1[:Scalar]" - tag2 = "x2[:Scalar]" - np1 = np.array([step + 1]).astype(np.float32) - np2 = np.array([step + 2]).astype(np.float32) - - dict1 = {} - dict1["name"] = tag1 - dict1["data"] = Tensor(np1) - - dict2 = {} - dict2["name"] = tag2 - dict2["data"] = Tensor(np2) - - test_data_list.append(dict1) - test_data_list.append(dict2) - - return test_data_list - - -# Test: shape = (1,) -def test_scalar_summary_sample_with_shape_1(): - """ test_scalar_summary_sample_with_shape_1 """ - with SummaryRecord(SUMMARY_DIR, file_suffix="_MS_SCALAR") as test_writer: - for i in range(1, 100): - test_data = get_test_data_shape_1(i) - _cache_summary_tensor_data(test_data) - test_writer.record(i) - - -# Test: test with ge -class SummaryDemo(nn.Cell): - """ SummaryDemo definition """ - - def __init__(self,): - super(SummaryDemo, self).__init__() - self.s = P.ScalarSummary() - self.histogram_summary = P.HistogramSummary() - self.add = P.Add() - - def construct(self, x, y): - self.s("x1", x) - z = self.add(x, y) - self.s("z1", z) - self.s("y1", y) - self.histogram_summary("histogram", z) - return z - - -def test_scalar_summary_with_ge(): - """ test_scalar_summary_with_ge """ - with SummaryRecord(SUMMARY_DIR, file_suffix="_MS_SCALAR") as test_writer: - net = SummaryDemo() - net.set_train() - - # step 2: create the Event - steps = 100 - for i in range(1, steps): - x = Tensor(np.array([1.1 + random.uniform(1, 10)]).astype(np.float32)) - y = Tensor(np.array([1.2 + random.uniform(1, 10)]).astype(np.float32)) - net(x, y) - test_writer.record(i) - - -# test the problem of two consecutive use cases going wrong -def test_scalar_summary_with_ge_2(): - """ test_scalar_summary_with_ge_2 """ - with SummaryRecord(SUMMARY_DIR, file_suffix="_MS_SCALAR") as test_writer: - net = SummaryDemo() - net.set_train() - - steps = 100 - for i in range(1, steps): - x = Tensor(np.array([1.1]).astype(np.float32)) - y = Tensor(np.array([1.2]).astype(np.float32)) - net(x, y) - test_writer.record(i) diff --git a/tests/ut/python/train/summary/test_summary_collector.py b/tests/ut/python/train/summary/test_summary_collector.py index 96011f1f73..9ca01cb188 100644 --- a/tests/ut/python/train/summary/test_summary_collector.py +++ b/tests/ut/python/train/summary/test_summary_collector.py @@ -294,11 +294,11 @@ class TestSummaryCollector: def test_collect_input_data_with_train_dataset_element_invalid(self): """Test the param 'train_dataset_element' in cb_params is invalid.""" cb_params = _InternalCallbackParam() - for invalid in (), [], None, [None]: + for invalid in (), [], None: cb_params.train_dataset_element = invalid - with SummaryCollector(tempfile.mkdtemp(dir=self.base_summary_dir)) as summary_collector: - summary_collector._collect_input_data(cb_params) - assert not summary_collector._collect_specified_data['collect_input_data'] + summary_collector = SummaryCollector(tempfile.mkdtemp(dir=self.base_summary_dir)) + summary_collector._collect_input_data(cb_params) + assert not summary_collector._collect_specified_data['collect_input_data'] @mock.patch.object(SummaryRecord, 'add_value') def test_collect_input_data_success(self, mock_add_value): @@ -342,8 +342,12 @@ class TestSummaryCollector: cb_params = _InternalCallbackParam() cb_params.net_outputs = net_output summary_collector = SummaryCollector((tempfile.mkdtemp(dir=self.base_summary_dir))) + summary_collector._get_loss(cb_params) - assert summary_collector._is_parse_loss_success + if expected_loss is None: + assert not summary_collector._is_parse_loss_success + else: + assert summary_collector._is_parse_loss_success def test_get_optimizer_from_cb_params_success(self): """Test get optimizer success from cb params.""" @@ -395,20 +399,18 @@ class TestSummaryCollector: assert optimizer is None assert summary_collector._temp_optimizer == 'Failed' - @pytest.mark.parametrize("histogram_regular, expected_names, expected_values", [ + @pytest.mark.parametrize("histogram_regular, expected_names", [ ( 'conv1|conv2', - ['conv1.weight1/auto', 'conv2.weight2/auto', 'conv1.bias1/auto'], - [1, 2, 3] + ['conv1.weight1/auto', 'conv2.weight2/auto', 'conv1.bias1/auto'] ), ( None, - ['conv1.weight1/auto', 'conv2.weight2/auto', 'conv1.bias1/auto', 'conv3.bias/auto', 'conv5.bias/auto'], - [1, 2, 3, 4, 5] + ['conv1.weight1/auto', 'conv2.weight2/auto', 'conv1.bias1/auto', 'conv3.bias/auto', 'conv5.bias/auto'] ) ]) @mock.patch.object(SummaryRecord, 'add_value') - def test_collect_histogram_from_regular(self, mock_add_value, histogram_regular, expected_names, expected_values): + def test_collect_histogram_from_regular(self, mock_add_value, histogram_regular, expected_names): """Test collect histogram from regular success.""" mock_add_value.side_effect = add_value cb_params = _InternalCallbackParam() diff --git a/tests/ut/python/train/summary/test_summary_record.py b/tests/ut/python/train/summary/test_summary_record.py index 4f317174b8..86fcc568f5 100644 --- a/tests/ut/python/train/summary/test_summary_record.py +++ b/tests/ut/python/train/summary/test_summary_record.py @@ -72,7 +72,7 @@ class TestSummaryRecord: assert "raise_exception" in str(exc.value) - @pytest.mark.parametrize("step", [False, 2.0, (1, 3), [2, 3], "str"]) + @pytest.mark.parametrize("step", ["str"]) def test_step_of_record_with_type_error(self, step): summary_dir = tempfile.mkdtemp(dir=self.base_summary_dir) with pytest.raises(TypeError): diff --git a/tests/ut/python/train/summary/test_tensor_summary.py b/tests/ut/python/train/summary/test_tensor_summary.py deleted file mode 100644 index cbf27873ca..0000000000 --- a/tests/ut/python/train/summary/test_tensor_summary.py +++ /dev/null @@ -1,145 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -""" -@File : test_tensor_summary.py -@Author: -@Date : 2019-07-4 -@Desc : test summary function -""" -import logging -import os -import numpy as np - -import mindspore.nn as nn -from mindspore.common.tensor import Tensor -from mindspore.ops import operations as P -from mindspore.train.summary.summary_record import SummaryRecord, _cache_summary_tensor_data - -CUR_DIR = os.getcwd() -SUMMARY_DIR = CUR_DIR + "/test_temp_summary_event_file/" - -log = logging.getLogger("test") -log.setLevel(level=logging.ERROR) - - -def get_test_data(step): - """ get_test_data """ - test_data_list = [] - - dict_x1 = {} - dict_x1["name"] = "x1[:Tensor]" - dict_x1["data"] = Tensor(np.array([[1, 2, step + 1], [2, 3, 4]]).astype(np.int8)) - test_data_list.append(dict_x1) - dict_x2 = {} - dict_x2["name"] = "x2[:Tensor]" - dict_x2["data"] = Tensor(np.array([[1, 2, step + 2], [2, 3, 4]]).astype(np.int16)) - test_data_list.append(dict_x2) - dict_x3 = {} - dict_x3["name"] = "x3[:Tensor]" - dict_x3["data"] = Tensor(np.array([[1, 2, step + 1], [2, 3, 4]]).astype(np.int32)) - test_data_list.append(dict_x3) - dict_x4 = {} - dict_x4["name"] = "x4[:Tensor]" - dict_x4["data"] = Tensor(np.array([[1, 2, step + 1], [2, 3, 4]]).astype(np.int64)) - test_data_list.append(dict_x4) - dict_x5 = {} - dict_x5["name"] = "x5[:Tensor]" - dict_x5["data"] = Tensor(np.array([[1, 2, step + 1], [2, 3, 4]]).astype(np.float)) - test_data_list.append(dict_x5) - dict_x6 = {} - dict_x6["name"] = "x6[:Tensor]" - dict_x6["data"] = Tensor(np.array([[1, 2, step + 1], [2, 3, 4]]).astype(np.float16)) - test_data_list.append(dict_x6) - dict_x7 = {} - dict_x7["name"] = "x7[:Tensor]" - dict_x7["data"] = Tensor(np.array([[1, 2, step + 1], [2, 3, 4]]).astype(np.float32)) - test_data_list.append(dict_x7) - dict_x8 = {} - dict_x8["name"] = "x8[:Tensor]" - dict_x8["data"] = Tensor(np.array([[1, 2, step + 1], [2, 3, 4]]).astype(np.float64)) - test_data_list.append(dict_x8) - - return test_data_list - - -# Test: call method on parse graph code -def test_tensor_summary_sample(): - """ test_tensor_summary_sample """ - log.debug("begin test_tensor_summary_sample") - # step 0: create the thread - with SummaryRecord(SUMMARY_DIR, file_suffix="_MS_TENSOR") as test_writer: - # step 1: create the Event - for i in range(1, 100): - test_data = get_test_data(i) - - _cache_summary_tensor_data(test_data) - test_writer.record(i) - - # step 2: accept the event and write the file - - log.debug("finished test_tensor_summary_sample") - - -def get_test_data_check(step): - """ get_test_data_check """ - test_data_list = [] - tag1 = "x1[:Tensor]" - np1 = np.array([[step, step, step], [2, 3, 4]]).astype(np.float32) - - dict1 = {} - dict1["name"] = tag1 - dict1["data"] = Tensor(np1) - test_data_list.append(dict1) - - return test_data_list - - -# Test: test with ge -class SummaryDemo(nn.Cell): - """ SummaryDemo definition """ - - def __init__(self,): - super(SummaryDemo, self).__init__() - self.s = P.TensorSummary() - self.add = P.Add() - - def construct(self, x, y): - self.s("x1", x) - z = self.add(x, y) - self.s("z1", z) - self.s("y1", y) - return z - -def test_tensor_summary_with_ge(): - """ test_tensor_summary_with_ge """ - log.debug("begin test_tensor_summary_with_ge") - - # step 0: create the thread - with SummaryRecord(SUMMARY_DIR) as test_writer: - # step 1: create the network for summary - x = Tensor(np.array([1.1]).astype(np.float32)) - y = Tensor(np.array([1.2]).astype(np.float32)) - net = SummaryDemo() - net.set_train() - - # step 2: create the Event - steps = 100 - for i in range(1, steps): - x = Tensor(np.array([[i], [i]]).astype(np.float32)) - y = Tensor(np.array([[i + 1], [i + 1]]).astype(np.float32)) - net(x, y) - test_writer.record(i) - - log.debug("finished test_tensor_summary_with_ge")