diff --git a/tests/ut/data/dataset/golden/linear_transformation_01_result.npz b/tests/ut/data/dataset/golden/linear_transformation_01_result.npz new file mode 100644 index 0000000000..ef54a007b5 Binary files /dev/null and b/tests/ut/data/dataset/golden/linear_transformation_01_result.npz differ diff --git a/tests/ut/data/dataset/golden/to_pil_01_result.npz b/tests/ut/data/dataset/golden/to_pil_01_result.npz new file mode 100644 index 0000000000..0f9a34dd41 Binary files /dev/null and b/tests/ut/data/dataset/golden/to_pil_01_result.npz differ diff --git a/tests/ut/data/dataset/golden/to_pil_02_result.npz b/tests/ut/data/dataset/golden/to_pil_02_result.npz new file mode 100644 index 0000000000..67faba626a Binary files /dev/null and b/tests/ut/data/dataset/golden/to_pil_02_result.npz differ diff --git a/tests/ut/data/dataset/golden/to_type_01_result.npz b/tests/ut/data/dataset/golden/to_type_01_result.npz new file mode 100644 index 0000000000..de5d925a1e Binary files /dev/null and b/tests/ut/data/dataset/golden/to_type_01_result.npz differ diff --git a/tests/ut/data/dataset/golden/to_type_02_result.npz b/tests/ut/data/dataset/golden/to_type_02_result.npz new file mode 100644 index 0000000000..a568b3a995 Binary files /dev/null and b/tests/ut/data/dataset/golden/to_type_02_result.npz differ diff --git a/tests/ut/python/dataset/test_linear_transformation.py b/tests/ut/python/dataset/test_linear_transformation.py new file mode 100644 index 0000000000..b79f3718bb --- /dev/null +++ b/tests/ut/python/dataset/test_linear_transformation.py @@ -0,0 +1,225 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +Testing LinearTransformation op in DE +""" +import numpy as np +import mindspore.dataset as ds +import mindspore.dataset.transforms.vision.py_transforms as py_vision +from mindspore import log as logger +from util import diff_mse, visualize, save_and_check_md5 + +GENERATE_GOLDEN = False + +DATA_DIR = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"] +SCHEMA_DIR = "../data/dataset/test_tf_file_3_images/datasetSchema.json" + + +def test_linear_transformation_op(plot=False): + """ + Test LinearTransformation op: verify if images transform correctly + """ + logger.info("test_linear_transformation_01") + + # Initialize parameters + height = 50 + weight = 50 + dim = 3 * height * weight + transformation_matrix = np.eye(dim) + mean_vector = np.zeros(dim) + + # Define operations + transforms = [ + py_vision.Decode(), + py_vision.CenterCrop([height, weight]), + py_vision.ToTensor() + ] + transform = py_vision.ComposeOp(transforms) + + # First dataset + data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + data1 = data1.map(input_columns=["image"], operations=transform()) + # Note: if transformation matrix is diagonal matrix with all 1 in diagonal, + # the output matrix in expected to be the same as the input matrix. + data1 = data1.map(input_columns=["image"], + operations=py_vision.LinearTransformation(transformation_matrix, mean_vector)) + + # Second dataset + data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + data2 = data2.map(input_columns=["image"], operations=transform()) + + image_transformed = [] + image = [] + for item1, item2 in zip(data1.create_dict_iterator(), data2.create_dict_iterator()): + image1 = (item1["image"].transpose(1, 2, 0) * 255).astype(np.uint8) + image2 = (item2["image"].transpose(1, 2, 0) * 255).astype(np.uint8) + image_transformed.append(image1) + image.append(image2) + + mse = diff_mse(image1, image2) + assert mse == 0 + if plot: + visualize(image, image_transformed) + +def test_linear_transformation_md5_01(): + """ + Test LinearTransformation op: valid params (transformation_matrix, mean_vector) + Expected to pass + """ + logger.info("test_linear_transformation_md5_01") + + # Initialize parameters + height = 50 + weight = 50 + dim = 3 * height * weight + transformation_matrix = np.ones([dim, dim]) + mean_vector = np.zeros(dim) + + # Generate dataset + data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + transforms = [ + py_vision.Decode(), + py_vision.CenterCrop([height, weight]), + py_vision.ToTensor(), + py_vision.LinearTransformation(transformation_matrix, mean_vector) + ] + transform = py_vision.ComposeOp(transforms) + data1 = data1.map(input_columns=["image"], operations=transform()) + + # Compare with expected md5 from images + filename = "linear_transformation_01_result.npz" + save_and_check_md5(data1, filename, generate_golden=GENERATE_GOLDEN) + +def test_linear_transformation_md5_02(): + """ + Test LinearTransformation op: transformation_matrix is not provided + Expected to raise ValueError + """ + logger.info("test_linear_transformation_md5_02") + + # Initialize parameters + height = 50 + weight = 50 + dim = 3 * height * weight + mean_vector = np.zeros(dim) + + # Generate dataset + data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + try: + transforms = [ + py_vision.Decode(), + py_vision.CenterCrop([height, weight]), + py_vision.ToTensor(), + py_vision.LinearTransformation(None, mean_vector) + ] + transform = py_vision.ComposeOp(transforms) + data1 = data1.map(input_columns=["image"], operations=transform()) + except ValueError as e: + logger.info("Got an exception in DE: {}".format(str(e))) + assert "not provided" in str(e) + +def test_linear_transformation_md5_03(): + """ + Test LinearTransformation op: mean_vector is not provided + Expected to raise ValueError + """ + logger.info("test_linear_transformation_md5_03") + + # Initialize parameters + height = 50 + weight = 50 + dim = 3 * height * weight + transformation_matrix = np.ones([dim, dim]) + + # Generate dataset + data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + try: + transforms = [ + py_vision.Decode(), + py_vision.CenterCrop([height, weight]), + py_vision.ToTensor(), + py_vision.LinearTransformation(transformation_matrix, None) + ] + transform = py_vision.ComposeOp(transforms) + data1 = data1.map(input_columns=["image"], operations=transform()) + except ValueError as e: + logger.info("Got an exception in DE: {}".format(str(e))) + assert "not provided" in str(e) + +def test_linear_transformation_md5_04(): + """ + Test LinearTransformation op: transformation_matrix is not a square matrix + Expected to raise ValueError + """ + logger.info("test_linear_transformation_md5_04") + + # Initialize parameters + height = 50 + weight = 50 + dim = 3 * height * weight + transformation_matrix = np.ones([dim, dim - 1]) + mean_vector = np.zeros(dim) + + # Generate dataset + data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + try: + transforms = [ + py_vision.Decode(), + py_vision.CenterCrop([height, weight]), + py_vision.ToTensor(), + py_vision.LinearTransformation(transformation_matrix, mean_vector) + ] + transform = py_vision.ComposeOp(transforms) + data1 = data1.map(input_columns=["image"], operations=transform()) + except ValueError as e: + logger.info("Got an exception in DE: {}".format(str(e))) + assert "square matrix" in str(e) + +def test_linear_transformation_md5_05(): + """ + Test LinearTransformation op: mean_vector does not match dimension of transformation_matrix + Expected to raise ValueError + """ + logger.info("test_linear_transformation_md5_05") + + # Initialize parameters + height = 50 + weight = 50 + dim = 3 * height * weight + transformation_matrix = np.ones([dim, dim]) + mean_vector = np.zeros(dim-1) + + # Generate dataset + data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + try: + transforms = [ + py_vision.Decode(), + py_vision.CenterCrop([height, weight]), + py_vision.ToTensor(), + py_vision.LinearTransformation(transformation_matrix, mean_vector) + ] + transform = py_vision.ComposeOp(transforms) + data1 = data1.map(input_columns=["image"], operations=transform()) + except ValueError as e: + logger.info("Got an exception in DE: {}".format(str(e))) + assert "should match" in str(e) + +if __name__ == '__main__': + test_linear_transformation_op(True) + test_linear_transformation_md5_01() + test_linear_transformation_md5_02() + test_linear_transformation_md5_03() + test_linear_transformation_md5_04() + test_linear_transformation_md5_05() diff --git a/tests/ut/python/dataset/test_to_pil.py b/tests/ut/python/dataset/test_to_pil.py new file mode 100644 index 0000000000..2e260e5517 --- /dev/null +++ b/tests/ut/python/dataset/test_to_pil.py @@ -0,0 +1,79 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +Testing ToPIL op in DE +""" +import mindspore.dataset as ds +import mindspore.dataset.transforms.vision.c_transforms as c_vision +import mindspore.dataset.transforms.vision.py_transforms as py_vision +from mindspore import log as logger +from util import save_and_check_md5 + +GENERATE_GOLDEN = False + +DATA_DIR = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"] +SCHEMA_DIR = "../data/dataset/test_tf_file_3_images/datasetSchema.json" + + +def test_to_pil_01(): + """ + Test ToPIL Op with md5 comparison: input is already PIL image + Expected to pass + """ + logger.info("test_to_pil_01") + + # Generate dataset + data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + transforms = [ + py_vision.Decode(), + # If input is already PIL image. + py_vision.ToPIL(), + py_vision.CenterCrop(375), + py_vision.ToTensor() + ] + transform = py_vision.ComposeOp(transforms) + data1 = data1.map(input_columns=["image"], operations=transform()) + + # Compare with expected md5 from images + filename = "to_pil_01_result.npz" + save_and_check_md5(data1, filename, generate_golden=GENERATE_GOLDEN) + +def test_to_pil_02(): + """ + Test ToPIL Op with md5 comparison: input is not PIL image + Expected to pass + """ + logger.info("test_to_pil_02") + + # Generate dataset + data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + decode_op = c_vision.Decode() + transforms = [ + # If input type is not PIL. + py_vision.ToPIL(), + py_vision.CenterCrop(375), + py_vision.ToTensor() + ] + transform = py_vision.ComposeOp(transforms) + data1 = data1.map(input_columns=["image"], operations=decode_op) + data1 = data1.map(input_columns=["image"], operations=transform()) + + # Compare with expected md5 from images + filename = "to_pil_02_result.npz" + save_and_check_md5(data1, filename, generate_golden=GENERATE_GOLDEN) + +if __name__ == "__main__": + test_to_pil_01() + test_to_pil_02() diff --git a/tests/ut/python/dataset/test_to_type.py b/tests/ut/python/dataset/test_to_type.py new file mode 100644 index 0000000000..29d3a8afe8 --- /dev/null +++ b/tests/ut/python/dataset/test_to_type.py @@ -0,0 +1,180 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +Testing ToType op in DE +""" +import numpy as np +import mindspore.dataset as ds +import mindspore.dataset.transforms.vision.py_transforms as py_vision +from mindspore import log as logger +from util import save_and_check_md5 + +GENERATE_GOLDEN = False + +DATA_DIR = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"] +SCHEMA_DIR = "../data/dataset/test_tf_file_3_images/datasetSchema.json" + + +def test_to_type_op(): + """ + Test ToType Op + """ + logger.info("test_to_type_op") + + # First dataset + data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + transforms1 = [ + py_vision.Decode(), + py_vision.ToTensor(), + # Note: Convert the datatype from float32 to int16 + py_vision.ToType(np.int16) + ] + transform1 = py_vision.ComposeOp(transforms1) + data1 = data1.map(input_columns=["image"], operations=transform1()) + + # Second dataset + data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + transforms2 = [ + py_vision.Decode(), + py_vision.ToTensor() + ] + transform2 = py_vision.ComposeOp(transforms2) + data2 = data2.map(input_columns=["image"], operations=transform2()) + + for item1, item2 in zip(data1.create_dict_iterator(), data2.create_dict_iterator()): + image1 = item1["image"] + image2 = item2["image"] + + assert isinstance(image1, np.ndarray) + assert isinstance(image2, np.ndarray) + assert image1.dtype == np.int16 + assert image2.dtype == np.float32 + assert image1.shape == image2.shape + +def test_to_type_01(): + """ + Test ToType Op with md5 comparison: valid input (Numpy dtype) + Expect to pass + """ + logger.info("test_to_type_01") + + # Generate dataset + data = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + transforms = [ + py_vision.Decode(), + py_vision.ToTensor(), + # Note: Convert the datatype from float32 to int32 + py_vision.ToType(np.int32) + ] + transform = py_vision.ComposeOp(transforms) + data = data.map(input_columns=["image"], operations=transform()) + + # Compare with expected md5 from images + filename = "to_type_01_result.npz" + save_and_check_md5(data, filename, generate_golden=GENERATE_GOLDEN) + +def test_to_type_02(): + """ + Test ToType Op with md5 comparison: valid input (str) + Expect to pass + """ + logger.info("test_to_type_02") + # Generate dataset + data = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + transforms = [ + py_vision.Decode(), + py_vision.ToTensor(), + # Note: Convert to type int + py_vision.ToType('int') + ] + transform = py_vision.ComposeOp(transforms) + data = data.map(input_columns=["image"], operations=transform()) + + # Compare with expected md5 from images + filename = "to_type_02_result.npz" + save_and_check_md5(data, filename, generate_golden=GENERATE_GOLDEN) + +def test_to_type_03(): + """ + Test ToType Op: invalid input image type + Expect to raise error + """ + logger.info("test_to_type_03") + + try: + # Generate dataset + data = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + transforms = [ + py_vision.Decode(), + # Note: If the object is not numpy, e.g. PIL image, TypeError will raise + py_vision.ToType(np.int32) + ] + transform = py_vision.ComposeOp(transforms) + data = data.map(input_columns=["image"], operations=transform()) + except BaseException as e: + logger.info("Got an exception in DE: {}".format(str(e))) + assert "Numpy" in str(e) + +def test_to_type_04(): + """ + Test ToType Op: no output_type given + Expect to raise error + """ + logger.info("test_to_type_04") + + try: + # Generate dataset + data = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + transforms = [ + py_vision.Decode(), + py_vision.ToTensor(), + # Note: if output_type is not explicitly given + py_vision.ToType() + ] + transform = py_vision.ComposeOp(transforms) + data = data.map(input_columns=["image"], operations=transform()) + except BaseException as e: + logger.info("Got an exception in DE: {}".format(str(e))) + assert "missing" in str(e) + +def test_to_type_05(): + """ + Test ToType Op: invalid output_type + Expect to raise error + """ + logger.info("test_to_type_05") + + try: + # Generate dataset + data = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + transforms = [ + py_vision.Decode(), + py_vision.ToTensor(), + # Note: if output_type is not explicitly given + py_vision.ToType('invalid') + ] + transform = py_vision.ComposeOp(transforms) + data = data.map(input_columns=["image"], operations=transform()) + except BaseException as e: + logger.info("Got an exception in DE: {}".format(str(e))) + assert "data type" in str(e) + +if __name__ == "__main__": + test_to_type_op() + test_to_type_01() + test_to_type_02() + test_to_type_03() + test_to_type_04() + test_to_type_05()