zzy34407230
/
mindspore2022
Not watched
1
0
Fork 0
Code
Releases 22 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: cdd3e50e98
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'cdd3e50e98'
${ noResults }
mindspore2022/tests/ut/python/dataset/test_fast_text.py

236 lines
12 kB
Raw Blame History 

  1. # Copyright 2021 Huawei Technologies Co., Ltd

  2. #

  3. # Licensed under the Apache License, Version 2.0 (the "License");

  4. # you may not use this file except in compliance with the License.

  5. # You may obtain a copy of the License at

  6. #

  7. # http://www.apache.org/licenses/LICENSE-2.0

  8. #

  9. # Unless required by applicable law or agreed to in writing, software

  10. # distributed under the License is distributed on an "AS IS" BASIS,

  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

  12. # See the License for the specific language governing permissions and

  13. # limitations under the License.

  14. # ==============================================================================

  15. 

  16. import numpy as np

  17. import pytest

  18. 

  19. from mindspore import log

  20. import mindspore.dataset as ds

  21. import mindspore.dataset.text as text

  22. import mindspore.dataset.text.transforms as T

  23. 

  24. DATASET_ROOT_PATH = "../data/dataset/test_fast_text/"

  25. 

  26. 

  27. def test_fast_text_all_build_from_file_params():

  28.     """

  29.     Feature: FastText

  30.     Description: test with all parameters which include `path` and `max_vector` in function BuildFromFile

  31.     Expectation: output is equal to the expected value

  32.     """

  33.     vectors = text.FastText.from_file(DATASET_ROOT_PATH + "fast_text.vec", max_vectors=100)

  34.     to_vectors = text.ToVectors(vectors)

  35.     data = ds.TextFileDataset(DATASET_ROOT_PATH + "words.txt", shuffle=False)

  36.     data = data.map(operations=to_vectors, input_columns=["text"])

  37.     ind = 0

  38.     res = [[0.418, 0.24968, -0.41242, 0.1217, 0.34527, -0.04445718411],

  39.            [0, 0, 0, 0, 0, 0],

  40.            [0.15164, 0.30177, -0.16763, 0.17684, 0.31719, 0.33973],

  41.            [0.70853, 0.57088, -0.4716, 0.18048, 0.54449, 0.72603],

  42.            [0.68047, -0.039263, 0.30186, -0.17792, 0.42962, 0.032246],

  43.            [0.26818, 0.14346, -0.27877, 0.016257, 0.11384, 0.69923],

  44.            [0, 0, 0, 0, 0, 0]]

  45.     for d in data.create_dict_iterator(num_epochs=1, output_numpy=True):

  46.         res_array = np.array(res[ind], dtype=np.float32)

  47.         assert np.array_equal(res_array, d["text"]), ind

  48.         ind += 1

  49. 

  50. 

  51. def test_fast_text_all_build_from_file_params_eager():

  52.     """

  53.     Feature: FastText

  54.     Description: test with all parameters which include `path` and `max_vector` in function BuildFromFile in eager mode

  55.     Expectation: output is equal to the expected value

  56.     """

  57.     vectors = text.FastText.from_file(DATASET_ROOT_PATH + "fast_text.vec", max_vectors=4)

  58.     to_vectors = T.ToVectors(vectors)

  59.     result1 = to_vectors("ok")

  60.     result2 = to_vectors("!")

  61.     result3 = to_vectors("this")

  62.     result4 = to_vectors("is")

  63.     result5 = to_vectors("my")

  64.     result6 = to_vectors("home")

  65.     result7 = to_vectors("none")

  66.     res = [[0.418, 0.24968, -0.41242, 0.1217, 0.34527, -0.04445718411],

  67.            [0.013441, 0.23682, -0.16899, 0.40951, 0.63812, 0.47709],

  68.            [0.15164, 0.30177, -0.16763, 0.17684, 0.31719, 0.33973],

  69.            [0.70853, 0.57088, -0.4716, 0.18048, 0.54449, 0.72603],

  70.            [0, 0, 0, 0, 0, 0],

  71.            [0, 0, 0, 0, 0, 0],

  72.            [0, 0, 0, 0, 0, 0]]

  73.     res_array = np.array(res, dtype=np.float32)

  74. 

  75.     assert np.array_equal(result1, res_array[0])

  76.     assert np.array_equal(result2, res_array[1])

  77.     assert np.array_equal(result3, res_array[2])

  78.     assert np.array_equal(result4, res_array[3])

  79.     assert np.array_equal(result5, res_array[4])

  80.     assert np.array_equal(result6, res_array[5])

  81.     assert np.array_equal(result7, res_array[6])

  82. 

  83. 

  84. def test_fast_text_all_to_vectors_params_eager():

  85.     """

  86.     Feature: FastText

  87.     Description: test with all parameters which include `unk_init` and `lower_case_backup` in function ToVectors

  88.         in eager mode

  89.     Expectation: output is equal to the expected value

  90.     """

  91.     vectors = text.FastText.from_file(DATASET_ROOT_PATH + "fast_text.vec", max_vectors=4)

  92.     my_unk = [-1, -1, -1, -1, -1, -1]

  93.     to_vectors = T.ToVectors(vectors, unk_init=my_unk, lower_case_backup=True)

  94.     result1 = to_vectors("Ok")

  95.     result2 = to_vectors("!")

  96.     result3 = to_vectors("This")

  97.     result4 = to_vectors("is")

  98.     result5 = to_vectors("my")

  99.     result6 = to_vectors("home")

  100.     result7 = to_vectors("none")

  101.     res = [[0.418, 0.24968, -0.41242, 0.1217, 0.34527, -0.04445718411],

  102.            [0.013441, 0.23682, -0.16899, 0.40951, 0.63812, 0.47709],

  103.            [0.15164, 0.30177, -0.16763, 0.17684, 0.31719, 0.33973],

  104.            [0.70853, 0.57088, -0.4716, 0.18048, 0.54449, 0.72603],

  105.            [-1, -1, -1, -1, -1, -1],

  106.            [-1, -1, -1, -1, -1, -1],

  107.            [-1, -1, -1, -1, -1, -1]]

  108.     res_array = np.array(res, dtype=np.float32)

  109. 

  110.     assert np.array_equal(result1, res_array[0])

  111.     assert np.array_equal(result2, res_array[1])

  112.     assert np.array_equal(result3, res_array[2])

  113.     assert np.array_equal(result4, res_array[3])

  114.     assert np.array_equal(result5, res_array[4])

  115.     assert np.array_equal(result6, res_array[5])

  116.     assert np.array_equal(result7, res_array[6])

  117. 

  118. 

  119. def test_fast_text_build_from_file():

  120.     """

  121.     Feature: FastText

  122.     Description: test with only default parameter

  123.     Expectation: output is equal to the expected value

  124.     """

  125.     vectors = text.FastText.from_file(DATASET_ROOT_PATH + "fast_text.vec")

  126.     to_vectors = text.ToVectors(vectors)

  127.     data = ds.TextFileDataset(DATASET_ROOT_PATH + "words.txt", shuffle=False)

  128.     data = data.map(operations=to_vectors, input_columns=["text"])

  129.     ind = 0

  130.     res = [[0.418, 0.24968, -0.41242, 0.1217, 0.34527, -0.04445718411],

  131.            [0, 0, 0, 0, 0, 0],

  132.            [0.15164, 0.30177, -0.16763, 0.17684, 0.31719, 0.33973],

  133.            [0.70853, 0.57088, -0.4716, 0.18048, 0.54449, 0.72603],

  134.            [0.68047, -0.039263, 0.30186, -0.17792, 0.42962, 0.032246],

  135.            [0.26818, 0.14346, -0.27877, 0.016257, 0.11384, 0.69923],

  136.            [0, 0, 0, 0, 0, 0]]

  137.     for d in data.create_dict_iterator(num_epochs=1, output_numpy=True):

  138.         res_array = np.array(res[ind], dtype=np.float32)

  139.         assert np.array_equal(res_array, d["text"]), ind

  140.         ind += 1

  141. 

  142. 

  143. def test_fast_text_build_from_file_eager():

  144.     """

  145.     Feature: FastText

  146.     Description: test with only default parameter in eager mode

  147.     Expectation: output is equal to the expected value

  148.     """

  149.     vectors = text.FastText.from_file(DATASET_ROOT_PATH + "fast_text.vec")

  150.     to_vectors = T.ToVectors(vectors)

  151.     result1 = to_vectors("ok")

  152.     result2 = to_vectors("!")

  153.     result3 = to_vectors("this")

  154.     result4 = to_vectors("is")

  155.     result5 = to_vectors("my")

  156.     result6 = to_vectors("home")

  157.     result7 = to_vectors("none")

  158.     res = [[0.418, 0.24968, -0.41242, 0.1217, 0.34527, -0.04445718411],

  159.            [0.013441, 0.23682, -0.16899, 0.40951, 0.63812, 0.47709],

  160.            [0.15164, 0.30177, -0.16763, 0.17684, 0.31719, 0.33973],

  161.            [0.70853, 0.57088, -0.4716, 0.18048, 0.54449, 0.72603],

  162.            [0.68047, -0.039263, 0.30186, -0.17792, 0.42962, 0.032246],

  163.            [0.26818, 0.14346, -0.27877, 0.016257, 0.11384, 0.69923],

  164.            [0, 0, 0, 0, 0, 0]]

  165.     res_array = np.array(res, dtype=np.float32)

  166. 

  167.     assert np.array_equal(result1, res_array[0])

  168.     assert np.array_equal(result2, res_array[1])

  169.     assert np.array_equal(result3, res_array[2])

  170.     assert np.array_equal(result4, res_array[3])

  171.     assert np.array_equal(result5, res_array[4])

  172.     assert np.array_equal(result6, res_array[5])

  173.     assert np.array_equal(result7, res_array[6])

  174. 

  175. 

  176. def test_fast_text_invalid_input():

  177.     """

  178.     Feature: FastText

  179.     Description: test the validate function with invalid parameters

  180.     Expectation: output is equal to the expected error

  181.     """

  182.     def test_invalid_input(test_name, file_path, error, error_msg, max_vectors=None, unk_init=None,

  183.                            lower_case_backup=False, token="ok"):

  184.         log.info("Test Vectors with wrong input: {0}".format(test_name))

  185.         with pytest.raises(error) as error_info:

  186.             vectors = text.FastText.from_file(file_path, max_vectors=max_vectors)

  187.             to_vectors = T.ToVectors(vectors, unk_init=unk_init, lower_case_backup=lower_case_backup)

  188.             to_vectors(token)

  189.         assert error_msg in str(error_info.value)

  190. 

  191.     test_invalid_input("Not all vectors have the same number of dimensions",

  192.                        DATASET_ROOT_PATH + "fast_text_dim_different.vec", error=RuntimeError,

  193.                        error_msg="all vectors must have the same number of dimensions, " \

  194.                        "but got dim 5 while expecting 6")

  195.     test_invalid_input("the file is empty.", DATASET_ROOT_PATH + "fast_text_empty.vec",

  196.                        error=RuntimeError, error_msg="invalid file, file is empty.")

  197.     test_invalid_input("the count of `unknown_init`'s element is different with word vector.",

  198.                        DATASET_ROOT_PATH + "fast_text.vec",

  199.                        error=RuntimeError,

  200.                        error_msg="unk_init must be the same length as vectors, but got unk_init",

  201.                        unk_init=[-1, -1])

  202.     test_invalid_input("The file not exist", DATASET_ROOT_PATH + "not_exist.vec", RuntimeError,

  203.                        error_msg="FastText: invalid file")

  204.     test_invalid_input("The token is 1-dimensional", DATASET_ROOT_PATH + "fast_text_with_wrong_info.vec",

  205.                        error=RuntimeError, error_msg="token with 1-dimensional vector.")

  206.     test_invalid_input("max_vectors parameter must be greater than 0", DATASET_ROOT_PATH + "fast_text.vec",

  207.                        error=ValueError, error_msg="Input max_vectors is not within the required interval",

  208.                        max_vectors=-1)

  209.     test_invalid_input("invalid max_vectors parameter type as a float", DATASET_ROOT_PATH + "fast_text.vec",

  210.                        error=TypeError, error_msg="Argument max_vectors with value 1.0 is not of type [<class 'int'>],"

  211.                        " but got <class 'float'>.", max_vectors=1.0)

  212.     test_invalid_input("invalid max_vectors parameter type as a string", DATASET_ROOT_PATH + "fast_text.vec",

  213.                        error=TypeError, error_msg="Argument max_vectors with value 1 is not of type [<class 'int'>],"

  214.                        " but got <class 'str'>.", max_vectors="1")

  215.     test_invalid_input("invalid token parameter type as a float", DATASET_ROOT_PATH + "fast_text.vec",

  216.                        error=RuntimeError, error_msg="input tensor type should be string.", token=1.0)

  217.     test_invalid_input("invalid lower_case_backup parameter type as a string", DATASET_ROOT_PATH + "fast_text.vec",

  218.                        error=TypeError, error_msg="Argument lower_case_backup with value True is " \

  219.                        "not of type [<class 'bool'>],"

  220.                        " but got <class 'str'>.", lower_case_backup="True")

  221.     test_invalid_input("invalid lower_case_backup parameter type as a string", DATASET_ROOT_PATH + "fast_text.vec",

  222.                        error=TypeError, error_msg="Argument lower_case_backup with value True is " \

  223.                        "not of type [<class 'bool'>],"

  224.                        " but got <class 'str'>.", lower_case_backup="True")

  225.     test_invalid_input("the suffix of pre-training set must be `*.vec`", DATASET_ROOT_PATH + "fast_text.txt",

  226.                        error=RuntimeError, error_msg="FastText: invalid file, can not find file '*.vec'")

  227. 

  228. 

  229. if __name__ == '__main__':

  230.     test_fast_text_all_build_from_file_params()

  231.     test_fast_text_all_build_from_file_params_eager()

  232.     test_fast_text_all_to_vectors_params_eager()

  233.     test_fast_text_build_from_file()

  234.     test_fast_text_build_from_file_eager()

  235.     test_fast_text_invalid_input()