zzy34407230
/
mindspore2022
Not watched
1
0
Fork 0
Code
Releases 22 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 9cee0d2143
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '9cee0d2143'
${ noResults }
mindspore2022/tests/ut/python/parallel/test_element_wise_function.py

956 lines
35 kB
Raw Blame History 

  1. # Copyright 2019 Huawei Technologies Co., Ltd

  2. #

  3. # Licensed under the Apache License, Version 2.0 (the "License");

  4. # you may not use this file except in compliance with the License.

  5. # You may obtain a copy of the License at

  6. #

  7. # http://www.apache.org/licenses/LICENSE-2.0

  8. #

  9. # Unless required by applicable law or agreed to in writing, software

  10. # distributed under the License is distributed on an "AS IS" BASIS,

  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

  12. # See the License for the specific language governing permissions and

  13. # limitations under the License.

  14. 

  15. import numpy as np

  16. 

  17. import mindspore as ms

  18. import mindspore.nn as nn

  19. from mindspore import Tensor

  20. from mindspore import context

  21. from mindspore.common.api import _executor

  22. from mindspore.ops import composite as C

  23. from mindspore.ops import operations as P

  24. from tests.ut.python.ops.test_math_ops import VirtualLoss

  25. 

  26. 

  27. grad_all = C.GradOperation(get_all=True)

  28. 

  29. 

  30. class NetWithLoss(nn.Cell):

  31.     def __init__(self, network):

  32.         super(NetWithLoss, self).__init__()

  33.         self.loss = VirtualLoss()

  34.         self.network = network

  35. 

  36.     def construct(self, x, y, b):

  37.         predict = self.network(x, y, b)

  38.         return self.loss(predict)

  39. 

  40. 

  41. class GradWrap(nn.Cell):

  42.     def __init__(self, network):

  43.         super(GradWrap, self).__init__()

  44.         self.network = network

  45. 

  46.     def construct(self, x, y, b):

  47.         return grad_all(self.network)(x, y, b)

  48. 

  49. 

  50. def compile_net(net, x, y, b):

  51.     net.set_auto_parallel()

  52.     _executor.compile(net, x, y, b)

  53. 

  54. 

  55. def test_matmul_pow():

  56.     class Net(nn.Cell):

  57.         def __init__(self, strategy1, strategy2):

  58.             super().__init__()

  59.             self.matmul = P.MatMul().shard(strategy1)

  60.             self.pow = P.Pow().shard(strategy2)

  61.             self.matmul2 = P.MatMul().shard(strategy1)

  62. 

  63.         def construct(self, x, y, b):

  64.             out = self.matmul(x, y)

  65.             out = self.pow(out, 2.0)

  66.             out = self.matmul2(out, b)

  67.             return out

  68. 

  69.     context.set_auto_parallel_context(device_num=8, global_rank=0)

  70.     strategy1 = ((2, 2), (2, 2))

  71.     strategy2 = ((4, 2), ())

  72.     net = GradWrap(NetWithLoss(Net(strategy1, strategy2)))

  73.     context.set_auto_parallel_context(parallel_mode="semi_auto_parallel")

  74. 

  75.     x = Tensor(np.ones([128, 32]), dtype=ms.float32)

  76.     y = Tensor(np.ones([32, 64]), dtype=ms.float32)

  77.     b = Tensor(np.ones([64, 64]), dtype=ms.float32)

  78.     compile_net(net, x, y, b)

  79. 

  80. 

  81. def test_matmul_exp():

  82.     class Net(nn.Cell):

  83.         def __init__(self, strategy1, strategy2):

  84.             super().__init__()

  85.             self.matmul = P.MatMul().shard(strategy1)

  86.             self.exp = P.Exp().shard(strategy2)

  87.             self.matmul2 = P.MatMul().shard(strategy1)

  88. 

  89.         def construct(self, x, y, b):

  90.             out = self.matmul(x, y)

  91.             out = self.exp(out)

  92.             out = self.matmul2(out, b)

  93.             return out

  94. 

  95.     context.set_auto_parallel_context(device_num=8, global_rank=0)

  96.     strategy1 = ((2, 2), (2, 2))

  97.     strategy2 = ((4, 2),)

  98.     net = GradWrap(NetWithLoss(Net(strategy1, strategy2)))

  99.     context.set_auto_parallel_context(parallel_mode="semi_auto_parallel")

  100. 

  101.     x = Tensor(np.ones([128, 32]), dtype=ms.float32)

  102.     y = Tensor(np.ones([32, 64]), dtype=ms.float32)

  103.     b = Tensor(np.ones([64, 64]), dtype=ms.float32)

  104.     compile_net(net, x, y, b)

  105. 

  106. 

  107. def test_matmul_log():

  108.     class Net(nn.Cell):

  109.         def __init__(self, strategy1, strategy2):

  110.             super().__init__()

  111.             self.matmul = P.MatMul().shard(strategy1)

  112.             self.log = P.Log().shard(strategy2)

  113.             self.matmul2 = P.MatMul().shard(strategy1)

  114. 

  115.         def construct(self, x, y, b):

  116.             out = self.matmul(x, y)

  117.             out = self.log(out)

  118.             out = self.matmul2(out, b)

  119.             return out

  120. 

  121.     context.set_auto_parallel_context(device_num=8, global_rank=0)

  122.     strategy1 = ((2, 2), (2, 2))

  123.     strategy2 = ((4, 2),)

  124.     net = GradWrap(NetWithLoss(Net(strategy1, strategy2)))

  125.     context.set_auto_parallel_context(parallel_mode="semi_auto_parallel")

  126. 

  127.     x = Tensor(np.ones([128, 32]), dtype=ms.float32)

  128.     y = Tensor(np.ones([32, 64]), dtype=ms.float32)

  129.     b = Tensor(np.ones([64, 64]), dtype=ms.float32)

  130.     compile_net(net, x, y, b)

  131. 

  132. def test_matmul_abs():

  133.     class Net(nn.Cell):

  134.         def __init__(self, strategy1, strategy2):

  135.             super().__init__()

  136.             self.matmul = P.MatMul().shard(strategy1)

  137.             self.abs = P.Abs().shard(strategy2)

  138.             self.matmul2 = P.MatMul().shard(strategy1)

  139. 

  140.         def construct(self, x, y, b):

  141.             out = self.matmul(x, y)

  142.             out = self.abs(out)

  143.             out = self.matmul2(out, b)

  144.             return out

  145. 

  146.     context.set_auto_parallel_context(device_num=8, global_rank=0)

  147.     strategy1 = ((2, 2), (2, 2))

  148.     strategy2 = ((4, 2),)

  149.     net = GradWrap(NetWithLoss(Net(strategy1, strategy2)))

  150.     context.set_auto_parallel_context(parallel_mode="semi_auto_parallel")

  151. 

  152.     x = Tensor(np.random.uniform(-5, 5, size=(128, 32)), dtype=ms.float32)

  153.     y = Tensor(np.random.uniform(-5, 5, size=(32, 64)), dtype=ms.float32)

  154.     b = Tensor(np.random.uniform(-5, 5, size=(64, 64)), dtype=ms.float32)

  155.     compile_net(net, x, y, b)

  156. 

  157. def test_matmul_sign():

  158.     class Net(nn.Cell):

  159.         def __init__(self, strategy1, strategy2):

  160.             super().__init__()

  161.             self.matmul = P.MatMul().shard(strategy1)

  162.             self.sign = P.Sign().shard(strategy2)

  163.             self.matmul2 = P.MatMul().shard(strategy1)

  164. 

  165.         def construct(self, x, y, b):

  166.             out = self.matmul(x, y)

  167.             out = self.sign(out)

  168.             out = self.matmul2(out, b)

  169.             return out

  170. 

  171.     context.set_auto_parallel_context(device_num=8, global_rank=0)

  172.     strategy1 = ((2, 2), (2, 2))

  173.     strategy2 = ((4, 2),)

  174.     net = GradWrap(NetWithLoss(Net(strategy1, strategy2)))

  175.     context.set_auto_parallel_context(parallel_mode="semi_auto_parallel")

  176. 

  177.     x = Tensor(np.random.uniform(-5, 5, size=(128, 32)), dtype=ms.float32)

  178.     y = Tensor(np.random.uniform(-5, 5, size=(32, 64)), dtype=ms.float32)

  179.     b = Tensor(np.random.uniform(-5, 5, size=(64, 64)), dtype=ms.float32)

  180.     compile_net(net, x, y, b)

  181. 

  182. 

  183. def test_matmul_floor():

  184.     class Net(nn.Cell):

  185.         def __init__(self, strategy1, strategy2):

  186.             super().__init__()

  187.             self.matmul = P.MatMul().shard(strategy1)

  188.             self.floor = P.Floor().shard(strategy2)

  189.             self.matmul2 = P.MatMul().shard(strategy1)

  190. 

  191.         def construct(self, x, y, b):

  192.             out = self.matmul(x, y)

  193.             out = self.floor(out)

  194.             out = self.matmul2(out, b)

  195.             return out

  196. 

  197.     context.set_auto_parallel_context(device_num=8, global_rank=0)

  198.     strategy1 = ((2, 2), (2, 2))

  199.     strategy2 = ((4, 2),)

  200.     net = GradWrap(NetWithLoss(Net(strategy1, strategy2)))

  201.     context.set_auto_parallel_context(parallel_mode="semi_auto_parallel")

  202. 

  203.     x = Tensor(np.random.uniform(-5, 5, size=(128, 32)), dtype=ms.float32)

  204.     y = Tensor(np.random.uniform(-5, 5, size=(32, 64)), dtype=ms.float32)

  205.     b = Tensor(np.random.uniform(-5, 5, size=(64, 64)), dtype=ms.float32)

  206.     compile_net(net, x, y, b)

  207. 

  208. def test_matmul_round():

  209.     class Net(nn.Cell):

  210.         def __init__(self, strategy1, strategy2):

  211.             super().__init__()

  212.             self.matmul = P.MatMul().shard(strategy1)

  213.             self.round = P.Round().shard(strategy2)

  214.             self.matmul2 = P.MatMul().shard(strategy1)

  215. 

  216.         def construct(self, x, y, b):

  217.             out = self.matmul(x, y)

  218.             out = self.round(out)

  219.             out = self.matmul2(out, b)

  220.             return out

  221. 

  222.     context.set_auto_parallel_context(device_num=8, global_rank=0)

  223.     strategy1 = ((2, 2), (2, 2))

  224.     strategy2 = ((4, 2),)

  225.     net = GradWrap(NetWithLoss(Net(strategy1, strategy2)))

  226.     context.set_auto_parallel_context(parallel_mode="semi_auto_parallel")

  227. 

  228.     x = Tensor(np.random.uniform(-5, 5, size=(128, 32)), dtype=ms.float32)

  229.     y = Tensor(np.random.uniform(-5, 5, size=(32, 64)), dtype=ms.float32)

  230.     b = Tensor(np.random.uniform(-5, 5, size=(64, 64)), dtype=ms.float32)

  231.     compile_net(net, x, y, b)

  232. 

  233. 

  234. def test_matmul_reciprocal():

  235.     class Net(nn.Cell):

  236.         def __init__(self, strategy1, strategy2):

  237.             super().__init__()

  238.             self.matmul = P.MatMul().shard(strategy1)

  239.             self.reciprocal = P.Reciprocal().shard(strategy2)

  240.             self.matmul2 = P.MatMul().shard(strategy1)

  241. 

  242.         def construct(self, x, y, b):

  243.             out = self.matmul(x, y)

  244.             out = self.reciprocal(out)

  245.             out = self.matmul2(out, b)

  246.             return out

  247. 

  248.     context.set_auto_parallel_context(device_num=8, global_rank=0)

  249.     strategy1 = ((2, 2), (2, 2))

  250.     strategy2 = ((4, 2),)

  251.     net = GradWrap(NetWithLoss(Net(strategy1, strategy2)))

  252.     context.set_auto_parallel_context(parallel_mode="semi_auto_parallel")

  253. 

  254.     x = Tensor(np.random.uniform(-5, 5, size=(128, 32)), dtype=ms.float32)

  255.     y = Tensor(np.random.uniform(-5, 5, size=(32, 64)), dtype=ms.float32)

  256.     b = Tensor(np.random.uniform(-5, 5, size=(64, 64)), dtype=ms.float32)

  257.     compile_net(net, x, y, b)

  258. 

  259. 

  260. def test_matmul_inv():

  261.     class Net(nn.Cell):

  262.         def __init__(self, strategy1, strategy2):

  263.             super().__init__()

  264.             self.matmul = P.MatMul().shard(strategy1)

  265.             self.inv = P.Inv().shard(strategy2)

  266.             self.matmul2 = P.MatMul().shard(strategy1)

  267. 

  268.         def construct(self, x, y, b):

  269.             out = self.matmul(x, y)

  270.             out = self.inv(out)

  271.             out = self.matmul2(out, b)

  272.             return out

  273. 

  274.     context.set_auto_parallel_context(device_num=8, global_rank=0)

  275.     strategy1 = ((2, 2), (2, 2))

  276.     strategy2 = ((4, 2),)

  277.     net = GradWrap(NetWithLoss(Net(strategy1, strategy2)))

  278.     context.set_auto_parallel_context(parallel_mode="semi_auto_parallel")

  279. 

  280.     x = Tensor(np.random.uniform(-5, 5, size=(128, 32)), dtype=ms.float32)

  281.     y = Tensor(np.random.uniform(-5, 5, size=(32, 64)), dtype=ms.float32)

  282.     b = Tensor(np.random.uniform(-5, 5, size=(64, 64)), dtype=ms.float32)

  283.     compile_net(net, x, y, b)

  284. 

  285. 

  286. def test_matmul_rsqrt():

  287.     class Net(nn.Cell):

  288.         def __init__(self, strategy1, strategy2):

  289.             super().__init__()

  290.             self.matmul = P.MatMul().shard(strategy1)

  291.             self.rsqrt = P.Rsqrt().shard(strategy2)

  292.             self.matmul2 = P.MatMul().shard(strategy1)

  293. 

  294.         def construct(self, x, y, b):

  295.             out = self.matmul(x, y)

  296.             out = self.rsqrt(out)

  297.             out = self.matmul2(out, b)

  298.             return out

  299. 

  300.     context.set_auto_parallel_context(device_num=8, global_rank=0)

  301.     strategy1 = ((2, 2), (2, 2))

  302.     strategy2 = ((4, 2),)

  303.     net = GradWrap(NetWithLoss(Net(strategy1, strategy2)))

  304.     context.set_auto_parallel_context(parallel_mode="semi_auto_parallel")

  305. 

  306.     x = Tensor(np.random.uniform(-5, 5, size=(128, 32)), dtype=ms.float32)

  307.     y = Tensor(np.random.uniform(-5, 5, size=(32, 64)), dtype=ms.float32)

  308.     b = Tensor(np.random.uniform(-5, 5, size=(64, 64)), dtype=ms.float32)

  309.     compile_net(net, x, y, b)

  310. 

  311. 

  312. def test_matmul_tan():

  313.     class Net(nn.Cell):

  314.         def __init__(self, strategy1, strategy2):

  315.             super().__init__()

  316.             self.matmul = P.MatMul().shard(strategy1)

  317.             self.tan = P.Tan().shard(strategy2)

  318.             self.matmul2 = P.MatMul().shard(strategy1)

  319. 

  320.         def construct(self, x, y, b):

  321.             out = self.matmul(x, y)

  322.             out = self.tan(out)

  323.             out = self.matmul2(out, b)

  324.             return out

  325. 

  326.     context.set_auto_parallel_context(device_num=8, global_rank=0)

  327.     strategy1 = ((2, 2), (2, 2))

  328.     strategy2 = ((4, 2),)

  329.     net = GradWrap(NetWithLoss(Net(strategy1, strategy2)))

  330.     context.set_auto_parallel_context(parallel_mode="semi_auto_parallel")

  331. 

  332.     x = Tensor(np.random.uniform(-5, 5, size=(128, 32)), dtype=ms.float32)

  333.     y = Tensor(np.random.uniform(-5, 5, size=(32, 64)), dtype=ms.float32)

  334.     b = Tensor(np.random.uniform(-5, 5, size=(64, 64)), dtype=ms.float32)

  335.     compile_net(net, x, y, b)

  336. 

  337. 

  338. def test_matmul_sin():

  339.     class Net(nn.Cell):

  340.         def __init__(self, strategy1, strategy2):

  341.             super().__init__()

  342.             self.matmul = P.MatMul().shard(strategy1)

  343.             self.sin = P.Sin().shard(strategy2)

  344.             self.matmul2 = P.MatMul().shard(strategy1)

  345. 

  346.         def construct(self, x, y, b):

  347.             out = self.matmul(x, y)

  348.             out = self.sin(out)

  349.             out = self.matmul2(out, b)

  350.             return out

  351. 

  352.     context.set_auto_parallel_context(device_num=8, global_rank=0)

  353.     strategy1 = ((2, 2), (2, 2))

  354.     strategy2 = ((4, 2),)

  355.     net = GradWrap(NetWithLoss(Net(strategy1, strategy2)))

  356.     context.set_auto_parallel_context(parallel_mode="semi_auto_parallel")

  357. 

  358.     x = Tensor(np.random.uniform(-5, 5, size=(128, 32)), dtype=ms.float32)

  359.     y = Tensor(np.random.uniform(-5, 5, size=(32, 64)), dtype=ms.float32)

  360.     b = Tensor(np.random.uniform(-5, 5, size=(64, 64)), dtype=ms.float32)

  361.     compile_net(net, x, y, b)

  362. 

  363. 

  364. def test_matmul_sinh():

  365.     class Net(nn.Cell):

  366.         def __init__(self, strategy1, strategy2):

  367.             super().__init__()

  368.             self.matmul = P.MatMul().shard(strategy1)

  369.             self.sinh = P.Sinh().shard(strategy2)

  370.             self.matmul2 = P.MatMul().shard(strategy1)

  371. 

  372.         def construct(self, x, y, b):

  373.             out = self.matmul(x, y)

  374.             out = self.sinh(out)

  375.             out = self.matmul2(out, b)

  376.             return out

  377. 

  378.     context.set_auto_parallel_context(device_num=8, global_rank=0)

  379.     strategy1 = ((2, 2), (2, 2))

  380.     strategy2 = ((4, 2),)

  381.     net = GradWrap(NetWithLoss(Net(strategy1, strategy2)))

  382.     context.set_auto_parallel_context(parallel_mode="semi_auto_parallel")

  383. 

  384.     x = Tensor(np.random.uniform(-5, 5, size=(128, 32)), dtype=ms.float32)

  385.     y = Tensor(np.random.uniform(-5, 5, size=(32, 64)), dtype=ms.float32)

  386.     b = Tensor(np.random.uniform(-5, 5, size=(64, 64)), dtype=ms.float32)

  387.     compile_net(net, x, y, b)

  388. 

  389. 

  390. def test_matmul_log1p():

  391.     class Net(nn.Cell):

  392.         def __init__(self, strategy1, strategy2):

  393.             super().__init__()

  394.             self.matmul = P.MatMul().shard(strategy1)

  395.             self.log1p = P.Log1p().shard(strategy2)

  396.             self.matmul2 = P.MatMul().shard(strategy1)

  397. 

  398.         def construct(self, x, y, b):

  399.             out = self.matmul(x, y)

  400.             out = self.log1p(out)

  401.             out = self.matmul2(out, b)

  402.             return out

  403. 

  404.     context.set_auto_parallel_context(device_num=8, global_rank=0)

  405.     strategy1 = ((2, 2), (2, 2))

  406.     strategy2 = ((4, 2),)

  407.     net = GradWrap(NetWithLoss(Net(strategy1, strategy2)))

  408.     context.set_auto_parallel_context(parallel_mode="semi_auto_parallel")

  409. 

  410.     x = Tensor(np.random.uniform(-5, 5, size=(128, 32)), dtype=ms.float32)

  411.     y = Tensor(np.random.uniform(-5, 5, size=(32, 64)), dtype=ms.float32)

  412.     b = Tensor(np.random.uniform(-5, 5, size=(64, 64)), dtype=ms.float32)

  413.     compile_net(net, x, y, b)

  414. 

  415. 

  416. def test_matmul_expm1():

  417.     class Net(nn.Cell):

  418.         def __init__(self, strategy1, strategy2):

  419.             super().__init__()

  420.             self.matmul = P.MatMul().shard(strategy1)

  421.             self.expm1 = P.Expm1().shard(strategy2)

  422.             self.matmul2 = P.MatMul().shard(strategy1)

  423. 

  424.         def construct(self, x, y, b):

  425.             out = self.matmul(x, y)

  426.             out = self.expm1(out)

  427.             out = self.matmul2(out, b)

  428.             return out

  429. 

  430.     context.set_auto_parallel_context(device_num=8, global_rank=0)

  431.     strategy1 = ((2, 2), (2, 2))

  432.     strategy2 = ((4, 2),)

  433.     net = GradWrap(NetWithLoss(Net(strategy1, strategy2)))

  434.     context.set_auto_parallel_context(parallel_mode="semi_auto_parallel")

  435. 

  436.     x = Tensor(np.random.uniform(-5, 5, size=(128, 32)), dtype=ms.float32)

  437.     y = Tensor(np.random.uniform(-5, 5, size=(32, 64)), dtype=ms.float32)

  438.     b = Tensor(np.random.uniform(-5, 5, size=(64, 64)), dtype=ms.float32)

  439.     compile_net(net, x, y, b)

  440. 

  441. 

  442. def test_matmul_cosh():

  443.     class Net(nn.Cell):

  444.         def __init__(self, strategy1, strategy2):

  445.             super().__init__()

  446.             self.matmul = P.MatMul().shard(strategy1)

  447.             self.cosh = P.Cosh().shard(strategy2)

  448.             self.matmul2 = P.MatMul().shard(strategy1)

  449. 

  450.         def construct(self, x, y, b):

  451.             out = self.matmul(x, y)

  452.             out = self.cosh(out)

  453.             out = self.matmul2(out, b)

  454.             return out

  455. 

  456.     context.set_auto_parallel_context(device_num=8, global_rank=0)

  457.     strategy1 = ((2, 2), (2, 2))

  458.     strategy2 = ((4, 2),)

  459.     net = GradWrap(NetWithLoss(Net(strategy1, strategy2)))

  460.     context.set_auto_parallel_context(parallel_mode="semi_auto_parallel")

  461. 

  462.     x = Tensor(np.random.uniform(-5, 5, size=(128, 32)), dtype=ms.float32)

  463.     y = Tensor(np.random.uniform(-5, 5, size=(32, 64)), dtype=ms.float32)

  464.     b = Tensor(np.random.uniform(-5, 5, size=(64, 64)), dtype=ms.float32)

  465.     compile_net(net, x, y, b)

  466. 

  467. def test_matmul_erf():

  468.     class Net(nn.Cell):

  469.         def __init__(self, strategy1, strategy2):

  470.             super().__init__()

  471.             self.matmul = P.MatMul().shard(strategy1)

  472.             self.erf = P.Erf().shard(strategy2)

  473.             self.matmul2 = P.MatMul().shard(strategy1)

  474. 

  475.         def construct(self, x, y, b):

  476.             out = self.matmul(x, y)

  477.             out = self.erf(out)

  478.             out = self.matmul2(out, b)

  479.             return out

  480. 

  481.     context.set_auto_parallel_context(device_num=8, global_rank=0)

  482.     strategy1 = ((2, 2), (2, 2))

  483.     strategy2 = ((4, 2),)

  484.     net = GradWrap(NetWithLoss(Net(strategy1, strategy2)))

  485.     context.set_auto_parallel_context(parallel_mode="semi_auto_parallel")

  486. 

  487.     x = Tensor(np.random.uniform(1, 5, size=(128, 32)), dtype=ms.float32)

  488.     y = Tensor(np.random.uniform(1, 5, size=(32, 64)), dtype=ms.float32)

  489.     b = Tensor(np.random.uniform(1, 5, size=(64, 64)), dtype=ms.float32)

  490.     compile_net(net, x, y, b)

  491. 

  492. 

  493. def test_matmul_erfc():

  494.     class Net(nn.Cell):

  495.         def __init__(self, strategy1, strategy2):

  496.             super().__init__()

  497.             self.matmul = P.MatMul().shard(strategy1)

  498.             self.erfc = P.Erfc().shard(strategy2)

  499.             self.matmul2 = P.MatMul().shard(strategy1)

  500. 

  501.         def construct(self, x, y, b):

  502.             out = self.matmul(x, y)

  503.             out = self.erfc(out)

  504.             out = self.matmul2(out, b)

  505.             return out

  506. 

  507.     context.set_auto_parallel_context(device_num=8, global_rank=0)

  508.     strategy1 = ((2, 2), (2, 2))

  509.     strategy2 = ((4, 2),)

  510.     net = GradWrap(NetWithLoss(Net(strategy1, strategy2)))

  511.     context.set_auto_parallel_context(parallel_mode="semi_auto_parallel")

  512. 

  513.     x = Tensor(np.random.uniform(1, 5, size=(128, 32)), dtype=ms.float32)

  514.     y = Tensor(np.random.uniform(1, 5, size=(32, 64)), dtype=ms.float32)

  515.     b = Tensor(np.random.uniform(1, 5, size=(64, 64)), dtype=ms.float32)

  516.     compile_net(net, x, y, b)

  517. 

  518. 

  519. def test_matmul_zeroslike():

  520.     class Net(nn.Cell):

  521.         def __init__(self, strategy1, strategy2):

  522.             super().__init__()

  523.             self.matmul = P.MatMul().shard(strategy1)

  524.             self.zeroslike = P.ZerosLike().shard(strategy2)

  525.             self.matmul2 = P.MatMul().shard(strategy1)

  526. 

  527.         def construct(self, x, y, b):

  528.             out = self.matmul(x, y)

  529.             out = self.zeroslike(out)

  530.             out = self.matmul2(out, b)

  531.             return out

  532. 

  533.     context.set_auto_parallel_context(device_num=8, global_rank=0)

  534.     strategy1 = ((2, 2), (2, 2))

  535.     strategy2 = ((4, 2),)

  536.     net = GradWrap(NetWithLoss(Net(strategy1, strategy2)))

  537.     context.set_auto_parallel_context(parallel_mode="semi_auto_parallel")

  538. 

  539.     x = Tensor(np.random.uniform(1, 5, size=(128, 32)), dtype=ms.float32)

  540.     y = Tensor(np.random.uniform(1, 5, size=(32, 64)), dtype=ms.float32)

  541.     b = Tensor(np.random.uniform(1, 5, size=(64, 64)), dtype=ms.float32)

  542.     compile_net(net, x, y, b)

  543. 

  544. 

  545. def test_matmul_oneslike():

  546.     class Net(nn.Cell):

  547.         def __init__(self, strategy1, strategy2):

  548.             super().__init__()

  549.             self.matmul = P.MatMul().shard(strategy1)

  550.             self.oneslike = P.OnesLike().shard(strategy2)

  551.             self.matmul2 = P.MatMul().shard(strategy1)

  552. 

  553.         def construct(self, x, y, b):

  554.             out = self.matmul(x, y)

  555.             out = self.oneslike(out)

  556.             out = self.matmul2(out, b)

  557.             return out

  558. 

  559.     context.set_auto_parallel_context(device_num=8, global_rank=0)

  560.     strategy1 = ((2, 2), (2, 2))

  561.     strategy2 = ((4, 2),)

  562.     net = GradWrap(NetWithLoss(Net(strategy1, strategy2)))

  563.     context.set_auto_parallel_context(parallel_mode="semi_auto_parallel")

  564. 

  565.     x = Tensor(np.random.uniform(1, 5, size=(128, 32)), dtype=ms.float32)

  566.     y = Tensor(np.random.uniform(1, 5, size=(32, 64)), dtype=ms.float32)

  567.     b = Tensor(np.random.uniform(1, 5, size=(64, 64)), dtype=ms.float32)

  568.     compile_net(net, x, y, b)

  569. 

  570. 

  571. def test_matmul_BesselI0e():

  572.     class Net(nn.Cell):

  573.         def __init__(self, strategy1, strategy2):

  574.             super().__init__()

  575.             self.matmul = P.MatMul().shard(strategy1)

  576.             self.BesselI0e = P.BesselI0e().shard(strategy2)

  577.             self.matmul2 = P.MatMul().shard(strategy1)

  578. 

  579.         def construct(self, x, y, b):

  580.             out = self.matmul(x, y)

  581.             out = self.BesselI0e(out)

  582.             out = self.matmul2(out, b)

  583.             return out

  584. 

  585.     context.set_auto_parallel_context(device_num=8, global_rank=0)

  586.     strategy1 = ((2, 2), (2, 2))

  587.     strategy2 = ((4, 2),)

  588.     net = GradWrap(NetWithLoss(Net(strategy1, strategy2)))

  589.     context.set_auto_parallel_context(parallel_mode="semi_auto_parallel")

  590. 

  591.     x = Tensor(np.random.uniform(1, 5, size=(128, 32)), dtype=ms.float32)

  592.     y = Tensor(np.random.uniform(1, 5, size=(32, 64)), dtype=ms.float32)

  593.     b = Tensor(np.random.uniform(1, 5, size=(64, 64)), dtype=ms.float32)

  594.     compile_net(net, x, y, b)

  595. 

  596. 

  597. def test_matmul_BesselI1e():

  598.     class Net(nn.Cell):

  599.         def __init__(self, strategy1, strategy2):

  600.             super().__init__()

  601.             self.matmul = P.MatMul().shard(strategy1)

  602.             self.BesselI1e = P.BesselI1e().shard(strategy2)

  603.             self.matmul2 = P.MatMul().shard(strategy1)

  604. 

  605.         def construct(self, x, y, b):

  606.             out = self.matmul(x, y)

  607.             out = self.BesselI1e(out)

  608.             out = self.matmul2(out, b)

  609.             return out

  610. 

  611.     context.set_auto_parallel_context(device_num=8, global_rank=0)

  612.     strategy1 = ((2, 2), (2, 2))

  613.     strategy2 = ((4, 2),)

  614.     net = GradWrap(NetWithLoss(Net(strategy1, strategy2)))

  615.     context.set_auto_parallel_context(parallel_mode="semi_auto_parallel")

  616. 

  617.     x = Tensor(np.random.uniform(1, 5, size=(128, 32)), dtype=ms.float32)

  618.     y = Tensor(np.random.uniform(1, 5, size=(32, 64)), dtype=ms.float32)

  619.     b = Tensor(np.random.uniform(1, 5, size=(64, 64)), dtype=ms.float32)

  620.     compile_net(net, x, y, b)

  621. 

  622. 

  623. def test_matmul_ceil():

  624.     class Net(nn.Cell):

  625.         def __init__(self, strategy1, strategy2):

  626.             super().__init__()

  627.             self.matmul = P.MatMul().shard(strategy1)

  628.             self.Ceil = P.Ceil().shard(strategy2)

  629.             self.matmul2 = P.MatMul().shard(strategy1)

  630. 

  631.         def construct(self, x, y, b):

  632.             out = self.matmul(x, y)

  633.             out = self.Ceil(out)

  634.             out = self.matmul2(out, b)

  635.             return out

  636. 

  637.     context.set_auto_parallel_context(device_num=8, global_rank=0)

  638.     strategy1 = ((2, 2), (2, 2))

  639.     strategy2 = ((4, 2),)

  640.     net = GradWrap(NetWithLoss(Net(strategy1, strategy2)))

  641.     context.set_auto_parallel_context(parallel_mode="semi_auto_parallel")

  642. 

  643.     x = Tensor(np.random.uniform(-5, 5, size=(128, 32)), dtype=ms.float32)

  644.     y = Tensor(np.random.uniform(-5, 5, size=(32, 64)), dtype=ms.float32)

  645.     b = Tensor(np.random.uniform(-5, 5, size=(64, 64)), dtype=ms.float32)

  646.     compile_net(net, x, y, b)

  647. 

  648. 

  649. def test_matmul_atan():

  650.     class Net(nn.Cell):

  651.         def __init__(self, strategy1, strategy2):

  652.             super().__init__()

  653.             self.matmul = P.MatMul().shard(strategy1)

  654.             self.atan = P.Atan().shard(strategy2)

  655.             self.matmul2 = P.MatMul().shard(strategy1)

  656. 

  657.         def construct(self, x, y, b):

  658.             out = self.matmul(x, y)

  659.             out = self.atan(out)

  660.             out = self.matmul2(out, b)

  661.             return out

  662. 

  663.     context.set_auto_parallel_context(device_num=8, global_rank=0)

  664.     strategy1 = ((2, 2), (2, 2))

  665.     strategy2 = ((4, 2),)

  666.     net = GradWrap(NetWithLoss(Net(strategy1, strategy2)))

  667.     context.set_auto_parallel_context(parallel_mode="semi_auto_parallel")

  668. 

  669.     x = Tensor(np.random.uniform(-5, 5, size=(128, 32)), dtype=ms.float32)

  670.     y = Tensor(np.random.uniform(-5, 5, size=(32, 64)), dtype=ms.float32)

  671.     b = Tensor(np.random.uniform(-5, 5, size=(64, 64)), dtype=ms.float32)

  672.     compile_net(net, x, y, b)

  673. 

  674. 

  675. def test_matmul_Atanh():

  676.     class Net(nn.Cell):

  677.         def __init__(self, strategy1, strategy2):

  678.             super().__init__()

  679.             self.matmul = P.MatMul().shard(strategy1)

  680.             self.atanh = P.Atanh().shard(strategy2)

  681.             self.matmul2 = P.MatMul().shard(strategy1)

  682. 

  683.         def construct(self, x, y, b):

  684.             out = self.matmul(x, y)

  685.             out = self.atanh(out)

  686.             out = self.matmul2(out, b)

  687.             return out

  688. 

  689.     context.set_auto_parallel_context(device_num=8, global_rank=0)

  690.     strategy1 = ((2, 2), (2, 2))

  691.     strategy2 = ((4, 2),)

  692.     net = GradWrap(NetWithLoss(Net(strategy1, strategy2)))

  693.     context.set_auto_parallel_context(parallel_mode="semi_auto_parallel")

  694. 

  695.     x = Tensor(np.random.uniform(-5, 5, size=(128, 32)), dtype=ms.float32)

  696.     y = Tensor(np.random.uniform(-5, 5, size=(32, 64)), dtype=ms.float32)

  697.     b = Tensor(np.random.uniform(-5, 5, size=(64, 64)), dtype=ms.float32)

  698.     compile_net(net, x, y, b)

  699. 

  700. 

  701. def test_matmul_asin():

  702.     class Net(nn.Cell):

  703.         def __init__(self, strategy1, strategy2):

  704.             super().__init__()

  705.             self.matmul = P.MatMul().shard(strategy1)

  706.             self.asin = P.Asin().shard(strategy2)

  707.             self.matmul2 = P.MatMul().shard(strategy1)

  708. 

  709.         def construct(self, x, y, b):

  710.             out = self.matmul(x, y)

  711.             out = self.asin(out)

  712.             out = self.matmul2(out, b)

  713.             return out

  714. 

  715.     context.set_auto_parallel_context(device_num=8, global_rank=0)

  716.     strategy1 = ((2, 2), (2, 2))

  717.     strategy2 = ((4, 2),)

  718.     net = GradWrap(NetWithLoss(Net(strategy1, strategy2)))

  719.     context.set_auto_parallel_context(parallel_mode="semi_auto_parallel")

  720. 

  721.     x = Tensor(np.random.uniform(-5, 5, size=(128, 32)), dtype=ms.float32)

  722.     y = Tensor(np.random.uniform(-5, 5, size=(32, 64)), dtype=ms.float32)

  723.     b = Tensor(np.random.uniform(-5, 5, size=(64, 64)), dtype=ms.float32)

  724.     compile_net(net, x, y, b)

  725. 

  726. 

  727. def test_matmul_asinh():

  728.     class Net(nn.Cell):

  729.         def __init__(self, strategy1, strategy2):

  730.             super().__init__()

  731.             self.matmul = P.MatMul().shard(strategy1)

  732.             self.asinh = P.Asinh().shard(strategy2)

  733.             self.matmul2 = P.MatMul().shard(strategy1)

  734. 

  735.         def construct(self, x, y, b):

  736.             out = self.matmul(x, y)

  737.             out = self.asinh(out)

  738.             out = self.matmul2(out, b)

  739.             return out

  740. 

  741.     context.set_auto_parallel_context(device_num=8, global_rank=0)

  742.     strategy1 = ((2, 2), (2, 2))

  743.     strategy2 = ((4, 2),)

  744.     net = GradWrap(NetWithLoss(Net(strategy1, strategy2)))

  745.     context.set_auto_parallel_context(parallel_mode="semi_auto_parallel")

  746. 

  747.     x = Tensor(np.random.uniform(-5, 5, size=(128, 32)), dtype=ms.float32)

  748.     y = Tensor(np.random.uniform(-5, 5, size=(32, 64)), dtype=ms.float32)

  749.     b = Tensor(np.random.uniform(-5, 5, size=(64, 64)), dtype=ms.float32)

  750.     compile_net(net, x, y, b)

  751. 

  752. 

  753. def test_matmul_acosh():

  754.     class Net(nn.Cell):

  755.         def __init__(self, strategy1, strategy2):

  756.             super().__init__()

  757.             self.matmul = P.MatMul().shard(strategy1)

  758.             self.acosh = P.Acosh().shard(strategy2)

  759.             self.matmul2 = P.MatMul().shard(strategy1)

  760. 

  761.         def construct(self, x, y, b):

  762.             out = self.matmul(x, y)

  763.             out = self.acosh(out)

  764.             out = self.matmul2(out, b)

  765.             return out

  766. 

  767.     context.set_auto_parallel_context(device_num=8, global_rank=0)

  768.     strategy1 = ((2, 2), (2, 2))

  769.     strategy2 = ((4, 2),)

  770.     net = GradWrap(NetWithLoss(Net(strategy1, strategy2)))

  771.     context.set_auto_parallel_context(parallel_mode="semi_auto_parallel")

  772. 

  773.     x = Tensor(np.random.uniform(-5, 5, size=(128, 32)), dtype=ms.float32)

  774.     y = Tensor(np.random.uniform(-5, 5, size=(32, 64)), dtype=ms.float32)

  775.     b = Tensor(np.random.uniform(-5, 5, size=(64, 64)), dtype=ms.float32)

  776.     compile_net(net, x, y, b)

  777. 

  778. 

  779. def test_matmul_logical_not():

  780.     class Net(nn.Cell):

  781.         def __init__(self, strategy1, strategy2, strategy3):

  782.             super().__init__()

  783.             self.matmul = P.MatMul().shard(strategy1)

  784.             self.logicalnot = P.LogicalNot().shard(strategy2)

  785.             self.equal = P.Equal().shard(strategy3)

  786. 

  787.         def construct(self, x, y, b):

  788.             out = self.matmul(x, y)

  789.             out = self.equal(out, b)

  790.             out = self.logicalnot(out)

  791.             return out

  792. 

  793.     context.set_auto_parallel_context(device_num=8, global_rank=0)

  794.     strategy1 = ((2, 2), (2, 2))

  795.     strategy2 = ((4, 2),)

  796.     strategy3 = ((4, 2), (4, 2))

  797.     net = GradWrap(NetWithLoss(Net(strategy1, strategy2, strategy3)))

  798.     context.set_auto_parallel_context(parallel_mode="semi_auto_parallel")

  799. 

  800.     x = Tensor(np.ones([128, 32]), dtype=ms.float32)

  801.     y = Tensor(np.ones([32, 64]), dtype=ms.float32)

  802.     b = Tensor(np.ones([128, 64]), dtype=ms.float32)

  803.     compile_net(net, x, y, b)

  804. 

  805. 

  806. def test_matmul_cast():

  807.     class Net(nn.Cell):

  808.         def __init__(self, strategy1, strategy2, strategy3):

  809.             super().__init__()

  810.             self.matmul = P.MatMul().shard(strategy1)

  811.             self.cast = P.Cast().shard(strategy2)

  812.             self.matmul2 = P.MatMul().shard(strategy3)

  813. 

  814.         def construct(self, x, y, b):

  815.             out = self.matmul(x, y)

  816.             b = self.cast(b, ms.float32)

  817.             out = self.matmul2(out, b)

  818.             return out

  819. 

  820.     context.set_auto_parallel_context(device_num=8, global_rank=0)

  821.     strategy1 = ((2, 2), (2, 2))

  822.     strategy2 = ((4, 2),)

  823.     strategy3 = ((1, 4), (4, 2))

  824.     net = GradWrap(NetWithLoss(Net(strategy1, strategy2, strategy3)))

  825.     context.set_auto_parallel_context(parallel_mode="semi_auto_parallel")

  826. 

  827.     x = Tensor(np.ones([128, 32]), dtype=ms.float32)

  828.     y = Tensor(np.ones([32, 64]), dtype=ms.float32)

  829.     b = Tensor(np.ones([64, 64]), dtype=ms.int32)

  830.     compile_net(net, x, y, b)

  831. 

  832. 

  833. def test_gradient_fp32_sync():

  834.     class Net(nn.Cell):

  835.         def __init__(self, strategy1):

  836.             super().__init__()

  837.             self.matmul = P.MatMul().shard(strategy1)

  838.             self.cast = P.Cast()

  839. 

  840.         def construct(self, x, y, b):

  841.             out = self.matmul(x, y)

  842.             b = self.cast(b, ms.float32)

  843.             out = self.matmul(out, b)

  844.             return out

  845. 

  846.     context.set_auto_parallel_context(device_num=8, global_rank=0, gradient_fp32_sync=True)

  847.     strategy1 = ((2, 2), (2, 2))

  848.     net = GradWrap(NetWithLoss(Net(strategy1)))

  849.     context.set_auto_parallel_context(parallel_mode="semi_auto_parallel")

  850. 

  851.     x = Tensor(np.ones([128, 32]), dtype=ms.float32)

  852.     y = Tensor(np.ones([32, 64]), dtype=ms.float32)

  853.     b = Tensor(np.ones([64, 64]), dtype=ms.float16)

  854.     compile_net(net, x, y, b)

  855. 

  856. 

  857. def test_gradient_fp32_sync1():

  858.     class Net(nn.Cell):

  859.         def __init__(self, strategy1):

  860.             super().__init__()

  861.             self.matmul = P.MatMul().shard(strategy1)

  862.             self.cast = P.Cast()

  863. 

  864.         def construct(self, x, y, b):

  865.             out = self.matmul(x, y)

  866.             b = self.cast(b, ms.float16)

  867.             out = self.matmul(out, b)

  868.             return out

  869. 

  870.     context.set_auto_parallel_context(device_num=8, global_rank=0, gradient_fp32_sync=True)

  871.     strategy1 = ((2, 2), (2, 2))

  872.     net = GradWrap(NetWithLoss(Net(strategy1)))

  873.     context.set_auto_parallel_context(parallel_mode="semi_auto_parallel")

  874. 

  875.     x = Tensor(np.ones([128, 32]), dtype=ms.float16)

  876.     y = Tensor(np.ones([32, 64]), dtype=ms.float16)

  877.     b = Tensor(np.ones([64, 64]), dtype=ms.float32)

  878.     compile_net(net, x, y, b)

  879. 

  880. 

  881. def test_gradient_fp32_sync2():

  882.     class Net(nn.Cell):

  883.         def __init__(self, strategy1):

  884.             super().__init__()

  885.             self.matmul = P.MatMul().shard(strategy1)

  886.             self.cast = P.Cast()

  887. 

  888.         def construct(self, x, y, b):

  889.             out = self.matmul(x, y)

  890.             b = self.cast(b, ms.float16)

  891.             out = self.matmul(out, b)

  892.             return out

  893. 

  894.     context.set_auto_parallel_context(device_num=8, global_rank=0, gradient_fp32_sync=False)

  895.     strategy1 = ((2, 2), (2, 2))

  896.     net = GradWrap(NetWithLoss(Net(strategy1)))

  897.     context.set_auto_parallel_context(parallel_mode="semi_auto_parallel")

  898. 

  899.     x = Tensor(np.ones([128, 32]), dtype=ms.float16)

  900.     y = Tensor(np.ones([32, 64]), dtype=ms.float16)

  901.     b = Tensor(np.ones([64, 64]), dtype=ms.float32)

  902.     compile_net(net, x, y, b)

  903. 

  904. 

  905. def test_gradient_fp32_sync3():

  906.     class Net(nn.Cell):

  907.         def __init__(self, strategy1):

  908.             super().__init__()

  909.             self.matmul = P.MatMul().shard(strategy1)

  910.             self.cast = P.Cast()

  911. 

  912.         def construct(self, x, y, b):

  913.             out = self.matmul(x, y)

  914.             b = self.cast(b, ms.float16)

  915.             out = self.matmul(out, b)

  916.             return out

  917. 

  918.     context.set_auto_parallel_context(device_num=8, global_rank=0)

  919.     strategy1 = ((2, 2), (2, 2))

  920.     net = GradWrap(NetWithLoss(Net(strategy1)))

  921.     context.set_auto_parallel_context(parallel_mode="semi_auto_parallel")

  922. 

  923.     x = Tensor(np.ones([128, 32]), dtype=ms.float16)

  924.     y = Tensor(np.ones([32, 64]), dtype=ms.float16)

  925.     b = Tensor(np.ones([64, 64]), dtype=ms.float32)

  926.     compile_net(net, x, y, b)

  927. 

  928. 

  929. def test_mul_two_cast():

  930.     class Net(nn.Cell):

  931.         def __init__(self, strategy1, strategy2, strategy3):

  932.             super().__init__()

  933.             self.mul = P.Mul().shard(strategy1)

  934.             self.mul2 = P.Mul().shard(strategy2)

  935.             self.cast = P.Cast().shard(strategy3)

  936.             self.cast2 = P.Cast().shard(strategy3)

  937. 

  938.         def construct(self, x, y, b):

  939.             out = self.mul(x, y)

  940.             out = self.mul2(out, b)

  941.             out = self.cast(out, ms.int32)

  942.             out = self.cast2(out, ms.bool_)

  943.             return out

  944. 

  945.     context.set_auto_parallel_context(device_num=8, global_rank=0)

  946.     strategy1 = ((2, 2), (2, 2))

  947.     strategy2 = ((8, 1), (8, 1))

  948.     strategy3 = ((8, 1),)

  949.     net = GradWrap(Net(strategy1, strategy2, strategy3))

  950.     context.set_auto_parallel_context(parallel_mode="semi_auto_parallel")

  951. 

  952.     x = Tensor(np.ones([128, 32]), dtype=ms.float32)

  953.     y = Tensor(np.ones([128, 32]), dtype=ms.float32)

  954.     b = Tensor(np.ones([128, 32]), dtype=ms.float32)

  955.     compile_net(net, x, y, b)