Huawei_Technology
/
MindSpore-AKG
Not watched
1
0
Fork 0
Code
Releases 1 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 7da00a1f10
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '7da00a1f10'
${ noResults }
MindSpore-AKG/tests/common/test_run/focalloss_ad_run.py

171 lines
8.0 kB
Raw Blame History 

  1. # Copyright 2019 Huawei Technologies Co., Ltd

  2. #

  3. # Licensed under the Apache License, Version 2.0 (the "License");

  4. # you may not use this file except in compliance with the License.

  5. # You may obtain a copy of the License at

  6. #

  7. # http://www.apache.org/licenses/LICENSE-2.0

  8. #

  9. # Unless required by applicable law or agreed to in writing, software

  10. # distributed under the License is distributed on an "AS IS" BASIS,

  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

  12. # See the License for the specific language governing permissions and

  13. # limitations under the License.

  14. 

  15. import numpy as np

  16. import akg

  17. import akg.tvm

  18. from akg.utils import kernel_exec as utils

  19. from tests.common.test_op import focalloss_grad

  20. from tests.common.test_run.focal_loss_run import softmax as softmax_np

  21. from tests.common.test_run.focal_loss_run import logsoftmax as logsoftmax_np

  22. from tests.common.test_op.focalloss_ad import focalloss_ad

  23. from tests.common.tensorio import compare_tensor

  24. from tests.common.gen_random import random_gaussian

  25. 

  26. def softmax(x):

  27.     mv = np.max(x, axis=-1, keepdims=True)

  28.     v = x - mv

  29.     s = np.exp(v) / np.sum(np.exp(v), axis=-1, keepdims=True)

  30.     return s

  31. 

  32. 

  33. def logsoftmax(x):

  34.     mv = np.max(x, axis=-1, keepdims=True)

  35.     v = x - mv

  36.     exp_x = np.exp(v)

  37.     Z = np.sum(exp_x, axis=-1, keepdims=True)

  38.     return v - np.log(Z)

  39. 

  40. 

  41. def benchmark(x, y, gamma):

  42.     y_pred = softmax(x)

  43.     expect = -y * ((1 - y_pred) ** gamma) * logsoftmax(x)

  44.     res = np.sum(expect, axis=-1)

  45.     return res

  46. 

  47. 

  48. def RANGEFILL(shape, offset=0):

  49.     size = np.prod([d for d in shape])

  50.     return np.arange(0 + offset, size + offset, dtype).reshape(shape)

  51. 

  52. 

  53. def focalloss_ad_run(shape, p_dtype, t_dtype, gamma, kernel_name, attrs):

  54.     head = np.random.rand(*shape[:2]).astype(p_dtype)

  55. 

  56.     if 'tuning' in attrs.keys():

  57.         t = attrs.get("tuning", False)

  58.         kernel_name = attrs.get("kernel_name", False)

  59.         mod = utils.op_build_test(focalloss_ad, [head.shape, shape, shape], [p_dtype, p_dtype, t_dtype],

  60.                                   op_attrs=[gamma], kernel_name=kernel_name, attrs=attrs, tuning=t)

  61.         if t:

  62.             expect, output, pred, targ = gen_data(gamma, head, p_dtype, shape, t_dtype)

  63.             return mod, expect, (head, pred, targ, output)

  64.         else:

  65.             return mod

  66.     else:

  67.         mod = utils.op_build_test(focalloss_ad, [head.shape, shape, shape], [p_dtype, p_dtype, t_dtype],

  68.                                   op_attrs=[gamma], kernel_name=kernel_name, attrs=attrs)

  69.         expect, output, pred, targ = gen_data(gamma, head, p_dtype, shape, t_dtype)

  70.         output = utils.mod_launch(mod, [head, pred, targ, output], expect=expect)

  71.         return [head, pred, targ, gamma], output, expect, compare_tensor(output, expect, rtol=5e-03, atol=5e-03,

  72.                                                                          equal_nan=True)

  73. 

  74. 

  75. def gen_data(gamma, head, p_dtype, shape, t_dtype):

  76.     pred = np.random.rand(*shape).astype(p_dtype)

  77.     targ = (np.eye(shape[-1])[np.random.randint(0, shape[-1], size=shape[:-1])]).astype(t_dtype)

  78.     y = benchmark(pred, targ, gamma)

  79.     expect_labels, expect_logits = get_expect(pred, targ, head, gamma)

  80.     expect = expect_logits

  81.     out_shape = expect.shape

  82.     output = np.full(out_shape, 0, p_dtype)

  83.     return expect, output, pred, targ

  84. 

  85. 

  86. def focalloss_ad_run2(shape, dtype, attrs):

  87.     logits_pld = akg.tvm.placeholder(shape, dtype=dtype, name='logits')

  88.     labels_pld = akg.tvm.placeholder(shape, dtype='int32', name='labels')

  89.     d_labels, d_logits, head = focalloss_ad.focalloss_ad(labels_pld, logits_pld)

  90.     print("autodiff d_logits:\n", akg.tvm.PrintTensorRecursively(d_logits))

  91.     print("autodiff d_labels:\n", akg.tvm.PrintTensorRecursively(d_labels))

  92. 

  93.     # build autodiff kernels

  94.     io = [labels_pld, logits_pld, head, d_labels, d_logits]

  95.     s = akg.tvm.create_schedule([e.op for e in io])

  96.     kernel_name = utils.gen_name_kernel("focalloss_ad", dtype, (shape[0], shape[1],))

  97.     with akg.build_config(add_lower_pass=utils.debug_mode(0), dump_pass_ir=True):

  98.         mod = akg.build(s, io, "cce", name=kernel_name, attrs=attrs, polyhedral=True)

  99. 

  100.     labels_np = RANGEFILL((batchsize,))

  101.     logits_np = RANGEFILL((batchsize,), 2)

  102.     head_np = RANGEFILL((batchsize,), 2)

  103.     output = np.full(expect.shape, np.nan, dtype)

  104.     output = utils.mod_launch(mod, (labels_np, logits_np, head_np, output), expect=output)

  105.     expect = output  # hack

  106. 

  107.     return (input_np, head_np), output, expect, compare_tensor(output, expect, atol=0.1)

  108. 

  109. 

  110. def get_expect(prediction, labels, grad, gamma):

  111.     if str(prediction.dtype) == "float16":

  112.         eps = 1e-4

  113.     else:

  114.         eps = 1e-8

  115.     pred = softmax_np(prediction).astype(prediction.dtype)

  116.     log_p = logsoftmax_np(prediction).astype(pred.dtype)

  117.     neg_pred_pow = np.exp(gamma * np.log(1 - pred + eps)).astype(pred.dtype)

  118. 

  119.     d_labels = -neg_pred_pow * log_p * np.expand_dims(grad, -1).astype(pred.dtype)

  120. 

  121.     d_logits1 = (-labels * (-log_p * gamma * neg_pred_pow * pred + neg_pred_pow * (1 - pred))).astype(pred.dtype)

  122. 

  123.     d_logits2 = np.zeros(pred.shape, dtype=pred.dtype)

  124.     for k in range(pred.shape[2]):

  125.         d_logits2[:, :, k] += np.sum(-labels * (gamma * neg_pred_pow / (1 - pred + eps) *

  126.                                                 np.expand_dims(pred[:, :, k], -1) * pred *

  127.                                                 log_p - neg_pred_pow * np.expand_dims(pred[:, :, k], -1)),

  128.                                      axis=2)

  129.     d_logits = np.expand_dims(grad, -1) * (labels * d_logits1 + (1 - labels) * d_logits2)

  130.     return d_labels, d_logits

  131. 

  132. 

  133. def focalloss_grad_run(shape, dtype, label_dtype, gamma, attrs):

  134.     kernel_name = utils.gen_name_kernel("focalloss_grad", dtype, shape)

  135.     if 'tuning' in attrs.keys():

  136.         t = attrs.get("tuning", False)

  137.         kernel_name = attrs.get("kernel_name", False)

  138.         mod = utils.op_build_test(focalloss_grad.focal_loss_bwd,

  139.                                   input_shapes=[shape, shape, shape[:2]],

  140.                                   input_types=[dtype, label_dtype, dtype],

  141.                                   op_attrs=[gamma], kernel_name=kernel_name, attrs=attrs, tuning=t)

  142.         if t:

  143.             d_logits, expect_logits, grad_np, labels_np, logits_np = gen_grad_data(attrs, dtype, gamma, label_dtype,

  144.                                                                                    shape)

  145.             return mod, expect_logits, (logits_np, labels_np, grad_np, d_logits)

  146.         else:

  147.             return mod

  148.     else:

  149.         mod = utils.op_build_test(focalloss_grad.focal_loss_bwd,

  150.                                   input_shapes=[shape, shape, shape[:2]],

  151.                                   input_types=[dtype, label_dtype, dtype],

  152.                                   op_attrs=[gamma], kernel_name=kernel_name, attrs=attrs)

  153.         d_logits, expect_logits, grad_np, labels_np, logits_np = gen_grad_data(attrs, dtype, gamma, label_dtype, shape)

  154.         d_logits = utils.mod_launch(mod, (logits_np, labels_np, grad_np, d_logits), expect=expect_logits)

  155.         result_logits = compare_tensor(d_logits, expect_logits, rtol=2e-2, atol=1e-4)

  156. 

  157.         return (labels_np, logits_np, grad_np), d_logits, expect_logits, result_logits

  158. 

  159. 

  160. def gen_grad_data(attrs, dtype, gamma, label_dtype, shape):

  161.     if not utils.product_is_mini():

  162.         attrs['enable_align_fix'] = True

  163.         attrs['enable_multicore'] = True

  164. 

  165.     logits_np = random_gaussian(shape, miu=50, sigma=80).astype(dtype)

  166.     labels_np = (np.eye(shape[-1])[np.random.randint(0, shape[-1], size=shape[:-1])]).astype(label_dtype)

  167.     grad_np = np.ones(shape[:2]).astype(dtype)

  168.     expect_labels, expect_logits = get_expect(logits_np, labels_np, grad_np, gamma)

  169.     d_logits = np.full(expect_logits.shape, np.nan, dtype)

  170.     return d_logits, expect_logits, grad_np, labels_np, logits_np