hummingbird
/
graphengine
Not watched
2
0
Fork 0
Code
Releases 13 Wiki Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 0c33e9d125
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '0c33e9d125'
${ noResults }
graphengine/third_party/fwkacllib/inc/ops/nn_norm_ops.h

492 lines
20 kB
Raw Blame History 

  1. /**

  2.  * Copyright 2019-2020 Huawei Technologies Co., Ltd

  3.  *

  4.  * Licensed under the Apache License, Version 2.0 (the "License");

  5.  * you may not use this file except in compliance with the License.

  6.  * You may obtain a copy of the License at

  7.  *

  8.  * http://www.apache.org/licenses/LICENSE-2.0

  9.  *

  10.  * Unless required by applicable law or agreed to in writing, software

  11.  * distributed under the License is distributed on an "AS IS" BASIS,

  12.  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

  13.  * See the License for the specific language governing permissions and

  14.  * limitations under the License.

  15.  */

  16. 

  17. #ifndef GE_OP_NN_NORM_OPS_H

  18. #define GE_OP_NN_NORM_OPS_H

  19. 

  20. #include "../graph/operator_reg.h"

  21. namespace ge {

  22. 

  23. /**

  24. *@brief Computes the gradient for log softmax activations.

  25. 

  26. *@par Inputs:

  27. *@li grad: A Tensor. Must be one of the following types: float16, float32.

  28. *@li x: A Tensor. Must be one of the following types: float16, float32.

  29. 

  30. *@par Attributes:

  31. * axis: An optional list of ints. Defaults to "{-1}".

  32. 

  33. *@par Outputs:

  34. * y: A Tensor. Has the same type as "grad".

  35. */

  36. 

  37. REG_OP(LogSoftmaxGrad)

  38.     .INPUT(grad, TensorType({DT_FLOAT16, DT_FLOAT}))

  39.     .INPUT(x, TensorType({DT_FLOAT16, DT_FLOAT}))

  40.     .OUTPUT(y, TensorType({DT_FLOAT16, DT_FLOAT}))

  41.     .ATTR(axis, ListInt, {-1})

  42.     .OP_END_FACTORY_REG(LogSoftmaxGrad)

  43. 

  44. REG_OP(SparseSoftmaxCrossEntropyWithLogitsCCE)

  45.     .INPUT(features, TensorType{DT_FLOAT})

  46.     .INPUT(labels, TensorType{DT_FLOAT})

  47.     .OUTPUT(out, TensorType{DT_FLOAT})

  48.     .OUTPUT(non, TensorType{DT_FLOAT})

  49.     .ATTR(cross_entropy_is_grad, Bool, 0)

  50.     .ATTR(cross_entropy_mode, Int, 1)

  51.     .ATTR(softmax_cross_entropy_lossscale_div_batch, Float, 1.0)

  52.     .OP_END_FACTORY_REG(SparseSoftmaxCrossEntropyWithLogitsCCE)

  53. 

  54. /**

  55. *@brief Computes sparse softmax cross entropy cost and gradients to backpropagate.

  56. 

  57. *@par Inputs:

  58. *Two inputs, including:

  59. * @li features: A Tensor. Must be one of the following types: half, float32, double.

  60. *    A "batch_size * num_classes" matrix.

  61. * @li labels: A Tensor of the same type as "features". batch_size vector with values in [0, num_classes).

  62. 

  63. 

  64. *@par Outputs:

  65. *loss: A Tensor for per example loss (a "batch_size" vector). Has the same type as "features".

  66. *backprop: A Tensor for the backpropagated gradients (a batch_size * num_classes matrix). Has the same type as "features".

  67. */

  68. REG_OP(SparseSoftmaxCrossEntropyWithLogits)

  69.     .INPUT(features, TensorType({DT_FLOAT16,DT_FLOAT}))

  70.     .INPUT(labels, TensorType({DT_INT32, DT_INT64}))

  71.     .OUTPUT(loss, TensorType({DT_FLOAT16,DT_FLOAT}))

  72.     .OUTPUT(backprop, TensorType({DT_FLOAT16,DT_FLOAT}))

  73.     .OP_END_FACTORY_REG(SparseSoftmaxCrossEntropyWithLogits)

  74. 

  75. /**

  76. *@brief Computes softmax cross entropy cost and gradients to backpropagate.

  77. 

  78. *@par Inputs:

  79. *Two inputs, including:

  80. * @li features: A Tensor. Must be one of the following types: half, float32, double.

  81. *    A "batch_size * num_classes" matrix.

  82. * @li labels: A Tensor of the same type as "features". A "batch_size * num_classes" matrix.

  83. 

  84. *@par Outputs:

  85. *loss: A Tensor for per example loss (a "batch_size" vector). Has the same type as "features".

  86. *backprop: A Tensor for the backpropagated gradients (a batch_size * num_classes matrix). Has the same type as "features".

  87. */

  88. REG_OP(SoftmaxCrossEntropyWithLogits)

  89.     .INPUT(features, TensorType({DT_DOUBLE,DT_FLOAT16,DT_FLOAT}))

  90.     .INPUT(labels, TensorType({DT_DOUBLE,DT_FLOAT16,DT_FLOAT}))

  91.     .OUTPUT(loss, TensorType({DT_DOUBLE,DT_FLOAT16,DT_FLOAT}))

  92.     .OUTPUT(backprop, TensorType({DT_DOUBLE,DT_FLOAT16,DT_FLOAT}))

  93.     .OP_END_FACTORY_REG(SoftmaxCrossEntropyWithLogits)

  94. 

  95. REG_OP(SoftmaxGrad)

  96.     .INPUT(softmax, TensorType({DT_FLOAT16,DT_FLOAT,DT_INT32,DT_INT8,DT_UINT8}))

  97.     .INPUT(grad_softmax, TensorType({DT_FLOAT16,DT_FLOAT,DT_INT32,DT_INT8,DT_UINT8}))

  98.     .OUTPUT(grad_x, TensorType({DT_FLOAT16,DT_FLOAT,DT_INT32,DT_INT8,DT_UINT8}))

  99.     .OP_END_FACTORY_REG(SoftmaxGrad)

  100. 

  101. REG_OP(SigmoidCrossEntropyWithLogitsGrad)

  102.     .INPUT(predict, TensorType({DT_FLOAT16, DT_FLOAT}))

  103.     .INPUT(target, TensorType({DT_FLOAT16, DT_FLOAT}))

  104.     .INPUT(dout, TensorType({DT_FLOAT16, DT_FLOAT}))

  105.     .OUTPUT(gradient, TensorType({DT_FLOAT16, DT_FLOAT}))

  106.     .OP_END_FACTORY_REG(SigmoidCrossEntropyWithLogitsGrad)

  107. 

  108. REG_OP(SigmoidCrossEntropyWithLogits)

  109.     .INPUT(predict, TensorType({DT_FLOAT16, DT_FLOAT}))

  110.     .INPUT(target, TensorType({DT_FLOAT16, DT_FLOAT}))

  111.     .OUTPUT(loss, TensorType({DT_FLOAT16, DT_FLOAT}))

  112.     .OP_END_FACTORY_REG(SigmoidCrossEntropyWithLogits)

  113. 

  114. REG_OP(SmoothL1Loss)

  115.     .INPUT(predict, TensorType({DT_FLOAT16, DT_FLOAT}))

  116.     .INPUT(label, TensorType({DT_FLOAT16, DT_FLOAT}))

  117.     .OUTPUT(loss, TensorType({DT_FLOAT16, DT_FLOAT}))

  118.     .ATTR(sigma, Float, 1.0)

  119.     .OP_END_FACTORY_REG(SmoothL1Loss)

  120. 

  121. REG_OP(SmoothL1LossGrad)

  122.     .INPUT(predict, TensorType({DT_FLOAT16, DT_FLOAT}))

  123.     .INPUT(label, TensorType({DT_FLOAT16, DT_FLOAT}))

  124.     .INPUT(dout, TensorType({DT_FLOAT16, DT_FLOAT}))

  125.     .OUTPUT(gradient, TensorType({DT_FLOAT16, DT_FLOAT}))

  126.     .ATTR(sigma, Float, 1.0)

  127.     .OP_END_FACTORY_REG(SmoothL1LossGrad)

  128. 

  129. REG_OP(BinaryCrossEntropy)

  130.     .INPUT(x, TensorType({DT_FLOAT, DT_FLOAT16}))

  131.     .INPUT(y, TensorType({DT_FLOAT, DT_FLOAT16}))

  132.     .OPTIONAL_INPUT(weight, TensorType({DT_FLOAT, DT_FLOAT16}))

  133.     .OUTPUT(output, TensorType({DT_FLOAT, DT_FLOAT16}))

  134.     .ATTR(reduction, String, "mean")

  135.     .OP_END_FACTORY_REG(BinaryCrossEntropy)

  136. 

  137. REG_OP(BinaryCrossEntropyGrad)

  138.     .INPUT(x, TensorType({DT_FLOAT, DT_FLOAT16}))

  139.     .INPUT(y, TensorType({DT_FLOAT, DT_FLOAT16}))

  140.     .INPUT(grad_output, TensorType({DT_FLOAT, DT_FLOAT16}))

  141.     .OPTIONAL_INPUT(weight, TensorType({DT_FLOAT, DT_FLOAT16}))

  142.     .OUTPUT(output, TensorType({DT_FLOAT, DT_FLOAT16}))

  143.     .ATTR(reduction, String, "mean")

  144.     .OP_END_FACTORY_REG(BinaryCrossEntropyGrad)

  145. 

  146. /**

  147. *@brief Applies the Softmax function to an n-dimensional input Tensor rescaling them \n so 

  148. that the elements of the n-dimensional output Tensor lie in the range [0,1] and sum to 1.

  149. 

  150. *@par Inputs:

  151. *One input:

  152. *x: A mutable Tensor. Must be one of the following types: float16,

  153. *float32, double. Should be a Variable Tensor.

  154. 

  155. *@par Attributes:

  156. *axis: A list of ints. The dimension softmax would be performed on.

  157. 

  158. *@par Outputs:

  159. *y: A Tensor. Has the same dimensionality and shape as the "x" with values in the range [0, 1]. Must be one of the following types: float16, float32, int32.

  160. */

  161. REG_OP(Softmax)

  162.     .INPUT(x, TensorType({DT_DOUBLE, DT_FLOAT16, DT_FLOAT}))

  163.     .OUTPUT(y, TensorType({DT_DOUBLE, DT_FLOAT16, DT_FLOAT}))

  164.     .ATTR(axis, ListInt, {-1})

  165.     .OP_END_FACTORY_REG(Softmax)

  166. 

  167. /**

  168. *@brief Computes log softmax activations.

  169. 

  170. *@par Inputs:

  171. *One input:

  172. * logits: A Tensor. Must be one of the following types: double, float16, float32.

  173. 

  174. *@par Attributes:

  175. * axis: An optional list of ints. Defaults to "{-1}".

  176. 

  177. *@par Outputs:

  178. * logsoftmax: A Tensor. Has the same type as "logits".

  179. */

  180. REG_OP(LogSoftmax)

  181.     .INPUT(logits, TensorType({DT_DOUBLE, DT_FLOAT16, DT_FLOAT}))

  182.     .OUTPUT(logsoftmax, TensorType({DT_DOUBLE, DT_FLOAT16, DT_FLOAT}))

  183.     .ATTR(axis, ListInt, {-1})

  184.     .OP_END_FACTORY_REG(LogSoftmax)

  185. 

  186. REG_OP(FusedBatchNormV2)

  187.     .INPUT(x, TensorType{DT_FLOAT})                  /* Input data tensor from the previous operator"" */

  188.     .INPUT(scale, TensorType{DT_FLOAT})              /* If spatial is true, the dimension of bias is (C) If spatial is false, the dimensions of scale are (C x D1 x ... x Dn)*/

  189.     .INPUT(b, TensorType{DT_FLOAT})                  /* If spatial is true, the dimension of bias is (C) If spatial is false, the dimensions of scale are (C x D1 x ... x Dn)*/

  190.     .OPTIONAL_INPUT(mean, TensorType{DT_FLOAT})               /* If spatial is true, the dimension of the running mean (training) or the estimated mean (testing) is (C).If spatial is false, the dimensions of the running mean (training) or the estimated mean (testing) are (C x D1 x ... x Dn)*/

  191.     .OPTIONAL_INPUT(variance, TensorType{DT_FLOAT})           /* If spatial is true, the dimension of the running variance(training) or the estimated variance (testing) is (C). If spatial is false, the dimensions of the running variance(training) or the estimated variance (testing) are (C x D1 x ... x Dn).*/

  192.     .OUTPUT(y, TensorType{DT_FLOAT})                 /* The output tensor of the same shape as X */

  193.     .ATTR(momentum, Float, 0.9)            // Factor used in computing the running mean and variance.

  194.     .ATTR(epsilon, Float, 1e-5f)           // The epsilon value to use to avoid division by zero

  195.     .ATTR(mode, Int, 1)                    // 1 means using "CC_BATCHNORM_SPATIAL"; 0 means using "CC_BATCHNORM_PER_ACTIVATION"; only support 1 now

  196.     .ATTR(use_global_stats, Bool, true)

  197.     .ATTR(alpha, Float, 1)

  198.     .ATTR(beta, Float, 0)

  199.     .OP_END_FACTORY_REG(FusedBatchNormV2)

  200. 

  201. REG_OP(Scale)

  202.     .INPUT(x, TensorType{DT_FLOAT})

  203.     .OPTIONAL_INPUT(w, TensorType{DT_FLOAT})

  204.     .OPTIONAL_INPUT(b, TensorType{DT_FLOAT})

  205.     .OUTPUT(y, TensorType{DT_FLOAT})

  206.     .ATTR(bias_term, Bool, false)

  207.     .ATTR(axis, Int, 1)

  208.     .ATTR(num_axis, Int, 1)

  209.     .ATTR(alpha, Float, 1.0)

  210.     .ATTR(beta, Float, 0.0)

  211.     .OP_END_FACTORY_REG(Scale)

  212. 

  213. REG_OP(SoftmaxGradExt)

  214.   .INPUT(grad, TensorType({DT_FLOAT16,DT_FLOAT}))

  215.   .INPUT(x1, TensorType({DT_FLOAT16,DT_FLOAT}))

  216.   .INPUT(x2, TensorType({DT_FLOAT16,DT_FLOAT}))

  217.   .OUTPUT(y, TensorType({DT_FLOAT16,DT_FLOAT}))

  218.   .ATTR(axis, ListInt, {-1})

  219.   .ATTR(keep_dims, Bool, false)

  220.   .OP_END_FACTORY_REG(SoftmaxGradExt)

  221. 

  222. /**

  223. *@brief Confuse mul, sum and sub.

  224. 

  225. *@par Inputs:

  226. *Two inputs, including:

  227. * @li grad: A Tensor. Must be one of the following types: float16, float32.

  228. * @li x: A Tensor. Must be one of the following types: float16, float32.

  229. 

  230. *@par Outputs:

  231. * y: A Tensor of the same type as "grad".

  232. 

  233. */

  234. REG_OP(ConfusionSoftmaxGrad)

  235.   .INPUT(grad, TensorType({DT_FLOAT16,DT_FLOAT}))

  236.   .INPUT(x, TensorType({DT_FLOAT16,DT_FLOAT}))

  237.   .OUTPUT(y, TensorType({DT_FLOAT16,DT_FLOAT}))

  238.   .OP_END_FACTORY_REG(ConfusionSoftmaxGrad)

  239. 

  240. /**

  241. *@brief Layernorm operator interface implementation

  242. *  calculating: x, gamma, beta

  243. *  mean  = np.mean(x, reduce_axis, keepdims=True)

  244. *  variance = np.mean(np.power((x - mean),2), reduce_axis, keepdims=True)

  245. *  y = gamma*((x - mean) / np.sqrt(variance + 0.001)) + beta

  246. 

  247. *@par Inputs:

  248. *Three inputs, including:

  249. * @li x: A Tensor. Must be one of the following types: float16, float32.

  250. * @li gamma: A Tensor. Must be one of the following types: float16, float32.

  251. * @li beta: A Tensor. Must be one of the following types: float16, float32.

  252. 

  253. *@par Attributes:

  254. * @li begin_norm_axis: A required attribute, the type is int32.

  255. * @li begin_params_axis: A required attribute,the type is int32.

  256. 

  257. *@par Outputs:

  258. *Three outputs, including:

  259. * @li y: A Tensor. Must be one of the following types: float16, float32.

  260. * @li mean: A Tensor. Must be one of the following types: float16, float32.

  261. * @li variance: A Tensor. Must be one of the following types: float16, float32.

  262. */

  263. REG_OP(LayerNorm)

  264.     .INPUT(x, TensorType({DT_FLOAT, DT_FLOAT16}))

  265.     .INPUT(gamma, TensorType({DT_FLOAT, DT_FLOAT16}))

  266.     .INPUT(beta, TensorType({DT_FLOAT, DT_FLOAT16}))

  267.     .OUTPUT(y, TensorType({DT_FLOAT, DT_FLOAT16}))

  268.     .OUTPUT(mean, TensorType({DT_FLOAT, DT_FLOAT16}))

  269.     .OUTPUT(variance, TensorType({DT_FLOAT, DT_FLOAT16}))

  270.     .ATTR(begin_norm_axis, Int, 0)

  271.     .ATTR(begin_params_axis, Int, 0)

  272.     .ATTR(epsilon, Float, 0.0000001)

  273.     .OP_END_FACTORY_REG(LayerNorm)

  274. 

  275. /**

  276. *@brief LayerNormGrad operator interface implementation

  277. *  calculating: dy, x, variance, mean, gamma

  278. *  pd_xl = data_dy*data_gamma

  279. *  pd_var = np.sum(((-0.5)*pd_xl*(data_x - data_mean)

  280. *           np.power((data_variance + EPSLON), (-1.5))),

  281. *           reduce_axis, keepdims=True)

  282. *  pd_mean = np.sum(((-1.0)*pd_xl

  283. *            np.power((data_variance + EPSLON), (-0.5))),

  284. *            reduce_axis, keepdims=True)

  285. *            + pd_var*(1.0/m)

  286. *            np.sum(((-2.0)*(data_x - data_mean)), reduce_axis, keepdims=True)

  287. *  pd_x = pd_xl*np.power((data_variance + EPSLON), (-0.5)) +

  288. *         pd_var*(2.0/m)*(data_x - data_mean) + pd_mean*(1.0/m)

  289. *  pd_gamma = np.sum((data_dy*(data_x - data_mean)

  290. *             np.power((data_variance + EPSLON), (-0.5))), param_axis, keepdims=True)

  291. *  pd_beta = np.sum(data_dy, param_axis, keepdims=True)

  292. 

  293. *@par Inputs:

  294. *Three inputs, including:

  295. * @li dy: A Tensor. Must be one of the following types: float16, float32.

  296. * @li x: A Tensor. Must be one of the following types: float16, float32.

  297. * @li variance: A Tensor. Must be one of the following types: float16, float32.

  298. * @li mean: A Tensor. Must be one of the following types: float16, float32.

  299. * @li gamma: A Tensor. Must be one of the following types: float16, float32.

  300. 

  301. *@par Outputs:

  302. *Three outputs, including:

  303. * @li pd_x: A Tensor. Must be one of the following types: float16, float32.

  304. * @li pd_gamma: A Tensor. Must be one of the following types: float16, float32.

  305. * @li pd_beta: A Tensor. Must be one of the following types: float16, float32.

  306. */

  307. REG_OP(LayerNormGrad)

  308.     .INPUT(dy, TensorType({DT_FLOAT, DT_FLOAT16}))

  309.     .INPUT(x, TensorType({DT_FLOAT, DT_FLOAT16}))

  310.     .INPUT(variance, TensorType({DT_FLOAT, DT_FLOAT16}))

  311.     .INPUT(mean, TensorType({DT_FLOAT, DT_FLOAT16}))

  312.     .INPUT(gamma, TensorType({DT_FLOAT, DT_FLOAT16}))

  313.     .OUTPUT(pd_x, TensorType({DT_FLOAT, DT_FLOAT16}))

  314.     .OUTPUT(pd_gamma, TensorType({DT_FLOAT, DT_FLOAT16}))

  315.     .OUTPUT(pd_beta, TensorType({DT_FLOAT, DT_FLOAT16}))

  316.     .OP_END_FACTORY_REG(LayerNormGrad)

  317. 

  318. /**

  319. *@brief LayerNormXBackprop operator interface implementation

  320. *  calculating: dy, x, variance, mean, gamma

  321. *  pd_xl = data_dy*data_gamma

  322. *  pd_var = np.sum(((-0.5)*pd_xl*(data_x - data_mean)

  323. *           np.power((data_variance + EPSLON), (-1.5))),

  324. *           reduce_axis, keepdims=True)

  325. *  pd_mean = np.sum(((-1.0)*pd_xl

  326. *            np.power((data_variance + EPSLON), (-0.5))),

  327. *            reduce_axis, keepdims=True)

  328. *            + pd_var*(1.0/m)

  329. *            np.sum(((-2.0)*(data_x - data_mean)), reduce_axis, keepdims=True)

  330. *  pd_x = pd_xl*np.power((data_variance + EPSLON), (-0.5)) +

  331. *         pd_var*(2.0/m)*(data_x - data_mean) + pd_mean*(1.0/m)

  332. *  pd_gamma = np.sum((data_dy*(data_x - data_mean)

  333. *             np.power((data_variance + EPSLON), (-0.5))), param_axis, keepdims=True)

  334. *  pd_beta = np.sum(data_dy, param_axis, keepdims=True)

  335. 

  336. *@par Inputs:

  337. *Three inputs, including:

  338. * @li dy: A Tensor. Must be one of the following types: float16, float32.

  339. * @li x: A Tensor. Must be one of the following types: float16, float32.

  340. * @li variance: A Tensor. Must be one of the following types: float16, float32.

  341. * @li mean: A Tensor. Must be one of the following types: float16, float32.

  342. * @li gamma: A Tensor. Must be one of the following types: float16, float32.

  343. 

  344. *@par Outputs:

  345. *Three outputs, including:

  346. * @li pd_x: A Tensor. Must be one of the following types: float16, float32.

  347. */

  348. REG_OP(LayerNormXBackprop)

  349.     .INPUT(dy, TensorType({DT_FLOAT, DT_FLOAT16}))

  350.     .INPUT(x, TensorType({DT_FLOAT, DT_FLOAT16}))

  351.     .INPUT(variance, TensorType({DT_FLOAT, DT_FLOAT16}))

  352.     .INPUT(mean, TensorType({DT_FLOAT, DT_FLOAT16}))

  353.     .INPUT(gamma, TensorType({DT_FLOAT, DT_FLOAT16}))

  354.     .OUTPUT(pd_x, TensorType({DT_FLOAT, DT_FLOAT16}))

  355.     .OP_END_FACTORY_REG(LayerNormXBackprop)

  356. 

  357. /**

  358. *@brief LayerNormBetaGammaBackprop operator interface implementation

  359. *  calculating: dy, x, variance, mean

  360. *  pd_xl = data_dy*data_gamma

  361. *  pd_var = np.sum(((-0.5)*pd_xl*(data_x - data_mean)

  362. *           np.power((data_variance + EPSLON), (-1.5))),

  363. *           reduce_axis, keepdims=True)

  364. *  pd_mean = np.sum(((-1.0)*pd_xl

  365. *            np.power((data_variance + EPSLON), (-0.5))),

  366. *            reduce_axis, keepdims=True)

  367. *            + pd_var*(1.0/m)

  368. *            np.sum(((-2.0)*(data_x - data_mean)), reduce_axis, keepdims=True)

  369. *  pd_x = pd_xl*np.power((data_variance + EPSLON), (-0.5)) +

  370. *         pd_var*(2.0/m)*(data_x - data_mean) + pd_mean*(1.0/m)

  371. *  pd_gamma = np.sum((data_dy*(data_x - data_mean)

  372. *             np.power((data_variance + EPSLON), (-0.5))), param_axis, keepdims=True)

  373. *  pd_beta = np.sum(data_dy, param_axis, keepdims=True)

  374. 

  375. *@par Inputs:

  376. *Three inputs, including:

  377. * @li dy: A Tensor. Must be one of the following types: float16, float32.

  378. * @li x: A Tensor. Must be one of the following types: float16, float32.

  379. * @li variance: A Tensor. Must be one of the following types: float16, float32.

  380. * @li mean: A Tensor. Must be one of the following types: float16, float32.

  381. 

  382. *@par Outputs:

  383. *Three outputs, including:

  384. * @li pd_gamma: A Tensor. Must be one of the following types: float16, float32.

  385. * @li pd_beta: A Tensor. Must be one of the following types: float16, float32.

  386. */

  387. REG_OP(LayerNormBetaGammaBackprop)

  388.     .INPUT(dy, TensorType({DT_FLOAT, DT_FLOAT16}))

  389.     .INPUT(x, TensorType({DT_FLOAT, DT_FLOAT16}))

  390.     .INPUT(variance, TensorType({DT_FLOAT, DT_FLOAT16}))

  391.     .INPUT(mean, TensorType({DT_FLOAT, DT_FLOAT16}))

  392.     .OUTPUT(pd_gamma, TensorType({DT_FLOAT, DT_FLOAT16}))

  393.     .OUTPUT(pd_beta, TensorType({DT_FLOAT, DT_FLOAT16}))

  394.     .REQUIRED_ATTR(shape_gamma, ListInt)

  395.     .OP_END_FACTORY_REG(LayerNormBetaGammaBackprop)

  396. 

  397. /**

  398. *@brief Return "output" according to the algorithm of dropout_do_mask: \n

  399. *  scale_x = x *(1 / keep_prob)

  400. *  output = select(mask == 1, scale_x, 0)

  401. 

  402. *@par Inputs:

  403. *Three inputs, including: \n

  404. * @li x: A mutable Tensor. Must be one of the following types:

  405. *     float16, float32

  406. * @li mask: A mutable Tensor. Must met all of the following rules:

  407. *     shape of mask should be 1D.

  408. *     dtype of mask should be uint8.

  409. *     value of shape should met the following algorithm:

  410. *     value = (size(x) + 128 - 1) // 128 * 128 //8

  411. * @li keep_prob: A mutable Tensor. Must met all of the following rules:

  412. *     shape of "keep_prob" should be (1,) or [1,].

  413. *     Has the same type as "x".

  414. 

  415. *@par Output:

  416. *y: A mutable Tensor. Has the same type as "x".

  417. */

  418. REG_OP(DropOutDoMask)

  419.     .INPUT(x, TensorType({DT_FLOAT, DT_FLOAT16}))

  420.     .INPUT(mask, TensorType({DT_UINT8}))

  421.     .INPUT(keep_prob, TensorType({DT_FLOAT, DT_FLOAT16}))

  422.     .OUTPUT(y, TensorType({DT_FLOAT, DT_FLOAT16}))

  423.     .OP_END_FACTORY_REG(DropOutDoMask)

  424. 

  425. /**

  426. *@brief Local Response Normalization.

  427. 

  428. *@par Inputs:

  429. *One input, including:

  430. *@li x: A Tensor. Must be 4-D shape, and only support the following types: float16, float32.

  431. 

  432. *@par Attributes:

  433. *@li depth_radius: An optional int, specifying the half-width of the

  434. * normalization window. Defaults to "5".

  435. *@li bias: An optional float32. An offset, usually > 0 to avoid dividing by 0.

  436. * Defaults to "1".

  437. *@li alpha: An optional float32. A scaling factor, usually positive.

  438. * Defaults to "1".

  439. *@li beta: An optional float32. An exponent. Defaults to "0.5".

  440. *@li norm_region: An optional string. A mode option. Defaults to "ACROSS_CHANNELS".

  441. 

  442. *@par Outputs:

  443. *y: A Tensor. Has the same data type and shape as "x".

  444. */

  445. REG_OP(LRN)

  446.     .INPUT(x, TensorType({DT_FLOAT16,DT_FLOAT}))

  447.     .OUTPUT(y, TensorType({DT_FLOAT16,DT_FLOAT}))

  448.     .ATTR(depth_radius, Int, 5)

  449.     .ATTR(bias, Float, 1.0)

  450.     .ATTR(alpha, Float, 1.0)

  451.     .ATTR(beta, Float, 0.5)

  452.     .ATTR(norm_region, String, "ACROSS_CHANNELS")

  453.     .OP_END_FACTORY_REG(LRN)

  454. 

  455. /**

  456. * @brief Computes the gradient for Local Response Normalization.

  457. 

  458. * @par Inputs:

  459. * @li grads: A 4D Tensor of type float16 or float32.

  460. * @li x: A 4D Tensor of type float16 or float32.

  461. * @li y: A 4D Tensor of type float16 or float32.

  462. 

  463. * @par Attributes:

  464. * @li depth_radius: An optional int, specifying the half-width of the

  465. * normalization window. Defaults to "5".

  466. * @li bias: An optional float32. An offset, usually > 0 to avoid dividing by 0.

  467. * Defaults to "1".

  468. * @li alpha: An optional float32. A scaling factor, usually positive.

  469. * Defaults to "1".

  470. * @li beta: An optional float32. An exponent. Defaults to "0.5".

  471. 

  472. * @par Outputs:

  473. * z: A Tensor. Has the same type and shape as "grads".

  474. 

  475. * @attention Constraints:

  476. * "x" and "y" must have the same shape and type as "grads".

  477. */

  478. REG_OP(LRNGrad)

  479.     .INPUT(grads, TensorType({DT_FLOAT16,DT_FLOAT}))

  480.     .INPUT(x, TensorType({DT_FLOAT16,DT_FLOAT}))

  481.     .INPUT(y, TensorType({DT_FLOAT16,DT_FLOAT}))

  482.     .OUTPUT(z, TensorType({DT_FLOAT16,DT_FLOAT}))

  483.     .ATTR(depth_radius, Int, 5)

  484.     .ATTR(bias, Float, 1.0)

  485.     .ATTR(alpha, Float, 1.0)

  486.     .ATTR(beta, Float, 0.5)

  487.     .OP_END_FACTORY_REG(LRNGrad)

  488. 

  489. }  // namespace ge

  490. 

  491. #endif  //GE_OP_NN_NORM_OPS_H