OpenI
/
MegEngine
Not watched
1
0
Fork 0
Code
Releases 31 Wiki Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 946a340c3d
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '946a340c3d'
${ noResults }
MegEngine/dnn/src/arm_common/conv_bias/postprocess_helper.h

381 lines
22 kB
Raw Blame History 

  1. /**

  2.  * \file dnn/src/arm_common/conv_bias/postprocess_helper.h

  3.  * MegEngine is Licensed under the Apache License, Version 2.0 (the "License")

  4.  *

  5.  * Copyright (c) 2014-2020 Megvii Inc. All rights reserved.

  6.  *

  7.  * Unless required by applicable law or agreed to in writing,

  8.  * software distributed under the License is distributed on an

  9.  * "AS IS" BASIS, WITHOUT ARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

  10.  */

  11. 

  12. #pragma once

  13. 

  14. #include "megdnn/basic_types.h"

  15. #include "src/arm_common/elemwise_helper/kimpl/op_base.h"

  16. #include "src/arm_common/elemwise_op.h"

  17. #include "src/fallback/conv_bias/opr_impl.h"

  18. 

  19. #include "midout.h"

  20. 

  21. MIDOUT_DECL(arm_common_conv_bias_postprocess_helper)

  22. 

  23. namespace {

  24. 

  25. 

  26. #define CONCAT_OP(_name) megdnn::arm_common::_name

  27. #define CONCAT_NL(_name) megdnn::NonlineMode::_name

  28. 

  29. #define CB(_caller, _op, _mode, midout_tag)                                    \

  30.     case _mode:                                                                \

  31.         MIDOUT_BEGIN(arm_common_conv_bias_postprocess_helper, 1, midout_tag) { \

  32.             _caller(_op);                                                      \

  33.         }                                                                      \

  34.         MIDOUT_END();                                                          \

  35.         break;

  36. 

  37. #define DEFAULT                                 \

  38.     default:                                    \

  39.         megdnn_throw("unsupported nolinemode"); \

  40.         break;

  41. 

  42. #define HANDLE_IDENTITY()               \

  43.     case megdnn::NonlineMode::IDENTITY: \

  44.         break;

  45. 

  46. #define FOR_NONLINEAR_UNARY(_op)                                             \

  47.     megdnn::arm_common::OpCallerUnary<_op<ctype>, megdnn::arm_common::VEC>:: \

  48.             run(static_cast<ctype*>(conv_dst_ptr),                           \

  49.                 reinterpret_cast<ctype*>(dst_ptr), bias_type, dst_type,      \

  50.                 N* OC* OH* OW* pack_oc_size);

  51. 

  52. #define FOR_NONLINEAR_BINARY_BROADCAST(_op)                                    \

  53.     megdnn::arm_common::                                                       \

  54.             OpCallerBinary<_op<ctype>, megdnn::arm_common::VEC_BCAST101>::run( \

  55.                     static_cast<ctype*>(conv_dst_ptr),                         \

  56.                     reinterpret_cast<const ctype*>(bias_ptr),                  \

  57.                     reinterpret_cast<ctype*>(dst_ptr), bias_type, bias_type,   \

  58.                     dst_type, N, OC, OH* OW);

  59. 

  60. #define FOR_NONLINEAR_BINARY_BROADCAST_NCHW44(_op)                           \

  61.     megdnn::arm_common::OpCallerBinary<_op<ctype>,                           \

  62.                                        megdnn::arm_common::VEC_BCAST101x4>:: \

  63.             run(static_cast<ctype*>(conv_dst_ptr),                           \

  64.                 reinterpret_cast<const ctype*>(bias_ptr),                    \

  65.                 reinterpret_cast<ctype*>(dst_ptr), bias_type, bias_type,     \

  66.                 dst_type, N, OC, OH* OW, pack_oc_size);

  67. 

  68. #define FOR_NONLINEAR_BINARY(_op)                                            \

  69.     megdnn::arm_common::                                                     \

  70.             OpCallerBinary<_op<ctype>, megdnn::arm_common::VEC_VEC>::run(    \

  71.                     static_cast<ctype*>(conv_dst_ptr),                       \

  72.                     reinterpret_cast<const ctype*>(bias_ptr),                \

  73.                     reinterpret_cast<ctype*>(dst_ptr), bias_type, bias_type, \

  74.                     dst_type, N* OC* OH* OW* pack_oc_size);

  75. 

  76. #define FOR_BIAS(_mode)                                                   \

  77.     switch (_mode) {                                                      \

  78.         case megdnn::BiasMode::NO_BIAS:                                   \

  79.             MIDOUT_BEGIN(arm_common_conv_bias_postprocess_helper, 0, 0) { \

  80.                 FOR_NONLINEAR_NOBIAS(FOR_NONLINEAR_UNARY);                \

  81.             }                                                             \

  82.             MIDOUT_END();                                                 \

  83.             break;                                                        \

  84.         case megdnn::BiasMode::BROADCAST_CHANNEL_BIAS:                    \

  85.             MIDOUT_BEGIN(arm_common_conv_bias_postprocess_helper, 0, 1) { \

  86.                 if (pack_oc_size == 1) {                                  \

  87.                     FOR_NONLINEAR(FOR_NONLINEAR_BINARY_BROADCAST);        \

  88.                 } else {                                                  \

  89.                     megdnn_assert(pack_oc_size == 4,                      \

  90.                                   "Only support nchw44 in ARM");          \

  91.                     FOR_NONLINEAR(FOR_NONLINEAR_BINARY_BROADCAST_NCHW44); \

  92.                 }                                                         \

  93.             }                                                             \

  94.             MIDOUT_END();                                                 \

  95.             break;                                                        \

  96.         case megdnn::BiasMode::BIAS:                                      \

  97.             MIDOUT_BEGIN(arm_common_conv_bias_postprocess_helper, 0, 2) { \

  98.                 FOR_NONLINEAR(FOR_NONLINEAR_BINARY);                      \

  99.             }                                                             \

  100.             MIDOUT_END();                                                 \

  101.             break;                                                        \

  102.         default:                                                          \

  103.             megdnn_throw("no quantized unsupported biasmode");            \

  104.             break;                                                        \

  105.     }

  106. 

  107. #define FOR_NONLINEAR(_caller)                                          \

  108.     switch (nonlineMode) {                                              \

  109.         CB(_caller, CONCAT_OP(AddOp), CONCAT_NL(IDENTITY), 3)           \

  110.         CB(_caller, CONCAT_OP(FuseAddReluOp), CONCAT_NL(RELU), 4)       \

  111.         CB(_caller, CONCAT_OP(FuseAddSigmoidOp), CONCAT_NL(SIGMOID), 5) \

  112.         CB(_caller, CONCAT_OP(FuseAddHSwishOp), CONCAT_NL(H_SWISH), 6)  \

  113.         DEFAULT                                                         \

  114.     }

  115. 

  116. #define FOR_NONLINEAR_NOBIAS(_caller)                             \

  117.     switch (nonlineMode) {                                        \

  118.         HANDLE_IDENTITY()                                         \

  119.         CB(_caller, CONCAT_OP(ReluOp), CONCAT_NL(RELU), 7);       \

  120.         CB(_caller, CONCAT_OP(SigmoidOp), CONCAT_NL(SIGMOID), 8); \

  121.         CB(_caller, CONCAT_OP(HSwishOp), CONCAT_NL(H_SWISH), 9);  \

  122.         DEFAULT                                                   \

  123.     }

  124. 

  125. template <typename ctype, typename dtype = ctype,

  126.           megdnn::PostprocessMode postprocess_mode =

  127.                   megdnn::PostprocessMode::FLOAT>

  128. struct PostProcess {

  129.     static void run(void* conv_dst_ptr, const void* bias_ptr, void* dst_ptr,

  130.                     megdnn::BiasMode bias_mode, megdnn::NonlineMode nonlineMode,

  131.                     megdnn::DType bias_type, megdnn::DType dst_type, size_t N,

  132.                     size_t OC, size_t OH, size_t OW, size_t pack_oc_size = 1) {

  133.         FOR_BIAS(bias_mode)

  134.     }

  135. };

  136. 

  137. template <typename ctype, typename dtype>

  138. struct PostProcess<ctype, dtype, megdnn::PostprocessMode::NO_PROCESS> {

  139.     static void run(void* conv_dst_ptr, void* bias_ptr, void* dst_ptr,

  140.                     megdnn::BiasMode bias_mode, megdnn::NonlineMode nonlineMode,

  141.                     megdnn::DType bias_type, megdnn::DType dst_type, size_t N,

  142.                     size_t OC, size_t OH, size_t OW, size_t pack_oc_size = 1) {

  143.         MEGDNN_MARK_USED_VAR(conv_dst_ptr);

  144.         MEGDNN_MARK_USED_VAR(bias_ptr);

  145.         MEGDNN_MARK_USED_VAR(dst_ptr);

  146.         MEGDNN_MARK_USED_VAR(bias_mode);

  147.         MEGDNN_MARK_USED_VAR(nonlineMode);

  148.         MEGDNN_MARK_USED_VAR(bias_type);

  149.         MEGDNN_MARK_USED_VAR(dst_type);

  150.         MEGDNN_MARK_USED_VAR(N);

  151.         MEGDNN_MARK_USED_VAR(OC);

  152.         MEGDNN_MARK_USED_VAR(OH);

  153.         MEGDNN_MARK_USED_VAR(OW);

  154.         MEGDNN_MARK_USED_VAR(pack_oc_size);

  155.         megdnn_assert(bias_mode == megdnn::BiasMode::NO_BIAS &&

  156.                       nonlineMode == megdnn::NonlineMode::IDENTITY);

  157.     }

  158. };

  159. 

  160. #undef FOR_NONLINEAR_UNARY

  161. #undef FOR_NONLINEAR_BINARY_BROADCAST

  162. #undef FOR_NONLINEAR_BINARY_BROADCAST_NCHW44

  163. #undef FOR_NONLINEAR_BINARY

  164. #undef FOR_NONLINEAR_NOBIAS

  165. #undef FOR_NONLINEAR

  166. #undef FOR_BIAS

  167. #undef HANDLE_IDENTITY

  168. 

  169. #define FOR_NONLINEAR_UNARY(_op)                                               \

  170.     megdnn::arm_common::OpCallerUnary<                                         \

  171.             _op<opctype, opdtype>,                                             \

  172.             megdnn::arm_common::VEC>::run(static_cast<opctype*>(conv_dst_ptr), \

  173.                                           reinterpret_cast<opdtype*>(dst_ptr), \

  174.                                           bias_type, dst_type,                 \

  175.                                           N* OC* OH* OW* pack_oc_size);

  176. 

  177. #define FOR_NONLINEAR_BINARY_BROADCAST(_op)                                \

  178.     megdnn::arm_common::OpCallerBinary<_op<opctype, opdtype>,              \

  179.                                        megdnn::arm_common::VEC_BCAST101>:: \

  180.             run(static_cast<opctype*>(conv_dst_ptr),                       \

  181.                 reinterpret_cast<const opctype*>(bias_ptr),                \

  182.                 reinterpret_cast<opdtype*>(dst_ptr), bias_type, bias_type, \

  183.                 dst_type, N, OC, OH* OW);

  184. 

  185. #define FOR_NONLINEAR_BINARY_BROADCAST_NCHW44(_op)                           \

  186.     megdnn::arm_common::OpCallerBinary<_op<opctype, opdtype>,                \

  187.                                        megdnn::arm_common::VEC_BCAST101x4>:: \

  188.             run(static_cast<opctype*>(conv_dst_ptr),                         \

  189.                 reinterpret_cast<const opctype*>(bias_ptr),                  \

  190.                 reinterpret_cast<opdtype*>(dst_ptr), bias_type, bias_type,   \

  191.                 dst_type, N, OC, OH* OW, pack_oc_size);

  192. 

  193. #define HANDLE_IDENTITY(_caller, _op)              \

  194.     case megdnn::NonlineMode::IDENTITY:            \

  195.         _caller(_op) break;

  196. 

  197. #define FOR_NONLINEAR(_caller)                                          \

  198.     switch (nonlineMode) {                                              \

  199.         HANDLE_IDENTITY(_caller, CONCAT_OP(AddOp))                      \

  200.         CB(_caller, CONCAT_OP(FuseAddReluOp), CONCAT_NL(RELU), 10)      \

  201.         CB(_caller, CONCAT_OP(FuseAddHSwishOp), CONCAT_NL(H_SWISH), 11) \

  202.         DEFAULT                                                         \

  203.     }

  204. 

  205. #define FOR_NONLINEAR_NOBIAS(_caller)                            \

  206.     switch (nonlineMode) {                                       \

  207.         HANDLE_IDENTITY(_caller, CONCAT_OP(TypeCvtOp))           \

  208.         CB(_caller, CONCAT_OP(ReluOp), CONCAT_NL(RELU), 12)      \

  209.         CB(_caller, CONCAT_OP(HSwishOp), CONCAT_NL(H_SWISH), 13) \

  210.         DEFAULT                                                  \

  211.     }

  212. 

  213. #define FOR_BIAS(_bias_mode, OH, OW)                                  \

  214.     switch (_bias_mode) {                                             \

  215.         case megdnn::BiasMode::NO_BIAS:                               \

  216.             FOR_NONLINEAR_NOBIAS(FOR_NONLINEAR_UNARY);                \

  217.             break;                                                    \

  218.         case megdnn::BiasMode::BROADCAST_CHANNEL_BIAS:                \

  219.             if (pack_oc_size == 1) {                                  \

  220.                 FOR_NONLINEAR(FOR_NONLINEAR_BINARY_BROADCAST);        \

  221.             } else {                                                  \

  222.                 megdnn_assert(pack_oc_size == 4,                      \

  223.                               "Only support nchw44 in ARM");          \

  224.                 FOR_NONLINEAR(FOR_NONLINEAR_BINARY_BROADCAST_NCHW44); \

  225.             }                                                         \

  226.             break;                                                    \

  227.         default:                                                      \

  228.             if (OH * OW == 1) {                                       \

  229.                 FOR_NONLINEAR(FOR_NONLINEAR_BINARY_BROADCAST);        \

  230.                 break;                                                \

  231.             }                                                         \

  232.             megdnn_throw("quantized unsupported biasmode");           \

  233.             break;                                                    \

  234.     }

  235. 

  236. template <typename opctype, typename opdtype>

  237. struct PostProcess<opctype, opdtype, megdnn::PostprocessMode::QUANTIZED> {

  238.     static void run(void* conv_dst_ptr, const void* bias_ptr, void* dst_ptr,

  239.                     megdnn::BiasMode bias_mode, megdnn::NonlineMode nonlineMode,

  240.                     megdnn::DType bias_type, megdnn::DType dst_type, size_t N,

  241.                     size_t OC, size_t OH, size_t OW, size_t pack_oc_size = 1) {

  242.         //! when OH * OW = 1, the bias_mode will be BiasMode::BIAS. It is wrong,

  243.         //! we deal this case at default branch.

  244.         FOR_BIAS(bias_mode, OH, OW);

  245.     }

  246. };

  247. 

  248. #undef FOR_NONLINEAR_UNARY

  249. #undef FOR_NONLINEAR_BINARY_BROADCAST

  250. #undef FOR_NONLINEAR_BINARY_BROADCAST_NCHW44

  251. #undef FOR_NONLINEAR_BINARY

  252. #undef FOR_NONLINEAR_NOBIAS

  253. #undef FOR_NONLINEAR

  254. #undef FOR_BIAS

  255. #undef CB

  256. #undef CONCAT_OP

  257. #undef CONCAT_NL

  258. #undef DEFAULT

  259. #undef HANDLE_IDENTITY

  260. 

  261. #define DISPATCH_CONV_WINOGRAD_NONLINE(_midout_tag, cb, _bias_id, _src_type,  \

  262.                                        _dst_type, _bmode, _nonline_mode, ...) \

  263.     switch (_nonline_mode) {                                                  \

  264.         case param::ConvBias::NonlineMode::IDENTITY: {                        \

  265.             MIDOUT_BEGIN(_midout_tag, _bias_id, 0) {                          \

  266.                 cb(_bmode, NoneOp<_src_type MEGDNN_COMMA _dst_type>,          \

  267.                    __VA_ARGS__);                                              \

  268.             }                                                                 \

  269.             MIDOUT_END();                                                     \

  270.             break;                                                            \

  271.         }                                                                     \

  272.         case param::ConvBias::NonlineMode::RELU: {                            \

  273.             MIDOUT_BEGIN(_midout_tag, _bias_id, 1) {                          \

  274.                 cb(_bmode, ReluOp<_src_type MEGDNN_COMMA _dst_type>,          \

  275.                    __VA_ARGS__);                                              \

  276.             }                                                                 \

  277.             MIDOUT_END();                                                     \

  278.             break;                                                            \

  279.         }                                                                     \

  280.         case param::ConvBias::NonlineMode::SIGMOID: {                         \

  281.             MIDOUT_BEGIN(_midout_tag, _bias_id, 2) {                          \

  282.                 cb(_bmode, SigmoidOp<_src_type MEGDNN_COMMA _dst_type>,       \

  283.                    __VA_ARGS__);                                              \

  284.             }                                                                 \

  285.             MIDOUT_END();                                                     \

  286.             break;                                                            \

  287.         }                                                                     \

  288.         case param::ConvBias::NonlineMode::H_SWISH: {                         \

  289.             MIDOUT_BEGIN(_midout_tag, _bias_id, 3) {                          \

  290.                 cb(_bmode, HSwishOp<_src_type MEGDNN_COMMA _dst_type>,        \

  291.                    __VA_ARGS__);                                              \

  292.             }                                                                 \

  293.             MIDOUT_END();                                                     \

  294.             break;                                                            \

  295.         }                                                                     \

  296.         default:                                                              \

  297.             megdnn_assert(0);                                                 \

  298.             break;                                                            \

  299.     }

  300. 

  301. #define DISPATCH_CONV_WINOGRAD_NONLINE_QUANTIZED(_midout_tag, cb, _bias_id,    \

  302.                                                  _src_type, _dst_type, _bmode, \

  303.                                                  _nonline_mode, ...)           \

  304.     switch (_nonline_mode) {                                                   \

  305.         case param::ConvBias::NonlineMode::IDENTITY: {                         \

  306.             MIDOUT_BEGIN(_midout_tag, _bias_id, 0) {                           \

  307.                 cb(_bmode, TypeCvtOp<_src_type MEGDNN_COMMA _dst_type>,        \

  308.                    __VA_ARGS__);                                               \

  309.             }                                                                  \

  310.             MIDOUT_END();                                                      \

  311.             break;                                                             \

  312.         }                                                                      \

  313.         case param::ConvBias::NonlineMode::RELU: {                             \

  314.             MIDOUT_BEGIN(_midout_tag, _bias_id, 1) {                           \

  315.                 cb(_bmode, ReluOp<_src_type MEGDNN_COMMA _dst_type>,           \

  316.                    __VA_ARGS__);                                               \

  317.             }                                                                  \

  318.             MIDOUT_END();                                                      \

  319.             break;                                                             \

  320.         }                                                                      \

  321.         default:                                                               \

  322.             megdnn_assert(0);                                                  \

  323.             break;                                                             \

  324.     }

  325. 

  326. #define DISPATCH_CONV_WINOGRAD_BIAS(_midout_tag, cb, _src_type, _dst_type,   \

  327.                                     _bmode, _nonline_mode, ...)              \

  328.     switch (_bmode) {                                                        \

  329.         case BiasMode::BIAS: {                                               \

  330.             DISPATCH_CONV_WINOGRAD_NONLINE(_midout_tag, cb, 0, _src_type,    \

  331.                                            _dst_type, BiasMode::BIAS,        \

  332.                                            _nonline_mode, __VA_ARGS__)       \

  333.             break;                                                           \

  334.         }                                                                    \

  335.         case BiasMode::NO_BIAS: {                                            \

  336.             DISPATCH_CONV_WINOGRAD_NONLINE(_midout_tag, cb, 1, _src_type,    \

  337.                                            _dst_type, BiasMode::NO_BIAS,     \

  338.                                            _nonline_mode, __VA_ARGS__)       \

  339.             break;                                                           \

  340.         }                                                                    \

  341.         case BiasMode::BROADCAST_CHANNEL_BIAS: {                             \

  342.             DISPATCH_CONV_WINOGRAD_NONLINE(_midout_tag, cb, 2, _src_type,    \

  343.                                            _dst_type,                        \

  344.                                            BiasMode::BROADCAST_CHANNEL_BIAS, \

  345.                                            _nonline_mode, __VA_ARGS__)       \

  346.             break;                                                           \

  347.         }                                                                    \

  348.         default:                                                             \

  349.             megdnn_assert(0);                                                \

  350.             break;                                                           \

  351.     }

  352. 

  353. #define DISPATCH_CONV_WINOGRAD_BIAS_QUANTIZED(                                \

  354.         _midout_tag, cb, _src_type, _dst_type, _bmode, _nonline_mode, ...)    \

  355.     switch (_bmode) {                                                         \

  356.         case BiasMode::BIAS: {                                                \

  357.             DISPATCH_CONV_WINOGRAD_NONLINE_QUANTIZED(                         \

  358.                     _midout_tag, cb, 0, _src_type, _dst_type, BiasMode::BIAS, \

  359.                     _nonline_mode, __VA_ARGS__)                               \

  360.             break;                                                            \

  361.         }                                                                     \

  362.         case BiasMode::NO_BIAS: {                                             \

  363.             DISPATCH_CONV_WINOGRAD_NONLINE_QUANTIZED(                         \

  364.                     _midout_tag, cb, 1, _src_type, _dst_type,                 \

  365.                     BiasMode::NO_BIAS, _nonline_mode, __VA_ARGS__)            \

  366.             break;                                                            \

  367.         }                                                                     \

  368.         case BiasMode::BROADCAST_CHANNEL_BIAS: {                              \

  369.             DISPATCH_CONV_WINOGRAD_NONLINE_QUANTIZED(                         \

  370.                     _midout_tag, cb, 2, _src_type, _dst_type,                 \

  371.                     BiasMode::BROADCAST_CHANNEL_BIAS, _nonline_mode,          \

  372.                     __VA_ARGS__)                                              \

  373.             break;                                                            \

  374.         }                                                                     \

  375.         default:                                                              \

  376.             megdnn_assert(0);                                                 \

  377.             break;                                                            \

  378.     }

  379. 

  380. }  // namespace