OpenI
/
MegEngine
Not watched
1
0
Fork 0
Code
Releases 31 Wiki Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: d345c86277
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'd345c86277'
${ noResults }
MegEngine/dnn/src/x86/conv_bias/postprocess_helper.h

468 lines
25 kB
Raw Blame History 

  1. /**

  2.  * \file dnn/src/x86/conv_bias/postprocess_helper.h

  3.  * MegEngine is Licensed under the Apache License, Version 2.0 (the "License")

  4.  *

  5.  * Copyright (c) 2014-2020 Megvii Inc. All rights reserved.

  6.  *

  7.  * Unless required by applicable law or agreed to in writing,

  8.  * software distributed under the License is distributed on an

  9.  * "AS IS" BASIS, WITHOUT ARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

  10.  */

  11. 

  12. #pragma once

  13. 

  14. #include "megdnn/opr_param_defs.h"

  15. #include "src/fallback/conv_bias/common.h"

  16. #include "src/x86/elemwise_op.h"

  17. #include "src/x86/utils.h"

  18. #include "src/fallback/conv_bias/opr_impl.h"

  19. 

  20. namespace megdnn {

  21. namespace x86 {

  22. 

  23. #define BIAS_CASE(mode)                                             \

  24.     case megdnn::param::ConvBias::NonlineMode::mode:                \

  25.         elem_mode = megdnn::param::Elemwise::Mode::FUSE_ADD_##mode; \

  26.         break;

  27. 

  28. #define NOBIAS_CASE(mode)                                \

  29.     case megdnn::param::ConvBias::NonlineMode::mode:     \

  30.         elem_mode = megdnn::param::Elemwise::Mode::mode; \

  31.         break;

  32. 

  33. #define IDENTITY_CASE(mode)                          \

  34.     case megdnn::param::ConvBias::NonlineMode::mode: \

  35.         break;

  36. 

  37. #define DEFAULT_CASE                            \

  38.     default:                                    \

  39.         megdnn_throw("unsupported nolinemode"); \

  40.         break;

  41. 

  42. #define CALL_UNARY(_op, _simd_type)                                           \

  43.     thin_function<void(const ctype*, ctype*, DType, DType, size_t)> run =     \

  44.             OpCallerUnary<_op<_simd_type, ctype, ctype>, _simd_type>::run;    \

  45.     run(static_cast<ctype*>(conv_dst_ptr), reinterpret_cast<ctype*>(dst_ptr), \

  46.         bias_type, dst_type, N* OC* OH* OW);

  47. 

  48. #define CALL_BINARY_BROADCAST(_op, _simd_type)                                \

  49.     thin_function<void(const ctype*, const ctype*, ctype*, DType, DType,      \

  50.                        DType, size_t, size_t, size_t)>                        \

  51.             run = OpCallerBinary<_op<_simd_type, ctype, ctype>, _simd_type,   \

  52.                                  megdnn::x86::BcastType::VEC_BCAST101>::run;  \

  53.     run(static_cast<ctype*>(conv_dst_ptr), static_cast<ctype*>(bias_ptr),     \

  54.         reinterpret_cast<ctype*>(dst_ptr), bias_type, bias_type, dst_type, N, \

  55.         OC, OH* OW);

  56. 

  57. #define CALL_BINARY(_op, _simd_type)                                        \

  58.     thin_function<void(const ctype*, const ctype*, ctype*, DType, DType,    \

  59.                        DType, size_t)>                                      \

  60.             run = OpCallerBinary<_op<_simd_type, ctype, ctype>, _simd_type, \

  61.                                  megdnn::x86::BcastType::VEC_VEC>::run;     \

  62.     run(static_cast<ctype*>(conv_dst_ptr), static_cast<ctype*>(bias_ptr),   \

  63.         reinterpret_cast<ctype*>(dst_ptr), bias_type, bias_type, dst_type,  \

  64.         N* OC* OH* OW);

  65. 

  66. #define cb_unary(_simd_type)                                          \

  67.     if (elem_mode == megdnn::param::Elemwise::Mode::RELU) {           \

  68.         CALL_UNARY(ReluOp, _simd_type);                               \

  69.     } else if (elem_mode == megdnn::param::Elemwise::Mode::SIGMOID) { \

  70.         CALL_UNARY(SigmoidOp, _simd_type);                            \

  71.     } else if (elem_mode == megdnn::param::Elemwise::Mode::H_SWISH) { \

  72.         CALL_UNARY(HSwishOp, _simd_type);                             \

  73.     }

  74. 

  75. #define FOR_NONLINEAR_NOBIAS()                    \

  76.     if (is_supported(SIMDType::AVX2)) {           \

  77.         cb_unary(SIMDType::AVX2)                  \

  78.     } else if (is_supported(SIMDType::SSE4_2)) {  \

  79.         cb_unary(SIMDType::SSE4_2)                \

  80.     } else {                                      \

  81.         cb_unary(SIMDType::NONE)                  \

  82.     }

  83. 

  84. #define cb_binary(_caller, _simd_type)                                         \

  85.     if (elem_mode == megdnn::param::Elemwise::Mode::ADD) {                     \

  86.         _caller(AddOp, _simd_type);                                            \

  87.     } else if (elem_mode == megdnn::param::Elemwise::Mode::FUSE_ADD_SIGMOID) { \

  88.         _caller(FuseAddSigmoidOp, _simd_type);                                 \

  89.     } else if (elem_mode == megdnn::param::Elemwise::Mode::FUSE_ADD_RELU) {    \

  90.         _caller(FuseAddReluOp, _simd_type);                                    \

  91.     } else if (elem_mode == megdnn::param::Elemwise::Mode::FUSE_ADD_H_SWISH) { \

  92.         _caller(FuseAddHSwishOp, _simd_type);                                  \

  93.     }

  94. 

  95. #define FOR_NONLINEAR(CALLER)                     \

  96.     if (is_supported(SIMDType::AVX2)) {           \

  97.         cb_binary(CALLER, SIMDType::AVX2)         \

  98.     } else if (is_supported(SIMDType::SSE4_2)) {  \

  99.         cb_binary(CALLER, SIMDType::SSE4_2)       \

  100.     } else {                                      \

  101.         cb_binary(CALLER, SIMDType::NONE)         \

  102.     }

  103. 

  104. #define FOR_BIAS(bias_mode)                       \

  105.     switch (bias_mode) {                          \

  106.         case BiasMode::NO_BIAS:                   \

  107.             FOR_NONLINEAR_NOBIAS();               \

  108.             break;                                \

  109.         case BiasMode::BROADCAST_CHANNEL_BIAS:    \

  110.             FOR_NONLINEAR(CALL_BINARY_BROADCAST); \

  111.             break;                                \

  112.         case BiasMode::BIAS:                      \

  113.             FOR_NONLINEAR(CALL_BINARY);           \

  114.             break;                                \

  115.         default:                                  \

  116.             break;                                \

  117.     }

  118. 

  119. template <typename ctype, typename dtype = ctype,

  120.           megdnn::PostprocessMode postprocess_mode =

  121.                   megdnn::PostprocessMode::FLOAT>

  122. struct PostProcess {

  123.     static void run(void* conv_dst_ptr, void* bias_ptr, void* dst_ptr,

  124.                     megdnn::ConvBiasForward::BiasMode bias_mode,

  125.                     megdnn::param::ConvBias::NonlineMode nonlineMode,

  126.                     DType bias_type, DType dst_type, size_t N, size_t OC,

  127.                     size_t OH, size_t OW, size_t pack_oc_size = 1) {

  128.         MEGDNN_MARK_USED_VAR(pack_oc_size);

  129.         megdnn::param::Elemwise::Mode elem_mode =

  130.                 megdnn::param::Elemwise::Mode::ADD;

  131.         if (bias_mode != megdnn::ConvBiasForward::BiasMode::NO_BIAS) {

  132.             switch (nonlineMode) {

  133.                 BIAS_CASE(RELU);

  134.                 BIAS_CASE(SIGMOID);

  135.                 BIAS_CASE(H_SWISH);

  136.                 IDENTITY_CASE(IDENTITY);

  137.                 DEFAULT_CASE;

  138.             }

  139.         } else {

  140.             switch (nonlineMode) {

  141.                 NOBIAS_CASE(RELU);

  142.                 NOBIAS_CASE(SIGMOID);

  143.                 NOBIAS_CASE(H_SWISH);

  144.                 IDENTITY_CASE(IDENTITY);

  145.                 DEFAULT_CASE;

  146.             }

  147.         }

  148.         FOR_BIAS(bias_mode);

  149.     }

  150. };

  151. 

  152. template <typename ctype, typename dtype>

  153. struct PostProcess<ctype, dtype, megdnn::PostprocessMode::FLOAT> {

  154.     static void run(void* conv_dst_ptr, void* bias_ptr, void* dst_ptr,

  155.                     megdnn::ConvBiasForward::BiasMode bias_mode,

  156.                     megdnn::param::ConvBias::NonlineMode nonlineMode,

  157.                     DType bias_type, DType dst_type, size_t N, size_t OC,

  158.                     size_t OH, size_t OW, size_t pack_oc_size=1) {

  159.         MEGDNN_MARK_USED_VAR(pack_oc_size);

  160.         megdnn::param::Elemwise::Mode elem_mode =

  161.                 megdnn::param::Elemwise::Mode::ADD;

  162.         if (bias_mode != megdnn::ConvBiasForward::BiasMode::NO_BIAS) {

  163.             switch (nonlineMode) {

  164.                 BIAS_CASE(RELU);

  165.                 BIAS_CASE(SIGMOID);

  166.                 BIAS_CASE(H_SWISH);

  167.                 IDENTITY_CASE(IDENTITY);

  168.                 DEFAULT_CASE;

  169.             }

  170.         } else {

  171.             switch (nonlineMode) {

  172.                 NOBIAS_CASE(RELU);

  173.                 NOBIAS_CASE(SIGMOID);

  174.                 NOBIAS_CASE(H_SWISH);

  175.                 IDENTITY_CASE(IDENTITY);

  176.                 DEFAULT_CASE;

  177.             }

  178.         }

  179. 

  180.         FOR_BIAS(bias_mode);

  181.     }

  182. };

  183. 

  184. template <typename ctype, typename dtype>

  185. struct PostProcess<ctype, dtype, megdnn::PostprocessMode::NO_PROCESS> {

  186.     static void run(void* conv_dst_ptr, void* bias_ptr, void* dst_ptr,

  187.                     megdnn::ConvBiasForward::BiasMode bias_mode,

  188.                     megdnn::param::ConvBias::NonlineMode nonlineMode,

  189.                     DType bias_type, DType dst_type, size_t N, size_t OC,

  190.                     size_t OH, size_t OW,size_t pack_oc_size = 1) {

  191.         MEGDNN_MARK_USED_VAR(pack_oc_size);

  192.         MEGDNN_MARK_USED_VAR(conv_dst_ptr);

  193.         MEGDNN_MARK_USED_VAR(bias_ptr);

  194.         MEGDNN_MARK_USED_VAR(dst_ptr);

  195.         MEGDNN_MARK_USED_VAR(bias_mode);

  196.         MEGDNN_MARK_USED_VAR(nonlineMode);

  197.         MEGDNN_MARK_USED_VAR(bias_type);

  198.         MEGDNN_MARK_USED_VAR(dst_type);

  199.         MEGDNN_MARK_USED_VAR(N);

  200.         MEGDNN_MARK_USED_VAR(OC);

  201.         MEGDNN_MARK_USED_VAR(OH);

  202.         MEGDNN_MARK_USED_VAR(OW);

  203.     }

  204. };

  205. #undef FOR_NONLINEAR_NOBIAS

  206. #undef FOR_NONLINEAR

  207. #undef FOR_BIAS

  208. 

  209. #undef cb_binary

  210. #undef cb_unary

  211. #undef CALL_UNARY

  212. #undef CALL_BINARY_BROADCAST

  213. 

  214. #define CALL_UNARY(_op, _simd_type)                                           \

  215.     thin_function<void(const ctype*, dtype*, DType, DType, size_t)> run =     \

  216.             OpCallerUnary<_op<_simd_type, ctype, dtype>, _simd_type>::run;    \

  217.     run(static_cast<ctype*>(conv_dst_ptr), reinterpret_cast<dtype*>(dst_ptr), \

  218.         bias_type, dst_type, N* OC* OH* OW);

  219. 

  220. #define CALL_BINARY_BROADCAST(_op, _simd_type)                                \

  221.     thin_function<void(const ctype*, const ctype*, dtype*, DType, DType,      \

  222.                        DType, size_t, size_t, size_t)>                        \

  223.             run = OpCallerBinary<_op<_simd_type, ctype, dtype>, _simd_type,   \

  224.                                  megdnn::x86::BcastType::VEC_BCAST101>::run;  \

  225.     run(static_cast<ctype*>(conv_dst_ptr), static_cast<ctype*>(bias_ptr),     \

  226.         reinterpret_cast<dtype*>(dst_ptr), bias_type, bias_type, dst_type, N, \

  227.         OC, OH* OW);

  228. 

  229. #define cb_unary(_simd_type)                                                 \

  230.     if (elem_mode == megdnn::param::Elemwise::Mode::RELU) {                  \

  231.         CALL_UNARY(ReluOp, _simd_type);                                      \

  232.     } else if (elem_mode == megdnn::param::Elemwise::Mode::H_SWISH) {        \

  233.         CALL_UNARY(HSwishOp, _simd_type);                                    \

  234.     } else {                                                                 \

  235.         if (nonlineMode == megdnn::param::ConvBias::NonlineMode::IDENTITY) { \

  236.             CALL_UNARY(TypeCvtOp, _simd_type);                               \

  237.         } else {                                                             \

  238.             megdnn_throw("not supported nonlinemode\n");                     \

  239.         }                                                                    \

  240.     }

  241. 

  242. #define FOR_NONLINEAR_NOBIAS()                                            \

  243.     if (is_supported(SIMDType::AVX2)) {                                   \

  244.         if (elem_mode == megdnn::param::Elemwise::Mode::RELU) {           \

  245.             CALL_UNARY(ReluOp, SIMDType::AVX2);                           \

  246.         } else if (elem_mode == megdnn::param::Elemwise::Mode::H_SWISH) { \

  247.             CALL_UNARY(HSwishOp, SIMDType::NONE);                         \

  248.         } else {                                                          \

  249.             if (nonlineMode ==                                            \

  250.                 megdnn::param::ConvBias::NonlineMode::IDENTITY) {         \

  251.                 CALL_UNARY(TypeCvtOp, SIMDType::NONE);                    \

  252.             } else {                                                      \

  253.                 megdnn_throw("not supported nonlinemode\n");              \

  254.             }                                                             \

  255.         }                                                                 \

  256.     } else if (is_supported(SIMDType::SSE4_2)) {                          \

  257.         cb_unary(SIMDType::SSE4_2)                                        \

  258.     } else {                                                              \

  259.         cb_unary(SIMDType::NONE)                                          \

  260.     }

  261. 

  262. #define cb_binary(_caller, _simd_type)                                         \

  263.     if (elem_mode == megdnn::param::Elemwise::Mode::ADD) {                     \

  264.         _caller(AddOp, _simd_type);                                            \

  265.     } else if (elem_mode == megdnn::param::Elemwise::Mode::FUSE_ADD_RELU) {    \

  266.         _caller(FuseAddReluOp, _simd_type);                                    \

  267.     } else if (elem_mode == megdnn::param::Elemwise::Mode::FUSE_ADD_H_SWISH) { \

  268.         _caller(FuseAddHSwishOp, _simd_type);                                  \

  269.     }

  270. 

  271. #define FOR_NONLINEAR(CALLER)                     \

  272.     if (is_supported(SIMDType::AVX2)) {           \

  273.         cb_binary(CALLER, SIMDType::AVX2)         \

  274.     } else if (!is_supported(SIMDType::SSE4_2)) { \

  275.         cb_binary(CALLER, SIMDType::SSE4_2)       \

  276.     } else {                                      \

  277.         cb_binary(CALLER, SIMDType::NONE)         \

  278.     }

  279. 

  280. #define FOR_BIAS(bias_mode)                              \

  281.     switch (bias_mode) {                                 \

  282.         case BiasMode::NO_BIAS:                          \

  283.             FOR_NONLINEAR_NOBIAS();                      \

  284.             break;                                       \

  285.         case BiasMode::BROADCAST_CHANNEL_BIAS:           \

  286.             FOR_NONLINEAR(CALL_BINARY_BROADCAST);        \

  287.             break;                                       \

  288.         default:                                         \

  289.             break;                                       \

  290.     }

  291. 

  292. template <typename ctype, typename dtype>

  293. struct PostProcess<ctype, dtype, megdnn::PostprocessMode::QUANTIZED> {

  294.     static void run(void* conv_dst_ptr, void* bias_ptr, void* dst_ptr,

  295.                     megdnn::ConvBiasForward::BiasMode bias_mode,

  296.                     megdnn::param::ConvBiasV0::NonlineMode nonlineMode,

  297.                     DType bias_type, DType dst_type, size_t N, size_t OC,

  298.                     size_t OH, size_t OW, size_t pack_oc_size = 1) {

  299.         MEGDNN_MARK_USED_VAR(pack_oc_size);

  300.         megdnn::param::Elemwise::Mode elem_mode =

  301.                 megdnn::param::Elemwise::Mode::ADD;

  302.         if (bias_mode != megdnn::ConvBiasForward::BiasMode::NO_BIAS) {

  303.             switch (nonlineMode) {

  304.                 BIAS_CASE(RELU);

  305.                 BIAS_CASE(H_SWISH);

  306.                 IDENTITY_CASE(IDENTITY);

  307.                 DEFAULT_CASE;

  308.             }

  309.         } else {

  310.             switch (nonlineMode) {

  311.                 NOBIAS_CASE(RELU);

  312.                 NOBIAS_CASE(H_SWISH);

  313.                 IDENTITY_CASE(IDENTITY);

  314.                 DEFAULT_CASE;

  315.             }

  316.         }

  317. 

  318.         FOR_BIAS(bias_mode);

  319. 

  320. #undef FOR_NONLINEAR_NOBIAS

  321. #undef FOR_NONLINEAR

  322. #undef FOR_BIAS

  323.     }

  324. };

  325. #undef cb_unary

  326. #undef cb_binary

  327. #undef BIAS_CASE

  328. #undef NOBIAS_CASE

  329. #undef DEFAULT_CASE

  330. #undef CALL_UNARY

  331. #undef CALL_BINARY

  332. #undef CALL_BINARY_BROADCAST

  333. 

  334. #define DISPATCH_CONV_WINOGRAD_NONLINE(_midout_tag, cb, _bias_id, _simd_type, \

  335.                                        _src_type, _dst_type, _bmode,          \

  336.                                        _nonline_mode, ...)                    \

  337.     switch (_nonline_mode) {                                                  \

  338.         case param::ConvBias::NonlineMode::IDENTITY:                          \

  339.             MIDOUT_BEGIN(_midout_tag, _bias_id, 0) {                          \

  340.                 cb(_bmode,                                                    \

  341.                    NoneOp<_simd_type MEGDNN_COMMA _src_type MEGDNN_COMMA      \

  342.                                   _dst_type>,                                 \

  343.                    __VA_ARGS__);                                              \

  344.             }                                                                 \

  345.             MIDOUT_END();                                                     \

  346.             break;                                                            \

  347.         case param::ConvBias::NonlineMode::RELU: {                            \

  348.             MIDOUT_BEGIN(_midout_tag, _bias_id, 1) {                          \

  349.                 cb(_bmode,                                                    \

  350.                    ReluOp<_simd_type MEGDNN_COMMA _src_type MEGDNN_COMMA      \

  351.                                   _dst_type>,                                 \

  352.                    __VA_ARGS__);                                              \

  353.             }                                                                 \

  354.             MIDOUT_END();                                                     \

  355.             break;                                                            \

  356.         }                                                                     \

  357.         case param::ConvBias::NonlineMode::SIGMOID: {                         \

  358.             MIDOUT_BEGIN(_midout_tag, _bias_id, 2) {                          \

  359.                 cb(_bmode,                                                    \

  360.                    SigmoidOp<_simd_type MEGDNN_COMMA _src_type MEGDNN_COMMA   \

  361.                                      _dst_type>,                              \

  362.                    __VA_ARGS__);                                              \

  363.             }                                                                 \

  364.             MIDOUT_END();                                                     \

  365.             break;                                                            \

  366.         }                                                                     \

  367.         case param::ConvBias::NonlineMode::H_SWISH: {                         \

  368.             MIDOUT_BEGIN(_midout_tag, _bias_id, 3) {                          \

  369.                 cb(_bmode,                                                    \

  370.                    HSwishOp<_simd_type MEGDNN_COMMA _src_type MEGDNN_COMMA    \

  371.                                     _dst_type>,                               \

  372.                    __VA_ARGS__);                                              \

  373.             }                                                                 \

  374.             MIDOUT_END();                                                     \

  375.             break;                                                            \

  376.         }                                                                     \

  377.         default:                                                              \

  378.             megdnn_assert(0);                                                 \

  379.             break;                                                            \

  380.     }

  381. 

  382. #define DISPATCH_CONV_WINOGRAD_BIAS(_midout_tag, cb, _simd_type, _src_type,  \

  383.                                     _dst_type, _bmode, _nonline_mode, ...)   \

  384.     switch (_bmode) {                                                        \

  385.         case BiasMode::BIAS: {                                               \

  386.             DISPATCH_CONV_WINOGRAD_NONLINE(                                  \

  387.                     _midout_tag, cb, 0, _simd_type, _src_type, _dst_type,    \

  388.                     BiasMode::BIAS, _nonline_mode, __VA_ARGS__)              \

  389.             break;                                                           \

  390.         }                                                                    \

  391.         case BiasMode::NO_BIAS: {                                            \

  392.             DISPATCH_CONV_WINOGRAD_NONLINE(                                  \

  393.                     _midout_tag, cb, 1, _simd_type, _src_type, _dst_type,    \

  394.                     BiasMode::NO_BIAS, _nonline_mode, __VA_ARGS__)           \

  395.             break;                                                           \

  396.         }                                                                    \

  397.         case BiasMode::BROADCAST_CHANNEL_BIAS: {                             \

  398.             DISPATCH_CONV_WINOGRAD_NONLINE(_midout_tag, cb, 2, _simd_type,   \

  399.                                            _src_type, _dst_type,             \

  400.                                            BiasMode::BROADCAST_CHANNEL_BIAS, \

  401.                                            _nonline_mode, __VA_ARGS__)       \

  402.             break;                                                           \

  403.         }                                                                    \

  404.         default:                                                             \

  405.             megdnn_assert(0);                                                \

  406.             break;                                                           \

  407.     }

  408. 

  409. #define DISPATCH_CONV_WINOGRAD_NONLINE_QUANTIZED(                            \

  410.         _midout_tag, cb, _bias_id, _simd_type, _src_type, _dst_type, _bmode, \

  411.         _nonline_mode, ...)                                                  \

  412.     switch (_nonline_mode) {                                                 \

  413.         case param::ConvBias::NonlineMode::IDENTITY: {                       \

  414.             MIDOUT_BEGIN(_midout_tag, _bias_id, 0) {                         \

  415.                 cb(_bmode,                                                   \

  416.                    TypeCvtOp<_simd_type MEGDNN_COMMA _src_type MEGDNN_COMMA  \

  417.                                      _dst_type>,                             \

  418.                    __VA_ARGS__);                                             \

  419.             }                                                                \

  420.             MIDOUT_END();                                                    \

  421.             break;                                                           \

  422.         }                                                                    \

  423.         case param::ConvBias::NonlineMode::RELU: {                           \

  424.             MIDOUT_BEGIN(_midout_tag, _bias_id, 1) {                         \

  425.                 cb(_bmode,                                                   \

  426.                    ReluOp<_simd_type MEGDNN_COMMA _src_type MEGDNN_COMMA     \

  427.                                   _dst_type>,                                \

  428.                    __VA_ARGS__);                                             \

  429.             }                                                                \

  430.             MIDOUT_END();                                                    \

  431.             break;                                                           \

  432.         }                                                                    \

  433.         default:                                                             \

  434.             megdnn_assert(0);                                                \

  435.             break;                                                           \

  436.     }

  437. 

  438. #define DISPATCH_CONV_WINOGRAD_BIAS_QUANTIZED(_midout_tag, cb, _simd_type,  \

  439.                                               _src_type, _dst_type, _bmode, \

  440.                                               _nonline_mode, ...)           \

  441.     switch (_bmode) {                                                       \

  442.         case BiasMode::BIAS: {                                              \

  443.             DISPATCH_CONV_WINOGRAD_NONLINE_QUANTIZED(                       \

  444.                     _midout_tag, cb, 0, _simd_type, _src_type, _dst_type,   \

  445.                     BiasMode::BIAS, _nonline_mode, __VA_ARGS__)             \

  446.             break;                                                          \

  447.         }                                                                   \

  448.         case BiasMode::NO_BIAS: {                                           \

  449.             DISPATCH_CONV_WINOGRAD_NONLINE_QUANTIZED(                       \

  450.                     _midout_tag, cb, 1, _simd_type, _src_type, _dst_type,   \

  451.                     BiasMode::NO_BIAS, _nonline_mode, __VA_ARGS__)          \

  452.             break;                                                          \

  453.         }                                                                   \

  454.         case BiasMode::BROADCAST_CHANNEL_BIAS: {                            \

  455.             DISPATCH_CONV_WINOGRAD_NONLINE_QUANTIZED(                       \

  456.                     _midout_tag, cb, 2, _simd_type, _src_type, _dst_type,   \

  457.                     BiasMode::BROADCAST_CHANNEL_BIAS, _nonline_mode,        \

  458.                     __VA_ARGS__)                                            \

  459.             break;                                                          \

  460.         }                                                                   \

  461.         default:                                                            \

  462.             megdnn_assert(0);                                               \

  463.             break;                                                          \

  464.     }

  465. 

  466. }  // namespace x86

  467. }  // namespace megdnn