OpenI
/
MegEngine
Not watched
1
0
Fork 0
Code
Releases 31 Wiki Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: ea93927d19
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'ea93927d19'
${ noResults }
MegEngine/dnn/src/fallback/conv_bias/im2col/algos.cpp

992 lines
51 kB
Raw Blame History 

  1. /**

  2.  * \file dnn/src/fallback/conv_bias/im2col/algos.cpp

  3.  * MegEngine is Licensed under the Apache License, Version 2.0 (the "License")

  4.  *

  5.  * Copyright (c) 2014-2020 Megvii Inc. All rights reserved.

  6.  *

  7.  * Unless required by applicable law or agreed to in writing,

  8.  * software distributed under the License is distributed on an

  9.  * "AS IS" BASIS, WITHOUT ARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

  10.  */

  11. 

  12. #include "src/fallback/conv_bias/im2col/algos.h"

  13. #include "megdnn/opr_param_defs.h"

  14. #include "src/common/opr_delegate.h"

  15. #include "src/fallback/conv_bias/common.h"

  16. #include "src/fallback/conv_bias/opr_impl.h"

  17. #include "src/fallback/conv_bias/winograd/strategy.h"

  18. #include "src/fallback/convolution/img2col_helper.h"

  19. #include "src/naive/convolution/helper.h"

  20. #if MEGDNN_X86

  21. #include "src/x86/conv_bias/postprocess_helper.h"

  22. #endif

  23. #include "midout.h"

  24. MIDOUT_DECL(megdnn_fallback_im2col)

  25. 

  26. using namespace megdnn;

  27. using namespace fallback;

  28. 

  29. #if MEGDNN_X86

  30. using namespace x86;

  31. #endif

  32. 

  33. /*======================== AlgoIm2col=======================*/

  34. /*!

  35.  *  *\brief The index of all parts workspace in im2col workspace bundel

  36.  *  *Through witch can convenient get the needed ptr

  37.  */

  38. struct Im2colBundelIndex {

  39.     static constexpr size_t BUNDLE_PADDING_INDEX = 0_z;

  40.     static constexpr size_t BUNDLE_PACKA_INDEX = 1_z;

  41.     static constexpr size_t BUNDLE_THREAD_INDEX = 2_z;

  42.     static constexpr size_t THREAD_BUNDLE_PACKB_INDEX = 0_z;

  43.     static constexpr size_t THREAD_BUNDLE_IM2COL_INDEX = 1_z;

  44.     static constexpr size_t THREAD_BUNDLE_MATMUL_DST_INDEX = 2_z;

  45.     static constexpr size_t THREAD_BUNDLE_BIAS_INDEX = 3_z;

  46.     static constexpr size_t THREAD_BUNDLE_COMPUTE_INDEX = 4_z;

  47. };

  48. 

  49. /*!

  50.  *  *\brief PtrGetter is get the im2col needed ptr according to the provided

  51.  *  *conditions

  52.  */

  53. class PtrGetter {

  54. public:

  55.     template <typename dtype>

  56.     static inline dtype* get_matmul_dst_ptr(

  57.             const ConvBiasImpl::NCBKernParam& param,

  58.             const WorkspaceBundle& bundle_thread, size_t bundle_id,

  59.             size_t oc_cur_index, size_t OHW, bool is_dst_8bit,

  60.             bool ohw_bigger_ohwblock, size_t batch_id, size_t group_id) {

  61.         if (is_dst_8bit || !ohw_bigger_ohwblock) {

  62.             return static_cast<dtype*>(bundle_thread.get(bundle_id));

  63.         } else {

  64.             dtype* dst =

  65.                     param.dst<dtype>(batch_id, group_id) + oc_cur_index * OHW;

  66.             return static_cast<dtype*>(dst);

  67.         }

  68.     }

  69. 

  70.     template <typename bias_ctype>

  71.     static inline bias_ctype* get_bias_temp_ptr(

  72.             const ConvBiasImpl::NCBKernParam& param,

  73.             const WorkspaceBundle& bundle_thread) {

  74.         bias_ctype* bias_tmp_ptr =

  75.                 param.bias_mode == megdnn::BiasMode::BIAS

  76.                         ? static_cast<bias_ctype*>(bundle_thread.get(

  77.                                   Im2colBundelIndex::THREAD_BUNDLE_BIAS_INDEX))

  78.                         : nullptr;

  79.         return bias_tmp_ptr;

  80.     }

  81. 

  82.     template <typename dtype>

  83.     static inline dtype* get_bundle_offset_byte_ptr(

  84.             const WorkspaceBundle& bundle, size_t bundle_id, size_t offset) {

  85.         return reinterpret_cast<dtype*>(

  86.                 reinterpret_cast<uintptr_t>(bundle.get(bundle_id)) + offset);

  87.     }

  88. };

  89. 

  90. using Pack_Mode=fallback::MatrixMulImpl::AlgoBase::PackMode;

  91. 

  92. //! Process one input channel copy padding

  93. template <typename src_ctype>

  94. static void copy_padding_kern(WorkspaceBundle bundle,

  95.                               const ConvBiasImpl::NCBKernParam& param,

  96.                               ConvBiasImpl::NCBKernIndex ncb_index) {

  97.     UNPACK_CONV_F32_NCB_KERN_SIZES(param);

  98.     MEGDNN_MARK_USED_VAR(N);

  99.     MEGDNN_MARK_USED_VAR(OC);

  100.     MEGDNN_MARK_USED_VAR(OH);

  101.     MEGDNN_MARK_USED_VAR(OW);

  102.     MEGDNN_MARK_USED_VAR(FH);

  103.     MEGDNN_MARK_USED_VAR(FW);

  104.     MEGDNN_MARK_USED_VAR(SH);

  105.     MEGDNN_MARK_USED_VAR(SW);

  106. 

  107.     size_t IW2 = IW + 2 * PW;

  108.     size_t IH2 = IH + 2 * PH;

  109.     size_t group_id = ncb_index.ndrange_id[0];

  110.     size_t batch_id = ncb_index.ndrange_id[1];

  111.     size_t channel_id = ncb_index.ndrange_id[2];

  112. 

  113.     size_t padding_group_size = IH2 * IW2 * IC;

  114.     size_t workspace_channel_offset = IH2 * IW2 * channel_id;

  115.     size_t workspace_group_offset = group_id * padding_group_size;

  116.     size_t workspace_batch_offset =

  117.             param.filter_meta.group * batch_id * padding_group_size;

  118.     bundle.set(param.workspace_ptr);

  119. 

  120.     src_ctype src_zp = static_cast<src_ctype>(0);

  121.     if (param.src_type.enumv() == DTypeEnum::Quantized8Asymm) {

  122.         src_zp = param.src_type.param<dtype::Quantized8Asymm>().zero_point;

  123.     }

  124.     src_ctype* src = const_cast<src_ctype*>(

  125.             param.src<src_ctype>(batch_id, group_id, channel_id));

  126.     src_ctype* src2;

  127.     src2 = static_cast<src_ctype*>(

  128.                    bundle.get(Im2colBundelIndex::BUNDLE_PADDING_INDEX)) +

  129.            workspace_group_offset + workspace_batch_offset +

  130.            workspace_channel_offset;

  131.     src_ctype* src2_ptr = src2;

  132.     const src_ctype* src_ptr = src;

  133.     if (PH != 0) {

  134.         std::memset(src2_ptr, src_zp, sizeof(src_ctype) * PH * IW2);

  135.         src2_ptr += PH * IW2;

  136.     }

  137.     rep(ih, IH) {

  138.         if (PW != 0)

  139.             rep(pw, PW) * (src2_ptr++) = src_zp;

  140.         std::memcpy(src2_ptr, src_ptr, sizeof(src_ctype) * IW);

  141.         src2_ptr += IW;

  142.         src_ptr += IW;

  143.         if (PW != 0)

  144.             rep(pw, PW) * (src2_ptr++) = src_zp;

  145.     }

  146.     if (PH != 0) {

  147.         std::memset(src2_ptr, src_zp, sizeof(src_ctype) * PH * IW2);

  148.         src2_ptr += PH * IW2;

  149.     }

  150. };

  151. 

  152. /*!

  153.  * *\brief Im2colKerns collects all the im2col kerns in it

  154.  */

  155. 

  156. #define COPY_BIAS()                                                         \

  157.     const bias_ctype* bias_ptr = static_cast<const bias_ctype*>(            \

  158.             param.bias<bias_ctype>(batch_id, group_id));                    \

  159.     bias_ctype* bias_temp_ptr =                                             \

  160.             PtrGetter::get_bias_temp_ptr<bias_ctype>(param, bundle_thread); \

  161.     if (param.bias_mode == megdnn::BiasMode::BIAS) {                        \

  162.         bias_ctype* copy_dst = bias_temp_ptr;                               \

  163.         const bias_ctype* copy_src =                                        \

  164.                 bias_ptr + oc_cur_index * OH * OW + ohw_cur_index;          \

  165.         for (size_t oc = oc_cur_index; oc < oc_end_index; oc++) {           \

  166.             std::memcpy(copy_dst, copy_src,                                 \

  167.                         sizeof(bias_ctype) * output_block_size);            \

  168.             copy_dst += output_block_size;                                  \

  169.             copy_src += OH * OW;                                            \

  170.         }                                                                   \

  171.     }

  172. 

  173. #define IM2COL()                                                               \

  174.     src_ctype* im2col_dst = nullptr;                                           \

  175.     src_ctype* no_padding_src =                                                \

  176.             const_cast<src_ctype*>(param.src<src_ctype>(batch_id, group_id)) + \

  177.             ohw_cur_index;                                                     \

  178.     if (!special_1x1) {                                                        \

  179.         size_t padding_group_size = IH2 * IW2 * IC * sizeof(src_ctype);        \

  180.         src_ctype* src2 = PtrGetter::get_bundle_offset_byte_ptr<src_ctype>(    \

  181.                 bundle, Im2colBundelIndex::BUNDLE_PADDING_INDEX,               \

  182.                 (ncb_index.ndrange_id[0] +                                     \

  183.                  param.filter_meta.group * ncb_index.ndrange_id[1]) *          \

  184.                         padding_group_size);                                   \

  185.         if (PH == 0 && PW == 0) {                                              \

  186.             src2 = const_cast<src_ctype*>(                                     \

  187.                     param.src<src_ctype>(batch_id, group_id));                 \

  188.         }                                                                      \

  189.         im2col_dst = static_cast<src_ctype*>(bundle_thread.get(                \

  190.                 Im2colBundelIndex::THREAD_BUNDLE_IM2COL_INDEX));               \

  191.         if (SH == 1 && SW == 1) {                                              \

  192.             if (is_xcorr) {                                                    \

  193.                 img2col<true>(src2, im2col_dst, OC, OH, OW, IC, IH2, IW2, FH,  \

  194.                               FW, ohw_cur_index, output_block_size);           \

  195.             } else {                                                           \

  196.                 img2col<false>(src2, im2col_dst, OC, OH, OW, IC, IH2, IW2, FH, \

  197.                                FW, ohw_cur_index, output_block_size);          \

  198.             }                                                                  \

  199.         } else {                                                               \

  200.             if (is_xcorr) {                                                    \

  201.                 img2col_stride<true>(src2, im2col_dst, OC, OH, OW, IC, IH2,    \

  202.                                      IW2, FH, FW, SH, SW, ohw_cur_index,       \

  203.                                      output_block_size);                       \

  204.             } else {                                                           \

  205.                 img2col_stride<false>(src2, im2col_dst, OC, OH, OW, IC, IH2,   \

  206.                                       IW2, FH, FW, SH, SW, ohw_cur_index,      \

  207.                                       output_block_size);                      \

  208.             }                                                                  \

  209.         }                                                                      \

  210.     }

  211. 

  212. #define POSTPROCESS_AND_COPYDST()                                            \

  213.     PostProcess<op_ctype, op_dtype, postprocess_mode>::run(                  \

  214.             matmul_dst,                                                      \

  215.             param.bias_mode == megdnn::BiasMode::BIAS                        \

  216.                     ? bias_temp_ptr                                          \

  217.                     : const_cast<bias_ctype*>(bias_ptr + oc_cur_index),      \

  218.             matmul_dst, param.bias_mode, param.nonlineMode, param.bias_type, \

  219.             param.dst_type, 1_z, output_block_oc_size, 1_z,                  \

  220.             output_block_size);                                              \

  221.     if (!skip_copy_dst) {                                                    \

  222.         dst_ctype* dst_tmp_ptr = reinterpret_cast<dst_ctype*>(matmul_dst);   \

  223.         dst_ctype* dst = param.dst<dst_ctype>(batch_id, group_id) +          \

  224.                          oc_cur_index * OHW + ohw_cur_index;                 \

  225.         for (size_t oc = 0; oc < output_block_oc_size; oc++) {               \

  226.             std::memcpy(dst, dst_tmp_ptr,                                    \

  227.                         sizeof(dst_ctype) * output_block_size);              \

  228.             dst_tmp_ptr += output_block_size;                                \

  229.             dst += OHW;                                                      \

  230.         }                                                                    \

  231.     }

  232. 

  233. #define PREPAR_MATMUL_DATA()                                                  \

  234.     size_t packA_per_oc_block_size =                                          \

  235.             round_up(matmul_param.K, matmul_algo->get_inner_block_size().k) * \

  236.             oc_tile_size * matmul_algo->get_packA_type_size();                \

  237.     size_t packA_group_size =                                                 \

  238.             matmul_algo->get_bundle(matmul_param).get_size(0);                \

  239.     src_ctype* a_panel = PtrGetter::get_bundle_offset_byte_ptr<src_ctype>(    \

  240.             bundle, Im2colBundelIndex::BUNDLE_PACKA_INDEX,                    \

  241.             ncb_index.ndrange_id[0] * packA_group_size +                      \

  242.                     ncb_index.ndrange_id[3] * packA_per_oc_block_size);       \

  243.     src_ctype* b_panel = PtrGetter::get_bundle_offset_byte_ptr<src_ctype>(    \

  244.             bundle_thread, Im2colBundelIndex::THREAD_BUNDLE_PACKB_INDEX, 0);  \

  245.     /*In pack mode, the matmul dst and im2col dst is the same workspace*/     \

  246.     bias_ctype* matmul_dst = PtrGetter::get_matmul_dst_ptr<bias_ctype>(       \

  247.             param, bundle_thread,                                             \

  248.             Im2colBundelIndex::THREAD_BUNDLE_IM2COL_INDEX, oc_cur_index, OHW, \

  249.             is_dst_8bit, is_ohw_size_bigger, batch_id, group_id);

  250. 

  251. #define MATMUL_COMPUTE()                                                      \

  252.     auto matmul_kern_naked = matmul_algo->get_kern_naked(matmul_param);       \

  253.     matmul_param.M = output_block_oc_size;                                    \

  254.     matmul_param.N = output_block_size;                                       \

  255.     matmul_param.LDB = special_1x1 ? OH * OW : output_block_size;             \

  256.     matmul_param.LDC = output_block_size;                                     \

  257.     matmul_param.A_ptr = a_panel;                                             \

  258.     matmul_param.B_ptr = im2col_dst ? im2col_dst : no_padding_src;            \

  259.     matmul_param.C_ptr = matmul_dst;                                          \

  260.     matmul_algo->pack_B(matmul_param, b_panel, 0, output_block_size);         \

  261.     matmul_kern_naked(matmul_param, a_panel, b_panel);

  262. 

  263. template <Pack_Mode packmode>

  264. class Im2colKerns;

  265. 

  266. template <>

  267. class Im2colKerns<Pack_Mode::DEFAULT> {

  268. public:

  269.     //! packA kern

  270.     template <typename src_ctype>

  271.     static void packA_kern(WorkspaceBundle bundle,

  272.                            const ConvBiasImpl::NCBKernParam& param,

  273.                            fallback::MatrixMulImpl::KernSizeParam matmulparam,

  274.                            fallback::MatrixMulImpl::AlgoBase* matmul_algo,

  275.                            ConvBiasImpl::NCBKernIndex ncb_index) {

  276.         bundle.set(param.workspace_ptr);

  277.         fallback::MatrixMulImpl::KernParam matmul_param;

  278.         size_t group_id = ncb_index.ndrange_id[0];

  279.         static_cast<fallback::MatrixMulImpl::KernSizeParam&>(matmul_param) =

  280.                 matmulparam;

  281.         size_t packA_group_size =

  282.                 matmul_algo->get_bundle(matmul_param).get_size(0);

  283.         size_t packed_per_oc_block_size =

  284.                 round_up(matmul_param.K,

  285.                          matmul_algo->get_inner_block_size().k) *

  286.                 matmul_algo->get_inner_block_size().m *

  287.                 matmul_algo->get_packA_type_size();

  288.         size_t a_panel_offset =

  289.                 ncb_index.ndrange_id[2] * packed_per_oc_block_size;

  290.         int8_t* a_panel = static_cast<int8_t*>(bundle.get(

  291.                                   Im2colBundelIndex::BUNDLE_PACKA_INDEX)) +

  292.                           group_id * packA_group_size + a_panel_offset;

  293.         matmul_param.A_ptr =

  294.                 const_cast<src_ctype*>(param.filter<src_ctype>(group_id));

  295.         matmul_algo->pack_A(matmul_param, a_panel, ncb_index.ndrange_id[2],

  296.                             matmul_algo->get_inner_block_size().m);

  297.     };

  298. 

  299.     //! conv kernel

  300.     template <typename src_ctype, typename bias_ctype, typename dst_ctype,

  301.               typename op_ctype, typename op_dtype,

  302.               PostprocessMode postprocess_mode>

  303.     static void kerns(

  304.             WorkspaceBundle bundle, WorkspaceBundle bundle_thread,

  305.             const ConvBiasImpl::NCBKernParam& param,

  306.             fallback::MatrixMulImpl::KernSizeParam matmul_kernsize_param,

  307.             fallback::MatrixMulImpl::AlgoBase* matmul_algo,

  308.             fallback::ConvBiasImpl::NCBKernIndex ncb_index,

  309.             size_t ohw_tile_size, size_t oc_tile_size) {

  310.         auto is_xcorr = !param.filter_meta.should_flip;

  311.         UNPACK_CONV_F32_NCB_KERN_SIZES(param);

  312.         MEGDNN_MARK_USED_VAR(N);

  313.         auto IH2 = IH + 2 * PH;

  314.         auto IW2 = IW + 2 * PW;

  315.         size_t OHW = OH * OW;

  316.         size_t group_id = ncb_index.ndrange_id[0];

  317.         size_t batch_id = ncb_index.ndrange_id[1];

  318.         size_t output_block_size = std::min(

  319.                 ohw_tile_size, OHW - ncb_index.ndrange_id[2] * ohw_tile_size);

  320.         size_t output_block_oc_size = std::min(

  321.                 oc_tile_size, OC - ncb_index.ndrange_id[3] * oc_tile_size);

  322. 

  323.         //! misc flags

  324.         bool special_1x1 = (FH == 1 && FW == 1 && SH == 1 && SW == 1 &&

  325.                             PH == 0 && PW == 0);

  326.         bool is_dst_8bit =

  327.                 (param.src_type.enumv() == DTypeEnum::QuantizedS8 &&

  328.                  param.dst_type.enumv() == DTypeEnum::QuantizedS8) ||

  329.                 (param.src_type.enumv() == DTypeEnum::Quantized8Asymm &&

  330.                  param.dst_type.enumv() == DTypeEnum::Quantized8Asymm);

  331.         bool is_ohw_size_bigger = (ohw_tile_size >= OHW);

  332.         bool skip_copy_dst = is_ohw_size_bigger && !is_dst_8bit;

  333. 

  334.         //! misc index

  335.         size_t ohw_cur_index = ncb_index.ndrange_id[2] * ohw_tile_size;

  336.         size_t oc_cur_index = ncb_index.ndrange_id[3] * oc_tile_size;

  337.         size_t oc_end_index = oc_cur_index + output_block_oc_size;

  338. 

  339.         bundle.set(param.workspace_ptr);

  340.         bundle_thread.set(PtrGetter::get_bundle_offset_byte_ptr<int8_t>(

  341.                 bundle, Im2colBundelIndex::BUNDLE_THREAD_INDEX,

  342.                 bundle_thread.total_size_in_bytes() * ncb_index.thread_id));

  343. 

  344.         fallback::MatrixMulImpl::KernParam matmul_param;

  345.         static_cast<fallback::MatrixMulImpl::KernSizeParam&>(matmul_param) =

  346.                 matmul_kernsize_param;

  347.         matmul_param.workspace_ptr = bundle_thread.get(

  348.                 Im2colBundelIndex::THREAD_BUNDLE_COMPUTE_INDEX);

  349. 

  350.         //! 1.Copy bias if need

  351.         COPY_BIAS();

  352. 

  353.         //! 2.Im2col

  354.         IM2COL();

  355. 

  356.         //! 3.packb and matmul compute

  357.         PREPAR_MATMUL_DATA();

  358.         MATMUL_COMPUTE();

  359. 

  360.         //! 4.postprocess and copy dst if need

  361.         POSTPROCESS_AND_COPYDST();

  362. #undef PREPAR_MATMUL_DATA

  363. #undef MATMUL_COMPUTE

  364.     }

  365. };

  366. 

  367. #define PREPAR_MATMUL_DATA()                                                   \

  368.     bias_ctype* matmul_dst = nullptr;                                          \

  369.     src_ctype* b_panel = nullptr;                                              \

  370.     size_t packA_group_size =                                                  \

  371.             bundle.get_size(Im2colBundelIndex::BUNDLE_PACKA_INDEX) /           \

  372.             param.filter_meta.group;                                           \

  373.     size_t a_panel_offset = ncb_index.ndrange_id[3] *                          \

  374.                             matmul_algo->get_bundle(matmul_param).get_size(0); \

  375.                                                                                \

  376.     src_ctype* a_panel = PtrGetter::get_bundle_offset_byte_ptr<src_ctype>(     \

  377.             bundle, Im2colBundelIndex::BUNDLE_PACKA_INDEX,                     \

  378.             group_id * packA_group_size + a_panel_offset);                     \

  379.     matmul_dst = PtrGetter::get_matmul_dst_ptr<bias_ctype>(                    \

  380.             param, bundle_thread,                                              \

  381.             Im2colBundelIndex::THREAD_BUNDLE_MATMUL_DST_INDEX, oc_cur_index,   \

  382.             OHW, is_dst_8bit, is_ohw_size_bigger, batch_id, group_id);

  383. 

  384. #define MATMUL_COMPUTE()                                                      \

  385.     auto matmul_kern_naked = matmul_algo->get_kern_naked(matmul_param);       \

  386.     matmul_param.M = output_block_oc_size;                                    \

  387.     matmul_param.N = output_block_size;                                       \

  388.     matmul_param.LDB = special_1x1 ? OH * OW : output_block_size;             \

  389.     matmul_param.LDC = output_block_size;                                     \

  390.     matmul_param.A_ptr = a_panel;                                             \

  391.     matmul_param.B_ptr = im2col_dst ? im2col_dst : no_padding_src;            \

  392.     matmul_param.C_ptr = matmul_dst;                                          \

  393.     matmul_kern_naked(matmul_param, a_panel, b_panel);

  394. 

  395. template <>

  396. class Im2colKerns<Pack_Mode::ONLY_PACKA> {

  397. public:

  398.     //! packA kern

  399.     template <typename src_ctype>

  400.     static void packA_kern(WorkspaceBundle bundle,

  401.                            const ConvBiasImpl::NCBKernParam& param,

  402.                            fallback::MatrixMulImpl::KernSizeParam matmulparam,

  403.                            fallback::MatrixMulImpl::AlgoBase* matmul_algo,

  404.                            ConvBiasImpl::NCBKernIndex ncb_index) {

  405.         bundle.set(param.workspace_ptr);

  406.         fallback::MatrixMulImpl::KernParam matmul_param;

  407.         static_cast<fallback::MatrixMulImpl::KernSizeParam&>(matmul_param) =

  408.                 matmulparam;

  409.         size_t OC = param.filter_meta.ocpg;

  410.         size_t oc_tile_size = matmul_param.M;

  411.         size_t group_id = ncb_index.ndrange_id[0];

  412.         size_t output_block_oc_size = std::min(

  413.                 oc_tile_size, OC - ncb_index.ndrange_id[2] * oc_tile_size);

  414.         size_t oc_cur_index = ncb_index.ndrange_id[2] * oc_tile_size;

  415.         size_t packA_group_size =

  416.                 bundle.get_size(Im2colBundelIndex::BUNDLE_PACKA_INDEX) /

  417.                 param.filter_meta.group;

  418.         size_t a_panel_offset =

  419.                 ncb_index.ndrange_id[2] *

  420.                 matmul_algo->get_bundle(matmul_param).get_size(0);

  421.         int8_t* a_panel = static_cast<int8_t*>(bundle.get(

  422.                                   Im2colBundelIndex::BUNDLE_PACKA_INDEX)) +

  423.                           group_id * packA_group_size + a_panel_offset;

  424.         matmul_param.A_ptr =

  425.                 const_cast<src_ctype*>(param.filter<src_ctype>(group_id)) +

  426.                 oc_cur_index * matmul_param.K;

  427.         matmul_param.M = output_block_oc_size;

  428.         matmul_algo->pack_A(matmul_param, a_panel, 0_z, 0_z);

  429.     };

  430. 

  431.     //! conv kernel

  432.     template <typename src_ctype, typename bias_ctype, typename dst_ctype,

  433.               typename op_ctype, typename op_dtype,

  434.               PostprocessMode postprocess_mode>

  435.     static void kerns(

  436.             WorkspaceBundle bundle, WorkspaceBundle bundle_thread,

  437.             const ConvBiasImpl::NCBKernParam& param,

  438.             fallback::MatrixMulImpl::KernSizeParam matmul_kernsize_param,

  439.             fallback::MatrixMulImpl::AlgoBase* matmul_algo,

  440.             fallback::ConvBiasImpl::NCBKernIndex ncb_index,

  441.             size_t ohw_tile_size, size_t oc_tile_size) {

  442.         auto is_xcorr = !param.filter_meta.should_flip;

  443.         UNPACK_CONV_F32_NCB_KERN_SIZES(param);

  444.         MEGDNN_MARK_USED_VAR(N);

  445.         auto IH2 = IH + 2 * PH;

  446.         auto IW2 = IW + 2 * PW;

  447.         size_t group_id = ncb_index.ndrange_id[0];

  448.         size_t batch_id = ncb_index.ndrange_id[1];

  449.         size_t OHW = OH * OW;

  450.         size_t output_block_size = std::min(

  451.                 ohw_tile_size, OHW - ncb_index.ndrange_id[2] * ohw_tile_size);

  452.         size_t output_block_oc_size = std::min(

  453.                 oc_tile_size, OC - ncb_index.ndrange_id[3] * oc_tile_size);

  454. 

  455.         //! misc flags

  456.         bool special_1x1 = (FH == 1 && FW == 1 && SH == 1 && SW == 1 &&

  457.                             PH == 0 && PW == 0);

  458.         bool is_dst_8bit =

  459.                 (param.src_type.enumv() == DTypeEnum::QuantizedS8 &&

  460.                  param.dst_type.enumv() == DTypeEnum::QuantizedS8) ||

  461.                 (param.src_type.enumv() == DTypeEnum::Quantized8Asymm &&

  462.                  param.dst_type.enumv() == DTypeEnum::Quantized8Asymm);

  463.         bool is_ohw_size_bigger = (ohw_tile_size >= OHW);

  464.         bool skip_copy_dst = is_ohw_size_bigger && !is_dst_8bit;

  465. 

  466.         //! misc index

  467.         size_t ohw_cur_index = ncb_index.ndrange_id[2] * ohw_tile_size;

  468.         size_t oc_cur_index = ncb_index.ndrange_id[3] * oc_tile_size;

  469.         size_t oc_end_index = oc_cur_index + output_block_oc_size;

  470. 

  471.         bundle.set(param.workspace_ptr);

  472.         bundle_thread.set(PtrGetter::get_bundle_offset_byte_ptr<int8_t>(

  473.                 bundle, Im2colBundelIndex::BUNDLE_THREAD_INDEX,

  474.                 bundle_thread.total_size_in_bytes() * ncb_index.thread_id));

  475. 

  476.         fallback::MatrixMulImpl::KernParam matmul_param;

  477.         static_cast<fallback::MatrixMulImpl::KernSizeParam&>(matmul_param) =

  478.                 matmul_kernsize_param;

  479.         matmul_param.workspace_ptr = bundle_thread.get(

  480.                 Im2colBundelIndex::THREAD_BUNDLE_COMPUTE_INDEX);

  481. 

  482.         //! 1.Copy bias if need

  483.         COPY_BIAS();

  484. 

  485.         //! 2.Im2col

  486.         IM2COL();

  487. 

  488.         //! 3.packb and matmul compute

  489.         PREPAR_MATMUL_DATA();

  490.         MATMUL_COMPUTE();

  491. 

  492.         //! 4.postprocess and copy dst if need

  493.         POSTPROCESS_AND_COPYDST();

  494. #undef PREPAR_MATMUL_DATA

  495. #undef MATMUL_COMPUTE

  496.     }

  497. };

  498. 

  499. #define PREPAR_MATMUL_DATA()                                                 \

  500.     bias_ctype* matmul_dst = nullptr;                                        \

  501.     const src_ctype* filter =                                                \

  502.             param.filter<src_ctype>(group_id) + oc_cur_index * IC * FH * FW; \

  503.     matmul_dst = PtrGetter::get_matmul_dst_ptr<bias_ctype>(                  \

  504.             param, bundle_thread,                                            \

  505.             Im2colBundelIndex::THREAD_BUNDLE_MATMUL_DST_INDEX, oc_cur_index, \

  506.             OHW, is_dst_8bit, is_ohw_size_bigger, batch_id, group_id);

  507. 

  508. #define MATMUL_COMPUTE()                                           \

  509.     matmul_param.M = output_block_oc_size;                         \

  510.     matmul_param.N = output_block_size;                            \

  511.     matmul_param.LDB = special_1x1 ? OH * OW : output_block_size;  \

  512.     matmul_param.LDC = output_block_size;                          \

  513.     matmul_param.A_ptr = filter;                                   \

  514.     matmul_param.B_ptr = im2col_dst ? im2col_dst : no_padding_src; \

  515.     matmul_param.C_ptr = matmul_dst;                               \

  516.     auto matmul_kern_t = matmul_algo->get_kern(matmul_param);      \

  517.     matmul_kern_t(matmul_param);

  518. 

  519. template <>

  520. class Im2colKerns<Pack_Mode::NO_PACK> {

  521. public:

  522.     //! conv kernel

  523.     template <typename src_ctype, typename bias_ctype, typename dst_ctype,

  524.               typename op_ctype, typename op_dtype,

  525.               PostprocessMode postprocess_mode>

  526.     static void kerns(

  527.             WorkspaceBundle bundle, WorkspaceBundle bundle_thread,

  528.             const ConvBiasImpl::NCBKernParam& param,

  529.             fallback::MatrixMulImpl::KernSizeParam matmul_kernsize_param,

  530.             fallback::MatrixMulImpl::AlgoBase* matmul_algo,

  531.             fallback::ConvBiasImpl::NCBKernIndex ncb_index,

  532.             size_t ohw_tile_size, size_t oc_tile_size) {

  533.         auto is_xcorr = !param.filter_meta.should_flip;

  534.         UNPACK_CONV_F32_NCB_KERN_SIZES(param);

  535.         MEGDNN_MARK_USED_VAR(N);

  536.         auto IH2 = IH + 2 * PH;

  537.         auto IW2 = IW + 2 * PW;

  538.         size_t group_id = ncb_index.ndrange_id[0];

  539.         size_t batch_id = ncb_index.ndrange_id[1];

  540.         size_t OHW = OH * OW;

  541.         size_t output_block_size = std::min(

  542.                 ohw_tile_size, OHW - ncb_index.ndrange_id[2] * ohw_tile_size);

  543.         size_t output_block_oc_size = std::min(

  544.                 oc_tile_size, OC - ncb_index.ndrange_id[3] * oc_tile_size);

  545.         //! misc flags

  546.         bool special_1x1 = (FH == 1 && FW == 1 && SH == 1 && SW == 1 &&

  547.                             PH == 0 && PW == 0);

  548.         bool is_dst_8bit =

  549.                 (param.src_type.enumv() == DTypeEnum::QuantizedS8 &&

  550.                  param.dst_type.enumv() == DTypeEnum::QuantizedS8) ||

  551.                 (param.src_type.enumv() == DTypeEnum::Quantized8Asymm &&

  552.                  param.dst_type.enumv() == DTypeEnum::Quantized8Asymm);

  553.         bool is_ohw_size_bigger = (ohw_tile_size >= OHW);

  554.         bool skip_copy_dst = is_ohw_size_bigger && !is_dst_8bit;

  555. 

  556.         //! misc index

  557.         size_t ohw_cur_index = ncb_index.ndrange_id[2] * ohw_tile_size;

  558.         size_t oc_cur_index = ncb_index.ndrange_id[3] * oc_tile_size;

  559.         size_t oc_end_index = oc_cur_index + output_block_oc_size;

  560. 

  561.         bundle.set(param.workspace_ptr);

  562.         bundle_thread.set(PtrGetter::get_bundle_offset_byte_ptr<int8_t>(

  563.                 bundle, Im2colBundelIndex::BUNDLE_THREAD_INDEX,

  564.                 bundle_thread.total_size_in_bytes() * ncb_index.thread_id));

  565. 

  566.         fallback::MatrixMulImpl::KernParam matmul_param;

  567.         static_cast<fallback::MatrixMulImpl::KernSizeParam&>(matmul_param) =

  568.                 matmul_kernsize_param;

  569.         matmul_param.workspace_ptr = bundle_thread.get(

  570.                 Im2colBundelIndex::THREAD_BUNDLE_COMPUTE_INDEX);

  571. 

  572.         //! 1.Copy bias if need

  573.         COPY_BIAS();

  574. 

  575.         //! 2.Im2col

  576.         IM2COL();

  577. 

  578.         //! 3.packb and matmul compute

  579.         PREPAR_MATMUL_DATA();

  580.         MATMUL_COMPUTE();

  581. 

  582.         //! 4.postprocess and copy dst if need

  583.         POSTPROCESS_AND_COPYDST();

  584. 

  585. #undef PREPAR_MATMUL_DATA

  586. #undef MATMUL_COMPUTE

  587.     }

  588. };

  589. 

  590. #undef COPY_BIAS

  591. #undef IM2COL

  592. #undef POSTPROCESS_AND_COPYDST

  593. fallback::MatrixMulImpl::KernSizeParam

  594. ConvBiasImpl::AlgoIm2col ::get_matmul_kern_param(const NCBKernSizeParam& param,

  595.                                                  size_t ohw_tile_size,

  596.                                                  size_t oc_tile_size) const {

  597.     size_t M = oc_tile_size;

  598.     size_t N = ohw_tile_size;

  599.     size_t K = param.filter_meta.icpg * param.filter_meta.spatial[0] *

  600.                param.filter_meta.spatial[1];

  601.     size_t LDA = K, LDB = N, LDC = N;

  602.     bool is_dst_8bit = (param.src_type.enumv() == DTypeEnum::QuantizedS8 &&

  603.                         param.dst_type.enumv() == DTypeEnum::QuantizedS8) ||

  604.                        (param.src_type.enumv() == DTypeEnum::Quantized8Asymm &&

  605.                         param.dst_type.enumv() == DTypeEnum::Quantized8Asymm);

  606.     return {param.filter_type,

  607.             param.src_type,

  608.             is_dst_8bit ? param.bias_type : param.dst_type,

  609.             M,

  610.             N,

  611.             K,

  612.             LDA,

  613.             LDB,

  614.             LDC,

  615.             false,

  616.             false,

  617.             param::MatrixMul::ComputeMode::DEFAULT,

  618.             param::MatrixMul::Format::DEFAULT};

  619. }

  620. 

  621. void ConvBiasImpl::AlgoIm2col::choice_ohw_oc_block(

  622.         const NCBKernSizeParam& param, size_t block_m, size_t block_n,

  623.         bool need_pack) const {

  624.     size_t nr_threads = param.nr_threads;

  625.     size_t OC = param.filter_meta.ocpg;

  626.     size_t ohw = param.osz[0] * param.osz[1];

  627.     //! pay attention please, should not change the 2 line code,

  628.     //! the opr use the same im2col algo, via choice_ohw_oc_block may change the

  629.     //! m_ohw_tile_size and m_oc_tile_size， if the two value changed, the

  630.     //! workspace size may change, will ocur workspace not match problem, so

  631.     //! should use the original data init them to avoid the problem

  632.     m_oc_tile_size = DEFAULT_OC_TILE_SIZE;

  633.     m_ohw_tile_size = m_ohw_tile_origin;

  634. 

  635.     m_oc_tile_size = std::min(m_oc_tile_size, OC);

  636.     m_ohw_tile_size = std::min(m_ohw_tile_size, ohw);

  637. 

  638.     if (nr_threads > 1) {

  639.         if (ohw / m_ohw_tile_size < nr_threads) {

  640.             m_ohw_tile_size = round_up(div_ceil(ohw, nr_threads), block_n);

  641.             if (m_ohw_tile_size < DEFAULT_OHW_MIN_TILE_SIZE) {

  642.                 m_ohw_tile_size = ohw;

  643.                 m_oc_tile_size = round_up(div_ceil(OC, nr_threads), block_m);

  644.                 if (m_oc_tile_size > DEFAULT_OC_MAX_TILE_SIZE) {

  645.                     m_oc_tile_size = DEFAULT_OC_MAX_TILE_SIZE;

  646.                 } else if (m_oc_tile_size < DEFAULT_OC_MIN_TILE_SIZE) {

  647.                     m_oc_tile_size = DEFAULT_OC_MIN_TILE_SIZE;

  648.                 }

  649.             }

  650.         }

  651.     } else {

  652.         if (!need_pack) {  //! no pack ,usually in x86 save memroy

  653.             m_ohw_tile_size = ohw;

  654.             m_oc_tile_size = OC;

  655.         }

  656.     }

  657. }

  658. 

  659. WorkspaceBundle ConvBiasImpl::AlgoIm2col::get_bundle(

  660.         const NCBKernSizeParam& param) const {

  661.     UNPACK_CONV_F32_NCB_KERN_SIZES(param);

  662.     MEGDNN_MARK_USED_VAR(OC);

  663.     MEGDNN_MARK_USED_VAR(OH);

  664.     MEGDNN_MARK_USED_VAR(OW);

  665.     MEGDNN_MARK_USED_VAR(FH);

  666.     MEGDNN_MARK_USED_VAR(FW);

  667.     MEGDNN_MARK_USED_VAR(SW);

  668.     MEGDNN_MARK_USED_VAR(SH);

  669. 

  670.     auto IW2 = IH + 2 * PH;

  671.     auto IH2 = IW + 2 * PW;

  672.     bool no_need_pading = (PH == 0 && PW == 0);

  673.     size_t padding = 0, packa_size = 0, packa_group_size = 0;

  674.     size_t nr_threads = param.nr_threads;

  675.     size_t GROUP = param.filter_meta.group;

  676.     bool need_pack = m_matmul_algo->packmode() == Pack_Mode::DEFAULT;

  677.     bool only_packA = m_matmul_algo->packmode() == Pack_Mode::ONLY_PACKA;

  678.     if (need_pack || only_packA) {

  679.         auto inner_block = m_matmul_algo->get_inner_block_size();

  680.         choice_ohw_oc_block(param, inner_block.m, inner_block.n, need_pack);

  681.         auto im2col_kern_param = get_matmul_kern_param(

  682.                 param, m_ohw_tile_size, only_packA ? m_oc_tile_size : OC);

  683.         size_t oc_parallel_times = div_ceil<size_t>(OC, m_oc_tile_size);

  684.         WorkspaceBundle wb = m_matmul_algo->get_bundle(im2col_kern_param);

  685.         packa_group_size = only_packA ? oc_parallel_times * wb.get_size(0)

  686.                                       : wb.get_size(0);

  687.     } else {  //! not support pack,not need pack

  688.         size_t nopack_default_blockm = 8;

  689.         size_t nopack_default_blockn = 16;

  690.         choice_ohw_oc_block(param, nopack_default_blockm, nopack_default_blockn,

  691.                             need_pack);

  692.         packa_group_size = 0;

  693.     }

  694.     if (no_need_pading) {

  695.         padding = 0;  //! not need  padding

  696.     } else {

  697.         padding = (GROUP * N * IC * IH2 * IW2) *

  698.                   sizeof(param.src_type);  //! for padding

  699.     }

  700.     packa_size = GROUP * packa_group_size;  //! for packA  size = GROUP * a_size

  701.     WorkspaceBundle ws = get_thread_bundle(param);

  702.     return {nullptr,

  703.             {padding, packa_size, ws.total_size_in_bytes() * nr_threads}};

  704. }

  705. 

  706. WorkspaceBundle ConvBiasImpl::AlgoIm2col::get_thread_bundle(

  707.         const NCBKernSizeParam& param) const {

  708.     size_t IC = param.filter_meta.icpg, FH = param.filter_meta.spatial[0],

  709.            FW = param.filter_meta.spatial[1];

  710.     size_t ohw = param.osz[0] * param.osz[1];

  711. 

  712.     size_t im2col = 0, packb = 0, matmul_dst = 0, bias_temp = 0,

  713.            matmul_compute = 0;

  714.     auto im2col_kern_param =

  715.             get_matmul_kern_param(param, m_ohw_tile_size, m_oc_tile_size);

  716.     bool default_pack = m_matmul_algo->packmode() == Pack_Mode::DEFAULT;

  717.     bool only_packA = m_matmul_algo->packmode() == Pack_Mode::ONLY_PACKA;

  718.     bool is_dst_8bit = (param.src_type.enumv() == DTypeEnum::QuantizedS8 &&

  719.                         param.dst_type.enumv() == DTypeEnum::QuantizedS8) ||

  720.                        (param.src_type.enumv() == DTypeEnum::Quantized8Asymm &&

  721.                         param.dst_type.enumv() == DTypeEnum::Quantized8Asymm);

  722.     size_t im2col_dst_size =

  723.             IC * FH * FW * m_ohw_tile_size * sizeof(param.src_type);

  724.     size_t matmul_dst_size =

  725.             m_oc_tile_size * m_ohw_tile_size * sizeof(param.bias_type);

  726.     if (default_pack || only_packA) {

  727.         //! matmul_dst and im2col_dst use the same memory

  728.         WorkspaceBundle wb = m_matmul_algo->get_bundle(im2col_kern_param);

  729.         packb = wb.get_size(1);

  730.         im2col = only_packA ? im2col_dst_size

  731.                             : std::max(im2col_dst_size, matmul_dst_size);

  732.         matmul_dst = only_packA ? matmul_dst_size : 0;

  733.     } else {

  734.         im2col = im2col_dst_size;

  735.         if (is_dst_8bit) {

  736.             matmul_dst = matmul_dst_size;

  737.         } else {

  738.             matmul_dst = m_ohw_tile_size >= ohw ? 0 : matmul_dst_size;

  739.         }

  740.         matmul_compute = m_matmul_algo->get_workspace(im2col_kern_param);

  741.     }

  742.     if (param.bias_mode == megdnn::BiasMode::BIAS) {

  743.         bias_temp = m_oc_tile_size * m_ohw_tile_size * sizeof(param.bias_type);

  744.     }

  745.     return {nullptr, {packb, im2col, matmul_dst, bias_temp, matmul_compute}};

  746. }

  747. 

  748. size_t ConvBiasImpl::AlgoIm2col::get_workspace(

  749.         ConvBiasImpl*, const NCBKernSizeParam& p) const {

  750.     MIDOUT_BEGIN(megdnn_fallback_im2col, 0, 0) {

  751.         return get_bundle(p).total_size_in_bytes();

  752.     }

  753.     MIDOUT_END();

  754.     return 0;

  755. }

  756. 

  757. SmallVector<ConvBiasImpl::NCBKern> ConvBiasImpl::AlgoIm2col::dispatch_kerns(

  758.         ConvBiasImpl*, const NCBKernSizeParam& param) const {

  759.     MIDOUT_BEGIN(megdnn_fallback_im2col, 0, 1) {

  760.         size_t ohw = param.osz[0] * param.osz[1];

  761.         size_t ohw_parallel_times = div_ceil(ohw, m_ohw_tile_size);

  762.         size_t GROUP = param.filter_meta.group;

  763.         size_t IC = param.filter_meta.icpg;

  764.         size_t OC = param.filter_meta.ocpg;

  765.         size_t PH = param.filter_meta.padding[0];

  766.         size_t PW = param.filter_meta.padding[1];

  767. 

  768.         WorkspaceBundle bundle = get_bundle(param);

  769.         WorkspaceBundle bundle_thread = get_thread_bundle(param);

  770. 

  771.         size_t oc_parallel_times = div_ceil(OC, m_oc_tile_size);

  772.         bool need_padding = (PH != 0 || PW != 0);

  773.         bool default_pack = m_matmul_algo->packmode() == Pack_Mode::DEFAULT;

  774.         bool no_pack = m_matmul_algo->packmode() == Pack_Mode::NO_PACK;

  775.         bool only_packA = m_matmul_algo->packmode() == Pack_Mode::ONLY_PACKA;

  776.         size_t packa_parallel_times = 0;

  777.         if (only_packA) {

  778.             packa_parallel_times = div_ceil(OC, m_oc_tile_size);

  779.         } else if (default_pack) {

  780.             packa_parallel_times =

  781.                     div_ceil(OC, m_matmul_algo->get_inner_block_size().m);

  782.         }

  783. 

  784.         auto matmul_param = get_matmul_kern_param(

  785.                 param, m_ohw_tile_size, only_packA ? m_oc_tile_size : OC);

  786. 

  787.         SmallVector<ConvBiasImpl::NCBKern> ret_kern;

  788. 

  789. #define RETURN_KERNS()                                                      \

  790.     if (default_pack) {                                                     \

  791.         ret_kern.push_back(                                                 \

  792.                 {kern_default_packA, {GROUP, 1_z, packa_parallel_times}});  \

  793.     }                                                                       \

  794.     if (only_packA) {                                                       \

  795.         ret_kern.push_back(                                                 \

  796.                 {kern_only_packA, {GROUP, 1_z, packa_parallel_times}});     \

  797.     }                                                                       \

  798.     if (need_padding) {                                                     \

  799.         ret_kern.push_back({kern_padding, {GROUP, param.n, IC}});           \

  800.     }                                                                       \

  801.     if (default_pack) {                                                     \

  802.         ret_kern.push_back(                                                 \

  803.                 {kern_compute_default,                                      \

  804.                  {GROUP, param.n, ohw_parallel_times, oc_parallel_times}}); \

  805.     }                                                                       \

  806.     if (no_pack) {                                                          \

  807.         ret_kern.push_back(                                                 \

  808.                 {kern_compute_nopack,                                       \

  809.                  {GROUP, param.n, ohw_parallel_times, oc_parallel_times}}); \

  810.     }                                                                       \

  811.     if (only_packA) {                                                       \

  812.         ret_kern.push_back(                                                 \

  813.                 {kern_compute_onlypackA,                                    \

  814.                  {GROUP, param.n, ohw_parallel_times, oc_parallel_times}}); \

  815.     }                                                                       \

  816.     return ret_kern;

  817. 

  818. #define COMPUTE_KERN(_name, _pack_mode, _dt, _post_ctype, _postprocess_mode) \

  819.     auto kern_compute_##_name = [bundle, bundle_thread, matmul_param,        \

  820.                                  matmul_algo = m_matmul_algo,                \

  821.                                  ohw_tile_size = m_ohw_tile_size,            \

  822.                                  oc_tile_size = m_oc_tile_size](             \

  823.                                         const NCBKernParam& param,           \

  824.                                         const NCBKernIndex& ncb_index) {     \

  825.         Im2colKerns<_pack_mode>::kerns<_dt, _dt, _dt, _post_ctype,           \

  826.                                        _post_ctype, _postprocess_mode>(      \

  827.                 bundle, bundle_thread, param, matmul_param, matmul_algo,     \

  828.                 ncb_index, ohw_tile_size, oc_tile_size);                     \

  829.     };

  830. 

  831. #define cb(_dt, _post_ctype, _postprocess_mode, _midout_tags)                 \

  832.     do {                                                                      \

  833.         if (param.filter_type.enumv() == DTypeTrait<_dt>::enumv) {            \

  834.             MIDOUT_BEGIN(megdnn_fallback_im2col, 0, 1, _midout_tags) {        \

  835.                 auto kern_padding = [bundle](const NCBKernParam& param,       \

  836.                                              const NCBKernIndex& ncb_index) { \

  837.                     copy_padding_kern<_dt>(bundle, param, ncb_index);         \

  838.                 };                                                            \

  839.                 auto kern_default_packA =                                     \

  840.                         [bundle, matmul_algo = m_matmul_algo, matmul_param](  \

  841.                                 const NCBKernParam& param,                    \

  842.                                 const NCBKernIndex& ncb_index) {              \

  843.                             Im2colKerns<Pack_Mode::DEFAULT>::packA_kern<_dt>( \

  844.                                     bundle, param, matmul_param, matmul_algo, \

  845.                                     ncb_index);                               \

  846.                         };                                                    \

  847.                 auto kern_only_packA = [bundle, matmul_algo = m_matmul_algo,  \

  848.                                         matmul_param](                        \

  849.                                                const NCBKernParam& param,     \

  850.                                                const NCBKernIndex&            \

  851.                                                        ncb_index) {           \

  852.                     Im2colKerns<Pack_Mode::ONLY_PACKA>::packA_kern<_dt>(      \

  853.                             bundle, param, matmul_param, matmul_algo,         \

  854.                             ncb_index);                                       \

  855.                 };                                                            \

  856.                 COMPUTE_KERN(default, Pack_Mode::DEFAULT, _dt, _post_ctype,   \

  857.                              _postprocess_mode);                              \

  858.                 COMPUTE_KERN(nopack, Pack_Mode::NO_PACK, _dt, _post_ctype,    \

  859.                              _postprocess_mode);                              \

  860.                 COMPUTE_KERN(onlypackA, Pack_Mode::ONLY_PACKA, _dt,           \

  861.                              _post_ctype, _postprocess_mode);                 \

  862.                 RETURN_KERNS();                                               \

  863.             }                                                                 \

  864.             MIDOUT_END();                                                     \

  865.             return {};                                                        \

  866.         }                                                                     \

  867.     } while (0);

  868. 

  869.         cb(dt_float32, dt_float32, PostprocessMode::FLOAT, 0);

  870. #if !MEGDNN_DISABLE_FLOAT16

  871.         cb(dt_float16, dt_float16, PostprocessMode::NO_PROCESS, 2);

  872. #endif

  873. #undef cb

  874. #undef COMPUTE_KERN

  875. 

  876. #define COMPUTE_KERN(_name, _pack_mode, _src_ctype, _bias_ctype, _dst_ctype, \

  877.                      _i_bias_type, _i_dst_type, _postprocess_mode)           \

  878.     auto kern_compute_##_name = [bundle, bundle_thread, matmul_param,        \

  879.                                  matmul_algo = m_matmul_algo,                \

  880.                                  ohw_tile_size = m_ohw_tile_size,            \

  881.                                  oc_tile_size = m_oc_tile_size](             \

  882.                                         const NCBKernParam& param,           \

  883.                                         const NCBKernIndex& ncb_index) {     \

  884.         Im2colKerns<_pack_mode>::kerns<_src_ctype, _bias_ctype, _dst_ctype,  \

  885.                                        DTypeTrait<_i_bias_type>::ctype,      \

  886.                                        DTypeTrait<_i_dst_type>::ctype,       \

  887.                                        _postprocess_mode>(                   \

  888.                 bundle, bundle_thread, param, matmul_param, matmul_algo,     \

  889.                 ncb_index, ohw_tile_size, oc_tile_size);                     \

  890.     };

  891. 

  892. #define cb(_i_src_type, _i_bias_type, _i_dst_type, _src_ctype, _bias_ctype,   \

  893.            _dst_ctype, _postprocess_mode, _midout_tags)                       \

  894.     do {                                                                      \

  895.         if (param.filter_type.enumv() == param.src_type.enumv() &&            \

  896.             param.src_type.enumv() == DTypeTrait<_i_src_type>::enumv &&       \

  897.             param.dst_type.enumv() == DTypeTrait<_i_dst_type>::enumv) {       \

  898.             MIDOUT_BEGIN(megdnn_fallback_im2col, 0, 1, _midout_tags) {        \

  899.                 auto kern_padding = [bundle](const NCBKernParam& param,       \

  900.                                              const NCBKernIndex& ncb_index) { \

  901.                     copy_padding_kern<_src_ctype>(bundle, param, ncb_index);  \

  902.                 };                                                            \

  903.                 auto kern_default_packA = [bundle,                            \

  904.                                            matmul_algo = m_matmul_algo,       \

  905.                                            matmul_param](                     \

  906.                                                   const NCBKernParam& param,  \

  907.                                                   const NCBKernIndex&         \

  908.                                                           ncb_index) {        \

  909.                     Im2colKerns<Pack_Mode::DEFAULT>::packA_kern<_src_ctype>(  \

  910.                             bundle, param, matmul_param, matmul_algo,         \

  911.                             ncb_index);                                       \

  912.                 };                                                            \

  913.                 auto kern_only_packA =                                        \

  914.                         [bundle, matmul_algo = m_matmul_algo, matmul_param](  \

  915.                                 const NCBKernParam& param,                    \

  916.                                 const NCBKernIndex& ncb_index) {              \

  917.                             Im2colKerns<Pack_Mode::ONLY_PACKA>::packA_kern<   \

  918.                                     _src_ctype>(bundle, param, matmul_param,  \

  919.                                                 matmul_algo, ncb_index);      \

  920.                         };                                                    \

  921.                 COMPUTE_KERN(default, Pack_Mode::DEFAULT, _src_ctype,         \

  922.                              _bias_ctype, _dst_ctype, _i_bias_type,           \

  923.                              _i_dst_type, _postprocess_mode);                 \

  924.                 COMPUTE_KERN(nopack, Pack_Mode::NO_PACK, _src_ctype,          \

  925.                              _bias_ctype, _dst_ctype, _i_bias_type,           \

  926.                              _i_dst_type, _postprocess_mode);                 \

  927.                 COMPUTE_KERN(onlypackA, Pack_Mode::ONLY_PACKA, _src_ctype,    \

  928.                              _bias_ctype, _dst_ctype, _i_bias_type,           \

  929.                              _i_dst_type, _postprocess_mode);                 \

  930.                 RETURN_KERNS();                                               \

  931.             }                                                                 \

  932.             MIDOUT_END();                                                     \

  933.             return {};                                                        \

  934.         }                                                                     \

  935.     } while (0);

  936. 

  937.         cb(dt_int8, dt_int32, dt_int32, dt_int8, dt_int32, dt_int32,

  938.            PostprocessMode::NO_PROCESS, 3);

  939. 

  940.         cb(dt_int8, dt_int16, dt_int16, dt_int8, dt_int16, dt_int16,

  941.            PostprocessMode::NO_PROCESS, 4);

  942. 

  943.         cb(dtype::QuantizedS8, dtype::QuantizedS32, dtype::QuantizedS32,

  944.            dt_int8, dt_int32, dt_int32, PostprocessMode::NO_PROCESS, 7);

  945. 

  946.         cb(dtype::QuantizedS8, dtype::QuantizedS32, dtype::QuantizedS8, dt_int8,

  947.            dt_int32, dt_int8, PostprocessMode::QUANTIZED, 8);

  948. #undef COMPUTE_KERN

  949. #undef RETURN_KERNS

  950. #undef cb

  951.         megdnn_throw("unsupported data type on im2col matmul algo");

  952.     }

  953.     MIDOUT_END();

  954.     return {};

  955. }

  956. 

  957. bool ConvBiasImpl::AlgoIm2col::usable(

  958.         ConvBiasImpl* opr, const NCBKernSizeParam& param,

  959.         AlgoSelectionStrategy /*algo_selection_strategy*/) const {

  960.     MIDOUT_BEGIN(megdnn_fallback_im2col, 0, 2) {

  961.         //! make sure 8x8x16 and 8x8x32 biasmode is  nobias and nonlineMode is

  962.         //! identity otherwise return false mean that 8x8x32 and 8x8x16 not support

  963.         //! PostProcess

  964.         if (param.src_type.enumv() == param.filter_type.enumv() &&

  965.             ((param.src_type.enumv() == DTypeEnum::Int8 &&

  966.               (param.dst_type.enumv() == DTypeEnum::Int16 ||

  967.                param.dst_type.enumv() == DTypeEnum::Int32)) ||

  968.              ((param.src_type.enumv() == DTypeEnum::QuantizedS8 ||

  969.                param.src_type.enumv() == DTypeEnum::Quantized8Asymm) &&

  970.               param.dst_type.enumv() == DTypeEnum::QuantizedS32)) &&

  971.             param.bias_mode != megdnn::BiasMode::NO_BIAS &&

  972.             param.nonlineMode != megdnn::NonlineMode::IDENTITY) {

  973.             return false;

  974.         }

  975.         fallback::MatrixMulImpl::KernSizeParam matmul_param =

  976.                 get_matmul_kern_param(param, m_ohw_tile_size, m_oc_tile_size);

  977.         bool matmulusable = m_matmul_algo->usable(matmul_param);

  978.         return matmulusable &&

  979.                (opr->param().format == param::ConvBias::Format::NCHW) &&

  980.                (param.filter_meta.spatial[0] == param.filter_meta.spatial[1] &&

  981.                 (param.filter_meta.spatial[0] <= 7)) &&

  982.                (param.filter_meta.dilation[0] ==

  983.                         param.filter_meta.dilation[1] &&

  984.                 param.filter_meta.dilation[0] == 1) &&

  985.                param.compute_mode == param::ConvBias::ComputeMode::DEFAULT;

  986.     }

  987.     MIDOUT_END();

  988.     return false;

  989. }

  990. 

  991. // vim: syntax=cpp.doxygen