OpenI
/
MegEngine
Not watched
1
0
Fork 0
Code
Releases 31 Wiki Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 48ac1e1abd
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '48ac1e1abd'
${ noResults }
MegEngine/dnn/src/fallback/conv_bias/im2col/strategy_default.cpp

210 lines
9.9 kB
Raw Blame History 

  1. /**

  2.  * \file dnn/src/fallback/conv_bias/im2col/strategy_default.cpp

  3.  * MegEngine is Licensed under the Apache License, Version 2.0 (the "License")

  4.  *

  5.  * Copyright (c) 2014-2020 Megvii Inc. All rights reserved.

  6.  *

  7.  * Unless required by applicable law or agreed to in writing,

  8.  * software distributed under the License is distributed on an

  9.  * "AS IS" BASIS, WITHOUT ARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

  10.  */

  11. #include "src/fallback/conv_bias/im2col/strategy_base.h"

  12. #include "src/fallback/convolution/img2col_helper.h"

  13. 

  14. namespace megdnn {

  15. 

  16. template <typename src_ctype, typename bias_ctype, typename dst_ctype,

  17.           typename op_ctype, typename op_dtype,

  18.           megdnn::PostprocessMode postprocess_mode>

  19. void Strategy<src_ctype, bias_ctype, dst_ctype, op_ctype, op_dtype,

  20.               postprocess_mode, PackMode::DEFAULT>::

  21.         packA_kern(WorkspaceBundle bundle,

  22.                    const fallback::ConvBiasImpl::NCBKernParam& param,

  23.                    fallback::MatrixMulImpl::KernSizeParam matmulparam,

  24.                    fallback::MatrixMulImpl::AlgoBase* matmul_algo,

  25.                    const fallback::ConvBiasImpl::NCBKernIndex& ncb_index,

  26.                    size_t) {

  27.     bundle.set(param.workspace_ptr);

  28.     fallback::MatrixMulImpl::KernParam matmul_param;

  29.     size_t group_id = ncb_index.ndrange_id[0];

  30.     static_cast<fallback::MatrixMulImpl::KernSizeParam&>(matmul_param) =

  31.             matmulparam;

  32.     size_t packA_group_size = matmul_algo->get_bundle(matmul_param).get_size(0);

  33.     size_t packed_per_oc_block_size =

  34.             round_up(matmul_param.K, matmul_algo->get_inner_block_size().k) *

  35.             matmul_algo->get_inner_block_size().m *

  36.             matmul_algo->get_packA_type_size();

  37.     size_t a_panel_offset = ncb_index.ndrange_id[1] * packed_per_oc_block_size;

  38.     int8_t* a_panel = static_cast<int8_t*>(bundle.get(BUNDLE_PACKA_INDEX)) +

  39.                       group_id * packA_group_size + a_panel_offset;

  40.     matmul_param.A_ptr =

  41.             const_cast<src_ctype*>(param.filter<src_ctype>(group_id));

  42.     matmul_algo->pack_A(matmul_param, a_panel, ncb_index.ndrange_id[1],

  43.                         matmul_algo->get_inner_block_size().m);

  44. }

  45. 

  46. template <typename src_ctype, typename bias_ctype, typename dst_ctype,

  47.           typename op_ctype, typename op_dtype,

  48.           megdnn::PostprocessMode postprocess_mode>

  49. void Strategy<src_ctype, bias_ctype, dst_ctype, op_ctype, op_dtype,

  50.               postprocess_mode, PackMode::DEFAULT>::

  51.         exec_im2col(WorkspaceBundle bundle, WorkspaceBundle bundle_thread,

  52.                     const StrategyParam& sparam,

  53.                     const fallback::ConvBiasImpl::NCBKernParam& param,

  54.                     fallback::MatrixMulImpl::KernParam matmul_param,

  55.                     fallback::MatrixMulImpl::AlgoBase* matmul_algo) {

  56.     size_t sh = param.filter_meta.stride[0];

  57.     size_t sw = param.filter_meta.stride[1];

  58.     size_t oc = param.filter_meta.ocpg;

  59.     size_t oh = param.osz[0];

  60.     size_t ow = param.osz[1];

  61.     size_t ic = param.filter_meta.icpg;

  62.     size_t ih = param.isz[0] + param.filter_meta.padding[0] * 2;

  63.     size_t iw = param.isz[1] + param.filter_meta.padding[1] * 2;

  64.     size_t fh = param.filter_meta.spatial[0];

  65.     size_t fw = param.filter_meta.spatial[1];

  66.     size_t is_xcorr = !param.filter_meta.should_flip;

  67. 

  68.     size_t input_offset =

  69.             ih * iw * ic *

  70.             (sparam.group_id + param.filter_meta.group * sparam.batch_id) *

  71.             sizeof(src_ctype);

  72. 

  73.     src_ctype* src2 = reinterpret_cast<src_ctype*>(

  74.             reinterpret_cast<uintptr_t>(bundle.get(BUNDLE_PADDING_INDEX)) +

  75.             input_offset);

  76.     bool is_phpwzero = param.filter_meta.padding[0] == 0 &&

  77.                        param.filter_meta.padding[1] == 0;

  78.     if (is_phpwzero) {

  79.         src2 = const_cast<src_ctype*>(

  80.                 param.src<src_ctype>(sparam.batch_id, sparam.group_id));

  81.     }

  82.     src_ctype* im2col_dst = static_cast<src_ctype*>(

  83.             bundle_thread.get(THREAD_BUNDLE_IM2COL_INDEX));

  84.     if (sh == 1 && sw == 1) {

  85.         if (is_xcorr) {

  86.             img2col<true>(src2, im2col_dst, oc, oh, ow, ic, ih, iw, fh, fw,

  87.                           sparam.ohw_cur_index, sparam.output_block_size);

  88.         } else {

  89.             img2col<false>(src2, im2col_dst, oc, oh, ow, ic, ih, iw, fh, fw,

  90.                            sparam.ohw_cur_index, sparam.output_block_size);

  91.         }

  92.     } else {

  93.         if (is_xcorr) {

  94.             img2col_stride<true>(src2, im2col_dst, oc, oh, ow, ic, ih, iw, fh,

  95.                                  fw, sh, sw, sparam.ohw_cur_index,

  96.                                  sparam.output_block_size);

  97.         } else {

  98.             img2col_stride<false>(src2, im2col_dst, oc, oh, ow, ic, ih, iw, fh,

  99.                                   fw, sh, sw, sparam.ohw_cur_index,

  100.                                   sparam.output_block_size);

  101.         }

  102.     }

  103.     matmul_param.M = sparam.output_block_oc_size;

  104.     matmul_param.N = sparam.output_block_size;

  105.     matmul_param.LDB = sparam.output_block_size;

  106.     matmul_param.LDC = sparam.output_block_size;

  107.     matmul_param.B_ptr = im2col_dst;

  108. 

  109.     src_ctype* b_panel =

  110.             reinterpret_cast<src_ctype*>(reinterpret_cast<uintptr_t>(

  111.                     bundle_thread.get(THREAD_BUNDLE_PACKB_INDEX)));

  112.     matmul_algo->pack_B(matmul_param, b_panel, 0, matmul_param.N);

  113. }

  114. 

  115. template <typename src_ctype, typename bias_ctype, typename dst_ctype,

  116.           typename op_ctype, typename op_dtype,

  117.           megdnn::PostprocessMode postprocess_mode>

  118. void* Strategy<src_ctype, bias_ctype, dst_ctype, op_ctype, op_dtype,

  119.                postprocess_mode, PackMode::DEFAULT>::

  120.         get_matmul_dst_ptr(const fallback::ConvBiasImpl::NCBKernParam& param,

  121.                            const WorkspaceBundle& bundle_thread,

  122.                            const StrategyParam& sparam) {

  123.     if (sparam.is_dst_8bit || !sparam.is_ohw_size_bigger) {

  124.         return static_cast<void*>(

  125.                 bundle_thread.get(THREAD_BUNDLE_IM2COL_INDEX));

  126.     } else {

  127.         bias_ctype* dst =

  128.                 param.dst<bias_ctype>(sparam.batch_id, sparam.group_id) +

  129.                 sparam.oc_cur_index * sparam.ohw;

  130.         return static_cast<void*>(dst);

  131.     }

  132. }

  133. 

  134. template <typename src_ctype, typename bias_ctype, typename dst_ctype,

  135.           typename op_ctype, typename op_dtype,

  136.           megdnn::PostprocessMode postprocess_mode>

  137. void Strategy<src_ctype, bias_ctype, dst_ctype, op_ctype, op_dtype,

  138.               postprocess_mode, PackMode::DEFAULT>::

  139.         exec_matmul(const fallback::ConvBiasImpl::NCBKernParam& param,

  140.                     const StrategyParam& sparam, WorkspaceBundle bundle,

  141.                     WorkspaceBundle bundle_thread,

  142.                     fallback::MatrixMulImpl::KernParam matmul_param,

  143.                     fallback::MatrixMulImpl::AlgoBase* matmul_algo,

  144.                     const fallback::ConvBiasImpl::NCBKernIndex& ncb_index) {

  145.     size_t packA_per_oc_block_size =

  146.             round_up(matmul_param.K, matmul_algo->get_inner_block_size().k) *

  147.             sparam.oc_tile_size * matmul_algo->get_packA_type_size();

  148.     size_t packA_group_size = matmul_algo->get_bundle(matmul_param).get_size(0);

  149.     size_t a_panel_offset = ncb_index.ndrange_id[1] * packA_group_size +

  150.                             ncb_index.ndrange_id[3] * packA_per_oc_block_size;

  151. 

  152.     void* matmul_dst = get_matmul_dst_ptr(param, bundle_thread, sparam);

  153. 

  154.     src_ctype* a_panel = reinterpret_cast<src_ctype*>(

  155.             reinterpret_cast<uintptr_t>(bundle.get(BUNDLE_PACKA_INDEX)) +

  156.             a_panel_offset);

  157.     src_ctype* b_panel =

  158.             reinterpret_cast<src_ctype*>(reinterpret_cast<uintptr_t>(

  159.                     bundle_thread.get(THREAD_BUNDLE_PACKB_INDEX)));

  160.     size_t pack_oc_size = sparam.pack_oc_size;

  161.     matmul_param.M = sparam.output_block_oc_size;

  162.     matmul_param.N = sparam.output_block_size;

  163.     matmul_param.LDB = pack_oc_size * sparam.output_block_size;

  164.     matmul_param.LDC = pack_oc_size * sparam.output_block_size;

  165.     matmul_param.C_ptr = matmul_dst;

  166. 

  167.     auto matmul_kern_naked = matmul_algo->get_kern_naked(matmul_param);

  168.     matmul_kern_naked(matmul_param, a_panel, b_panel);

  169. }

  170. 

  171. #define INSTANTIAL_CLASS(_src_ctype, _bias_ctype, _dst_ctype, _op_ctype,    \

  172.                          _op_dtype, _postprocess_mode)                      \

  173.     template class Strategy<_src_ctype, _bias_ctype, _dst_ctype, _op_ctype, \

  174.                             _op_dtype, _postprocess_mode, PackMode::DEFAULT>;

  175. 

  176. INSTANTIAL_CLASS(dt_float32, dt_float32, dt_float32, dt_float32, dt_float32,

  177.                  megdnn::PostprocessMode::FLOAT)

  178. 

  179. #if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC

  180. INSTANTIAL_CLASS(dt_float16, dt_float16, dt_float16, __fp16, __fp16,

  181.                  megdnn::PostprocessMode::FLOAT)

  182. #else

  183. #if !MEGDNN_DISABLE_FLOAT16

  184. INSTANTIAL_CLASS(dt_float16, dt_float16, dt_float16, dt_float16, dt_float16,

  185.                  megdnn::PostprocessMode::NO_PROCESS)

  186. #endif

  187. #endif

  188. 

  189. #if MEGDNN_AARCH64 || MEGDNN_ARMV7

  190. //! x86 do not have uint8 matmul so only armv7 armv8 support uint8

  191. INSTANTIAL_CLASS(dt_uint8, dt_int32, dt_uint8, dt_qint32, dt_quint8,

  192.                  megdnn::PostprocessMode::QUANTIZED)

  193. INSTANTIAL_CLASS(dt_uint8, dt_int32, dt_int32, dt_qint32, dt_qint32,

  194.                  megdnn::PostprocessMode::NO_PROCESS)

  195. #endif

  196. 

  197. INSTANTIAL_CLASS(dt_int8, dt_int32, dt_int8, dt_qint32, dt_qint8,

  198.                  megdnn::PostprocessMode::QUANTIZED)

  199. INSTANTIAL_CLASS(dt_int8, dt_int32, dt_int32, dt_int32, dt_int32,

  200.                  megdnn::PostprocessMode::NO_PROCESS)

  201. INSTANTIAL_CLASS(dt_int8, dt_int16, dt_int16, dt_int16, dt_int16,

  202.                  megdnn::PostprocessMode::NO_PROCESS)

  203. INSTANTIAL_CLASS(dt_int8, dt_int32, dt_int32, dt_qint32, dt_qint32,

  204.                  megdnn::PostprocessMode::NO_PROCESS)

  205. 

  206. #undef INSTANTIAL_CLASS

  207. }  // namespace megdnn

  208. 

  209. // vim: syntax=cpp.doxygen