OpenI
/
MegEngine
Not watched
1
0
Fork 0
Code
Releases 31 Wiki Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: a1877ee0fa
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'a1877ee0fa'
${ noResults }
MegEngine/dnn/src/arm_common/convolution/int8x8x32/conv_backdata_stride2.cpp

1274 lines
47 kB
Raw Blame History 

  1. /**

  2.  * \file dnn/src/arm_common/convolution/int8x8x32/conv_backdata_stride2.cpp

  3.  * MegEngine is Licensed under the Apache License, Version 2.0 (the "License")

  4.  *

  5.  * Copyright (c) 2014-2020 Megvii Inc. All rights reserved.

  6.  *

  7.  * Unless required by applicable law or agreed to in writing,

  8.  * software distributed under the License is distributed on an

  9.  * "AS IS" BASIS, WITHOUT ARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

  10.  */

  11. 

  12. #if __ARM_FEATURE_DOTPROD

  13. #include "src/arm_common/convolution/int8x8x32/conv_backdata_stride2.h"

  14. #include "src/common/utils.h"

  15. 

  16. #include <cstring>

  17. #include "src/arm_common/simd_macro/marm_neon.h"

  18. 

  19. using namespace megdnn;

  20. using namespace arm_common;

  21. using namespace deconv;

  22. 

  23. namespace {

  24. 

  25. bool need_dst_copy(const NCBKernSizeParam& param) {

  26.     if (param.osz[1] % 4 != 0) {

  27.         // If the size of output is not multiples of 4, we need to copy it.

  28.         return true;

  29.     }

  30.     return false;

  31. }

  32. 

  33. void get_rectified_size(size_t IH, size_t IW, size_t OH, size_t OW, size_t FH,

  34.                         size_t FW, size_t PH, size_t PW, size_t& IH2,

  35.                         size_t& IW2, size_t& OW2) {

  36.     MEGDNN_MARK_USED_VAR(OH);

  37.     MEGDNN_MARK_USED_VAR(IW);

  38.     //! OW should be a multiple of 4

  39.     OW2 = (OW + 3) & ~3;

  40.     IH2 = 2 * IH - 1 + 2 * (FH - PH - 1);

  41.     IW2 = (OW2 - FW + 2 * PW) / 2 + 1 + (FW - PW - 1) + 16;

  42. }

  43. 

  44. WorkspaceBundle get_bundle(const NCBKernSizeParam& param) {

  45.     UNPACK_CONV_F32_NCB_KERN_SIZES(param);

  46.     MEGDNN_MARK_USED_VAR(N);

  47.     MEGDNN_MARK_USED_VAR(OC);

  48.     MEGDNN_MARK_USED_VAR(SH);

  49.     MEGDNN_MARK_USED_VAR(SW);

  50.     size_t src_size = 0, dst_size = 0;

  51.     size_t IH2, IW2, OW2;

  52.     get_rectified_size(IH, IW, OH, OW, FH, FW, PH, PW, IH2, IW2, OW2);

  53.     src_size = sizeof(int8_t) * IH2 * IW2;

  54.     if (need_dst_copy(param)) {

  55.         dst_size = sizeof(int32_t) * IC * OH * OW2;

  56.     }

  57.     return WorkspaceBundle(nullptr, {src_size, dst_size});

  58. }

  59. 

  60. inline int8x16_t vqtbl1q_s8_common(int8x16_t a, uint8x16_t index) {

  61.     int8x8x2_t src;

  62.     src.val[0] = vget_low_s8(a);

  63.     src.val[1] = vget_high_s8(a);

  64.     uint8x8_t index_low = vget_low_u8(index);

  65.     uint8x8_t index_high = vget_high_u8(index);

  66.     int8x8_t r00 = vtbl2_s8(src, vreinterpret_s8_u8(index_low));

  67.     int8x8_t r01 = vtbl2_s8(src, vreinterpret_s8_u8(index_high));

  68.     int8x16_t r = vcombine_s8(r00, r01);

  69.     return r;

  70. }

  71. 

  72. #define CALC_0(_k_idx, _c_idx)                     \

  73.     _elem = vqtbl1q_s8_common(_tmp, _idx##_c_idx); \

  74.     _sum0##_c_idx = vdotq_s32(_sum0##_c_idx, _k##_k_idx, _elem);

  75. 

  76. #define CALC_1(_k_idx, _c_idx)                     \

  77.     _elem = vqtbl1q_s8_common(_tmp, _idx##_c_idx); \

  78.     _sum1##_c_idx = vdotq_s32(_sum1##_c_idx, _k##_k_idx, _elem);

  79. 

  80. #define CALC_2(_k1_idx, _k2_idx, _c_idx)                          \

  81.     _elem = vqtbl1q_s8_common(_tmp, _idx##_c_idx);                \

  82.     _sum0##_c_idx = vdotq_s32(_sum0##_c_idx, _k##_k1_idx, _elem); \

  83.     _sum1##_c_idx = vdotq_s32(_sum1##_c_idx, _k##_k2_idx, _elem);

  84. 

  85. template <bool even>

  86. void deconv_direct_2x2(const int8_t* src, const int8_t* filter, int32_t* dst,

  87.                        size_t IH, size_t IW, size_t OH, size_t OW, size_t IC) {

  88.     MEGDNN_MARK_USED_VAR(IH);

  89.     const size_t tail_step = IW - OW / 2;

  90. 

  91.     const uint8x16_t _idx0 = {0, 1, 16, 16, 1, 2, 16, 16,

  92.                               2, 3, 16, 16, 3, 4, 16, 16};

  93.     const uint8x16_t _idx1 = {4, 5, 16, 16, 5, 6, 16, 16,

  94.                               6, 7, 16, 16, 7, 8, 16, 16};

  95.     uint8x16_t _idx_r_0, _idx_r_1;

  96.     if (even) {

  97.         _idx_r_0 = {0, 16, 1, 16, 2, 16, 3, 16, 4, 16, 5, 16, 6, 16, 7, 16};

  98.         _idx_r_1 = {16, 1, 16, 2, 16, 3, 16, 4, 16, 5, 16, 6, 16, 7, 16, 8};

  99.     } else {

  100.         _idx_r_0 = {16, 0, 16, 1, 16, 2, 16, 3, 16, 4, 16, 5, 16, 6, 16, 7};

  101.         _idx_r_1 = {0, 16, 1, 16, 2, 16, 3, 16, 4, 16, 5, 16, 6, 16, 7, 16};

  102.     }

  103.     rep(ic, IC) {

  104.         const int8_t* src_ptr = src;

  105.         int32_t* dst_ptr = dst + OW * OH * ic;

  106.         int32_t* outptr = dst_ptr;

  107.         int32_t* outptr2 = dst_ptr + OW;

  108. 

  109.         const int8_t* r0 = src_ptr;

  110.         const int8_t* r1 = src_ptr + IW;

  111.         const int8_t* r2 = src_ptr + 2 * IW;

  112. 

  113.         const int8_t* k0 = filter;

  114. 

  115.         int8x16_t _k0 = vreinterpretq_s8_s32(

  116.                 vdupq_n_s32(*reinterpret_cast<const int32_t*>(k0)));

  117.         uint8x16_t _idx_k = {3, 2, 1, 0, 3, 2, 1, 0, 3, 2, 1, 0, 3, 2, 1, 0};

  118.         int8x16_t _k = vqtbl1q_s8_common(_k0, _idx_k);

  119.         uint8x16_t _idx = {0, 1, 16, 16, 0, 1, 16, 16,

  120.                            0, 1, 16, 16, 0, 1, 16, 16};

  121.         int8x16_t _k1 = vqtbl1q_s8_common(_k, _idx);

  122.         _idx = {2, 3, 16, 16, 2, 3, 16, 16, 2, 3, 16, 16, 2, 3, 16, 16};

  123.         int8x16_t _k23 = vqtbl1q_s8_common(_k, _idx);

  124. 

  125.         int8x16_t _tmp, _elem;

  126.         const int width = OW >> 2;

  127.         size_t h = 0;

  128.         for (; h + 1 < OH; h += 2) {

  129.             int w = 0;

  130.             for (; w + 4 < width; w += 4) {

  131.                 int32x4x2_t _sum00, _sum01, _sum10, _sum11;

  132.                 _sum00 = vld2q_s32(outptr);

  133.                 _sum01 = vld2q_s32(outptr + 8);

  134.                 _sum10 = vld2q_s32(outptr2);

  135.                 _sum11 = vld2q_s32(outptr2 + 8);

  136. 

  137.                 int8x16_t _r0_ori = vld1q_s8(r0);

  138.                 int8x16_t _r00 = vqtbl1q_s8_common(_r0_ori, _idx_r_0);

  139.                 int8x16_t _r01 = vqtbl1q_s8_common(_r0_ori, _idx_r_1);

  140.                 int8x16_t _r1_ori = vld1q_s8(r1);

  141.                 int8x16_t _r10 = vqtbl1q_s8_common(_r1_ori, _idx_r_0);

  142.                 int8x16_t _r11 = vqtbl1q_s8_common(_r1_ori, _idx_r_1);

  143.                 int8x16_t _r2_ori = vld1q_s8(r2);

  144.                 int8x16_t _r20 = vqtbl1q_s8_common(_r2_ori, _idx_r_0);

  145.                 int8x16_t _r21 = vqtbl1q_s8_common(_r2_ori, _idx_r_1);

  146. 

  147.                 int16x8x2_t r_00 = vzipq_s16(vreinterpretq_s16_s8(_r00),

  148.                                              vreinterpretq_s16_s8(_r10));

  149.                 int8x16_t _r0 = r_00.val[0];

  150.                 int8x16_t _r2 = r_00.val[1];

  151. 

  152.                 int16x8x2_t r_11 = vzipq_s16(vreinterpretq_s16_s8(_r01),

  153.                                              vreinterpretq_s16_s8(_r11));

  154.                 int8x16_t _r1 = r_11.val[0];

  155.                 int8x16_t _r3 = r_11.val[1];

  156. 

  157.                 _sum00.val[0] = vdotq_s32(_sum00.val[0], _k, _r0);

  158.                 _sum00.val[1] = vdotq_s32(_sum00.val[1], _k, _r1);

  159.                 _sum01.val[0] = vdotq_s32(_sum01.val[0], _k, _r2);

  160.                 _sum01.val[1] = vdotq_s32(_sum01.val[1], _k, _r3);

  161. 

  162.                 r_00 = vzipq_s16(vreinterpretq_s16_s8(_r10),

  163.                                  vreinterpretq_s16_s8(_r20));

  164.                 _r0 = r_00.val[0];

  165.                 _r2 = r_00.val[1];

  166. 

  167.                 r_11 = vzipq_s16(vreinterpretq_s16_s8(_r11),

  168.                                  vreinterpretq_s16_s8(_r21));

  169.                 _r1 = r_11.val[0];

  170.                 _r3 = r_11.val[1];

  171. 

  172.                 _sum10.val[0] = vdotq_s32(_sum10.val[0], _k, _r0);

  173.                 _sum10.val[1] = vdotq_s32(_sum10.val[1], _k, _r1);

  174.                 _sum11.val[0] = vdotq_s32(_sum11.val[0], _k, _r2);

  175.                 _sum11.val[1] = vdotq_s32(_sum11.val[1], _k, _r3);

  176. 

  177.                 vst2q_s32(outptr, _sum00);

  178.                 vst2q_s32(outptr + 8, _sum01);

  179.                 vst2q_s32(outptr2, _sum10);

  180.                 vst2q_s32(outptr2 + 8, _sum11);

  181. 

  182.                 r0 += 8;

  183.                 r1 += 8;

  184.                 r2 += 8;

  185.                 outptr += 16;

  186.                 outptr2 += 16;

  187.             }

  188.             for (; w + 2 < width; w += 2) {

  189.                 int32x4_t _sum00 = vld1q_s32(outptr);

  190.                 int32x4_t _sum01 = vld1q_s32(outptr + 4);

  191.                 int32x4_t _sum10 = vld1q_s32(outptr2);

  192.                 int32x4_t _sum11 = vld1q_s32(outptr2 + 4);

  193. 

  194.                 int8x16_t _r_ori = vld1q_s8(r0);

  195.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  196.                 CALC_0(1, 0);

  197.                 CALC_0(1, 1);

  198. 

  199.                 _r_ori = vld1q_s8(r1);

  200.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  201.                 CALC_2(23, 1, 0);

  202.                 CALC_2(23, 1, 1);

  203. 

  204.                 _r_ori = vld1q_s8(r2);

  205.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  206.                 CALC_1(23, 0);

  207.                 CALC_1(23, 1);

  208. 

  209.                 vst1q_s32(outptr, _sum00);

  210.                 vst1q_s32(outptr + 4, _sum01);

  211.                 vst1q_s32(outptr2, _sum10);

  212.                 vst1q_s32(outptr2 + 4, _sum11);

  213. 

  214.                 r0 += 4;

  215.                 r1 += 4;

  216.                 r2 += 4;

  217.                 outptr += 8;

  218.                 outptr2 += 8;

  219.             }

  220. 

  221.             for (; w < width; w++) {

  222.                 int32x4_t _sum00 = vld1q_s32(outptr);

  223.                 int32x4_t _sum10 = vld1q_s32(outptr2);

  224. 

  225.                 int8x16_t _r_ori = vtranslq_s8(vld1_s8(r0));

  226.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  227.                 CALC_0(1, 0);

  228. 

  229.                 _r_ori = vtranslq_s8(vld1_s8(r1));

  230.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  231.                 CALC_2(23, 1, 0);

  232. 

  233.                 _r_ori = vtranslq_s8(vld1_s8(r2));

  234.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  235.                 CALC_1(23, 0);

  236. 

  237.                 vst1q_s32(outptr, _sum00);

  238.                 vst1q_s32(outptr2, _sum10);

  239. 

  240.                 r0 += 2;

  241.                 r1 += 2;

  242.                 r2 += 2;

  243.                 outptr += 4;

  244.                 outptr2 += 4;

  245.             }

  246.             r0 += tail_step + IW;

  247.             r1 += tail_step + IW;

  248.             r2 += tail_step + IW;

  249.             outptr += OW;

  250.             outptr2 += OW;

  251.         }

  252. 

  253.         for (; h < OH; h++) {

  254.             int w = 0;

  255.             for (; w + 4 < width; w += 4) {

  256.                 int32x4x2_t _sum0, _sum1;

  257.                 _sum0 = vld2q_s32(outptr);

  258.                 _sum1 = vld2q_s32(outptr + 8);

  259. 

  260.                 int8x16_t _r0_ori = vld1q_s8(r0);

  261.                 int8x16_t _r00 = vqtbl1q_s8_common(_r0_ori, _idx_r_0);

  262.                 int8x16_t _r01 = vqtbl1q_s8_common(_r0_ori, _idx_r_1);

  263.                 int8x16_t _r1_ori = vld1q_s8(r1);

  264.                 int8x16_t _r10 = vqtbl1q_s8_common(_r1_ori, _idx_r_0);

  265.                 int8x16_t _r11 = vqtbl1q_s8_common(_r1_ori, _idx_r_1);

  266. 

  267.                 int16x8x2_t r_00 = vzipq_s16(vreinterpretq_s16_s8(_r00),

  268.                                              vreinterpretq_s16_s8(_r10));

  269.                 int8x16_t _r0 = r_00.val[0];

  270.                 int8x16_t _r2 = r_00.val[1];

  271. 

  272.                 int16x8x2_t r_11 = vzipq_s16(vreinterpretq_s16_s8(_r01),

  273.                                              vreinterpretq_s16_s8(_r11));

  274.                 int8x16_t _r1 = r_11.val[0];

  275.                 int8x16_t _r3 = r_11.val[1];

  276. 

  277.                 _sum0.val[0] = vdotq_s32(_sum0.val[0], _k, _r0);

  278.                 _sum0.val[1] = vdotq_s32(_sum0.val[1], _k, _r1);

  279.                 _sum1.val[0] = vdotq_s32(_sum1.val[0], _k, _r2);

  280.                 _sum1.val[1] = vdotq_s32(_sum1.val[1], _k, _r3);

  281. 

  282.                 vst2q_s32(outptr, _sum0);

  283.                 vst2q_s32(outptr + 8, _sum1);

  284. 

  285.                 r0 += 8;

  286.                 r1 += 8;

  287.                 outptr += 16;

  288.             }

  289.             for (; w + 2 < width; w += 2) {

  290.                 int32x4_t _sum00 = vld1q_s32(outptr);

  291.                 int32x4_t _sum01 = vld1q_s32(outptr + 4);

  292. 

  293.                 int8x16_t _r_ori = vld1q_s8(r0);

  294.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  295.                 CALC_0(1, 0);

  296.                 CALC_0(1, 1);

  297. 

  298.                 _r_ori = vld1q_s8(r1);

  299.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  300.                 CALC_0(23, 0);

  301.                 CALC_0(23, 1);

  302. 

  303.                 vst1q_s32(outptr, _sum00);

  304.                 vst1q_s32(outptr + 4, _sum01);

  305. 

  306.                 r0 += 4;

  307.                 r1 += 4;

  308.                 outptr += 8;

  309.             }

  310. 

  311.             for (; w < width; w++) {

  312.                 int32x4_t _sum00 = vld1q_s32(outptr);

  313. 

  314.                 int8x16_t _r_ori = vtranslq_s8(vld1_s8(r0));

  315.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  316.                 CALC_0(1, 0);

  317. 

  318.                 _r_ori = vtranslq_s8(vld1_s8(r1));

  319.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  320.                 CALC_0(23, 0);

  321. 

  322.                 vst1q_s32(outptr, _sum00);

  323. 

  324.                 r0 += 2;

  325.                 r1 += 2;

  326.                 outptr += 4;

  327.             }

  328.             r0 += tail_step;

  329.             r1 += tail_step;

  330.         }

  331. 

  332.         filter += 4;

  333.     }

  334. }

  335. 

  336. template <bool even>

  337. void deconv_direct_3x3(const int8_t* src, const int8_t* filter, int32_t* dst,

  338.                        size_t IH, size_t IW, size_t OH, size_t OW, size_t IC) {

  339.     MEGDNN_MARK_USED_VAR(IH);

  340.     const size_t tail_step = IW - OW / 2;

  341. 

  342.     const uint8x16_t _idx0 = {0, 1, 2, 16, 1, 2, 3, 16,

  343.                               2, 3, 4, 16, 3, 4, 5, 16};

  344.     const uint8x16_t _idx1 = {4, 5, 6, 16, 5, 6, 7, 16,

  345.                               6, 7, 8, 16, 7, 8, 9, 16};

  346.     const uint8x16_t _idx2 = {8,  9,  10, 16, 9,  10, 11, 16,

  347.                               10, 11, 12, 16, 11, 12, 13, 16};

  348. 

  349.     uint8x16_t _idx_r_0;

  350.     if (even) {

  351.         _idx_r_0 = {0, 16, 1, 16, 2, 16, 3, 16, 4, 16, 5, 16, 6, 16, 7, 16};

  352.     } else {

  353.         _idx_r_0 = {16, 0, 16, 1, 16, 2, 16, 3, 16, 4, 16, 5, 16, 6, 16, 7};

  354.     }

  355.     rep(ic, IC) {

  356.         const int8_t* src_ptr = src;

  357.         int32_t* dst_ptr = dst + OW * OH * ic;

  358.         int32_t* outptr = dst_ptr;

  359.         int32_t* outptr2 = outptr + OW;

  360. 

  361.         const int8_t* r0 = src_ptr;

  362.         const int8_t* r1 = src_ptr + IW;

  363.         const int8_t* r2 = src_ptr + IW * 2;

  364.         const int8_t* r3 = src_ptr + IW * 3;

  365. 

  366.         const int8_t* k0 = filter;

  367. 

  368.         int8x16_t _k_tmp = vcombine_s8(vld1_s8(k0), vdup_n_s8(k0[8]));

  369.         uint8x16_t _idx = {8, 7, 6, 16, 8, 7, 6, 16, 8, 7, 6, 16, 8, 7, 6, 16};

  370.         int8x16_t _k12 = vqtbl1q_s8_common(_k_tmp, _idx);

  371.         _idx = {5, 4, 3, 16, 5, 4, 3, 16, 5, 4, 3, 16, 5, 4, 3, 16};

  372.         int8x16_t _k345 = vqtbl1q_s8_common(_k_tmp, _idx);

  373.         _idx = {2, 1, 0, 16, 2, 1, 0, 16, 2, 1, 0, 16, 2, 1, 0, 16};

  374.         int8x16_t _k678 = vqtbl1q_s8_common(_k_tmp, _idx);

  375. 

  376.         int8x16_t _tmp, _elem;

  377.         size_t h = 0;

  378.         for (; h + 1 < OH; h += 2) {

  379.             int width = OW >> 2;

  380. 

  381.             int w = 0;

  382.             for (; w + 3 < width; w += 3) {

  383.                 //! As the inner kernel read 16 elements, and IW is times of 16

  384.                 int32x4_t _sum00 = vld1q_s32(outptr);

  385.                 int32x4_t _sum01 = vld1q_s32(outptr + 4);

  386.                 int32x4_t _sum02 = vld1q_s32(outptr + 8);

  387.                 int32x4_t _sum10 = vld1q_s32(outptr2);

  388.                 int32x4_t _sum11 = vld1q_s32(outptr2 + 4);

  389.                 int32x4_t _sum12 = vld1q_s32(outptr2 + 8);

  390. 

  391.                 int8x16_t _r_ori = vld1q_s8(r0);

  392.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  393.                 CALC_0(12, 0);

  394.                 CALC_0(12, 1);

  395.                 CALC_0(12, 2);

  396. 

  397.                 _r_ori = vld1q_s8(r1);

  398.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  399.                 CALC_2(345, 12, 0);

  400.                 CALC_2(345, 12, 1);

  401.                 CALC_2(345, 12, 2);

  402. 

  403.                 _r_ori = vld1q_s8(r2);

  404.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  405.                 CALC_2(678, 345, 0);

  406.                 CALC_2(678, 345, 1);

  407.                 CALC_2(678, 345, 2);

  408. 

  409.                 _r_ori = vld1q_s8(r3);

  410.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  411.                 CALC_1(678, 0);

  412.                 CALC_1(678, 1);

  413.                 CALC_1(678, 2);

  414. 

  415.                 vst1q_s32(outptr, _sum00);

  416.                 vst1q_s32(outptr + 4, _sum01);

  417.                 vst1q_s32(outptr + 8, _sum02);

  418.                 vst1q_s32(outptr2, _sum10);

  419.                 vst1q_s32(outptr2 + 4, _sum11);

  420.                 vst1q_s32(outptr2 + 8, _sum12);

  421. 

  422.                 r0 += 6;

  423.                 r1 += 6;

  424.                 r2 += 6;

  425.                 r3 += 6;

  426.                 outptr += 12;

  427.                 outptr2 += 12;

  428.             }

  429.             for (; w < width; w++) {

  430.                 int32x4_t _sum00 = vld1q_s32(outptr);

  431.                 int32x4_t _sum10 = vld1q_s32(outptr2);

  432. 

  433.                 int8x16_t _r_ori = vtranslq_s8(vld1_s8(r0));

  434.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  435.                 CALC_0(12, 0);

  436. 

  437.                 _r_ori = vtranslq_s8(vld1_s8(r1));

  438.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  439.                 CALC_2(345, 12, 0);

  440. 

  441.                 _r_ori = vtranslq_s8(vld1_s8(r2));

  442.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  443.                 CALC_2(678, 345, 0);

  444. 

  445.                 _r_ori = vtranslq_s8(vld1_s8(r3));

  446.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  447.                 CALC_1(678, 0);

  448. 

  449.                 vst1q_s32(outptr, _sum00);

  450.                 vst1q_s32(outptr2, _sum10);

  451. 

  452.                 r0 += 2;

  453.                 r1 += 2;

  454.                 r2 += 2;

  455.                 r3 += 2;

  456.                 outptr += 4;

  457.                 outptr2 += 4;

  458.             }

  459. 

  460.             r0 += tail_step + IW;

  461.             r1 += tail_step + IW;

  462.             r2 += tail_step + IW;

  463.             r3 += tail_step + IW;

  464. 

  465.             outptr += OW;

  466.             outptr2 += OW;

  467.         }

  468. 

  469.         for (; h < OH; h++) {

  470.             int width = OW >> 2;

  471. 

  472.             int w = 0;

  473.             for (; w + 3 < width; w += 3) {

  474.                 int32x4_t _sum00 = vld1q_s32(outptr);

  475.                 int32x4_t _sum01 = vld1q_s32(outptr + 4);

  476.                 int32x4_t _sum02 = vld1q_s32(outptr + 8);

  477. 

  478.                 int8x16_t _r_ori = vld1q_s8(r0);

  479.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  480.                 CALC_0(12, 0);

  481.                 CALC_0(12, 1);

  482.                 CALC_0(12, 2);

  483. 

  484.                 _r_ori = vld1q_s8(r1);

  485.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  486.                 CALC_0(345, 0);

  487.                 CALC_0(345, 1);

  488.                 CALC_0(345, 2);

  489. 

  490.                 _r_ori = vld1q_s8(r2);

  491.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  492.                 CALC_0(678, 0);

  493.                 CALC_0(678, 1);

  494.                 CALC_0(678, 2);

  495. 

  496.                 vst1q_s32(outptr, _sum00);

  497.                 vst1q_s32(outptr + 4, _sum01);

  498.                 vst1q_s32(outptr + 8, _sum02);

  499. 

  500.                 r0 += 6;

  501.                 r1 += 6;

  502.                 r2 += 6;

  503.                 outptr += 12;

  504.             }

  505.             for (; w < width; w++) {

  506.                 int32x4_t _sum00 = vld1q_s32(outptr);

  507. 

  508.                 int8x16_t _r_ori = vtranslq_s8(vld1_s8(r0));

  509.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  510.                 CALC_0(12, 0);

  511. 

  512.                 _r_ori = vtranslq_s8(vld1_s8(r1));

  513.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  514.                 CALC_0(345, 0);

  515. 

  516.                 _r_ori = vtranslq_s8(vld1_s8(r2));

  517.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  518.                 CALC_0(678, 0);

  519. 

  520.                 vst1q_s32(outptr, _sum00);

  521. 

  522.                 r0 += 2;

  523.                 r1 += 2;

  524.                 r2 += 2;

  525.                 outptr += 4;

  526.             }

  527.             r0 += tail_step;

  528.             r1 += tail_step;

  529.             r2 += tail_step;

  530.         }

  531. 

  532.         filter += 9;

  533.     }

  534. }

  535. 

  536. #undef CALC_0

  537. #undef CALC_1

  538. #undef CALC_2

  539. 

  540. #define CALC_0(_k00_idx, _k01_idx, _c_idx)                         \

  541.     _elem = vqtbl1q_s8_common(_tmp, _idx##_c_idx##0);              \

  542.     _sum0##_c_idx = vdotq_s32(_sum0##_c_idx, _k##_k00_idx, _elem); \

  543.     _elem = vqtbl1q_s8_common(_tmp, _idx##_c_idx##1);              \

  544.     _sum0##_c_idx = vdotq_s32(_sum0##_c_idx, _k##_k01_idx, _elem);

  545. 

  546. #define CALC_1(_k00_idx, _k01_idx, _c_idx)                         \

  547.     _elem = vqtbl1q_s8_common(_tmp, _idx##_c_idx##0);              \

  548.     _sum1##_c_idx = vdotq_s32(_sum1##_c_idx, _k##_k00_idx, _elem); \

  549.     _elem = vqtbl1q_s8_common(_tmp, _idx##_c_idx##1);              \

  550.     _sum1##_c_idx = vdotq_s32(_sum1##_c_idx, _k##_k01_idx, _elem);

  551. 

  552. #define CALC_2(_k00_idx, _k01_idx, _k10_idx, _k11_idx, _c_idx)     \

  553.     _elem = vqtbl1q_s8_common(_tmp, _idx##_c_idx##0);              \

  554.     _sum0##_c_idx = vdotq_s32(_sum0##_c_idx, _k##_k00_idx, _elem); \

  555.     _sum1##_c_idx = vdotq_s32(_sum1##_c_idx, _k##_k10_idx, _elem); \

  556.     _elem = vqtbl1q_s8_common(_tmp, _idx##_c_idx##1);              \

  557.     _sum0##_c_idx = vdotq_s32(_sum0##_c_idx, _k##_k01_idx, _elem); \

  558.     _sum1##_c_idx = vdotq_s32(_sum1##_c_idx, _k##_k11_idx, _elem);

  559. 

  560. template <bool even>

  561. void deconv_direct_5x5(const int8_t* src, const int8_t* filter, int32_t* dst,

  562.                        size_t IH, size_t IW, size_t OH, size_t OW, size_t IC) {

  563.     MEGDNN_MARK_USED_VAR(IH);

  564.     const size_t tail_step = IW - OW / 2;

  565. 

  566.     const uint8x16_t _idx00 = {0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6};

  567.     const uint8x16_t _idx01 = {4, 16, 16, 16, 5, 16, 16, 16,

  568.                                6, 16, 16, 16, 7, 16, 16, 16};

  569.     const uint8x16_t _idx10 = {4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10};

  570.     const uint8x16_t _idx11 = {8,  16, 16, 16, 9,  16, 16, 16,

  571.                                10, 16, 16, 16, 11, 16, 16, 16};

  572.     const uint8x16_t _idx20 = {8,  9,  10, 11, 9,  10, 11, 12,

  573.                                10, 11, 12, 13, 11, 12, 13, 14};

  574.     const uint8x16_t _idx21 = {12, 16, 16, 16, 13, 16, 16, 16,

  575.                                14, 16, 16, 16, 15, 16, 16, 16};

  576. 

  577.     uint8x16_t _idx_r_0;

  578.     if (even) {

  579.         _idx_r_0 = {0, 16, 1, 16, 2, 16, 3, 16, 4, 16, 5, 16, 6, 16, 7, 16};

  580.     } else {

  581.         _idx_r_0 = {16, 0, 16, 1, 16, 2, 16, 3, 16, 4, 16, 5, 16, 6, 16, 7};

  582.     }

  583.     int8x16_t _tmp, _elem;

  584.     rep(ic, IC) {

  585.         const int8_t* src_ptr = src;

  586.         int32_t* dst_ptr = dst + OW * OH * ic;

  587.         int32_t* outptr = dst_ptr;

  588.         int32_t* outptr2 = outptr + OW;

  589. 

  590.         const int8_t* r0 = src_ptr;

  591.         const int8_t* r1 = src_ptr + IW;

  592.         const int8_t* r2 = src_ptr + IW * 2;

  593.         const int8_t* r3 = src_ptr + IW * 3;

  594.         const int8_t* r4 = src_ptr + IW * 4;

  595.         const int8_t* r5 = src_ptr + IW * 5;

  596. 

  597.         const int8_t* k0 = filter;

  598. 

  599.         int8x16_t _k = vld1q_s8(k0 + 9);

  600.         //! filter row 1

  601.         uint8x16_t _idx = {15, 14, 13, 12, 15, 14, 13, 12,

  602.                            15, 14, 13, 12, 15, 14, 13, 12};

  603.         int8x16_t _k123 = vqtbl1q_s8_common(_k, _idx);

  604.         _idx = {11, 16, 16, 16, 11, 16, 16, 16, 11, 16, 16, 16, 11, 16, 16, 16};

  605.         int8x16_t _k4 = vqtbl1q_s8_common(_k, _idx);

  606.         //! filter row 2

  607.         _idx = {10, 9, 8, 7, 10, 9, 8, 7, 10, 9, 8, 7, 10, 9, 8, 7};

  608.         int8x16_t _k5678 = vqtbl1q_s8_common(_k, _idx);

  609.         _idx = {6, 16, 16, 16, 6, 16, 16, 16, 6, 16, 16, 16, 6, 16, 16, 16};

  610.         int8x16_t _k9 = vqtbl1q_s8_common(_k, _idx);

  611.         //! filter row 3

  612.         _idx = {5, 4, 3, 2, 5, 4, 3, 2, 5, 4, 3, 2, 5, 4, 3, 2};

  613.         int8x16_t _k10111213 = vqtbl1q_s8_common(_k, _idx);

  614.         _idx = {1, 16, 16, 16, 1, 16, 16, 16, 1, 16, 16, 16, 1, 16, 16, 16};

  615.         int8x16_t _k14 = vqtbl1q_s8_common(_k, _idx);

  616.         //! 9 10 11 12 -> 13 14 15 16 -> 17 18 19 20 -> 21 22 23 24

  617.         _k = vld1q_s8(k0);

  618.         //! filter row 4

  619.         _idx = {9, 8, 7, 6, 9, 8, 7, 6, 9, 8, 7, 6, 9, 8, 7, 6};

  620.         int8x16_t _k15161718 = vqtbl1q_s8_common(_k, _idx);

  621.         _idx = {5, 16, 16, 16, 5, 16, 16, 16, 5, 16, 16, 16, 5, 16, 16, 16};

  622.         int8x16_t _k19 = vqtbl1q_s8_common(_k, _idx);

  623.         //! filter row 5

  624.         _idx = {4, 3, 2, 1, 4, 3, 2, 1, 4, 3, 2, 1, 4, 3, 2, 1};

  625.         int8x16_t _k20212223 = vqtbl1q_s8_common(_k, _idx);

  626.         _idx = {0, 16, 16, 16, 0, 16, 16, 16, 0, 16, 16, 16, 0, 16, 16, 16};

  627.         int8x16_t _k24 = vqtbl1q_s8_common(_k, _idx);

  628. 

  629.         const int width = OW >> 2;

  630.         size_t h = 0;

  631.         for (; h + 1 < OH; h += 2) {

  632.             int w = 0;

  633.             for (; w + 3 < width; w += 3) {

  634.                 //! As the inner kernel read 16 elements, and IW is times of 16

  635.                 int32x4_t _sum00 = vld1q_s32(outptr);

  636.                 int32x4_t _sum01 = vld1q_s32(outptr + 4);

  637.                 int32x4_t _sum02 = vld1q_s32(outptr + 8);

  638.                 int32x4_t _sum10 = vld1q_s32(outptr2);

  639.                 int32x4_t _sum11 = vld1q_s32(outptr2 + 4);

  640.                 int32x4_t _sum12 = vld1q_s32(outptr2 + 8);

  641. 

  642.                 int8x16_t _r_ori = vld1q_s8(r0);

  643.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  644.                 CALC_0(123, 4, 0);

  645.                 CALC_0(123, 4, 1);

  646.                 CALC_0(123, 4, 2);

  647. 

  648.                 _r_ori = vld1q_s8(r1);

  649.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  650.                 CALC_2(5678, 9, 123, 4, 0);

  651.                 CALC_2(5678, 9, 123, 4, 1);

  652.                 CALC_2(5678, 9, 123, 4, 2);

  653. 

  654.                 _r_ori = vld1q_s8(r2);

  655.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  656.                 CALC_2(10111213, 14, 5678, 9, 0);

  657.                 CALC_2(10111213, 14, 5678, 9, 1);

  658.                 CALC_2(10111213, 14, 5678, 9, 2);

  659. 

  660.                 _r_ori = vld1q_s8(r3);

  661.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  662.                 CALC_2(15161718, 19, 10111213, 14, 0);

  663.                 CALC_2(15161718, 19, 10111213, 14, 1);

  664.                 CALC_2(15161718, 19, 10111213, 14, 2);

  665. 

  666.                 _r_ori = vld1q_s8(r4);

  667.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  668.                 CALC_2(20212223, 24, 15161718, 19, 0);

  669.                 CALC_2(20212223, 24, 15161718, 19, 1);

  670.                 CALC_2(20212223, 24, 15161718, 19, 2);

  671. 

  672.                 _r_ori = vld1q_s8(r5);

  673.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  674.                 CALC_1(20212223, 24, 0);

  675.                 CALC_1(20212223, 24, 1);

  676.                 CALC_1(20212223, 24, 2);

  677. 

  678.                 vst1q_s32(outptr, _sum00);

  679.                 vst1q_s32(outptr + 4, _sum01);

  680.                 vst1q_s32(outptr + 8, _sum02);

  681.                 vst1q_s32(outptr2, _sum10);

  682.                 vst1q_s32(outptr2 + 4, _sum11);

  683.                 vst1q_s32(outptr2 + 8, _sum12);

  684. 

  685.                 r0 += 6;

  686.                 r1 += 6;

  687.                 r2 += 6;

  688.                 r3 += 6;

  689.                 r4 += 6;

  690.                 r5 += 6;

  691.                 outptr += 12;

  692.                 outptr2 += 12;

  693.             }

  694.             for (; w < width; w++) {

  695.                 int32x4_t _sum00 = vld1q_s32(outptr);

  696.                 int32x4_t _sum10 = vld1q_s32(outptr2);

  697. 

  698.                 int8x16_t _r_ori = vtranslq_s8(vld1_s8(r0));

  699.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  700.                 CALC_0(123, 4, 0);

  701. 

  702.                 _r_ori = vtranslq_s8(vld1_s8(r1));

  703.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  704.                 CALC_2(5678, 9, 123, 4, 0);

  705. 

  706.                 _r_ori = vtranslq_s8(vld1_s8(r2));

  707.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  708.                 CALC_2(10111213, 14, 5678, 9, 0);

  709. 

  710.                 _r_ori = vtranslq_s8(vld1_s8(r3));

  711.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  712.                 CALC_2(15161718, 19, 10111213, 14, 0);

  713. 

  714.                 _r_ori = vtranslq_s8(vld1_s8(r4));

  715.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  716.                 CALC_2(20212223, 24, 15161718, 19, 0);

  717. 

  718.                 _r_ori = vtranslq_s8(vld1_s8(r5));

  719.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  720.                 CALC_1(20212223, 24, 0);

  721. 

  722.                 vst1q_s32(outptr, _sum00);

  723.                 vst1q_s32(outptr2, _sum10);

  724. 

  725.                 r0 += 2;

  726.                 r1 += 2;

  727.                 r2 += 2;

  728.                 r3 += 2;

  729.                 r4 += 2;

  730.                 r5 += 2;

  731.                 outptr += 4;

  732.                 outptr2 += 4;

  733.             }

  734. 

  735.             r0 += tail_step + IW;

  736.             r1 += tail_step + IW;

  737.             r2 += tail_step + IW;

  738.             r3 += tail_step + IW;

  739.             r4 += tail_step + IW;

  740.             r5 += tail_step + IW;

  741. 

  742.             outptr += OW;

  743.             outptr2 += OW;

  744.         }

  745. 

  746.         for (; h < OH; h++) {

  747.             int w = 0;

  748.             for (; w + 3 < width; w += 3) {

  749.                 int32x4_t _sum00 = vld1q_s32(outptr);

  750.                 int32x4_t _sum01 = vld1q_s32(outptr + 4);

  751.                 int32x4_t _sum02 = vld1q_s32(outptr + 8);

  752. 

  753.                 int8x16_t _r_ori = vld1q_s8(r0);

  754.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  755.                 CALC_0(123, 4, 0);

  756.                 CALC_0(123, 4, 1);

  757.                 CALC_0(123, 4, 2);

  758. 

  759.                 _r_ori = vld1q_s8(r1);

  760.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  761.                 CALC_0(5678, 9, 0);

  762.                 CALC_0(5678, 9, 1);

  763.                 CALC_0(5678, 9, 2);

  764. 

  765.                 _r_ori = vld1q_s8(r2);

  766.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  767.                 CALC_0(10111213, 14, 0);

  768.                 CALC_0(10111213, 14, 1);

  769.                 CALC_0(10111213, 14, 2);

  770. 

  771.                 _r_ori = vld1q_s8(r3);

  772.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  773.                 CALC_0(15161718, 19, 0);

  774.                 CALC_0(15161718, 19, 1);

  775.                 CALC_0(15161718, 19, 2);

  776. 

  777.                 _r_ori = vld1q_s8(r4);

  778.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  779.                 CALC_0(20212223, 24, 0);

  780.                 CALC_0(20212223, 24, 1);

  781.                 CALC_0(20212223, 24, 2);

  782. 

  783.                 vst1q_s32(outptr, _sum00);

  784.                 vst1q_s32(outptr + 4, _sum01);

  785.                 vst1q_s32(outptr + 8, _sum02);

  786. 

  787.                 r0 += 6;

  788.                 r1 += 6;

  789.                 r2 += 6;

  790.                 r3 += 6;

  791.                 r4 += 6;

  792.                 outptr += 12;

  793.             }

  794.             for (; w < width; w++) {

  795.                 int32x4_t _sum00 = vld1q_s32(outptr);

  796. 

  797.                 int8x16_t _r_ori = vtranslq_s8(vld1_s8(r0));

  798.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  799.                 CALC_0(123, 4, 0);

  800. 

  801.                 _r_ori = vtranslq_s8(vld1_s8(r1));

  802.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  803.                 CALC_0(5678, 9, 0);

  804. 

  805.                 _r_ori = vtranslq_s8(vld1_s8(r2));

  806.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  807.                 CALC_0(10111213, 14, 0);

  808. 

  809.                 _r_ori = vtranslq_s8(vld1_s8(r3));

  810.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  811.                 CALC_0(15161718, 19, 0);

  812. 

  813.                 _r_ori = vtranslq_s8(vld1_s8(r4));

  814.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  815.                 CALC_0(20212223, 24, 0);

  816. 

  817.                 vst1q_s32(outptr, _sum00);

  818. 

  819.                 r0 += 2;

  820.                 r1 += 2;

  821.                 r2 += 2;

  822.                 r3 += 2;

  823.                 r4 += 2;

  824.                 outptr += 4;

  825.             }

  826.             r0 += tail_step;

  827.             r1 += tail_step;

  828.             r2 += tail_step;

  829.             r3 += tail_step;

  830.             r4 += tail_step;

  831.         }

  832. 

  833.         filter += 25;

  834.     }

  835. }

  836. 

  837. template <bool even>

  838. void deconv_direct_7x7(const int8_t* src, const int8_t* filter, int32_t* dst,

  839.                        size_t IH, size_t IW, size_t OH, size_t OW, size_t IC) {

  840.     MEGDNN_MARK_USED_VAR(IH);

  841.     const size_t tail_step = IW - OW / 2;

  842. 

  843.     const uint8x16_t _idx00 = {0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6};

  844.     const uint8x16_t _idx01 = {4, 5, 6, 16, 5, 6, 7, 16,

  845.                                6, 7, 8, 16, 7, 8, 9, 16};

  846.     const uint8x16_t _idx10 = {4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10};

  847.     const uint8x16_t _idx11 = {8,  9,  10, 16, 9,  10, 11, 16,

  848.                                10, 11, 12, 16, 11, 12, 13, 16};

  849. 

  850.     uint8x16_t _idx_r_0;

  851.     if (even) {

  852.         _idx_r_0 = {0, 16, 1, 16, 2, 16, 3, 16, 4, 16, 5, 16, 6, 16, 7, 16};

  853.     } else {

  854.         _idx_r_0 = {16, 0, 16, 1, 16, 2, 16, 3, 16, 4, 16, 5, 16, 6, 16, 7};

  855.     }

  856.     int8x16_t _tmp, _elem;

  857.     rep(ic, IC) {

  858.         const int8_t* src_ptr = src;

  859.         int32_t* dst_ptr = dst + OW * OH * ic;

  860.         int32_t* outptr = dst_ptr;

  861.         int32_t* outptr2 = outptr + OW;

  862. 

  863.         const int8_t* r0 = src_ptr;

  864.         const int8_t* r1 = src_ptr + IW;

  865.         const int8_t* r2 = src_ptr + IW * 2;

  866.         const int8_t* r3 = src_ptr + IW * 3;

  867.         const int8_t* r4 = src_ptr + IW * 4;

  868.         const int8_t* r5 = src_ptr + IW * 5;

  869.         const int8_t* r6 = src_ptr + IW * 6;

  870.         const int8_t* r7 = src_ptr + IW * 7;

  871. 

  872.         const int8_t* k0 = filter;

  873. 

  874.         int8x16_t _k = vld1q_s8(k0 + 33);

  875.         //! filter row 1

  876.         uint8x16_t _idx = {15, 14, 13, 12, 15, 14, 13, 12,

  877.                            15, 14, 13, 12, 15, 14, 13, 12};

  878.         int8x16_t _k123 = vqtbl1q_s8_common(_k, _idx);

  879.         _idx = {11, 10, 9, 16, 11, 10, 9, 16, 11, 10, 9, 16, 11, 10, 9, 16};

  880.         int8x16_t _k456 = vqtbl1q_s8_common(_k, _idx);

  881.         //! filter row 2

  882.         _idx = {8, 7, 6, 5, 8, 7, 6, 5, 8, 7, 6, 5, 8, 7, 6, 5};

  883.         int8x16_t _k78910 = vqtbl1q_s8_common(_k, _idx);

  884.         _idx = {4, 3, 2, 16, 4, 3, 2, 16, 4, 3, 2, 16, 4, 3, 2, 16};

  885.         int8x16_t _k111213 = vqtbl1q_s8_common(_k, _idx);

  886. 

  887.         //! 12 13 14 15 -> 16 17 18 19 -> 20 21 22 23 -> 24 25 26 27

  888.         _k = vld1q_s8(k0 + 19);

  889.         //! filter row 3

  890.         _idx = {15, 14, 13, 12, 15, 14, 13, 12, 15, 14, 13, 12, 15, 14, 13, 12};

  891.         int8x16_t _k14151617 = vqtbl1q_s8_common(_k, _idx);

  892.         _idx = {11, 10, 9, 16, 11, 10, 9, 16, 11, 10, 9, 16, 11, 10, 9, 16};

  893.         int8x16_t _k181920 = vqtbl1q_s8_common(_k, _idx);

  894.         //! filter row 4

  895.         _idx = {8, 7, 6, 5, 8, 7, 6, 5, 8, 7, 6, 5, 8, 7, 6, 5};

  896.         int8x16_t _k21222324 = vqtbl1q_s8_common(_k, _idx);

  897.         _idx = {4, 3, 2, 16, 4, 3, 2, 16, 4, 3, 2, 16, 4, 3, 2, 16};

  898.         int8x16_t _k252627 = vqtbl1q_s8_common(_k, _idx);

  899. 

  900.         //! 24 25 26 27->28 29 30 31 -> 32 33 34 35 -> 36 37 38 39

  901.         _k = vld1q_s8(k0 + 5);

  902.         //! filter row 5

  903.         _idx = {15, 14, 13, 12, 15, 14, 13, 12, 15, 14, 13, 12, 15, 14, 13, 12};

  904.         int8x16_t _k28293031 = vqtbl1q_s8_common(_k, _idx);

  905.         _idx = {11, 10, 9, 16, 11, 10, 9, 16, 11, 10, 9, 16, 11, 10, 9, 16};

  906.         int8x16_t _k323334 = vqtbl1q_s8_common(_k, _idx);

  907. 

  908.         //! 33 34 35 36 -> 37 38 39 40 -> 41 42 43 44 -> 45 46 47 48

  909.         _k = vld1q_s8(k0);

  910.         //! filter row 6

  911.         _idx = {13, 12, 11, 10, 13, 12, 11, 10, 13, 12, 11, 10, 13, 12, 11, 10};

  912.         int8x16_t _k35363738 = vqtbl1q_s8_common(_k, _idx);

  913.         _idx = {9, 8, 7, 16, 9, 8, 7, 16, 9, 8, 7, 16, 9, 8, 7, 16};

  914.         int8x16_t _k394041 = vqtbl1q_s8_common(_k, _idx);

  915. 

  916.         //! filter row 7

  917.         _idx = {6, 5, 4, 3, 6, 5, 4, 3, 6, 5, 4, 3, 6, 5, 4, 3};

  918.         int8x16_t _k42434445 = vqtbl1q_s8_common(_k, _idx);

  919.         _idx = {2, 1, 0, 16, 2, 1, 0, 16, 2, 1, 0, 16, 2, 1, 0, 16};

  920.         int8x16_t _k464748 = vqtbl1q_s8_common(_k, _idx);

  921. 

  922.         const int width = OW >> 2;

  923.         size_t h = 0;

  924.         for (; h + 1 < OH; h += 2) {

  925.             int w = 0;

  926.             for (; w + 2 < width; w += 2) {

  927.                 //! As the inner kernel read 16 elements, and IW is times of 16

  928.                 int32x4_t _sum00 = vld1q_s32(outptr);

  929.                 int32x4_t _sum01 = vld1q_s32(outptr + 4);

  930.                 int32x4_t _sum10 = vld1q_s32(outptr2);

  931.                 int32x4_t _sum11 = vld1q_s32(outptr2 + 4);

  932. 

  933.                 int8x16_t _r_ori = vld1q_s8(r0);

  934.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  935.                 CALC_0(123, 456, 0);

  936.                 CALC_0(123, 456, 1);

  937. 

  938.                 _r_ori = vld1q_s8(r1);

  939.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  940.                 CALC_2(78910, 111213, 123, 456, 0);

  941.                 CALC_2(78910, 111213, 123, 456, 1);

  942. 

  943.                 _r_ori = vld1q_s8(r2);

  944.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  945.                 CALC_2(14151617, 181920, 78910, 111213, 0);

  946.                 CALC_2(14151617, 181920, 78910, 111213, 1);

  947. 

  948.                 _r_ori = vld1q_s8(r3);

  949.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  950.                 CALC_2(21222324, 252627, 14151617, 181920, 0);

  951.                 CALC_2(21222324, 252627, 14151617, 181920, 1);

  952. 

  953.                 _r_ori = vld1q_s8(r4);

  954.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  955.                 CALC_2(28293031, 323334, 21222324, 252627, 0);

  956.                 CALC_2(28293031, 323334, 21222324, 252627, 1);

  957. 

  958.                 _r_ori = vld1q_s8(r5);

  959.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  960.                 CALC_2(35363738, 394041, 28293031, 323334, 0);

  961.                 CALC_2(35363738, 394041, 28293031, 323334, 1);

  962. 

  963.                 _r_ori = vld1q_s8(r6);

  964.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  965.                 CALC_2(42434445, 464748, 35363738, 394041, 0);

  966.                 CALC_2(42434445, 464748, 35363738, 394041, 1);

  967. 

  968.                 _r_ori = vld1q_s8(r7);

  969.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  970.                 CALC_1(42434445, 464748, 0);

  971.                 CALC_1(42434445, 464748, 1);

  972. 

  973.                 vst1q_s32(outptr, _sum00);

  974.                 vst1q_s32(outptr + 4, _sum01);

  975.                 vst1q_s32(outptr2, _sum10);

  976.                 vst1q_s32(outptr2 + 4, _sum11);

  977. 

  978.                 r0 += 4;

  979.                 r1 += 4;

  980.                 r2 += 4;

  981.                 r3 += 4;

  982.                 r4 += 4;

  983.                 r5 += 4;

  984.                 r6 += 4;

  985.                 r7 += 4;

  986.                 outptr += 8;

  987.                 outptr2 += 8;

  988.             }

  989.             for (; w < width; w++) {

  990.                 int32x4_t _sum00 = vld1q_s32(outptr);

  991.                 int32x4_t _sum10 = vld1q_s32(outptr2);

  992. 

  993.                 int8x16_t _r_ori = vld1q_s8(r0);

  994.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  995.                 CALC_0(123, 456, 0);

  996. 

  997.                 _r_ori = vld1q_s8(r1);

  998.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  999.                 CALC_2(78910, 111213, 123, 456, 0);

  1000. 

  1001.                 _r_ori = vld1q_s8(r2);

  1002.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  1003.                 CALC_2(14151617, 181920, 78910, 111213, 0);

  1004. 

  1005.                 _r_ori = vld1q_s8(r3);

  1006.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  1007.                 CALC_2(21222324, 252627, 14151617, 181920, 0);

  1008. 

  1009.                 _r_ori = vld1q_s8(r4);

  1010.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  1011.                 CALC_2(28293031, 323334, 21222324, 252627, 0);

  1012. 

  1013.                 _r_ori = vld1q_s8(r5);

  1014.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  1015.                 CALC_2(35363738, 394041, 28293031, 323334, 0);

  1016. 

  1017.                 _r_ori = vld1q_s8(r6);

  1018.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  1019.                 CALC_2(42434445, 464748, 35363738, 394041, 0);

  1020. 

  1021.                 _r_ori = vld1q_s8(r7);

  1022.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  1023.                 CALC_1(42434445, 464748, 0);

  1024. 

  1025.                 vst1q_s32(outptr, _sum00);

  1026.                 vst1q_s32(outptr2, _sum10);

  1027. 

  1028.                 r0 += 2;

  1029.                 r1 += 2;

  1030.                 r2 += 2;

  1031.                 r3 += 2;

  1032.                 r4 += 2;

  1033.                 r5 += 2;

  1034.                 r6 += 2;

  1035.                 r7 += 2;

  1036.                 outptr += 4;

  1037.                 outptr2 += 4;

  1038.             }

  1039. 

  1040.             r0 += tail_step + IW;

  1041.             r1 += tail_step + IW;

  1042.             r2 += tail_step + IW;

  1043.             r3 += tail_step + IW;

  1044.             r4 += tail_step + IW;

  1045.             r5 += tail_step + IW;

  1046.             r6 += tail_step + IW;

  1047.             r7 += tail_step + IW;

  1048. 

  1049.             outptr += OW;

  1050.             outptr2 += OW;

  1051.         }

  1052. 

  1053.         for (; h < OH; h++) {

  1054.             int w = 0;

  1055.             for (; w + 2 < width; w += 2) {

  1056.                 int32x4_t _sum00 = vld1q_s32(outptr);

  1057.                 int32x4_t _sum01 = vld1q_s32(outptr + 4);

  1058. 

  1059.                 int8x16_t _r_ori = vld1q_s8(r0);

  1060.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  1061.                 CALC_0(123, 456, 0);

  1062.                 CALC_0(123, 456, 1);

  1063. 

  1064.                 _r_ori = vld1q_s8(r1);

  1065.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  1066.                 CALC_0(78910, 111213, 0);

  1067.                 CALC_0(78910, 111213, 1);

  1068. 

  1069.                 _r_ori = vld1q_s8(r2);

  1070.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  1071.                 CALC_0(14151617, 181920, 0);

  1072.                 CALC_0(14151617, 181920, 1);

  1073. 

  1074.                 _r_ori = vld1q_s8(r3);

  1075.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  1076.                 CALC_0(21222324, 252627, 0);

  1077.                 CALC_0(21222324, 252627, 1);

  1078. 

  1079.                 _r_ori = vld1q_s8(r4);

  1080.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  1081.                 CALC_0(28293031, 323334, 0);

  1082.                 CALC_0(28293031, 323334, 1);

  1083. 

  1084.                 _r_ori = vld1q_s8(r5);

  1085.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  1086.                 CALC_0(35363738, 394041, 0);

  1087.                 CALC_0(35363738, 394041, 1);

  1088. 

  1089.                 _r_ori = vld1q_s8(r6);

  1090.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  1091.                 CALC_0(42434445, 464748, 0);

  1092.                 CALC_0(42434445, 464748, 1);

  1093. 

  1094.                 vst1q_s32(outptr, _sum00);

  1095.                 vst1q_s32(outptr + 4, _sum01);

  1096. 

  1097.                 r0 += 4;

  1098.                 r1 += 4;

  1099.                 r2 += 4;

  1100.                 r3 += 4;

  1101.                 r4 += 4;

  1102.                 r5 += 4;

  1103.                 r6 += 4;

  1104.                 outptr += 8;

  1105.             }

  1106.             for (; w < width; w++) {

  1107.                 int32x4_t _sum00 = vld1q_s32(outptr);

  1108. 

  1109.                 int8x16_t _r_ori = vld1q_s8(r0);

  1110.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  1111.                 CALC_0(123, 456, 0);

  1112. 

  1113.                 _r_ori = vld1q_s8(r1);

  1114.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  1115.                 CALC_0(78910, 111213, 0);

  1116. 

  1117.                 _r_ori = vld1q_s8(r2);

  1118.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  1119.                 CALC_0(14151617, 181920, 0);

  1120. 

  1121.                 _r_ori = vld1q_s8(r3);

  1122.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  1123.                 CALC_0(21222324, 252627, 0);

  1124. 

  1125.                 _r_ori = vld1q_s8(r4);

  1126.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  1127.                 CALC_0(28293031, 323334, 0);

  1128. 

  1129.                 _r_ori = vld1q_s8(r5);

  1130.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  1131.                 CALC_0(35363738, 394041, 0);

  1132. 

  1133.                 _r_ori = vld1q_s8(r6);

  1134.                 _tmp = vqtbl1q_s8_common(_r_ori, _idx_r_0);

  1135.                 CALC_0(42434445, 464748, 0);

  1136. 

  1137.                 vst1q_s32(outptr, _sum00);

  1138. 

  1139.                 r0 += 2;

  1140.                 r1 += 2;

  1141.                 r2 += 2;

  1142.                 r3 += 2;

  1143.                 r4 += 2;

  1144.                 r5 += 2;

  1145.                 r6 += 2;

  1146.                 outptr += 4;

  1147.             }

  1148.             r0 += tail_step;

  1149.             r1 += tail_step;

  1150.             r2 += tail_step;

  1151.             r3 += tail_step;

  1152.             r4 += tail_step;

  1153.             r5 += tail_step;

  1154.             r6 += tail_step;

  1155.         }

  1156. 

  1157.         filter += 49;

  1158.     }

  1159. }

  1160. 

  1161. #undef CALC_0

  1162. #undef CALC_1

  1163. #undef CALC_2

  1164. 

  1165. }  // anonymous namespace

  1166. 

  1167. size_t deconv::get_workspace_in_bytes_stride2_int8x8x32_dot(

  1168.         const NCBKernSizeParam& param) {

  1169.     return get_bundle(param).total_size_in_bytes();

  1170. }

  1171. 

  1172. bool deconv::can_stride2_int8x8x32_dot(const NCBKernSizeParam& param) {

  1173.     auto&& fm = param.filter_meta;

  1174.     auto FH = fm.spatial[0], FW = fm.spatial[1], OC = fm.ocpg,

  1175.          PH = fm.padding[0], PW = fm.padding[1];

  1176.     bool avaiable = fm.format == param::Convolution::Format::NCHW &&

  1177.                     !fm.should_flip && fm.spatial_ndim == 2 &&

  1178.                     fm.dilation[0] == 1 && fm.dilation[1] == 1 &&

  1179.                     fm.stride[0] == 2 && fm.stride[1] == 2 && FH == FW &&

  1180.                     (FH == 2 || FH == 3 || FH == 5 || FH == 7) &&

  1181.                     FH >= PH + 1 && FW >= PW + 1;

  1182. 

  1183.     avaiable &= (param.filter_type.enumv() == DTypeEnum::QuantizedS8 ||

  1184.                  param.filter_type.enumv() == DTypeEnum::Int8) &&

  1185.                 (param.grad_type.enumv() == DTypeEnum::QuantizedS32 ||

  1186.                  param.grad_type.enumv() == DTypeEnum::Int32);

  1187.     return avaiable && ((FH == 2 && OC <= 4) || (FH == 3 && OC <= 8) ||

  1188.                         (FH == 5 && OC <= 16) || (FH == 7 && OC < 32));

  1189. }

  1190. 

  1191. void deconv::stride2_int8x8x32_dot(const NCBKernParam& param) {

  1192.     auto bundle = get_bundle(param);

  1193.     bundle.set(param.workspace_ptr);

  1194.     UNPACK_CONV_F32_NCB_KERN_SIZES(param);

  1195.     MEGDNN_MARK_USED_VAR(SH);

  1196.     MEGDNN_MARK_USED_VAR(SW);

  1197.     size_t IH2, IW2, OW2;

  1198.     int padding_h = FH - PH - 1, padding_w = FW - PW - 1;

  1199. 

  1200.     get_rectified_size(IH, IW, OH, OW, FH, FW, PH, PW, IH2, IW2, OW2);

  1201. 

  1202.     using Func = std::function<void(const int8_t*, const int8_t*, int32_t*,

  1203.                                     size_t, size_t, size_t, size_t, size_t)>;

  1204.     Func conv = nullptr;

  1205.     if (FH == 2) {

  1206.         if ((padding_w & 1) == 0)

  1207.             conv = deconv_direct_2x2<true>;

  1208.         else

  1209.             conv = deconv_direct_2x2<false>;

  1210.     } else if (FH == 3) {

  1211.         if ((padding_w & 1) == 0)

  1212.             conv = deconv_direct_3x3<true>;

  1213.         else

  1214.             conv = deconv_direct_3x3<false>;

  1215.     } else if (FH == 5) {

  1216.         if ((padding_w & 1) == 0)

  1217.             conv = deconv_direct_5x5<true>;

  1218.         else

  1219.             conv = deconv_direct_5x5<false>;

  1220.     } else if (FH == 7) {

  1221.         if ((padding_w & 1) == 0)

  1222.             conv = deconv_direct_7x7<true>;

  1223.         else

  1224.             conv = deconv_direct_7x7<false>;

  1225.     } else {

  1226.         megdnn_assert(0);

  1227.     }

  1228. 

  1229.     bool need_dst_copy_var = need_dst_copy(param);

  1230.     int8_t* base_src_ptr = const_cast<int8_t*>(param.diff<int8_t>());

  1231.     int32_t* base_dst_ptr = param.grad<int32_t>();

  1232.     const int8_t* fptr = param.filter<int8_t>();

  1233. 

  1234.     for (size_t n = 0; n < N; ++n) {

  1235.         int32_t* dptr_copied = static_cast<int32_t*>(bundle.get(1));

  1236.         int32_t* dptr_ori = base_dst_ptr + n * param.out_bs;

  1237.         int32_t* dptr = nullptr;

  1238.         size_t OW_real = OW;

  1239.         if (need_dst_copy_var) {

  1240.             dptr = dptr_copied;

  1241.             OW_real = OW2;

  1242.         } else {

  1243.             dptr = dptr_ori;

  1244.         }

  1245.         std::memset(dptr, 0, sizeof(int32_t) * IC * OH * OW_real);

  1246. 

  1247.         int8_t* sptr_ori = base_src_ptr + n * param.inp_bs;

  1248.         int8_t* sptr_copied = static_cast<int8_t*>(bundle.get(0));

  1249.         int8_t* sptr = nullptr;

  1250.         rep(oc, OC) {

  1251.             std::memset(sptr_copied, 0, sizeof(int8_t) * IH2 * IW2);

  1252.             copy_plane_in_bytes(sptr_copied + padding_h * IW2 + padding_w / 2,

  1253.                                 sptr_ori + oc * IH * IW, IH,

  1254.                                 IW * sizeof(int8_t), 2 * IW2 * sizeof(int8_t),

  1255.                                 IW * sizeof(int8_t));

  1256.             sptr = sptr_copied;

  1257. 

  1258.             conv(sptr, fptr + oc * IC * FH * FW, dptr, IH2, IW2, OH, OW_real,

  1259.                  IC);

  1260.         }

  1261.         if (need_dst_copy_var) {

  1262.             for (size_t ic = 0; ic < IC; ++ic) {

  1263.                 copy_plane_in_bytes(dptr_ori + ic * OH * OW,

  1264.                                     dptr + ic * OH * OW2, OH,

  1265.                                     OW * sizeof(int32_t), OW * sizeof(int32_t),

  1266.                                     OW2 * sizeof(int32_t));

  1267.             }

  1268.         }

  1269.     }

  1270. }

  1271. 

  1272. #endif

  1273. // vim: syntax=cpp.doxygen