OpenI
/
MegEngine
Not watched
1
0
Fork 0
Code
Releases 31 Wiki Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 90ca85541e
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '90ca85541e'
${ noResults }
MegEngine/dnn/src/x86/conv_bias/int8/avx2_chanwise_kern.cpp

1594 lines
68 kB
Raw Blame History 

  1. /**

  2.  * \file src/x86/conv_bias/int8/avx2_chanwise_kern.cpp

  3.  * MegEngine is Licensed under the Apache License, Version 2.0 (the "License")

  4.  *

  5.  * Copyright (c) 2014-2020 Megvii Inc. All rights reserved.

  6.  *

  7.  * Unless required by applicable law or agreed to in writing,

  8.  * software distributed under the License is distributed on an

  9.  * "AS IS" BASIS, WITHOUT ARRANTIES OR CONDITIONS OF ANY KIND, either express or

  10.  * implied.

  11.  */

  12. 

  13. #include "src/x86/conv_bias/int8/avx2_chanwise_kern.h"

  14. #include <immintrin.h>

  15. #include "src/common/unroll_macro.h"

  16. #include "src/x86/conv_bias/int8/common_helper.h"

  17. #include "src/x86/elemwise_op.h"

  18. #ifdef WIN32CMAKE

  19. #include <smmintrin.h>

  20. #endif

  21. 

  22. namespace megdnn {

  23. namespace x86 {

  24. namespace avx2_chanwise_stride1 {

  25. 

  26. #define load_filter(i) __m128i k_##i = _mm_set1_epi8(*(filter + i));

  27. #define load_src0(i) \

  28.     __m256i cvt16_src##i##0 = _mm256_cvtepi8_epi16_from_ptr(r##i);

  29. #define load_src1(i) \

  30.     __m256i cvt16_src##i##1 = _mm256_cvtepi8_epi16_from_ptr(r##i + 1);

  31. #define load_src2(i) \

  32.     __m256i cvt16_src##i##2 = _mm256_cvtepi8_epi16_from_ptr(r##i + 2);

  33. #define load_src3(i) \

  34.     __m256i cvt16_src##i##3 = _mm256_cvtepi8_epi16_from_ptr(r##i + 3);

  35. #define load_src4(i) \

  36.     __m256i cvt16_src##i##4 = _mm256_cvtepi8_epi16_from_ptr(r##i + 4);

  37. #define load_src5(i) \

  38.     __m256i cvt16_src##i##5 = _mm256_cvtepi8_epi16_from_ptr(r##i + 5);

  39. #define load_src6(i) \

  40.     __m256i cvt16_src##i##6 = _mm256_cvtepi8_epi16_from_ptr(r##i + 6);

  41. #define load_src7(i) \

  42.     __m256i cvt16_src##i##7 = _mm256_cvtepi8_epi16_from_ptr(r##i + 7);

  43. 

  44. template <BiasMode bias_mode, bool is_quantized, typename Op>

  45. void avx2_chanwise_direct_stride1_2x2_int8(const int8_t* src,

  46.                                            const int8_t* filter,

  47.                                            const int32_t* bias, int32_t* temp,

  48.                                            int8_t* dst, const size_t IH,

  49.                                            const size_t IW, const size_t OH,

  50.                                            const size_t OW, const Op& op) {

  51.     size_t tail_step = IW - OW;

  52.     int8_t* dst0 = dst;

  53.     int8_t* dst1 = dst + OW;

  54.     int32_t* out_ptr0 = temp;

  55.     int32_t* out_ptr1 = temp + OW;

  56.     const int8_t* r0 = src;

  57.     const int8_t* r1 = src + IW;

  58.     const int8_t* r2 = src + 2 * IW;

  59. 

  60.     UNROLL_CALL0(4, load_filter)

  61. 

  62. #define pack_filter(i, j) __m128i k_##i##j = _mm_unpacklo_epi8(k_##i, k_##j)

  63.     pack_filter(0, 1);

  64.     pack_filter(2, 3);

  65. 

  66.     __m256i bias_val;

  67.     if (bias_mode == BiasMode::BROADCAST_CHANNEL_BIAS) {

  68.         bias_val = _mm256_set1_epi32(*(bias));

  69.     } else {

  70.         bias_val = _mm256_set1_epi32(0);

  71.     }

  72. #define cvt_filter(i, j) __m256i filter_##i##j = _mm256_cvtepi8_epi16(k_##i##j)

  73.     cvt_filter(0, 1);

  74.     cvt_filter(2, 3);

  75. 

  76.     size_t width = OW >> 4;

  77.     size_t h = 0;

  78.     for (; h + 1 < OH; h += 2) {

  79.         size_t w = 0;

  80.         for (; w < width; w++) {

  81.             UNROLL_CALL0(3, load_src0)

  82.             UNROLL_CALL0(3, load_src1)

  83. 

  84.             __m256i sum0_odd, sum0_even, sum1_odd, sum1_even;

  85.             __m256i tmp0_odd, tmp0_even, tmp1_odd, tmp1_even, tmp2_odd,

  86.                     tmp2_even, tmp3_odd, tmp3_even;

  87. 

  88.             tmp0_odd = _mm256_madd_epi16(cvt16_src00, filter_01);

  89.             tmp0_even = _mm256_madd_epi16(cvt16_src01, filter_01);

  90. 

  91.             tmp1_odd = _mm256_madd_epi16(cvt16_src10, filter_23);

  92.             tmp1_even = _mm256_madd_epi16(cvt16_src11, filter_23);

  93. 

  94.             tmp3_odd = _mm256_madd_epi16(cvt16_src10, filter_01);

  95.             tmp3_even = _mm256_madd_epi16(cvt16_src11, filter_01);

  96. 

  97.             tmp2_odd = _mm256_madd_epi16(cvt16_src20, filter_23);

  98.             tmp2_even = _mm256_madd_epi16(cvt16_src21, filter_23);

  99. 

  100.             sum0_odd = _mm256_add_epi32(tmp0_odd, tmp1_odd);

  101.             sum0_even = _mm256_add_epi32(tmp0_even, tmp1_even);

  102. 

  103.             __m256i sum_odd = _mm256_unpacklo_epi32(sum0_odd, sum0_even);

  104.             __m256i sum_even = _mm256_unpackhi_epi32(sum0_odd, sum0_even);

  105. 

  106.             //! switch_mask_low   = {00100000} = 32

  107.             //! switch_mask_high  = {00110001} = 49

  108.             __m256i sum_left = _mm256_permute2f128_si256(sum_odd, sum_even, 32);

  109.             __m256i sum_right =

  110.                     _mm256_permute2f128_si256(sum_odd, sum_even, 49);

  111. 

  112.             sum_left = _mm256_add_epi32(sum_left, bias_val);

  113.             sum_right = _mm256_add_epi32(sum_right, bias_val);

  114. 

  115.             if (is_quantized) {

  116.                 op({{sum_left, sum_right}}, reinterpret_cast<dt_qint8*>(dst0));

  117. 

  118.             } else {

  119.                 _mm256_storeu_si256((__m256i*)(out_ptr0), sum_left);

  120.                 _mm256_storeu_si256((__m256i*)(out_ptr0 + 8), sum_right);

  121.             }

  122. 

  123.             sum1_odd = _mm256_add_epi32(tmp3_odd, tmp2_odd);

  124.             sum1_even = _mm256_add_epi32(tmp3_even, tmp2_even);

  125. 

  126.             __m256i sum_1_odd = _mm256_unpacklo_epi32(sum1_odd, sum1_even);

  127.             __m256i sum_1_even = _mm256_unpackhi_epi32(sum1_odd, sum1_even);

  128. 

  129.             __m256i sum_1_left =

  130.                     _mm256_permute2f128_si256(sum_1_odd, sum_1_even, 32);

  131.             __m256i sum_1_right =

  132.                     _mm256_permute2f128_si256(sum_1_odd, sum_1_even, 49);

  133. 

  134.             sum_1_left = _mm256_add_epi32(sum_1_left, bias_val);

  135.             sum_1_right = _mm256_add_epi32(sum_1_right, bias_val);

  136. 

  137.             if (is_quantized) {

  138.                 op({{sum_1_left, sum_1_right}},

  139.                    reinterpret_cast<dt_qint8*>(dst1));

  140.             } else {

  141.                 _mm256_storeu_si256((__m256i*)(out_ptr1), sum_1_left);

  142.                 _mm256_storeu_si256((__m256i*)(out_ptr1 + 8), sum_1_right);

  143.             }

  144.             r0 += 16;

  145.             r1 += 16;

  146.             r2 += 16;

  147.             dst0 += 16;

  148.             dst1 += 16;

  149.             out_ptr0 += 16;

  150.             out_ptr1 += 16;

  151.         }

  152.         r0 += tail_step + IW;

  153.         r1 += tail_step + IW;

  154.         r2 += tail_step + IW;

  155. 

  156.         dst0 += OW;

  157.         dst1 += OW;

  158.         out_ptr0 += OW;

  159.         out_ptr1 += OW;

  160.     }

  161. 

  162.     for (; h < OH; h++) {

  163.         size_t w = 0;

  164.         for (; w < width; w++) {

  165.             UNROLL_CALL0(2, load_src0)

  166.             UNROLL_CALL0(2, load_src1)

  167. 

  168.             __m256i sum0_odd, sum0_even;

  169.             __m256i tmp0_odd, tmp0_even, tmp1_odd, tmp1_even;

  170. 

  171.             tmp0_odd = _mm256_madd_epi16(cvt16_src00, filter_01);

  172.             tmp0_even = _mm256_madd_epi16(cvt16_src01, filter_01);

  173. 

  174.             tmp1_odd = _mm256_madd_epi16(cvt16_src10, filter_23);

  175.             tmp1_even = _mm256_madd_epi16(cvt16_src11, filter_23);

  176. 

  177.             sum0_odd = _mm256_add_epi32(tmp0_odd, tmp1_odd);

  178.             sum0_even = _mm256_add_epi32(tmp0_even, tmp1_even);

  179. 

  180.             __m256i sum_odd = _mm256_unpacklo_epi32(sum0_odd, sum0_even);

  181.             __m256i sum_even = _mm256_unpackhi_epi32(sum0_odd, sum0_even);

  182. 

  183.             __m256i sum_left = _mm256_permute2f128_si256(sum_odd, sum_even, 32);

  184.             __m256i sum_right =

  185.                     _mm256_permute2f128_si256(sum_odd, sum_even, 49);

  186. 

  187.             sum_left = _mm256_add_epi32(sum_left, bias_val);

  188.             sum_right = _mm256_add_epi32(sum_right, bias_val);

  189. 

  190.             if (is_quantized) {

  191.                 op({{sum_left, sum_right}}, reinterpret_cast<dt_qint8*>(dst0));

  192.             } else {

  193.                 _mm256_storeu_si256((__m256i*)(out_ptr0), sum_left);

  194.                 _mm256_storeu_si256((__m256i*)(out_ptr0 + 8), sum_right);

  195.             }

  196. 

  197.             r0 += 16;

  198.             r1 += 16;

  199.             dst0 += 16;

  200.             out_ptr0 += 16;

  201.         }

  202.         r0 += tail_step;

  203.         r1 += tail_step;

  204.     }

  205.     MEGDNN_MARK_USED_VAR(IH);

  206. #undef pack_filter

  207. #undef cvt_filter

  208. }

  209. 

  210. template <BiasMode bias_mode, bool is_quantized, typename Op>

  211. void avx2_chanwise_direct_stride1_3x3_int8(const int8_t* src,

  212.                                            const int8_t* filter,

  213.                                            const int32_t* bias, int32_t* temp,

  214.                                            int8_t* dst, const size_t IH,

  215.                                            const size_t IW, const size_t OH,

  216.                                            const size_t OW, const Op& op) {

  217.     MEGDNN_MARK_USED_VAR(IH);

  218.     size_t tail_step = IW - OW;

  219.     int32_t* out_ptr0 = temp;

  220.     int32_t* out_ptr1 = temp + OW;

  221.     int8_t* dst0 = dst;

  222.     int8_t* dst1 = dst + OW;

  223.     const int8_t* r0 = src;

  224.     const int8_t* r1 = src + IW;

  225.     const int8_t* r2 = src + 2 * IW;

  226.     const int8_t* r3 = src + 3 * IW;

  227. 

  228.     uint8_t fill_zero = 0;

  229.     UNROLL_CALL0(9, load_filter)

  230. 

  231.     __m128i k_fill = _mm_set1_epi8(fill_zero);

  232. 

  233.     __m128i k01 = _mm_unpacklo_epi8(k_0, k_1);

  234.     __m128i k20 = _mm_unpacklo_epi8(k_2, k_fill);

  235. 

  236.     __m128i k34 = _mm_unpacklo_epi8(k_3, k_4);

  237.     __m128i k50 = _mm_unpacklo_epi8(k_5, k_fill);

  238. 

  239.     __m128i k67 = _mm_unpacklo_epi8(k_6, k_7);

  240.     __m128i k80 = _mm_unpacklo_epi8(k_8, k_fill);

  241. 

  242.     __m256i bias_val;

  243.     if (bias_mode == BiasMode::BROADCAST_CHANNEL_BIAS) {

  244.         bias_val = _mm256_set1_epi32(*(bias));

  245.     } else {

  246.         bias_val = _mm256_set1_epi32(0);

  247.     }

  248. 

  249.     //! cvt i8 --> i16

  250.     __m256i filter_01 = _mm256_cvtepi8_epi16(k01);

  251.     __m256i filter_20 = _mm256_cvtepi8_epi16(k20);

  252.     __m256i filter_34 = _mm256_cvtepi8_epi16(k34);

  253.     __m256i filter_50 = _mm256_cvtepi8_epi16(k50);

  254.     __m256i filter_67 = _mm256_cvtepi8_epi16(k67);

  255.     __m256i filter_80 = _mm256_cvtepi8_epi16(k80);

  256. 

  257.     size_t width = OW >> 4;

  258.     size_t h = 0;

  259.     for (; h + 1 < OH; h += 2) {

  260.         size_t w = 0;

  261.         for (; w < width; w++) {

  262.             UNROLL_CALL0(4, load_src0)

  263.             UNROLL_CALL0(4, load_src1)

  264.             UNROLL_CALL0(4, load_src2)

  265.             UNROLL_CALL0(4, load_src3)

  266. 

  267.             __m256i sum00_odd, sum00_even, sum11_odd, sum11_even, sum22_odd,

  268.                     sum22_even;

  269.             __m256i sum11_odd_01, sum11_even_01, sum22_odd_01, sum22_even_01,

  270.                     sum33_odd, sum33_even;

  271.             __m256i temp0, temp1;

  272. 

  273.             temp0 = _mm256_madd_epi16(cvt16_src00, filter_01);

  274.             temp1 = _mm256_madd_epi16(cvt16_src02, filter_20);

  275.             sum00_odd = _mm256_add_epi32(temp0, temp1);

  276. 

  277.             temp0 = _mm256_madd_epi16(cvt16_src01, filter_01);

  278.             temp1 = _mm256_madd_epi16(cvt16_src03, filter_20);

  279.             sum00_even = _mm256_add_epi32(temp0, temp1);

  280. 

  281.             temp0 = _mm256_madd_epi16(cvt16_src10, filter_34);

  282.             temp1 = _mm256_madd_epi16(cvt16_src12, filter_50);

  283.             sum11_odd = _mm256_add_epi32(temp0, temp1);

  284. 

  285.             temp0 = _mm256_madd_epi16(cvt16_src11, filter_34);

  286.             temp1 = _mm256_madd_epi16(cvt16_src13, filter_50);

  287.             sum11_even = _mm256_add_epi32(temp0, temp1);

  288. 

  289.             temp0 = _mm256_madd_epi16(cvt16_src10, filter_01);

  290.             temp1 = _mm256_madd_epi16(cvt16_src12, filter_20);

  291.             sum11_odd_01 = _mm256_add_epi32(temp0, temp1);

  292. 

  293.             temp0 = _mm256_madd_epi16(cvt16_src11, filter_01);

  294.             temp1 = _mm256_madd_epi16(cvt16_src13, filter_20);

  295.             sum11_even_01 = _mm256_add_epi32(temp0, temp1);

  296. 

  297.             temp0 = _mm256_madd_epi16(cvt16_src20, filter_67);

  298.             temp1 = _mm256_madd_epi16(cvt16_src22, filter_80);

  299.             sum22_odd = _mm256_add_epi32(temp0, temp1);

  300. 

  301.             temp0 = _mm256_madd_epi16(cvt16_src21, filter_67);

  302.             temp1 = _mm256_madd_epi16(cvt16_src23, filter_80);

  303.             sum22_even = _mm256_add_epi32(temp0, temp1);

  304. 

  305.             temp0 = _mm256_madd_epi16(cvt16_src20, filter_34);

  306.             temp1 = _mm256_madd_epi16(cvt16_src22, filter_50);

  307.             sum22_odd_01 = _mm256_add_epi32(temp0, temp1);

  308. 

  309.             temp0 = _mm256_madd_epi16(cvt16_src21, filter_34);

  310.             temp1 = _mm256_madd_epi16(cvt16_src23, filter_50);

  311.             sum22_even_01 = _mm256_add_epi32(temp0, temp1);

  312. 

  313.             temp0 = _mm256_madd_epi16(cvt16_src30, filter_67);

  314.             temp1 = _mm256_madd_epi16(cvt16_src32, filter_80);

  315.             sum33_odd = _mm256_add_epi32(temp0, temp1);

  316. 

  317.             temp0 = _mm256_madd_epi16(cvt16_src31, filter_67);

  318.             temp1 = _mm256_madd_epi16(cvt16_src33, filter_80);

  319.             sum33_even = _mm256_add_epi32(temp0, temp1);

  320. 

  321.             sum00_odd = _mm256_add_epi32(sum00_odd, sum11_odd);

  322.             sum00_odd = _mm256_add_epi32(sum00_odd, sum22_odd);

  323. 

  324.             sum00_even = _mm256_add_epi32(sum00_even, sum11_even);

  325.             sum00_even = _mm256_add_epi32(sum00_even, sum22_even);

  326. 

  327.             __m256i sum_odd = _mm256_unpacklo_epi32(sum00_odd, sum00_even);

  328.             __m256i sum_even = _mm256_unpackhi_epi32(sum00_odd, sum00_even);

  329. 

  330.             __m256i sum_left = _mm256_permute2f128_si256(sum_odd, sum_even, 32);

  331.             __m256i sum_right =

  332.                     _mm256_permute2f128_si256(sum_odd, sum_even, 49);

  333. 

  334.             sum_left = _mm256_add_epi32(sum_left, bias_val);

  335.             sum_right = _mm256_add_epi32(sum_right, bias_val);

  336. 

  337.             if (is_quantized) {

  338.                 op({{sum_left, sum_right}}, reinterpret_cast<dt_qint8*>(dst0));

  339.             } else {

  340.                 _mm256_storeu_si256((__m256i*)(out_ptr0), sum_left);

  341.                 _mm256_storeu_si256((__m256i*)(out_ptr0 + 8), sum_right);

  342.             }

  343. 

  344.             sum11_odd_01 = _mm256_add_epi32(sum11_odd_01, sum22_odd_01);

  345.             sum11_odd_01 = _mm256_add_epi32(sum11_odd_01, sum33_odd);

  346. 

  347.             sum11_even_01 = _mm256_add_epi32(sum11_even_01, sum22_even_01);

  348.             sum11_even_01 = _mm256_add_epi32(sum11_even_01, sum33_even);

  349. 

  350.             __m256i sum_oh1_odd =

  351.                     _mm256_unpacklo_epi32(sum11_odd_01, sum11_even_01);

  352.             __m256i sum_oh1_even =

  353.                     _mm256_unpackhi_epi32(sum11_odd_01, sum11_even_01);

  354. 

  355.             __m256i sum1_left =

  356.                     _mm256_permute2f128_si256(sum_oh1_odd, sum_oh1_even, 32);

  357.             __m256i sum1_right =

  358.                     _mm256_permute2f128_si256(sum_oh1_odd, sum_oh1_even, 49);

  359. 

  360.             sum1_left = _mm256_add_epi32(sum1_left, bias_val);

  361.             sum1_right = _mm256_add_epi32(sum1_right, bias_val);

  362. 

  363.             if (is_quantized) {

  364.                 op({{sum1_left, sum1_right}},

  365.                    reinterpret_cast<dt_qint8*>(dst1));

  366.             } else {

  367.                 _mm256_storeu_si256((__m256i*)(out_ptr1), sum1_left);

  368.                 _mm256_storeu_si256((__m256i*)(out_ptr1 + 8), sum1_right);

  369.             }

  370. 

  371.             r0 += 16;

  372.             r1 += 16;

  373.             r2 += 16;

  374.             r3 += 16;

  375.             dst0 += 16;

  376.             dst1 += 16;

  377.             out_ptr0 += 16;

  378.             out_ptr1 += 16;

  379.         }

  380.         r0 += tail_step + IW;

  381.         r1 += tail_step + IW;

  382.         r2 += tail_step + IW;

  383.         r3 += tail_step + IW;

  384. 

  385.         dst0 += OW;

  386.         dst1 += OW;

  387.         out_ptr0 += OW;

  388.         out_ptr1 += OW;

  389.     }

  390. 

  391.     for (; h < OH; h++) {

  392.         size_t w = 0;

  393.         for (; w < width; w++) {

  394.             UNROLL_CALL0(3, load_src0)

  395.             UNROLL_CALL0(3, load_src1)

  396.             UNROLL_CALL0(3, load_src2)

  397.             UNROLL_CALL0(3, load_src3)

  398. 

  399.             __m256i sum00_odd, sum00_even, sum11_odd, sum11_even, sum22_odd,

  400.                     sum22_even;

  401.             __m256i temp0, temp1;

  402. 

  403.             temp0 = _mm256_madd_epi16(cvt16_src00, filter_01);

  404.             temp1 = _mm256_madd_epi16(cvt16_src02, filter_20);

  405.             sum00_odd = _mm256_add_epi32(temp0, temp1);

  406. 

  407.             temp0 = _mm256_madd_epi16(cvt16_src01, filter_01);

  408.             temp1 = _mm256_madd_epi16(cvt16_src03, filter_20);

  409.             sum00_even = _mm256_add_epi32(temp0, temp1);

  410. 

  411.             temp0 = _mm256_madd_epi16(cvt16_src10, filter_34);

  412.             temp1 = _mm256_madd_epi16(cvt16_src12, filter_50);

  413.             sum11_odd = _mm256_add_epi32(temp0, temp1);

  414. 

  415.             temp0 = _mm256_madd_epi16(cvt16_src11, filter_34);

  416.             temp1 = _mm256_madd_epi16(cvt16_src13, filter_50);

  417.             sum11_even = _mm256_add_epi32(temp0, temp1);

  418. 

  419.             temp0 = _mm256_madd_epi16(cvt16_src20, filter_67);

  420.             temp1 = _mm256_madd_epi16(cvt16_src22, filter_80);

  421.             sum22_odd = _mm256_add_epi32(temp0, temp1);

  422. 

  423.             temp0 = _mm256_madd_epi16(cvt16_src21, filter_67);

  424.             temp1 = _mm256_madd_epi16(cvt16_src23, filter_80);

  425.             sum22_even = _mm256_add_epi32(temp0, temp1);

  426. 

  427.             sum00_odd = _mm256_add_epi32(sum00_odd, sum11_odd);

  428.             sum00_odd = _mm256_add_epi32(sum00_odd, sum22_odd);

  429. 

  430.             sum00_even = _mm256_add_epi32(sum00_even, sum11_even);

  431.             sum00_even = _mm256_add_epi32(sum00_even, sum22_even);

  432. 

  433.             __m256i sum_odd = _mm256_unpacklo_epi32(sum00_odd, sum00_even);

  434.             __m256i sum_even = _mm256_unpackhi_epi32(sum00_odd, sum00_even);

  435. 

  436.             __m256i sum_left = _mm256_permute2f128_si256(sum_odd, sum_even, 32);

  437.             __m256i sum_right =

  438.                     _mm256_permute2f128_si256(sum_odd, sum_even, 49);

  439. 

  440.             sum_left = _mm256_add_epi32(sum_left, bias_val);

  441.             sum_right = _mm256_add_epi32(sum_right, bias_val);

  442. 

  443.             if (is_quantized) {

  444.                 op({{sum_left, sum_right}}, reinterpret_cast<dt_qint8*>(dst0));

  445.             } else {

  446.                 _mm256_storeu_si256((__m256i*)(out_ptr0), sum_left);

  447.                 _mm256_storeu_si256((__m256i*)(out_ptr0 + 8), sum_right);

  448.             }

  449. 

  450.             r0 += 16;

  451.             r1 += 16;

  452.             r2 += 16;

  453.             dst0 += 16;

  454.             out_ptr0 += 16;

  455.         }

  456.         r0 += tail_step;

  457.         r1 += tail_step;

  458.         r2 += tail_step;

  459.     }

  460. }

  461. 

  462. template <BiasMode bias_mode, bool is_quantized, typename Op>

  463. void avx2_chanwise_direct_stride1_5x5_int8(const int8_t* src,

  464.                                            const int8_t* filter,

  465.                                            const int32_t* bias, int32_t* temp,

  466.                                            int8_t* dst, const size_t IH,

  467.                                            const size_t IW, const size_t OH,

  468.                                            const size_t OW, const Op& op) {

  469.     MEGDNN_MARK_USED_VAR(IH);

  470.     size_t tail_step = IW - OW;

  471.     int8_t* dst0 = dst;

  472.     int8_t* dst1 = dst + OW;

  473.     int32_t* out_ptr0 = temp;

  474.     int32_t* out_ptr1 = temp + OW;

  475.     const int8_t* r0 = src;

  476.     const int8_t* r1 = src + IW;

  477.     const int8_t* r2 = src + 2 * IW;

  478.     const int8_t* r3 = src + 3 * IW;

  479.     const int8_t* r4 = src + 4 * IW;

  480.     const int8_t* r5 = src + 5 * IW;

  481. 

  482.     uint8_t fill_zero = 0;

  483.     UNROLL_CALL0(25, load_filter)

  484. 

  485.     __m128i k_fill = _mm_set1_epi8(fill_zero);

  486. 

  487.     __m128i k01 = _mm_unpacklo_epi8(k_0, k_1);

  488.     __m128i k23 = _mm_unpacklo_epi8(k_2, k_3);

  489.     __m128i k40 = _mm_unpacklo_epi8(k_4, k_fill);

  490. 

  491.     __m128i k56 = _mm_unpacklo_epi8(k_5, k_6);

  492.     __m128i k78 = _mm_unpacklo_epi8(k_7, k_8);

  493.     __m128i k90 = _mm_unpacklo_epi8(k_9, k_fill);

  494. 

  495.     __m128i k1011 = _mm_unpacklo_epi8(k_10, k_11);

  496.     __m128i k1213 = _mm_unpacklo_epi8(k_12, k_13);

  497.     __m128i k140 = _mm_unpacklo_epi8(k_14, k_fill);

  498. 

  499.     __m128i k1516 = _mm_unpacklo_epi8(k_15, k_16);

  500.     __m128i k1718 = _mm_unpacklo_epi8(k_17, k_18);

  501.     __m128i k190 = _mm_unpacklo_epi8(k_19, k_fill);

  502. 

  503.     __m128i k2021 = _mm_unpacklo_epi8(k_20, k_21);

  504.     __m128i k2223 = _mm_unpacklo_epi8(k_22, k_23);

  505.     __m128i k240 = _mm_unpacklo_epi8(k_24, k_fill);

  506. 

  507.     __m256i bias_val;

  508.     //! load bias

  509.     if (bias_mode == BiasMode::BROADCAST_CHANNEL_BIAS) {

  510.         bias_val = _mm256_set1_epi32(*(bias));

  511.     } else {

  512.         bias_val = _mm256_set1_epi32(0);

  513.     }

  514. 

  515.     //! cvt i8 --> i16

  516.     __m256i filter_01 = _mm256_cvtepi8_epi16(k01);

  517.     __m256i filter_23 = _mm256_cvtepi8_epi16(k23);

  518.     __m256i filter_40 = _mm256_cvtepi8_epi16(k40);

  519. 

  520.     __m256i filter_56 = _mm256_cvtepi8_epi16(k56);

  521.     __m256i filter_78 = _mm256_cvtepi8_epi16(k78);

  522.     __m256i filter_90 = _mm256_cvtepi8_epi16(k90);

  523. 

  524.     __m256i filter_1011 = _mm256_cvtepi8_epi16(k1011);

  525.     __m256i filter_1213 = _mm256_cvtepi8_epi16(k1213);

  526.     __m256i filter_140 = _mm256_cvtepi8_epi16(k140);

  527. 

  528.     __m256i filter_1516 = _mm256_cvtepi8_epi16(k1516);

  529.     __m256i filter_1718 = _mm256_cvtepi8_epi16(k1718);

  530.     __m256i filter_190 = _mm256_cvtepi8_epi16(k190);

  531. 

  532.     __m256i filter_2021 = _mm256_cvtepi8_epi16(k2021);

  533.     __m256i filter_2223 = _mm256_cvtepi8_epi16(k2223);

  534.     __m256i filter_240 = _mm256_cvtepi8_epi16(k240);

  535. 

  536.     size_t width = OW >> 4;

  537.     size_t h = 0;

  538.     for (; h + 1 < OH; h += 2) {

  539.         size_t w = 0;

  540.         for (; w < width; w++) {

  541.             UNROLL_CALL0(6, load_src0)

  542.             UNROLL_CALL0(6, load_src1)

  543.             UNROLL_CALL0(6, load_src2)

  544.             UNROLL_CALL0(6, load_src3)

  545.             UNROLL_CALL0(6, load_src4)

  546.             UNROLL_CALL0(6, load_src5)

  547. 

  548.             __m256i sum0_odd, sum0_even, sum1_odd, sum1_even, sum2_odd,

  549.                     sum2_even, sum3_odd, sum3_even, sum4_odd, sum4_even;

  550. 

  551.             __m256i sum10_odd, sum10_even, sum20_odd, sum20_even, sum30_odd,

  552.                     sum30_even, sum40_odd, sum40_even, sum5_odd, sum5_even;

  553. 

  554.             //! cal src0

  555.             __m256i dot1, dot2, dot3;

  556.             dot1 = _mm256_madd_epi16(cvt16_src00, filter_01);

  557.             dot2 = _mm256_madd_epi16(cvt16_src02, filter_23);

  558.             dot3 = _mm256_madd_epi16(cvt16_src04, filter_40);

  559.             sum0_odd = _mm256_add_epi32(dot1, dot2);

  560.             sum0_odd = _mm256_add_epi32(sum0_odd, dot3);

  561. 

  562.             dot1 = _mm256_madd_epi16(cvt16_src01, filter_01);

  563.             dot2 = _mm256_madd_epi16(cvt16_src03, filter_23);

  564.             dot3 = _mm256_madd_epi16(cvt16_src05, filter_40);

  565.             sum0_even = _mm256_add_epi32(dot1, dot2);

  566.             sum0_even = _mm256_add_epi32(sum0_even, dot3);

  567. 

  568.             //! cal src1

  569.             dot1 = _mm256_madd_epi16(cvt16_src10, filter_56);

  570.             dot2 = _mm256_madd_epi16(cvt16_src12, filter_78);

  571.             dot3 = _mm256_madd_epi16(cvt16_src14, filter_90);

  572.             sum1_odd = _mm256_add_epi32(dot1, dot2);

  573.             sum1_odd = _mm256_add_epi32(sum1_odd, dot3);

  574. 

  575.             dot1 = _mm256_madd_epi16(cvt16_src11, filter_56);

  576.             dot2 = _mm256_madd_epi16(cvt16_src13, filter_78);

  577.             dot3 = _mm256_madd_epi16(cvt16_src15, filter_90);

  578.             sum1_even = _mm256_add_epi32(dot1, dot2);

  579.             sum1_even = _mm256_add_epi32(sum1_even, dot3);

  580. 

  581.             dot1 = _mm256_madd_epi16(cvt16_src10, filter_01);

  582.             dot2 = _mm256_madd_epi16(cvt16_src12, filter_23);

  583.             dot3 = _mm256_madd_epi16(cvt16_src14, filter_40);

  584.             sum10_odd = _mm256_add_epi32(dot1, dot2);

  585.             sum10_odd = _mm256_add_epi32(sum10_odd, dot3);

  586. 

  587.             dot1 = _mm256_madd_epi16(cvt16_src11, filter_01);

  588.             dot2 = _mm256_madd_epi16(cvt16_src13, filter_23);

  589.             dot3 = _mm256_madd_epi16(cvt16_src15, filter_40);

  590.             sum10_even = _mm256_add_epi32(dot1, dot2);

  591.             sum10_even = _mm256_add_epi32(sum10_even, dot3);

  592. 

  593.             //! cal src2

  594.             dot1 = _mm256_madd_epi16(cvt16_src20, filter_1011);

  595.             dot2 = _mm256_madd_epi16(cvt16_src22, filter_1213);

  596.             dot3 = _mm256_madd_epi16(cvt16_src24, filter_140);

  597.             sum2_odd = _mm256_add_epi32(dot1, dot2);

  598.             sum2_odd = _mm256_add_epi32(sum2_odd, dot3);

  599. 

  600.             dot1 = _mm256_madd_epi16(cvt16_src21, filter_1011);

  601.             dot2 = _mm256_madd_epi16(cvt16_src23, filter_1213);

  602.             dot3 = _mm256_madd_epi16(cvt16_src25, filter_140);

  603.             sum2_even = _mm256_add_epi32(dot1, dot2);

  604.             sum2_even = _mm256_add_epi32(sum2_even, dot3);

  605. 

  606.             dot1 = _mm256_madd_epi16(cvt16_src20, filter_56);

  607.             dot2 = _mm256_madd_epi16(cvt16_src22, filter_78);

  608.             dot3 = _mm256_madd_epi16(cvt16_src24, filter_90);

  609.             sum20_odd = _mm256_add_epi32(dot1, dot2);

  610.             sum20_odd = _mm256_add_epi32(sum20_odd, dot3);

  611. 

  612.             dot1 = _mm256_madd_epi16(cvt16_src21, filter_56);

  613.             dot2 = _mm256_madd_epi16(cvt16_src23, filter_78);

  614.             dot3 = _mm256_madd_epi16(cvt16_src25, filter_90);

  615.             sum20_even = _mm256_add_epi32(dot1, dot2);

  616.             sum20_even = _mm256_add_epi32(sum20_even, dot3);

  617. 

  618.             //! cal src3

  619.             dot1 = _mm256_madd_epi16(cvt16_src30, filter_1516);

  620.             dot2 = _mm256_madd_epi16(cvt16_src32, filter_1718);

  621.             dot3 = _mm256_madd_epi16(cvt16_src34, filter_190);

  622.             sum3_odd = _mm256_add_epi32(dot1, dot2);

  623.             sum3_odd = _mm256_add_epi32(sum3_odd, dot3);

  624. 

  625.             dot1 = _mm256_madd_epi16(cvt16_src31, filter_1516);

  626.             dot2 = _mm256_madd_epi16(cvt16_src33, filter_1718);

  627.             dot3 = _mm256_madd_epi16(cvt16_src35, filter_190);

  628.             sum3_even = _mm256_add_epi32(dot1, dot2);

  629.             sum3_even = _mm256_add_epi32(sum3_even, dot3);

  630. 

  631.             dot1 = _mm256_madd_epi16(cvt16_src30, filter_1011);

  632.             dot2 = _mm256_madd_epi16(cvt16_src32, filter_1213);

  633.             dot3 = _mm256_madd_epi16(cvt16_src34, filter_140);

  634.             sum30_odd = _mm256_add_epi32(dot1, dot2);

  635.             sum30_odd = _mm256_add_epi32(sum30_odd, dot3);

  636. 

  637.             dot1 = _mm256_madd_epi16(cvt16_src31, filter_1011);

  638.             dot2 = _mm256_madd_epi16(cvt16_src33, filter_1213);

  639.             dot3 = _mm256_madd_epi16(cvt16_src35, filter_140);

  640.             sum30_even = _mm256_add_epi32(dot1, dot2);

  641.             sum30_even = _mm256_add_epi32(sum30_even, dot3);

  642. 

  643.             //! cal src4

  644.             dot1 = _mm256_madd_epi16(cvt16_src40, filter_2021);

  645.             dot2 = _mm256_madd_epi16(cvt16_src42, filter_2223);

  646.             dot3 = _mm256_madd_epi16(cvt16_src44, filter_240);

  647.             sum4_odd = _mm256_add_epi32(dot1, dot2);

  648.             sum4_odd = _mm256_add_epi32(sum4_odd, dot3);

  649. 

  650.             dot1 = _mm256_madd_epi16(cvt16_src41, filter_2021);

  651.             dot2 = _mm256_madd_epi16(cvt16_src43, filter_2223);

  652.             dot3 = _mm256_madd_epi16(cvt16_src45, filter_240);

  653.             sum4_even = _mm256_add_epi32(dot1, dot2);

  654.             sum4_even = _mm256_add_epi32(sum4_even, dot3);

  655. 

  656.             dot1 = _mm256_madd_epi16(cvt16_src40, filter_1516);

  657.             dot2 = _mm256_madd_epi16(cvt16_src42, filter_1718);

  658.             dot3 = _mm256_madd_epi16(cvt16_src44, filter_190);

  659.             sum40_odd = _mm256_add_epi32(dot1, dot2);

  660.             sum40_odd = _mm256_add_epi32(sum40_odd, dot3);

  661. 

  662.             dot1 = _mm256_madd_epi16(cvt16_src41, filter_1516);

  663.             dot2 = _mm256_madd_epi16(cvt16_src43, filter_1718);

  664.             dot3 = _mm256_madd_epi16(cvt16_src45, filter_190);

  665.             sum40_even = _mm256_add_epi32(dot1, dot2);

  666.             sum40_even = _mm256_add_epi32(sum40_even, dot3);

  667. 

  668.             //! cal src5

  669.             dot1 = _mm256_madd_epi16(cvt16_src50, filter_2021);

  670.             dot2 = _mm256_madd_epi16(cvt16_src52, filter_2223);

  671.             dot3 = _mm256_madd_epi16(cvt16_src54, filter_240);

  672.             sum5_odd = _mm256_add_epi32(dot1, dot2);

  673.             sum5_odd = _mm256_add_epi32(sum5_odd, dot3);

  674. 

  675.             dot1 = _mm256_madd_epi16(cvt16_src51, filter_2021);

  676.             dot2 = _mm256_madd_epi16(cvt16_src53, filter_2223);

  677.             dot3 = _mm256_madd_epi16(cvt16_src55, filter_240);

  678.             sum5_even = _mm256_add_epi32(dot1, dot2);

  679.             sum5_even = _mm256_add_epi32(sum5_even, dot3);

  680. 

  681.             __m256i sum_odd, sum_even;

  682. 

  683.             sum_odd = _mm256_add_epi32(sum0_odd, sum1_odd);

  684.             sum_odd = _mm256_add_epi32(sum_odd, sum2_odd);

  685.             sum_odd = _mm256_add_epi32(sum_odd, sum3_odd);

  686.             sum_odd = _mm256_add_epi32(sum_odd, sum4_odd);

  687. 

  688.             sum_even = _mm256_add_epi32(sum0_even, sum1_even);

  689.             sum_even = _mm256_add_epi32(sum_even, sum2_even);

  690.             sum_even = _mm256_add_epi32(sum_even, sum3_even);

  691.             sum_even = _mm256_add_epi32(sum_even, sum4_even);

  692. 

  693.             __m256i sum_odd_0 = _mm256_unpacklo_epi32(sum_odd, sum_even);

  694.             __m256i sum_even_0 = _mm256_unpackhi_epi32(sum_odd, sum_even);

  695. 

  696.             __m256i sum_left =

  697.                     _mm256_permute2f128_si256(sum_odd_0, sum_even_0, 32);

  698.             __m256i sum_right =

  699.                     _mm256_permute2f128_si256(sum_odd_0, sum_even_0, 49);

  700. 

  701.             sum_left = _mm256_add_epi32(sum_left, bias_val);

  702.             sum_right = _mm256_add_epi32(sum_right, bias_val);

  703. 

  704.             if (is_quantized) {

  705.                 op({{sum_left, sum_right}}, reinterpret_cast<dt_qint8*>(dst0));

  706.             } else {

  707.                 _mm256_storeu_si256((__m256i*)(out_ptr0), sum_left);

  708.                 _mm256_storeu_si256((__m256i*)(out_ptr0 + 8), sum_right);

  709.             }

  710. 

  711.             __m256i sum_odd_oh1, sum_even_oh1;

  712. 

  713.             sum_odd_oh1 = _mm256_add_epi32(sum10_odd, sum20_odd);

  714.             sum_odd_oh1 = _mm256_add_epi32(sum_odd_oh1, sum30_odd);

  715.             sum_odd_oh1 = _mm256_add_epi32(sum_odd_oh1, sum40_odd);

  716.             sum_odd_oh1 = _mm256_add_epi32(sum_odd_oh1, sum5_odd);

  717. 

  718.             sum_even_oh1 = _mm256_add_epi32(sum10_even, sum20_even);

  719.             sum_even_oh1 = _mm256_add_epi32(sum_even_oh1, sum30_even);

  720.             sum_even_oh1 = _mm256_add_epi32(sum_even_oh1, sum40_even);

  721.             sum_even_oh1 = _mm256_add_epi32(sum_even_oh1, sum5_even);

  722. 

  723.             __m256i sum_odd_1 =

  724.                     _mm256_unpacklo_epi32(sum_odd_oh1, sum_even_oh1);

  725.             __m256i sum_even_1 =

  726.                     _mm256_unpackhi_epi32(sum_odd_oh1, sum_even_oh1);

  727. 

  728.             sum_left = _mm256_permute2f128_si256(sum_odd_1, sum_even_1, 32);

  729.             sum_right = _mm256_permute2f128_si256(sum_odd_1, sum_even_1, 49);

  730. 

  731.             sum_left = _mm256_add_epi32(sum_left, bias_val);

  732.             sum_right = _mm256_add_epi32(sum_right, bias_val);

  733. 

  734.             if (is_quantized) {

  735.                 op({{sum_left, sum_right}}, reinterpret_cast<dt_qint8*>(dst1));

  736.             } else {

  737.                 _mm256_storeu_si256((__m256i*)(out_ptr1), sum_left);

  738.                 _mm256_storeu_si256((__m256i*)(out_ptr1 + 8), sum_right);

  739.             }

  740. 

  741.             r0 += 16;

  742.             r1 += 16;

  743.             r2 += 16;

  744.             r3 += 16;

  745.             r4 += 16;

  746.             r5 += 16;

  747.             dst0 += 16;

  748.             dst1 += 16;

  749.             out_ptr0 += 16;

  750.             out_ptr1 += 16;

  751.         }

  752.         r0 += tail_step + IW;

  753.         r1 += tail_step + IW;

  754.         r2 += tail_step + IW;

  755.         r3 += tail_step + IW;

  756.         r4 += tail_step + IW;

  757.         r5 += tail_step + IW;

  758. 

  759.         dst0 += OW;

  760.         dst1 += OW;

  761.         out_ptr0 += OW;

  762.         out_ptr1 += OW;

  763.     }

  764. 

  765.     for (; h < OH; h++) {

  766.         size_t w = 0;

  767.         for (; w < width; w++) {

  768.             UNROLL_CALL0(5, load_src0)

  769.             UNROLL_CALL0(5, load_src1)

  770.             UNROLL_CALL0(5, load_src2)

  771.             UNROLL_CALL0(5, load_src3)

  772.             UNROLL_CALL0(5, load_src4)

  773.             UNROLL_CALL0(5, load_src5)

  774. 

  775.             __m256i sum0_odd, sum0_even, sum1_odd, sum1_even, sum2_odd,

  776.                     sum2_even, sum3_odd, sum3_even, sum4_odd, sum4_even;

  777. 

  778.             //! cal src0

  779.             __m256i dot1, dot2, dot3;

  780.             dot1 = _mm256_madd_epi16(cvt16_src00, filter_01);

  781.             dot2 = _mm256_madd_epi16(cvt16_src02, filter_23);

  782.             dot3 = _mm256_madd_epi16(cvt16_src04, filter_40);

  783.             sum0_odd = _mm256_add_epi32(dot1, dot2);

  784.             sum0_odd = _mm256_add_epi32(sum0_odd, dot3);

  785. 

  786.             dot1 = _mm256_madd_epi16(cvt16_src01, filter_01);

  787.             dot2 = _mm256_madd_epi16(cvt16_src03, filter_23);

  788.             dot3 = _mm256_madd_epi16(cvt16_src05, filter_40);

  789.             sum0_even = _mm256_add_epi32(dot1, dot2);

  790.             sum0_even = _mm256_add_epi32(sum0_even, dot3);

  791. 

  792.             //! cal src1

  793.             dot1 = _mm256_madd_epi16(cvt16_src10, filter_56);

  794.             dot2 = _mm256_madd_epi16(cvt16_src12, filter_78);

  795.             dot3 = _mm256_madd_epi16(cvt16_src14, filter_90);

  796.             sum1_odd = _mm256_add_epi32(dot1, dot2);

  797.             sum1_odd = _mm256_add_epi32(sum1_odd, dot3);

  798. 

  799.             dot1 = _mm256_madd_epi16(cvt16_src11, filter_56);

  800.             dot2 = _mm256_madd_epi16(cvt16_src13, filter_78);

  801.             dot3 = _mm256_madd_epi16(cvt16_src15, filter_90);

  802.             sum1_even = _mm256_add_epi32(dot1, dot2);

  803.             sum1_even = _mm256_add_epi32(sum1_even, dot3);

  804. 

  805.             //! cal src2

  806.             dot1 = _mm256_madd_epi16(cvt16_src20, filter_1011);

  807.             dot2 = _mm256_madd_epi16(cvt16_src22, filter_1213);

  808.             dot3 = _mm256_madd_epi16(cvt16_src24, filter_140);

  809.             sum2_odd = _mm256_add_epi32(dot1, dot2);

  810.             sum2_odd = _mm256_add_epi32(sum2_odd, dot3);

  811. 

  812.             dot1 = _mm256_madd_epi16(cvt16_src21, filter_1011);

  813.             dot2 = _mm256_madd_epi16(cvt16_src23, filter_1213);

  814.             dot3 = _mm256_madd_epi16(cvt16_src25, filter_140);

  815.             sum2_even = _mm256_add_epi32(dot1, dot2);

  816.             sum2_even = _mm256_add_epi32(sum2_even, dot3);

  817. 

  818.             //! cal src3

  819.             dot1 = _mm256_madd_epi16(cvt16_src30, filter_1516);

  820.             dot2 = _mm256_madd_epi16(cvt16_src32, filter_1718);

  821.             dot3 = _mm256_madd_epi16(cvt16_src34, filter_190);

  822.             sum3_odd = _mm256_add_epi32(dot1, dot2);

  823.             sum3_odd = _mm256_add_epi32(sum3_odd, dot3);

  824. 

  825.             dot1 = _mm256_madd_epi16(cvt16_src31, filter_1516);

  826.             dot2 = _mm256_madd_epi16(cvt16_src33, filter_1718);

  827.             dot3 = _mm256_madd_epi16(cvt16_src35, filter_190);

  828.             sum3_even = _mm256_add_epi32(dot1, dot2);

  829.             sum3_even = _mm256_add_epi32(sum3_even, dot3);

  830. 

  831.             //! cal src4

  832.             dot1 = _mm256_madd_epi16(cvt16_src40, filter_2021);

  833.             dot2 = _mm256_madd_epi16(cvt16_src42, filter_2223);

  834.             dot3 = _mm256_madd_epi16(cvt16_src44, filter_240);

  835.             sum4_odd = _mm256_add_epi32(dot1, dot2);

  836.             sum4_odd = _mm256_add_epi32(sum4_odd, dot3);

  837. 

  838.             dot1 = _mm256_madd_epi16(cvt16_src41, filter_2021);

  839.             dot2 = _mm256_madd_epi16(cvt16_src43, filter_2223);

  840.             dot3 = _mm256_madd_epi16(cvt16_src45, filter_240);

  841.             sum4_even = _mm256_add_epi32(dot1, dot2);

  842.             sum4_even = _mm256_add_epi32(sum4_even, dot3);

  843. 

  844.             __m256i sum_odd, sum_even;

  845. 

  846.             sum_odd = _mm256_add_epi32(sum0_odd, sum1_odd);

  847.             sum_odd = _mm256_add_epi32(sum_odd, sum2_odd);

  848.             sum_odd = _mm256_add_epi32(sum_odd, sum3_odd);

  849.             sum_odd = _mm256_add_epi32(sum_odd, sum4_odd);

  850. 

  851.             sum_even = _mm256_add_epi32(sum0_even, sum1_even);

  852.             sum_even = _mm256_add_epi32(sum_even, sum2_even);

  853.             sum_even = _mm256_add_epi32(sum_even, sum3_even);

  854.             sum_even = _mm256_add_epi32(sum_even, sum4_even);

  855. 

  856.             __m256i sum_odd_0 = _mm256_unpacklo_epi32(sum_odd, sum_even);

  857.             __m256i sum_even_0 = _mm256_unpackhi_epi32(sum_odd, sum_even);

  858. 

  859.             __m256i sum_left =

  860.                     _mm256_permute2f128_si256(sum_odd_0, sum_even_0, 32);

  861.             __m256i sum_right =

  862.                     _mm256_permute2f128_si256(sum_odd_0, sum_even_0, 49);

  863. 

  864.             sum_left = _mm256_add_epi32(sum_left, bias_val);

  865.             sum_right = _mm256_add_epi32(sum_right, bias_val);

  866. 

  867.             if (is_quantized) {

  868.                 op({{sum_left, sum_right}}, reinterpret_cast<dt_qint8*>(dst0));

  869.             } else {

  870.                 _mm256_storeu_si256((__m256i*)(out_ptr0), sum_left);

  871.                 _mm256_storeu_si256((__m256i*)(out_ptr0 + 8), sum_right);

  872.             }

  873. 

  874.             r0 += 16;

  875.             r1 += 16;

  876.             r2 += 16;

  877.             r3 += 16;

  878.             r4 += 16;

  879.             dst0 += 16;

  880.             out_ptr0 += 16;

  881.         }

  882.         r0 += tail_step;

  883.         r1 += tail_step;

  884.         r2 += tail_step;

  885.         r3 += tail_step;

  886.         r4 += tail_step;

  887.     }

  888. }

  889. 

  890. template <BiasMode bias_mode, bool is_quantized, typename Op>

  891. void avx2_chanwise_direct_stride1_7x7_int8(const int8_t* src,

  892.                                            const int8_t* filter,

  893.                                            const int32_t* bias, int32_t* temp,

  894.                                            int8_t* dst, const size_t IH,

  895.                                            const size_t IW, const size_t OH,

  896.                                            const size_t OW, const Op& op) {

  897.     MEGDNN_MARK_USED_VAR(IH);

  898.     size_t tail_step = IW - OW;

  899.     int8_t* dst0 = dst;

  900.     int8_t* dst1 = dst + OW;

  901.     int32_t* out_ptr0 = temp;

  902.     int32_t* out_ptr1 = temp + OW;

  903.     const int8_t* r0 = src;

  904.     const int8_t* r1 = src + IW;

  905.     const int8_t* r2 = src + 2 * IW;

  906.     const int8_t* r3 = src + 3 * IW;

  907.     const int8_t* r4 = src + 4 * IW;

  908.     const int8_t* r5 = src + 5 * IW;

  909.     const int8_t* r6 = src + 6 * IW;

  910.     const int8_t* r7 = src + 7 * IW;

  911. 

  912.     uint8_t fill_zero = 0;

  913.     UNROLL_CALL0(49, load_filter)

  914. 

  915.     __m128i k_fill = _mm_set1_epi8(fill_zero);

  916. 

  917.     __m128i k01 = _mm_unpacklo_epi8(k_0, k_1);

  918.     __m128i k23 = _mm_unpacklo_epi8(k_2, k_3);

  919.     __m128i k45 = _mm_unpacklo_epi8(k_4, k_5);

  920.     __m128i k60 = _mm_unpacklo_epi8(k_6, k_fill);

  921. 

  922.     __m128i k78 = _mm_unpacklo_epi8(k_7, k_8);

  923.     __m128i k910 = _mm_unpacklo_epi8(k_9, k_10);

  924.     __m128i k1112 = _mm_unpacklo_epi8(k_11, k_12);

  925.     __m128i k130 = _mm_unpacklo_epi8(k_13, k_fill);

  926. 

  927.     __m128i k1415 = _mm_unpacklo_epi8(k_14, k_15);

  928.     __m128i k1617 = _mm_unpacklo_epi8(k_16, k_17);

  929.     __m128i k1819 = _mm_unpacklo_epi8(k_18, k_19);

  930.     __m128i k200 = _mm_unpacklo_epi8(k_20, k_fill);

  931. 

  932.     __m128i k2122 = _mm_unpacklo_epi8(k_21, k_22);

  933.     __m128i k2324 = _mm_unpacklo_epi8(k_23, k_24);

  934.     __m128i k2526 = _mm_unpacklo_epi8(k_25, k_26);

  935.     __m128i k270 = _mm_unpacklo_epi8(k_27, k_fill);

  936. 

  937.     __m128i k2829 = _mm_unpacklo_epi8(k_28, k_29);

  938.     __m128i k3031 = _mm_unpacklo_epi8(k_30, k_31);

  939.     __m128i k3233 = _mm_unpacklo_epi8(k_32, k_33);

  940.     __m128i k340 = _mm_unpacklo_epi8(k_34, k_fill);

  941. 

  942.     __m128i k3536 = _mm_unpacklo_epi8(k_35, k_36);

  943.     __m128i k3738 = _mm_unpacklo_epi8(k_37, k_38);

  944.     __m128i k3940 = _mm_unpacklo_epi8(k_39, k_40);

  945.     __m128i k410 = _mm_unpacklo_epi8(k_41, k_fill);

  946. 

  947.     __m128i k4243 = _mm_unpacklo_epi8(k_42, k_43);

  948.     __m128i k4445 = _mm_unpacklo_epi8(k_44, k_45);

  949.     __m128i k4647 = _mm_unpacklo_epi8(k_46, k_47);

  950.     __m128i k480 = _mm_unpacklo_epi8(k_48, k_fill);

  951. 

  952.     __m256i bias_val;

  953.     //! load bias

  954.     if (bias_mode == BiasMode::BROADCAST_CHANNEL_BIAS) {

  955.         bias_val = _mm256_set1_epi32(*(bias));

  956.     } else {

  957.         bias_val = _mm256_set1_epi32(0);

  958.     }

  959. 

  960.     //! cvt i8 --> i16

  961.     __m256i filter_01 = _mm256_cvtepi8_epi16(k01);

  962.     __m256i filter_23 = _mm256_cvtepi8_epi16(k23);

  963.     __m256i filter_45 = _mm256_cvtepi8_epi16(k45);

  964.     __m256i filter_60 = _mm256_cvtepi8_epi16(k60);

  965. 

  966.     __m256i filter_78 = _mm256_cvtepi8_epi16(k78);

  967.     __m256i filter_910 = _mm256_cvtepi8_epi16(k910);

  968.     __m256i filter_1112 = _mm256_cvtepi8_epi16(k1112);

  969.     __m256i filter_130 = _mm256_cvtepi8_epi16(k130);

  970. 

  971.     __m256i filter_1415 = _mm256_cvtepi8_epi16(k1415);

  972.     __m256i filter_1617 = _mm256_cvtepi8_epi16(k1617);

  973.     __m256i filter_1819 = _mm256_cvtepi8_epi16(k1819);

  974.     __m256i filter_200 = _mm256_cvtepi8_epi16(k200);

  975. 

  976.     __m256i filter_2122 = _mm256_cvtepi8_epi16(k2122);

  977.     __m256i filter_2324 = _mm256_cvtepi8_epi16(k2324);

  978.     __m256i filter_2526 = _mm256_cvtepi8_epi16(k2526);

  979.     __m256i filter_270 = _mm256_cvtepi8_epi16(k270);

  980. 

  981.     __m256i filter_2829 = _mm256_cvtepi8_epi16(k2829);

  982.     __m256i filter_3031 = _mm256_cvtepi8_epi16(k3031);

  983.     __m256i filter_3233 = _mm256_cvtepi8_epi16(k3233);

  984.     __m256i filter_340 = _mm256_cvtepi8_epi16(k340);

  985. 

  986.     __m256i filter_3536 = _mm256_cvtepi8_epi16(k3536);

  987.     __m256i filter_3738 = _mm256_cvtepi8_epi16(k3738);

  988.     __m256i filter_3940 = _mm256_cvtepi8_epi16(k3940);

  989.     __m256i filter_410 = _mm256_cvtepi8_epi16(k410);

  990. 

  991.     __m256i filter_4243 = _mm256_cvtepi8_epi16(k4243);

  992.     __m256i filter_4445 = _mm256_cvtepi8_epi16(k4445);

  993.     __m256i filter_4647 = _mm256_cvtepi8_epi16(k4647);

  994.     __m256i filter_480 = _mm256_cvtepi8_epi16(k480);

  995. 

  996.     size_t width = OW >> 4;

  997.     size_t h = 0;

  998.     for (; h + 1 < OH; h += 2) {

  999.         size_t w = 0;

  1000.         for (; w < width; w++) {

  1001.             UNROLL_CALL0(8, load_src0)

  1002.             UNROLL_CALL0(8, load_src1)

  1003.             UNROLL_CALL0(8, load_src2)

  1004.             UNROLL_CALL0(8, load_src3)

  1005.             UNROLL_CALL0(8, load_src4)

  1006.             UNROLL_CALL0(8, load_src5)

  1007.             UNROLL_CALL0(8, load_src6)

  1008.             UNROLL_CALL0(8, load_src7)

  1009. 

  1010.             __m256i sum0_odd, sum0_even, sum1_odd, sum1_even, sum2_odd,

  1011.                     sum2_even, sum3_odd, sum3_even, sum4_odd, sum4_even,

  1012.                     sum5_odd, sum5_even, sum6_odd, sum6_even;

  1013. 

  1014.             __m256i sum10_odd, sum10_even, sum20_odd, sum20_even, sum30_odd,

  1015.                     sum30_even, sum40_odd, sum40_even, sum50_odd, sum50_even,

  1016.                     sum60_odd, sum60_even, sum7_odd, sum7_even;

  1017. 

  1018.             //! cal src0

  1019.             __m256i dot1, dot2, dot3, dot4;

  1020.             dot1 = _mm256_madd_epi16(cvt16_src00, filter_01);

  1021.             dot2 = _mm256_madd_epi16(cvt16_src02, filter_23);

  1022.             dot3 = _mm256_madd_epi16(cvt16_src04, filter_45);

  1023.             dot4 = _mm256_madd_epi16(cvt16_src06, filter_60);

  1024.             sum0_odd = _mm256_add_epi32(dot1, dot2);

  1025.             sum0_odd = _mm256_add_epi32(sum0_odd, dot3);

  1026.             sum0_odd = _mm256_add_epi32(sum0_odd, dot4);

  1027. 

  1028.             dot1 = _mm256_madd_epi16(cvt16_src01, filter_01);

  1029.             dot2 = _mm256_madd_epi16(cvt16_src03, filter_23);

  1030.             dot3 = _mm256_madd_epi16(cvt16_src05, filter_45);

  1031.             dot4 = _mm256_madd_epi16(cvt16_src07, filter_60);

  1032.             sum0_even = _mm256_add_epi32(dot1, dot2);

  1033.             sum0_even = _mm256_add_epi32(sum0_even, dot3);

  1034.             sum0_even = _mm256_add_epi32(sum0_even, dot4);

  1035. 

  1036.             //! cal src1

  1037.             dot1 = _mm256_madd_epi16(cvt16_src10, filter_78);

  1038.             dot2 = _mm256_madd_epi16(cvt16_src12, filter_910);

  1039.             dot3 = _mm256_madd_epi16(cvt16_src14, filter_1112);

  1040.             dot4 = _mm256_madd_epi16(cvt16_src16, filter_130);

  1041.             sum1_odd = _mm256_add_epi32(dot1, dot2);

  1042.             sum1_odd = _mm256_add_epi32(sum1_odd, dot3);

  1043.             sum1_odd = _mm256_add_epi32(sum1_odd, dot4);

  1044. 

  1045.             dot1 = _mm256_madd_epi16(cvt16_src11, filter_78);

  1046.             dot2 = _mm256_madd_epi16(cvt16_src13, filter_910);

  1047.             dot3 = _mm256_madd_epi16(cvt16_src15, filter_1112);

  1048.             dot4 = _mm256_madd_epi16(cvt16_src17, filter_130);

  1049.             sum1_even = _mm256_add_epi32(dot1, dot2);

  1050.             sum1_even = _mm256_add_epi32(sum1_even, dot3);

  1051.             sum1_even = _mm256_add_epi32(sum1_even, dot4);

  1052. 

  1053.             dot1 = _mm256_madd_epi16(cvt16_src10, filter_01);

  1054.             dot2 = _mm256_madd_epi16(cvt16_src12, filter_23);

  1055.             dot3 = _mm256_madd_epi16(cvt16_src14, filter_45);

  1056.             dot4 = _mm256_madd_epi16(cvt16_src16, filter_60);

  1057.             sum10_odd = _mm256_add_epi32(dot1, dot2);

  1058.             sum10_odd = _mm256_add_epi32(sum10_odd, dot3);

  1059.             sum10_odd = _mm256_add_epi32(sum10_odd, dot4);

  1060. 

  1061.             dot1 = _mm256_madd_epi16(cvt16_src11, filter_01);

  1062.             dot2 = _mm256_madd_epi16(cvt16_src13, filter_23);

  1063.             dot3 = _mm256_madd_epi16(cvt16_src15, filter_45);

  1064.             dot4 = _mm256_madd_epi16(cvt16_src17, filter_60);

  1065.             sum10_even = _mm256_add_epi32(dot1, dot2);

  1066.             sum10_even = _mm256_add_epi32(sum10_even, dot3);

  1067.             sum10_even = _mm256_add_epi32(sum10_even, dot4);

  1068. 

  1069.             //! cal src2

  1070.             dot1 = _mm256_madd_epi16(cvt16_src20, filter_1415);

  1071.             dot2 = _mm256_madd_epi16(cvt16_src22, filter_1617);

  1072.             dot3 = _mm256_madd_epi16(cvt16_src24, filter_1819);

  1073.             dot4 = _mm256_madd_epi16(cvt16_src26, filter_200);

  1074.             sum2_odd = _mm256_add_epi32(dot1, dot2);

  1075.             sum2_odd = _mm256_add_epi32(sum2_odd, dot3);

  1076.             sum2_odd = _mm256_add_epi32(sum2_odd, dot4);

  1077. 

  1078.             dot1 = _mm256_madd_epi16(cvt16_src21, filter_1415);

  1079.             dot2 = _mm256_madd_epi16(cvt16_src23, filter_1617);

  1080.             dot3 = _mm256_madd_epi16(cvt16_src25, filter_1819);

  1081.             dot4 = _mm256_madd_epi16(cvt16_src27, filter_200);

  1082.             sum2_even = _mm256_add_epi32(dot1, dot2);

  1083.             sum2_even = _mm256_add_epi32(sum2_even, dot3);

  1084.             sum2_even = _mm256_add_epi32(sum2_even, dot4);

  1085. 

  1086.             dot1 = _mm256_madd_epi16(cvt16_src20, filter_78);

  1087.             dot2 = _mm256_madd_epi16(cvt16_src22, filter_910);

  1088.             dot3 = _mm256_madd_epi16(cvt16_src24, filter_1112);

  1089.             dot4 = _mm256_madd_epi16(cvt16_src26, filter_130);

  1090.             sum20_odd = _mm256_add_epi32(dot1, dot2);

  1091.             sum20_odd = _mm256_add_epi32(sum20_odd, dot3);

  1092.             sum20_odd = _mm256_add_epi32(sum20_odd, dot4);

  1093. 

  1094.             dot1 = _mm256_madd_epi16(cvt16_src21, filter_78);

  1095.             dot2 = _mm256_madd_epi16(cvt16_src23, filter_910);

  1096.             dot3 = _mm256_madd_epi16(cvt16_src25, filter_1112);

  1097.             dot4 = _mm256_madd_epi16(cvt16_src27, filter_130);

  1098.             sum20_even = _mm256_add_epi32(dot1, dot2);

  1099.             sum20_even = _mm256_add_epi32(sum20_even, dot3);

  1100.             sum20_even = _mm256_add_epi32(sum20_even, dot4);

  1101. 

  1102.             //! cal src3

  1103.             dot1 = _mm256_madd_epi16(cvt16_src30, filter_2122);

  1104.             dot2 = _mm256_madd_epi16(cvt16_src32, filter_2324);

  1105.             dot3 = _mm256_madd_epi16(cvt16_src34, filter_2526);

  1106.             dot4 = _mm256_madd_epi16(cvt16_src36, filter_270);

  1107.             sum3_odd = _mm256_add_epi32(dot1, dot2);

  1108.             sum3_odd = _mm256_add_epi32(sum3_odd, dot3);

  1109.             sum3_odd = _mm256_add_epi32(sum3_odd, dot4);

  1110. 

  1111.             dot1 = _mm256_madd_epi16(cvt16_src31, filter_2122);

  1112.             dot2 = _mm256_madd_epi16(cvt16_src33, filter_2324);

  1113.             dot3 = _mm256_madd_epi16(cvt16_src35, filter_2526);

  1114.             dot4 = _mm256_madd_epi16(cvt16_src37, filter_270);

  1115.             sum3_even = _mm256_add_epi32(dot1, dot2);

  1116.             sum3_even = _mm256_add_epi32(sum3_even, dot3);

  1117.             sum3_even = _mm256_add_epi32(sum3_even, dot4);

  1118. 

  1119.             dot1 = _mm256_madd_epi16(cvt16_src30, filter_1415);

  1120.             dot2 = _mm256_madd_epi16(cvt16_src32, filter_1617);

  1121.             dot3 = _mm256_madd_epi16(cvt16_src34, filter_1819);

  1122.             dot4 = _mm256_madd_epi16(cvt16_src36, filter_200);

  1123.             sum30_odd = _mm256_add_epi32(dot1, dot2);

  1124.             sum30_odd = _mm256_add_epi32(sum30_odd, dot3);

  1125.             sum30_odd = _mm256_add_epi32(sum30_odd, dot4);

  1126. 

  1127.             dot1 = _mm256_madd_epi16(cvt16_src31, filter_1415);

  1128.             dot2 = _mm256_madd_epi16(cvt16_src33, filter_1617);

  1129.             dot3 = _mm256_madd_epi16(cvt16_src35, filter_1819);

  1130.             dot4 = _mm256_madd_epi16(cvt16_src37, filter_200);

  1131.             sum30_even = _mm256_add_epi32(dot1, dot2);

  1132.             sum30_even = _mm256_add_epi32(sum30_even, dot3);

  1133.             sum30_even = _mm256_add_epi32(sum30_even, dot4);

  1134. 

  1135.             //! cal src4

  1136.             dot1 = _mm256_madd_epi16(cvt16_src40, filter_2829);

  1137.             dot2 = _mm256_madd_epi16(cvt16_src42, filter_3031);

  1138.             dot3 = _mm256_madd_epi16(cvt16_src44, filter_3233);

  1139.             dot4 = _mm256_madd_epi16(cvt16_src46, filter_340);

  1140.             sum4_odd = _mm256_add_epi32(dot1, dot2);

  1141.             sum4_odd = _mm256_add_epi32(sum4_odd, dot3);

  1142.             sum4_odd = _mm256_add_epi32(sum4_odd, dot4);

  1143. 

  1144.             dot1 = _mm256_madd_epi16(cvt16_src41, filter_2829);

  1145.             dot2 = _mm256_madd_epi16(cvt16_src43, filter_3031);

  1146.             dot3 = _mm256_madd_epi16(cvt16_src45, filter_3233);

  1147.             dot4 = _mm256_madd_epi16(cvt16_src47, filter_340);

  1148.             sum4_even = _mm256_add_epi32(dot1, dot2);

  1149.             sum4_even = _mm256_add_epi32(sum4_even, dot3);

  1150.             sum4_even = _mm256_add_epi32(sum4_even, dot4);

  1151. 

  1152.             dot1 = _mm256_madd_epi16(cvt16_src40, filter_2122);

  1153.             dot2 = _mm256_madd_epi16(cvt16_src42, filter_2324);

  1154.             dot3 = _mm256_madd_epi16(cvt16_src44, filter_2526);

  1155.             dot4 = _mm256_madd_epi16(cvt16_src46, filter_270);

  1156.             sum40_odd = _mm256_add_epi32(dot1, dot2);

  1157.             sum40_odd = _mm256_add_epi32(sum40_odd, dot3);

  1158.             sum40_odd = _mm256_add_epi32(sum40_odd, dot4);

  1159. 

  1160.             dot1 = _mm256_madd_epi16(cvt16_src41, filter_2122);

  1161.             dot2 = _mm256_madd_epi16(cvt16_src43, filter_2324);

  1162.             dot3 = _mm256_madd_epi16(cvt16_src45, filter_2526);

  1163.             dot4 = _mm256_madd_epi16(cvt16_src47, filter_270);

  1164.             sum40_even = _mm256_add_epi32(dot1, dot2);

  1165.             sum40_even = _mm256_add_epi32(sum40_even, dot3);

  1166.             sum40_even = _mm256_add_epi32(sum40_even, dot4);

  1167. 

  1168.             //! cal src5

  1169.             dot1 = _mm256_madd_epi16(cvt16_src50, filter_3536);

  1170.             dot2 = _mm256_madd_epi16(cvt16_src52, filter_3738);

  1171.             dot3 = _mm256_madd_epi16(cvt16_src54, filter_3940);

  1172.             dot4 = _mm256_madd_epi16(cvt16_src56, filter_410);

  1173.             sum5_odd = _mm256_add_epi32(dot1, dot2);

  1174.             sum5_odd = _mm256_add_epi32(sum5_odd, dot3);

  1175.             sum5_odd = _mm256_add_epi32(sum5_odd, dot4);

  1176. 

  1177.             dot1 = _mm256_madd_epi16(cvt16_src51, filter_3536);

  1178.             dot2 = _mm256_madd_epi16(cvt16_src53, filter_3738);

  1179.             dot3 = _mm256_madd_epi16(cvt16_src55, filter_3940);

  1180.             dot4 = _mm256_madd_epi16(cvt16_src57, filter_410);

  1181.             sum5_even = _mm256_add_epi32(dot1, dot2);

  1182.             sum5_even = _mm256_add_epi32(sum5_even, dot3);

  1183.             sum5_even = _mm256_add_epi32(sum5_even, dot4);

  1184. 

  1185.             dot1 = _mm256_madd_epi16(cvt16_src50, filter_2829);

  1186.             dot2 = _mm256_madd_epi16(cvt16_src52, filter_3031);

  1187.             dot3 = _mm256_madd_epi16(cvt16_src54, filter_3233);

  1188.             dot4 = _mm256_madd_epi16(cvt16_src56, filter_340);

  1189.             sum50_odd = _mm256_add_epi32(dot1, dot2);

  1190.             sum50_odd = _mm256_add_epi32(sum50_odd, dot3);

  1191.             sum50_odd = _mm256_add_epi32(sum50_odd, dot4);

  1192. 

  1193.             dot1 = _mm256_madd_epi16(cvt16_src51, filter_2829);

  1194.             dot2 = _mm256_madd_epi16(cvt16_src53, filter_3031);

  1195.             dot3 = _mm256_madd_epi16(cvt16_src55, filter_3233);

  1196.             dot4 = _mm256_madd_epi16(cvt16_src57, filter_340);

  1197.             sum50_even = _mm256_add_epi32(dot1, dot2);

  1198.             sum50_even = _mm256_add_epi32(sum50_even, dot3);

  1199.             sum50_even = _mm256_add_epi32(sum50_even, dot4);

  1200. 

  1201.             //! cal src6

  1202.             dot1 = _mm256_madd_epi16(cvt16_src60, filter_4243);

  1203.             dot2 = _mm256_madd_epi16(cvt16_src62, filter_4445);

  1204.             dot3 = _mm256_madd_epi16(cvt16_src64, filter_4647);

  1205.             dot4 = _mm256_madd_epi16(cvt16_src66, filter_480);

  1206.             sum6_odd = _mm256_add_epi32(dot1, dot2);

  1207.             sum6_odd = _mm256_add_epi32(sum6_odd, dot3);

  1208.             sum6_odd = _mm256_add_epi32(sum6_odd, dot4);

  1209. 

  1210.             dot1 = _mm256_madd_epi16(cvt16_src61, filter_4243);

  1211.             dot2 = _mm256_madd_epi16(cvt16_src63, filter_4445);

  1212.             dot3 = _mm256_madd_epi16(cvt16_src65, filter_4647);

  1213.             dot4 = _mm256_madd_epi16(cvt16_src67, filter_480);

  1214.             sum6_even = _mm256_add_epi32(dot1, dot2);

  1215.             sum6_even = _mm256_add_epi32(sum6_even, dot3);

  1216.             sum6_even = _mm256_add_epi32(sum6_even, dot4);

  1217. 

  1218.             dot1 = _mm256_madd_epi16(cvt16_src60, filter_3536);

  1219.             dot2 = _mm256_madd_epi16(cvt16_src62, filter_3738);

  1220.             dot3 = _mm256_madd_epi16(cvt16_src64, filter_3940);

  1221.             dot4 = _mm256_madd_epi16(cvt16_src66, filter_410);

  1222.             sum60_odd = _mm256_add_epi32(dot1, dot2);

  1223.             sum60_odd = _mm256_add_epi32(sum60_odd, dot3);

  1224.             sum60_odd = _mm256_add_epi32(sum60_odd, dot4);

  1225. 

  1226.             dot1 = _mm256_madd_epi16(cvt16_src61, filter_3536);

  1227.             dot2 = _mm256_madd_epi16(cvt16_src63, filter_3738);

  1228.             dot3 = _mm256_madd_epi16(cvt16_src65, filter_3940);

  1229.             dot4 = _mm256_madd_epi16(cvt16_src67, filter_410);

  1230.             sum60_even = _mm256_add_epi32(dot1, dot2);

  1231.             sum60_even = _mm256_add_epi32(sum60_even, dot3);

  1232.             sum60_even = _mm256_add_epi32(sum60_even, dot4);

  1233. 

  1234.             dot1 = _mm256_madd_epi16(cvt16_src70, filter_4243);

  1235.             dot2 = _mm256_madd_epi16(cvt16_src72, filter_4445);

  1236.             dot3 = _mm256_madd_epi16(cvt16_src74, filter_4647);

  1237.             dot4 = _mm256_madd_epi16(cvt16_src76, filter_480);

  1238.             sum7_odd = _mm256_add_epi32(dot1, dot2);

  1239.             sum7_odd = _mm256_add_epi32(sum7_odd, dot3);

  1240.             sum7_odd = _mm256_add_epi32(sum7_odd, dot4);

  1241. 

  1242.             dot1 = _mm256_madd_epi16(cvt16_src71, filter_4243);

  1243.             dot2 = _mm256_madd_epi16(cvt16_src73, filter_4445);

  1244.             dot3 = _mm256_madd_epi16(cvt16_src75, filter_4647);

  1245.             dot4 = _mm256_madd_epi16(cvt16_src77, filter_480);

  1246.             sum7_even = _mm256_add_epi32(dot1, dot2);

  1247.             sum7_even = _mm256_add_epi32(sum7_even, dot3);

  1248.             sum7_even = _mm256_add_epi32(sum7_even, dot4);

  1249. 

  1250.             __m256i sum_odd, sum_even;

  1251. 

  1252.             //! add src0 ~ src6

  1253.             sum_odd = _mm256_add_epi32(sum0_odd, sum1_odd);

  1254.             sum_odd = _mm256_add_epi32(sum_odd, sum2_odd);

  1255.             sum_odd = _mm256_add_epi32(sum_odd, sum3_odd);

  1256.             sum_odd = _mm256_add_epi32(sum_odd, sum4_odd);

  1257.             sum_odd = _mm256_add_epi32(sum_odd, sum5_odd);

  1258.             sum_odd = _mm256_add_epi32(sum_odd, sum6_odd);

  1259. 

  1260.             sum_even = _mm256_add_epi32(sum0_even, sum1_even);

  1261.             sum_even = _mm256_add_epi32(sum_even, sum2_even);

  1262.             sum_even = _mm256_add_epi32(sum_even, sum3_even);

  1263.             sum_even = _mm256_add_epi32(sum_even, sum4_even);

  1264.             sum_even = _mm256_add_epi32(sum_even, sum5_even);

  1265.             sum_even = _mm256_add_epi32(sum_even, sum6_even);

  1266. 

  1267.             __m256i sum_odd_0 = _mm256_unpacklo_epi32(sum_odd, sum_even);

  1268.             __m256i sum_even_0 = _mm256_unpackhi_epi32(sum_odd, sum_even);

  1269. 

  1270.             __m256i sum_left =

  1271.                     _mm256_permute2f128_si256(sum_odd_0, sum_even_0, 32);

  1272.             __m256i sum_right =

  1273.                     _mm256_permute2f128_si256(sum_odd_0, sum_even_0, 49);

  1274. 

  1275.             sum_left = _mm256_add_epi32(sum_left, bias_val);

  1276.             sum_right = _mm256_add_epi32(sum_right, bias_val);

  1277. 

  1278.             if (is_quantized) {

  1279.                 op({{sum_left, sum_right}}, reinterpret_cast<dt_qint8*>(dst0));

  1280.             } else {

  1281.                 _mm256_storeu_si256((__m256i*)(out_ptr0), sum_left);

  1282.                 _mm256_storeu_si256((__m256i*)(out_ptr0 + 8), sum_right);

  1283.             }

  1284. 

  1285.             __m256i sum_odd_oh1, sum_even_oh1;

  1286. 

  1287.             //! add src1 ~ src7

  1288.             sum_odd_oh1 = _mm256_add_epi32(sum10_odd, sum20_odd);

  1289.             sum_odd_oh1 = _mm256_add_epi32(sum_odd_oh1, sum30_odd);

  1290.             sum_odd_oh1 = _mm256_add_epi32(sum_odd_oh1, sum40_odd);

  1291.             sum_odd_oh1 = _mm256_add_epi32(sum_odd_oh1, sum50_odd);

  1292.             sum_odd_oh1 = _mm256_add_epi32(sum_odd_oh1, sum60_odd);

  1293.             sum_odd_oh1 = _mm256_add_epi32(sum_odd_oh1, sum7_odd);

  1294. 

  1295.             sum_even_oh1 = _mm256_add_epi32(sum10_even, sum20_even);

  1296.             sum_even_oh1 = _mm256_add_epi32(sum_even_oh1, sum30_even);

  1297.             sum_even_oh1 = _mm256_add_epi32(sum_even_oh1, sum40_even);

  1298.             sum_even_oh1 = _mm256_add_epi32(sum_even_oh1, sum50_even);

  1299.             sum_even_oh1 = _mm256_add_epi32(sum_even_oh1, sum60_even);

  1300.             sum_even_oh1 = _mm256_add_epi32(sum_even_oh1, sum7_even);

  1301. 

  1302.             __m256i sum_odd_1 =

  1303.                     _mm256_unpacklo_epi32(sum_odd_oh1, sum_even_oh1);

  1304.             __m256i sum_even_1 =

  1305.                     _mm256_unpackhi_epi32(sum_odd_oh1, sum_even_oh1);

  1306. 

  1307.             sum_left = _mm256_permute2f128_si256(sum_odd_1, sum_even_1, 32);

  1308.             sum_right = _mm256_permute2f128_si256(sum_odd_1, sum_even_1, 49);

  1309. 

  1310.             sum_left = _mm256_add_epi32(sum_left, bias_val);

  1311.             sum_right = _mm256_add_epi32(sum_right, bias_val);

  1312. 

  1313.             if (is_quantized) {

  1314.                 op({{sum_left, sum_right}}, reinterpret_cast<dt_qint8*>(dst1));

  1315.             } else {

  1316.                 _mm256_storeu_si256((__m256i*)(out_ptr1), sum_left);

  1317.                 _mm256_storeu_si256((__m256i*)(out_ptr1 + 8), sum_right);

  1318.             }

  1319. 

  1320.             r0 += 16;

  1321.             r1 += 16;

  1322.             r2 += 16;

  1323.             r3 += 16;

  1324.             r4 += 16;

  1325.             r5 += 16;

  1326.             r6 += 16;

  1327.             r7 += 16;

  1328.             dst0 += 16;

  1329.             dst1 += 16;

  1330.             out_ptr0 += 16;

  1331.             out_ptr1 += 16;

  1332.         }

  1333.         r0 += tail_step + IW;

  1334.         r1 += tail_step + IW;

  1335.         r2 += tail_step + IW;

  1336.         r3 += tail_step + IW;

  1337.         r4 += tail_step + IW;

  1338.         r5 += tail_step + IW;

  1339.         r6 += tail_step + IW;

  1340.         r7 += tail_step + IW;

  1341. 

  1342.         dst0 += OW;

  1343.         dst1 += OW;

  1344.         out_ptr0 += OW;

  1345.         out_ptr1 += OW;

  1346.     }

  1347. 

  1348.     for (; h < OH; h++) {

  1349.         size_t w = 0;

  1350.         for (; w < width; w++) {

  1351.             UNROLL_CALL0(7, load_src0)

  1352.             UNROLL_CALL0(7, load_src1)

  1353.             UNROLL_CALL0(7, load_src2)

  1354.             UNROLL_CALL0(7, load_src3)

  1355.             UNROLL_CALL0(7, load_src4)

  1356.             UNROLL_CALL0(7, load_src5)

  1357.             UNROLL_CALL0(7, load_src6)

  1358.             UNROLL_CALL0(7, load_src7)

  1359.             __m256i sum0_odd, sum0_even, sum1_odd, sum1_even, sum2_odd,

  1360.                     sum2_even, sum3_odd, sum3_even, sum4_odd, sum4_even,

  1361.                     sum5_odd, sum5_even, sum6_odd, sum6_even;

  1362. 

  1363.             //! cal src0

  1364.             __m256i dot1, dot2, dot3, dot4;

  1365.             dot1 = _mm256_madd_epi16(cvt16_src00, filter_01);

  1366.             dot2 = _mm256_madd_epi16(cvt16_src02, filter_23);

  1367.             dot3 = _mm256_madd_epi16(cvt16_src04, filter_45);

  1368.             dot4 = _mm256_madd_epi16(cvt16_src06, filter_60);

  1369.             sum0_odd = _mm256_add_epi32(dot1, dot2);

  1370.             sum0_odd = _mm256_add_epi32(sum0_odd, dot3);

  1371.             sum0_odd = _mm256_add_epi32(sum0_odd, dot4);

  1372. 

  1373.             dot1 = _mm256_madd_epi16(cvt16_src01, filter_01);

  1374.             dot2 = _mm256_madd_epi16(cvt16_src03, filter_23);

  1375.             dot3 = _mm256_madd_epi16(cvt16_src05, filter_45);

  1376.             dot4 = _mm256_madd_epi16(cvt16_src07, filter_60);

  1377.             sum0_even = _mm256_add_epi32(dot1, dot2);

  1378.             sum0_even = _mm256_add_epi32(sum0_even, dot3);

  1379.             sum0_even = _mm256_add_epi32(sum0_even, dot4);

  1380. 

  1381.             //! cal src1

  1382.             dot1 = _mm256_madd_epi16(cvt16_src10, filter_78);

  1383.             dot2 = _mm256_madd_epi16(cvt16_src12, filter_910);

  1384.             dot3 = _mm256_madd_epi16(cvt16_src14, filter_1112);

  1385.             dot4 = _mm256_madd_epi16(cvt16_src16, filter_130);

  1386.             sum1_odd = _mm256_add_epi32(dot1, dot2);

  1387.             sum1_odd = _mm256_add_epi32(sum1_odd, dot3);

  1388.             sum1_odd = _mm256_add_epi32(sum1_odd, dot4);

  1389. 

  1390.             dot1 = _mm256_madd_epi16(cvt16_src11, filter_78);

  1391.             dot2 = _mm256_madd_epi16(cvt16_src13, filter_910);

  1392.             dot3 = _mm256_madd_epi16(cvt16_src15, filter_1112);

  1393.             dot4 = _mm256_madd_epi16(cvt16_src17, filter_130);

  1394.             sum1_even = _mm256_add_epi32(dot1, dot2);

  1395.             sum1_even = _mm256_add_epi32(sum1_even, dot3);

  1396.             sum1_even = _mm256_add_epi32(sum1_even, dot4);

  1397. 

  1398.             //! cal src2

  1399.             dot1 = _mm256_madd_epi16(cvt16_src20, filter_1415);

  1400.             dot2 = _mm256_madd_epi16(cvt16_src22, filter_1617);

  1401.             dot3 = _mm256_madd_epi16(cvt16_src24, filter_1819);

  1402.             dot4 = _mm256_madd_epi16(cvt16_src26, filter_200);

  1403.             sum2_odd = _mm256_add_epi32(dot1, dot2);

  1404.             sum2_odd = _mm256_add_epi32(sum2_odd, dot3);

  1405.             sum2_odd = _mm256_add_epi32(sum2_odd, dot4);

  1406. 

  1407.             dot1 = _mm256_madd_epi16(cvt16_src21, filter_1415);

  1408.             dot2 = _mm256_madd_epi16(cvt16_src23, filter_1617);

  1409.             dot3 = _mm256_madd_epi16(cvt16_src25, filter_1819);

  1410.             dot4 = _mm256_madd_epi16(cvt16_src27, filter_200);

  1411.             sum2_even = _mm256_add_epi32(dot1, dot2);

  1412.             sum2_even = _mm256_add_epi32(sum2_even, dot3);

  1413.             sum2_even = _mm256_add_epi32(sum2_even, dot4);

  1414. 

  1415.             //! cal src3

  1416.             dot1 = _mm256_madd_epi16(cvt16_src30, filter_2122);

  1417.             dot2 = _mm256_madd_epi16(cvt16_src32, filter_2324);

  1418.             dot3 = _mm256_madd_epi16(cvt16_src34, filter_2526);

  1419.             dot4 = _mm256_madd_epi16(cvt16_src36, filter_270);

  1420.             sum3_odd = _mm256_add_epi32(dot1, dot2);

  1421.             sum3_odd = _mm256_add_epi32(sum3_odd, dot3);

  1422.             sum3_odd = _mm256_add_epi32(sum3_odd, dot4);

  1423. 

  1424.             dot1 = _mm256_madd_epi16(cvt16_src31, filter_2122);

  1425.             dot2 = _mm256_madd_epi16(cvt16_src33, filter_2324);

  1426.             dot3 = _mm256_madd_epi16(cvt16_src35, filter_2526);

  1427.             dot4 = _mm256_madd_epi16(cvt16_src37, filter_270);

  1428.             sum3_even = _mm256_add_epi32(dot1, dot2);

  1429.             sum3_even = _mm256_add_epi32(sum3_even, dot3);

  1430.             sum3_even = _mm256_add_epi32(sum3_even, dot4);

  1431. 

  1432.             //! cal src4

  1433.             dot1 = _mm256_madd_epi16(cvt16_src40, filter_2829);

  1434.             dot2 = _mm256_madd_epi16(cvt16_src42, filter_3031);

  1435.             dot3 = _mm256_madd_epi16(cvt16_src44, filter_3233);

  1436.             dot4 = _mm256_madd_epi16(cvt16_src46, filter_340);

  1437.             sum4_odd = _mm256_add_epi32(dot1, dot2);

  1438.             sum4_odd = _mm256_add_epi32(sum4_odd, dot3);

  1439.             sum4_odd = _mm256_add_epi32(sum4_odd, dot4);

  1440. 

  1441.             dot1 = _mm256_madd_epi16(cvt16_src41, filter_2829);

  1442.             dot2 = _mm256_madd_epi16(cvt16_src43, filter_3031);

  1443.             dot3 = _mm256_madd_epi16(cvt16_src45, filter_3233);

  1444.             dot4 = _mm256_madd_epi16(cvt16_src47, filter_340);

  1445.             sum4_even = _mm256_add_epi32(dot1, dot2);

  1446.             sum4_even = _mm256_add_epi32(sum4_even, dot3);

  1447.             sum4_even = _mm256_add_epi32(sum4_even, dot4);

  1448. 

  1449.             //! cal src5

  1450.             dot1 = _mm256_madd_epi16(cvt16_src50, filter_3536);

  1451.             dot2 = _mm256_madd_epi16(cvt16_src52, filter_3738);

  1452.             dot3 = _mm256_madd_epi16(cvt16_src54, filter_3940);

  1453.             dot4 = _mm256_madd_epi16(cvt16_src56, filter_410);

  1454.             sum5_odd = _mm256_add_epi32(dot1, dot2);

  1455.             sum5_odd = _mm256_add_epi32(sum5_odd, dot3);

  1456.             sum5_odd = _mm256_add_epi32(sum5_odd, dot4);

  1457. 

  1458.             dot1 = _mm256_madd_epi16(cvt16_src51, filter_3536);

  1459.             dot2 = _mm256_madd_epi16(cvt16_src53, filter_3738);

  1460.             dot3 = _mm256_madd_epi16(cvt16_src55, filter_3940);

  1461.             dot4 = _mm256_madd_epi16(cvt16_src57, filter_410);

  1462.             sum5_even = _mm256_add_epi32(dot1, dot2);

  1463.             sum5_even = _mm256_add_epi32(sum5_even, dot3);

  1464.             sum5_even = _mm256_add_epi32(sum5_even, dot4);

  1465. 

  1466.             //! cal src6

  1467.             dot1 = _mm256_madd_epi16(cvt16_src60, filter_4243);

  1468.             dot2 = _mm256_madd_epi16(cvt16_src62, filter_4445);

  1469.             dot3 = _mm256_madd_epi16(cvt16_src64, filter_4647);

  1470.             dot4 = _mm256_madd_epi16(cvt16_src66, filter_480);

  1471.             sum6_odd = _mm256_add_epi32(dot1, dot2);

  1472.             sum6_odd = _mm256_add_epi32(sum6_odd, dot3);

  1473.             sum6_odd = _mm256_add_epi32(sum6_odd, dot4);

  1474. 

  1475.             dot1 = _mm256_madd_epi16(cvt16_src61, filter_4243);

  1476.             dot2 = _mm256_madd_epi16(cvt16_src63, filter_4445);

  1477.             dot3 = _mm256_madd_epi16(cvt16_src65, filter_4647);

  1478.             dot4 = _mm256_madd_epi16(cvt16_src67, filter_480);

  1479.             sum6_even = _mm256_add_epi32(dot1, dot2);

  1480.             sum6_even = _mm256_add_epi32(sum6_even, dot3);

  1481.             sum6_even = _mm256_add_epi32(sum6_even, dot4);

  1482. 

  1483.             __m256i sum_odd, sum_even;

  1484. 

  1485.             //! add src0 ~ src6

  1486.             sum_odd = _mm256_add_epi32(sum0_odd, sum1_odd);

  1487.             sum_odd = _mm256_add_epi32(sum_odd, sum2_odd);

  1488.             sum_odd = _mm256_add_epi32(sum_odd, sum3_odd);

  1489.             sum_odd = _mm256_add_epi32(sum_odd, sum4_odd);

  1490.             sum_odd = _mm256_add_epi32(sum_odd, sum5_odd);

  1491.             sum_odd = _mm256_add_epi32(sum_odd, sum6_odd);

  1492. 

  1493.             sum_even = _mm256_add_epi32(sum0_even, sum1_even);

  1494.             sum_even = _mm256_add_epi32(sum_even, sum2_even);

  1495.             sum_even = _mm256_add_epi32(sum_even, sum3_even);

  1496.             sum_even = _mm256_add_epi32(sum_even, sum4_even);

  1497.             sum_even = _mm256_add_epi32(sum_even, sum5_even);

  1498.             sum_even = _mm256_add_epi32(sum_even, sum6_even);

  1499. 

  1500.             __m256i sum_odd_0 = _mm256_unpacklo_epi32(sum_odd, sum_even);

  1501.             __m256i sum_even_0 = _mm256_unpackhi_epi32(sum_odd, sum_even);

  1502. 

  1503.             __m256i sum_left =

  1504.                     _mm256_permute2f128_si256(sum_odd_0, sum_even_0, 32);

  1505.             __m256i sum_right =

  1506.                     _mm256_permute2f128_si256(sum_odd_0, sum_even_0, 49);

  1507. 

  1508.             sum_left = _mm256_add_epi32(sum_left, bias_val);

  1509.             sum_right = _mm256_add_epi32(sum_right, bias_val);

  1510. 

  1511.             if (is_quantized) {

  1512.                 op({{sum_left, sum_right}}, reinterpret_cast<dt_qint8*>(dst0));

  1513.             } else {

  1514.                 _mm256_storeu_si256((__m256i*)(out_ptr0), sum_left);

  1515.                 _mm256_storeu_si256((__m256i*)(out_ptr0 + 8), sum_right);

  1516.             }

  1517. 

  1518.             r0 += 16;

  1519.             r1 += 16;

  1520.             r2 += 16;

  1521.             r3 += 16;

  1522.             r4 += 16;

  1523.             r5 += 16;

  1524.             r6 += 16;

  1525.             dst0 += 16;

  1526.             out_ptr0 += 16;

  1527.         }

  1528.         r0 += tail_step;

  1529.         r1 += tail_step;

  1530.         r2 += tail_step;

  1531.         r3 += tail_step;

  1532.         r4 += tail_step;

  1533.         r5 += tail_step;

  1534.         r6 += tail_step;

  1535.     }

  1536. }

  1537. #undef load_filter

  1538. #undef load_src0

  1539. #undef load_src1

  1540. #undef load_src2

  1541. #undef load_src3

  1542. #undef load_src4

  1543. #undef load_src5

  1544. #undef load_src6

  1545. #undef load_src7

  1546. 

  1547. #define INSTANTIATION(stride, i, bias, is_quantized, Op)                      \

  1548.     template void avx2_chanwise_direct_##stride##_##i##x##i##_int8<           \

  1549.             bias, is_quantized, Op>(const int8_t*, const int8_t*,             \

  1550.                                     const int32_t*, int32_t*, int8_t*,        \

  1551.                                     const size_t, const size_t, const size_t, \

  1552.                                     const size_t, const Op&);

  1553. 

  1554. #define FOR_OP(stride, i, is_quantized, bias)                                  \

  1555.     INSTANTIATION(stride, i, bias, is_quantized,                               \

  1556.                   TypeCvtOp<SIMDType::AVX2 MEGDNN_COMMA dt_qint32 MEGDNN_COMMA \

  1557.                                     dt_qint8>)                                 \

  1558.     INSTANTIATION(stride, i, bias, is_quantized,                               \

  1559.                   ReluOp<SIMDType::AVX2 MEGDNN_COMMA dt_qint32 MEGDNN_COMMA    \

  1560.                                  dt_qint8>)                                    \

  1561.     INSTANTIATION(stride, i, bias, is_quantized,                               \

  1562.                   HSwishOp<SIMDType::AVX2 MEGDNN_COMMA dt_qint32 MEGDNN_COMMA  \

  1563.                                    dt_qint8>)

  1564. 

  1565. #define FOR_BIAS(stride, i, is_quantized)              \

  1566.     FOR_OP(stride, i, is_quantized, BiasMode::NO_BIAS) \

  1567.     FOR_OP(stride, i, is_quantized, BiasMode::BROADCAST_CHANNEL_BIAS)

  1568. 

  1569. #define FOR_QUANTIZED(stride, i) \

  1570.     FOR_BIAS(stride, i, true)    \

  1571.     FOR_BIAS(stride, i, false)

  1572. 

  1573. #define FOR_FILTER(stride)   \

  1574.     FOR_QUANTIZED(stride, 2) \

  1575.     FOR_QUANTIZED(stride, 3) \

  1576.     FOR_QUANTIZED(stride, 5) \

  1577.     FOR_QUANTIZED(stride, 7)

  1578. 

  1579. #define FOR_STRIDE FOR_FILTER(stride1)

  1580. 

  1581. FOR_STRIDE

  1582. 

  1583. #undef FOR_STRIDE

  1584. #undef FOR_FILTER

  1585. #undef FOR_QUANTIZED

  1586. #undef FOR_BIAS

  1587. #undef FOR_OP

  1588. #undef INSTANTIATION

  1589. }  // namespace avx2_chanwise_stride1

  1590. }  // namespace x86

  1591. }  // namespace megdnn

  1592. 

  1593. // vim: syntax=cpp.doxygen