OpenI
/
MegEngine
Not watched
1
0
Fork 0
Code
Releases 31 Wiki Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: bfb30dcb81
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'bfb30dcb81'
${ noResults }
MegEngine/dnn/src/x86/convolution/avx/convolution_conv_fh7_avx.cpp

829 lines
50 kB
Raw Blame History 

  1. /**

  2.  * \file dnn/src/x86/convolution/avx/convolution_conv_fh7_avx.cpp

  3.  * MegEngine is Licensed under the Apache License, Version 2.0 (the "License")

  4.  *

  5.  * Copyright (c) 2014-2021 Megvii Inc. All rights reserved.

  6.  *

  7.  * Unless required by applicable law or agreed to in writing,

  8.  * software distributed under the License is distributed on an

  9.  * "AS IS" BASIS, WITHOUT ARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

  10.  */

  11. #define SIMD_H1                                               \

  12.     do {                                                      \

  13.         const size_t sh = dh;                                 \

  14.         const float* src_d = src + sh * src_w;                \

  15.         float* dst_d = dst + dh * dst_w;                      \

  16.         size_t dw = dst_w_beg;                                \

  17.         for (; dw < dst_w_end; dw += 8) {                     \

  18.             const size_t sw = dw;                             \

  19.             float* dst_dd = dst_d + dw;                       \

  20.             __m256 tmp0;                                      \

  21.             __m256 tmp1;                                      \

  22.             __m256 res0;                                      \

  23.             res0 = _mm256_loadu_ps(dst_dd + 0 * dst_w);       \

  24.             for (size_t fw = 0; fw < flt_w; ++fw) {           \

  25.                 const float* src_dd = src_d + sw + fw;        \

  26.                 __m256 vf0 = _mm256_broadcast_ss(             \

  27.                         &filter[0 * flt_w + flt_w - fw - 1]); \

  28.                 __m256 vf1 = _mm256_broadcast_ss(             \

  29.                         &filter[1 * flt_w + flt_w - fw - 1]); \

  30.                 __m256 vf2 = _mm256_broadcast_ss(             \

  31.                         &filter[2 * flt_w + flt_w - fw - 1]); \

  32.                 __m256 vf3 = _mm256_broadcast_ss(             \

  33.                         &filter[3 * flt_w + flt_w - fw - 1]); \

  34.                 __m256 vf4 = _mm256_broadcast_ss(             \

  35.                         &filter[4 * flt_w + flt_w - fw - 1]); \

  36.                 __m256 vf5 = _mm256_broadcast_ss(             \

  37.                         &filter[5 * flt_w + flt_w - fw - 1]); \

  38.                 __m256 vf6 = _mm256_broadcast_ss(             \

  39.                         &filter[6 * flt_w + flt_w - fw - 1]); \

  40.                 tmp0 = _mm256_loadu_ps(src_dd + 0 * src_w);   \

  41.                 tmp1 = _mm256_mul_ps(tmp0, vf6);              \

  42.                 res0 = _mm256_add_ps(res0, tmp1);             \

  43.                 tmp0 = _mm256_loadu_ps(src_dd + 1 * src_w);   \

  44.                 tmp1 = _mm256_mul_ps(tmp0, vf5);              \

  45.                 res0 = _mm256_add_ps(res0, tmp1);             \

  46.                 tmp0 = _mm256_loadu_ps(src_dd + 2 * src_w);   \

  47.                 tmp1 = _mm256_mul_ps(tmp0, vf4);              \

  48.                 res0 = _mm256_add_ps(res0, tmp1);             \

  49.                 tmp0 = _mm256_loadu_ps(src_dd + 3 * src_w);   \

  50.                 tmp1 = _mm256_mul_ps(tmp0, vf3);              \

  51.                 res0 = _mm256_add_ps(res0, tmp1);             \

  52.                 tmp0 = _mm256_loadu_ps(src_dd + 4 * src_w);   \

  53.                 tmp1 = _mm256_mul_ps(tmp0, vf2);              \

  54.                 res0 = _mm256_add_ps(res0, tmp1);             \

  55.                 tmp0 = _mm256_loadu_ps(src_dd + 5 * src_w);   \

  56.                 tmp1 = _mm256_mul_ps(tmp0, vf1);              \

  57.                 res0 = _mm256_add_ps(res0, tmp1);             \

  58.                 tmp0 = _mm256_loadu_ps(src_dd + 6 * src_w);   \

  59.                 tmp1 = _mm256_mul_ps(tmp0, vf0);              \

  60.                 res0 = _mm256_add_ps(res0, tmp1);             \

  61.             }                                                 \

  62.             _mm256_storeu_ps(dst_dd + 0 * dst_w, res0);       \

  63.         }                                                     \

  64.     } while (0)

  65. 

  66. #define SIMD_H2                                               \

  67.     do {                                                      \

  68.         const size_t sh = dh;                                 \

  69.         const float* src_d = src + sh * src_w;                \

  70.         float* dst_d = dst + dh * dst_w;                      \

  71.         size_t dw = dst_w_beg;                                \

  72.         for (; dw < dst_w_end; dw += 8) {                     \

  73.             const size_t sw = dw;                             \

  74.             float* dst_dd = dst_d + dw;                       \

  75.             __m256 tmp0;                                      \

  76.             __m256 tmp1;                                      \

  77.             __m256 res0;                                      \

  78.             res0 = _mm256_loadu_ps(dst_dd + 0 * dst_w);       \

  79.             __m256 res1;                                      \

  80.             res1 = _mm256_loadu_ps(dst_dd + 1 * dst_w);       \

  81.             for (size_t fw = 0; fw < flt_w; ++fw) {           \

  82.                 const float* src_dd = src_d + sw + fw;        \

  83.                 __m256 vf0 = _mm256_broadcast_ss(             \

  84.                         &filter[0 * flt_w + flt_w - fw - 1]); \

  85.                 __m256 vf1 = _mm256_broadcast_ss(             \

  86.                         &filter[1 * flt_w + flt_w - fw - 1]); \

  87.                 __m256 vf2 = _mm256_broadcast_ss(             \

  88.                         &filter[2 * flt_w + flt_w - fw - 1]); \

  89.                 __m256 vf3 = _mm256_broadcast_ss(             \

  90.                         &filter[3 * flt_w + flt_w - fw - 1]); \

  91.                 __m256 vf4 = _mm256_broadcast_ss(             \

  92.                         &filter[4 * flt_w + flt_w - fw - 1]); \

  93.                 __m256 vf5 = _mm256_broadcast_ss(             \

  94.                         &filter[5 * flt_w + flt_w - fw - 1]); \

  95.                 __m256 vf6 = _mm256_broadcast_ss(             \

  96.                         &filter[6 * flt_w + flt_w - fw - 1]); \

  97.                 tmp0 = _mm256_loadu_ps(src_dd + 0 * src_w);   \

  98.                 tmp1 = _mm256_mul_ps(tmp0, vf6);              \

  99.                 res0 = _mm256_add_ps(res0, tmp1);             \

  100.                 tmp0 = _mm256_loadu_ps(src_dd + 1 * src_w);   \

  101.                 tmp1 = _mm256_mul_ps(tmp0, vf5);              \

  102.                 res0 = _mm256_add_ps(res0, tmp1);             \

  103.                 tmp1 = _mm256_mul_ps(tmp0, vf6);              \

  104.                 res1 = _mm256_add_ps(res1, tmp1);             \

  105.                 tmp0 = _mm256_loadu_ps(src_dd + 2 * src_w);   \

  106.                 tmp1 = _mm256_mul_ps(tmp0, vf4);              \

  107.                 res0 = _mm256_add_ps(res0, tmp1);             \

  108.                 tmp1 = _mm256_mul_ps(tmp0, vf5);              \

  109.                 res1 = _mm256_add_ps(res1, tmp1);             \

  110.                 tmp0 = _mm256_loadu_ps(src_dd + 3 * src_w);   \

  111.                 tmp1 = _mm256_mul_ps(tmp0, vf3);              \

  112.                 res0 = _mm256_add_ps(res0, tmp1);             \

  113.                 tmp1 = _mm256_mul_ps(tmp0, vf4);              \

  114.                 res1 = _mm256_add_ps(res1, tmp1);             \

  115.                 tmp0 = _mm256_loadu_ps(src_dd + 4 * src_w);   \

  116.                 tmp1 = _mm256_mul_ps(tmp0, vf2);              \

  117.                 res0 = _mm256_add_ps(res0, tmp1);             \

  118.                 tmp1 = _mm256_mul_ps(tmp0, vf3);              \

  119.                 res1 = _mm256_add_ps(res1, tmp1);             \

  120.                 tmp0 = _mm256_loadu_ps(src_dd + 5 * src_w);   \

  121.                 tmp1 = _mm256_mul_ps(tmp0, vf1);              \

  122.                 res0 = _mm256_add_ps(res0, tmp1);             \

  123.                 tmp1 = _mm256_mul_ps(tmp0, vf2);              \

  124.                 res1 = _mm256_add_ps(res1, tmp1);             \

  125.                 tmp0 = _mm256_loadu_ps(src_dd + 6 * src_w);   \

  126.                 tmp1 = _mm256_mul_ps(tmp0, vf0);              \

  127.                 res0 = _mm256_add_ps(res0, tmp1);             \

  128.                 tmp1 = _mm256_mul_ps(tmp0, vf1);              \

  129.                 res1 = _mm256_add_ps(res1, tmp1);             \

  130.                 tmp0 = _mm256_loadu_ps(src_dd + 7 * src_w);   \

  131.                 tmp1 = _mm256_mul_ps(tmp0, vf0);              \

  132.                 res1 = _mm256_add_ps(res1, tmp1);             \

  133.             }                                                 \

  134.             _mm256_storeu_ps(dst_dd + 0 * dst_w, res0);       \

  135.             _mm256_storeu_ps(dst_dd + 1 * dst_w, res1);       \

  136.         }                                                     \

  137.     } while (0)

  138. 

  139. #define SIMD_H3                                               \

  140.     do {                                                      \

  141.         const size_t sh = dh;                                 \

  142.         const float* src_d = src + sh * src_w;                \

  143.         float* dst_d = dst + dh * dst_w;                      \

  144.         size_t dw = dst_w_beg;                                \

  145.         for (; dw < dst_w_end; dw += 8) {                     \

  146.             const size_t sw = dw;                             \

  147.             float* dst_dd = dst_d + dw;                       \

  148.             __m256 tmp0;                                      \

  149.             __m256 tmp1;                                      \

  150.             __m256 res0;                                      \

  151.             res0 = _mm256_loadu_ps(dst_dd + 0 * dst_w);       \

  152.             __m256 res1;                                      \

  153.             res1 = _mm256_loadu_ps(dst_dd + 1 * dst_w);       \

  154.             __m256 res2;                                      \

  155.             res2 = _mm256_loadu_ps(dst_dd + 2 * dst_w);       \

  156.             for (size_t fw = 0; fw < flt_w; ++fw) {           \

  157.                 const float* src_dd = src_d + sw + fw;        \

  158.                 __m256 vf0 = _mm256_broadcast_ss(             \

  159.                         &filter[0 * flt_w + flt_w - fw - 1]); \

  160.                 __m256 vf1 = _mm256_broadcast_ss(             \

  161.                         &filter[1 * flt_w + flt_w - fw - 1]); \

  162.                 __m256 vf2 = _mm256_broadcast_ss(             \

  163.                         &filter[2 * flt_w + flt_w - fw - 1]); \

  164.                 __m256 vf3 = _mm256_broadcast_ss(             \

  165.                         &filter[3 * flt_w + flt_w - fw - 1]); \

  166.                 __m256 vf4 = _mm256_broadcast_ss(             \

  167.                         &filter[4 * flt_w + flt_w - fw - 1]); \

  168.                 __m256 vf5 = _mm256_broadcast_ss(             \

  169.                         &filter[5 * flt_w + flt_w - fw - 1]); \

  170.                 __m256 vf6 = _mm256_broadcast_ss(             \

  171.                         &filter[6 * flt_w + flt_w - fw - 1]); \

  172.                 tmp0 = _mm256_loadu_ps(src_dd + 0 * src_w);   \

  173.                 tmp1 = _mm256_mul_ps(tmp0, vf6);              \

  174.                 res0 = _mm256_add_ps(res0, tmp1);             \

  175.                 tmp0 = _mm256_loadu_ps(src_dd + 1 * src_w);   \

  176.                 tmp1 = _mm256_mul_ps(tmp0, vf5);              \

  177.                 res0 = _mm256_add_ps(res0, tmp1);             \

  178.                 tmp1 = _mm256_mul_ps(tmp0, vf6);              \

  179.                 res1 = _mm256_add_ps(res1, tmp1);             \

  180.                 tmp0 = _mm256_loadu_ps(src_dd + 2 * src_w);   \

  181.                 tmp1 = _mm256_mul_ps(tmp0, vf4);              \

  182.                 res0 = _mm256_add_ps(res0, tmp1);             \

  183.                 tmp1 = _mm256_mul_ps(tmp0, vf5);              \

  184.                 res1 = _mm256_add_ps(res1, tmp1);             \

  185.                 tmp1 = _mm256_mul_ps(tmp0, vf6);              \

  186.                 res2 = _mm256_add_ps(res2, tmp1);             \

  187.                 tmp0 = _mm256_loadu_ps(src_dd + 3 * src_w);   \

  188.                 tmp1 = _mm256_mul_ps(tmp0, vf3);              \

  189.                 res0 = _mm256_add_ps(res0, tmp1);             \

  190.                 tmp1 = _mm256_mul_ps(tmp0, vf4);              \

  191.                 res1 = _mm256_add_ps(res1, tmp1);             \

  192.                 tmp1 = _mm256_mul_ps(tmp0, vf5);              \

  193.                 res2 = _mm256_add_ps(res2, tmp1);             \

  194.                 tmp0 = _mm256_loadu_ps(src_dd + 4 * src_w);   \

  195.                 tmp1 = _mm256_mul_ps(tmp0, vf2);              \

  196.                 res0 = _mm256_add_ps(res0, tmp1);             \

  197.                 tmp1 = _mm256_mul_ps(tmp0, vf3);              \

  198.                 res1 = _mm256_add_ps(res1, tmp1);             \

  199.                 tmp1 = _mm256_mul_ps(tmp0, vf4);              \

  200.                 res2 = _mm256_add_ps(res2, tmp1);             \

  201.                 tmp0 = _mm256_loadu_ps(src_dd + 5 * src_w);   \

  202.                 tmp1 = _mm256_mul_ps(tmp0, vf1);              \

  203.                 res0 = _mm256_add_ps(res0, tmp1);             \

  204.                 tmp1 = _mm256_mul_ps(tmp0, vf2);              \

  205.                 res1 = _mm256_add_ps(res1, tmp1);             \

  206.                 tmp1 = _mm256_mul_ps(tmp0, vf3);              \

  207.                 res2 = _mm256_add_ps(res2, tmp1);             \

  208.                 tmp0 = _mm256_loadu_ps(src_dd + 6 * src_w);   \

  209.                 tmp1 = _mm256_mul_ps(tmp0, vf0);              \

  210.                 res0 = _mm256_add_ps(res0, tmp1);             \

  211.                 tmp1 = _mm256_mul_ps(tmp0, vf1);              \

  212.                 res1 = _mm256_add_ps(res1, tmp1);             \

  213.                 tmp1 = _mm256_mul_ps(tmp0, vf2);              \

  214.                 res2 = _mm256_add_ps(res2, tmp1);             \

  215.                 tmp0 = _mm256_loadu_ps(src_dd + 7 * src_w);   \

  216.                 tmp1 = _mm256_mul_ps(tmp0, vf0);              \

  217.                 res1 = _mm256_add_ps(res1, tmp1);             \

  218.                 tmp1 = _mm256_mul_ps(tmp0, vf1);              \

  219.                 res2 = _mm256_add_ps(res2, tmp1);             \

  220.                 tmp0 = _mm256_loadu_ps(src_dd + 8 * src_w);   \

  221.                 tmp1 = _mm256_mul_ps(tmp0, vf0);              \

  222.                 res2 = _mm256_add_ps(res2, tmp1);             \

  223.             }                                                 \

  224.             _mm256_storeu_ps(dst_dd + 0 * dst_w, res0);       \

  225.             _mm256_storeu_ps(dst_dd + 1 * dst_w, res1);       \

  226.             _mm256_storeu_ps(dst_dd + 2 * dst_w, res2);       \

  227.         }                                                     \

  228.     } while (0)

  229. 

  230. #define SIMD_H4                                               \

  231.     do {                                                      \

  232.         const size_t sh = dh;                                 \

  233.         const float* src_d = src + sh * src_w;                \

  234.         float* dst_d = dst + dh * dst_w;                      \

  235.         size_t dw = dst_w_beg;                                \

  236.         for (; dw < dst_w_end; dw += 8) {                     \

  237.             const size_t sw = dw;                             \

  238.             float* dst_dd = dst_d + dw;                       \

  239.             __m256 tmp0;                                      \

  240.             __m256 tmp1;                                      \

  241.             __m256 res0;                                      \

  242.             res0 = _mm256_loadu_ps(dst_dd + 0 * dst_w);       \

  243.             __m256 res1;                                      \

  244.             res1 = _mm256_loadu_ps(dst_dd + 1 * dst_w);       \

  245.             __m256 res2;                                      \

  246.             res2 = _mm256_loadu_ps(dst_dd + 2 * dst_w);       \

  247.             __m256 res3;                                      \

  248.             res3 = _mm256_loadu_ps(dst_dd + 3 * dst_w);       \

  249.             for (size_t fw = 0; fw < flt_w; ++fw) {           \

  250.                 const float* src_dd = src_d + sw + fw;        \

  251.                 __m256 vf0 = _mm256_broadcast_ss(             \

  252.                         &filter[0 * flt_w + flt_w - fw - 1]); \

  253.                 __m256 vf1 = _mm256_broadcast_ss(             \

  254.                         &filter[1 * flt_w + flt_w - fw - 1]); \

  255.                 __m256 vf2 = _mm256_broadcast_ss(             \

  256.                         &filter[2 * flt_w + flt_w - fw - 1]); \

  257.                 __m256 vf3 = _mm256_broadcast_ss(             \

  258.                         &filter[3 * flt_w + flt_w - fw - 1]); \

  259.                 __m256 vf4 = _mm256_broadcast_ss(             \

  260.                         &filter[4 * flt_w + flt_w - fw - 1]); \

  261.                 __m256 vf5 = _mm256_broadcast_ss(             \

  262.                         &filter[5 * flt_w + flt_w - fw - 1]); \

  263.                 __m256 vf6 = _mm256_broadcast_ss(             \

  264.                         &filter[6 * flt_w + flt_w - fw - 1]); \

  265.                 tmp0 = _mm256_loadu_ps(src_dd + 0 * src_w);   \

  266.                 tmp1 = _mm256_mul_ps(tmp0, vf6);              \

  267.                 res0 = _mm256_add_ps(res0, tmp1);             \

  268.                 tmp0 = _mm256_loadu_ps(src_dd + 1 * src_w);   \

  269.                 tmp1 = _mm256_mul_ps(tmp0, vf5);              \

  270.                 res0 = _mm256_add_ps(res0, tmp1);             \

  271.                 tmp1 = _mm256_mul_ps(tmp0, vf6);              \

  272.                 res1 = _mm256_add_ps(res1, tmp1);             \

  273.                 tmp0 = _mm256_loadu_ps(src_dd + 2 * src_w);   \

  274.                 tmp1 = _mm256_mul_ps(tmp0, vf4);              \

  275.                 res0 = _mm256_add_ps(res0, tmp1);             \

  276.                 tmp1 = _mm256_mul_ps(tmp0, vf5);              \

  277.                 res1 = _mm256_add_ps(res1, tmp1);             \

  278.                 tmp1 = _mm256_mul_ps(tmp0, vf6);              \

  279.                 res2 = _mm256_add_ps(res2, tmp1);             \

  280.                 tmp0 = _mm256_loadu_ps(src_dd + 3 * src_w);   \

  281.                 tmp1 = _mm256_mul_ps(tmp0, vf3);              \

  282.                 res0 = _mm256_add_ps(res0, tmp1);             \

  283.                 tmp1 = _mm256_mul_ps(tmp0, vf4);              \

  284.                 res1 = _mm256_add_ps(res1, tmp1);             \

  285.                 tmp1 = _mm256_mul_ps(tmp0, vf5);              \

  286.                 res2 = _mm256_add_ps(res2, tmp1);             \

  287.                 tmp1 = _mm256_mul_ps(tmp0, vf6);              \

  288.                 res3 = _mm256_add_ps(res3, tmp1);             \

  289.                 tmp0 = _mm256_loadu_ps(src_dd + 4 * src_w);   \

  290.                 tmp1 = _mm256_mul_ps(tmp0, vf2);              \

  291.                 res0 = _mm256_add_ps(res0, tmp1);             \

  292.                 tmp1 = _mm256_mul_ps(tmp0, vf3);              \

  293.                 res1 = _mm256_add_ps(res1, tmp1);             \

  294.                 tmp1 = _mm256_mul_ps(tmp0, vf4);              \

  295.                 res2 = _mm256_add_ps(res2, tmp1);             \

  296.                 tmp1 = _mm256_mul_ps(tmp0, vf5);              \

  297.                 res3 = _mm256_add_ps(res3, tmp1);             \

  298.                 tmp0 = _mm256_loadu_ps(src_dd + 5 * src_w);   \

  299.                 tmp1 = _mm256_mul_ps(tmp0, vf1);              \

  300.                 res0 = _mm256_add_ps(res0, tmp1);             \

  301.                 tmp1 = _mm256_mul_ps(tmp0, vf2);              \

  302.                 res1 = _mm256_add_ps(res1, tmp1);             \

  303.                 tmp1 = _mm256_mul_ps(tmp0, vf3);              \

  304.                 res2 = _mm256_add_ps(res2, tmp1);             \

  305.                 tmp1 = _mm256_mul_ps(tmp0, vf4);              \

  306.                 res3 = _mm256_add_ps(res3, tmp1);             \

  307.                 tmp0 = _mm256_loadu_ps(src_dd + 6 * src_w);   \

  308.                 tmp1 = _mm256_mul_ps(tmp0, vf0);              \

  309.                 res0 = _mm256_add_ps(res0, tmp1);             \

  310.                 tmp1 = _mm256_mul_ps(tmp0, vf1);              \

  311.                 res1 = _mm256_add_ps(res1, tmp1);             \

  312.                 tmp1 = _mm256_mul_ps(tmp0, vf2);              \

  313.                 res2 = _mm256_add_ps(res2, tmp1);             \

  314.                 tmp1 = _mm256_mul_ps(tmp0, vf3);              \

  315.                 res3 = _mm256_add_ps(res3, tmp1);             \

  316.                 tmp0 = _mm256_loadu_ps(src_dd + 7 * src_w);   \

  317.                 tmp1 = _mm256_mul_ps(tmp0, vf0);              \

  318.                 res1 = _mm256_add_ps(res1, tmp1);             \

  319.                 tmp1 = _mm256_mul_ps(tmp0, vf1);              \

  320.                 res2 = _mm256_add_ps(res2, tmp1);             \

  321.                 tmp1 = _mm256_mul_ps(tmp0, vf2);              \

  322.                 res3 = _mm256_add_ps(res3, tmp1);             \

  323.                 tmp0 = _mm256_loadu_ps(src_dd + 8 * src_w);   \

  324.                 tmp1 = _mm256_mul_ps(tmp0, vf0);              \

  325.                 res2 = _mm256_add_ps(res2, tmp1);             \

  326.                 tmp1 = _mm256_mul_ps(tmp0, vf1);              \

  327.                 res3 = _mm256_add_ps(res3, tmp1);             \

  328.                 tmp0 = _mm256_loadu_ps(src_dd + 9 * src_w);   \

  329.                 tmp1 = _mm256_mul_ps(tmp0, vf0);              \

  330.                 res3 = _mm256_add_ps(res3, tmp1);             \

  331.             }                                                 \

  332.             _mm256_storeu_ps(dst_dd + 0 * dst_w, res0);       \

  333.             _mm256_storeu_ps(dst_dd + 1 * dst_w, res1);       \

  334.             _mm256_storeu_ps(dst_dd + 2 * dst_w, res2);       \

  335.             _mm256_storeu_ps(dst_dd + 3 * dst_w, res3);       \

  336.         }                                                     \

  337.     } while (0)

  338. 

  339. #define SIMD_H5                                               \

  340.     do {                                                      \

  341.         const size_t sh = dh;                                 \

  342.         const float* src_d = src + sh * src_w;                \

  343.         float* dst_d = dst + dh * dst_w;                      \

  344.         size_t dw = dst_w_beg;                                \

  345.         for (; dw < dst_w_end; dw += 8) {                     \

  346.             const size_t sw = dw;                             \

  347.             float* dst_dd = dst_d + dw;                       \

  348.             __m256 tmp0;                                      \

  349.             __m256 tmp1;                                      \

  350.             __m256 res0;                                      \

  351.             res0 = _mm256_loadu_ps(dst_dd + 0 * dst_w);       \

  352.             __m256 res1;                                      \

  353.             res1 = _mm256_loadu_ps(dst_dd + 1 * dst_w);       \

  354.             __m256 res2;                                      \

  355.             res2 = _mm256_loadu_ps(dst_dd + 2 * dst_w);       \

  356.             __m256 res3;                                      \

  357.             res3 = _mm256_loadu_ps(dst_dd + 3 * dst_w);       \

  358.             __m256 res4;                                      \

  359.             res4 = _mm256_loadu_ps(dst_dd + 4 * dst_w);       \

  360.             for (size_t fw = 0; fw < flt_w; ++fw) {           \

  361.                 const float* src_dd = src_d + sw + fw;        \

  362.                 __m256 vf0 = _mm256_broadcast_ss(             \

  363.                         &filter[0 * flt_w + flt_w - fw - 1]); \

  364.                 __m256 vf1 = _mm256_broadcast_ss(             \

  365.                         &filter[1 * flt_w + flt_w - fw - 1]); \

  366.                 __m256 vf2 = _mm256_broadcast_ss(             \

  367.                         &filter[2 * flt_w + flt_w - fw - 1]); \

  368.                 __m256 vf3 = _mm256_broadcast_ss(             \

  369.                         &filter[3 * flt_w + flt_w - fw - 1]); \

  370.                 __m256 vf4 = _mm256_broadcast_ss(             \

  371.                         &filter[4 * flt_w + flt_w - fw - 1]); \

  372.                 __m256 vf5 = _mm256_broadcast_ss(             \

  373.                         &filter[5 * flt_w + flt_w - fw - 1]); \

  374.                 __m256 vf6 = _mm256_broadcast_ss(             \

  375.                         &filter[6 * flt_w + flt_w - fw - 1]); \

  376.                 tmp0 = _mm256_loadu_ps(src_dd + 0 * src_w);   \

  377.                 tmp1 = _mm256_mul_ps(tmp0, vf6);              \

  378.                 res0 = _mm256_add_ps(res0, tmp1);             \

  379.                 tmp0 = _mm256_loadu_ps(src_dd + 1 * src_w);   \

  380.                 tmp1 = _mm256_mul_ps(tmp0, vf5);              \

  381.                 res0 = _mm256_add_ps(res0, tmp1);             \

  382.                 tmp1 = _mm256_mul_ps(tmp0, vf6);              \

  383.                 res1 = _mm256_add_ps(res1, tmp1);             \

  384.                 tmp0 = _mm256_loadu_ps(src_dd + 2 * src_w);   \

  385.                 tmp1 = _mm256_mul_ps(tmp0, vf4);              \

  386.                 res0 = _mm256_add_ps(res0, tmp1);             \

  387.                 tmp1 = _mm256_mul_ps(tmp0, vf5);              \

  388.                 res1 = _mm256_add_ps(res1, tmp1);             \

  389.                 tmp1 = _mm256_mul_ps(tmp0, vf6);              \

  390.                 res2 = _mm256_add_ps(res2, tmp1);             \

  391.                 tmp0 = _mm256_loadu_ps(src_dd + 3 * src_w);   \

  392.                 tmp1 = _mm256_mul_ps(tmp0, vf3);              \

  393.                 res0 = _mm256_add_ps(res0, tmp1);             \

  394.                 tmp1 = _mm256_mul_ps(tmp0, vf4);              \

  395.                 res1 = _mm256_add_ps(res1, tmp1);             \

  396.                 tmp1 = _mm256_mul_ps(tmp0, vf5);              \

  397.                 res2 = _mm256_add_ps(res2, tmp1);             \

  398.                 tmp1 = _mm256_mul_ps(tmp0, vf6);              \

  399.                 res3 = _mm256_add_ps(res3, tmp1);             \

  400.                 tmp0 = _mm256_loadu_ps(src_dd + 4 * src_w);   \

  401.                 tmp1 = _mm256_mul_ps(tmp0, vf2);              \

  402.                 res0 = _mm256_add_ps(res0, tmp1);             \

  403.                 tmp1 = _mm256_mul_ps(tmp0, vf3);              \

  404.                 res1 = _mm256_add_ps(res1, tmp1);             \

  405.                 tmp1 = _mm256_mul_ps(tmp0, vf4);              \

  406.                 res2 = _mm256_add_ps(res2, tmp1);             \

  407.                 tmp1 = _mm256_mul_ps(tmp0, vf5);              \

  408.                 res3 = _mm256_add_ps(res3, tmp1);             \

  409.                 tmp1 = _mm256_mul_ps(tmp0, vf6);              \

  410.                 res4 = _mm256_add_ps(res4, tmp1);             \

  411.                 tmp0 = _mm256_loadu_ps(src_dd + 5 * src_w);   \

  412.                 tmp1 = _mm256_mul_ps(tmp0, vf1);              \

  413.                 res0 = _mm256_add_ps(res0, tmp1);             \

  414.                 tmp1 = _mm256_mul_ps(tmp0, vf2);              \

  415.                 res1 = _mm256_add_ps(res1, tmp1);             \

  416.                 tmp1 = _mm256_mul_ps(tmp0, vf3);              \

  417.                 res2 = _mm256_add_ps(res2, tmp1);             \

  418.                 tmp1 = _mm256_mul_ps(tmp0, vf4);              \

  419.                 res3 = _mm256_add_ps(res3, tmp1);             \

  420.                 tmp1 = _mm256_mul_ps(tmp0, vf5);              \

  421.                 res4 = _mm256_add_ps(res4, tmp1);             \

  422.                 tmp0 = _mm256_loadu_ps(src_dd + 6 * src_w);   \

  423.                 tmp1 = _mm256_mul_ps(tmp0, vf0);              \

  424.                 res0 = _mm256_add_ps(res0, tmp1);             \

  425.                 tmp1 = _mm256_mul_ps(tmp0, vf1);              \

  426.                 res1 = _mm256_add_ps(res1, tmp1);             \

  427.                 tmp1 = _mm256_mul_ps(tmp0, vf2);              \

  428.                 res2 = _mm256_add_ps(res2, tmp1);             \

  429.                 tmp1 = _mm256_mul_ps(tmp0, vf3);              \

  430.                 res3 = _mm256_add_ps(res3, tmp1);             \

  431.                 tmp1 = _mm256_mul_ps(tmp0, vf4);              \

  432.                 res4 = _mm256_add_ps(res4, tmp1);             \

  433.                 tmp0 = _mm256_loadu_ps(src_dd + 7 * src_w);   \

  434.                 tmp1 = _mm256_mul_ps(tmp0, vf0);              \

  435.                 res1 = _mm256_add_ps(res1, tmp1);             \

  436.                 tmp1 = _mm256_mul_ps(tmp0, vf1);              \

  437.                 res2 = _mm256_add_ps(res2, tmp1);             \

  438.                 tmp1 = _mm256_mul_ps(tmp0, vf2);              \

  439.                 res3 = _mm256_add_ps(res3, tmp1);             \

  440.                 tmp1 = _mm256_mul_ps(tmp0, vf3);              \

  441.                 res4 = _mm256_add_ps(res4, tmp1);             \

  442.                 tmp0 = _mm256_loadu_ps(src_dd + 8 * src_w);   \

  443.                 tmp1 = _mm256_mul_ps(tmp0, vf0);              \

  444.                 res2 = _mm256_add_ps(res2, tmp1);             \

  445.                 tmp1 = _mm256_mul_ps(tmp0, vf1);              \

  446.                 res3 = _mm256_add_ps(res3, tmp1);             \

  447.                 tmp1 = _mm256_mul_ps(tmp0, vf2);              \

  448.                 res4 = _mm256_add_ps(res4, tmp1);             \

  449.                 tmp0 = _mm256_loadu_ps(src_dd + 9 * src_w);   \

  450.                 tmp1 = _mm256_mul_ps(tmp0, vf0);              \

  451.                 res3 = _mm256_add_ps(res3, tmp1);             \

  452.                 tmp1 = _mm256_mul_ps(tmp0, vf1);              \

  453.                 res4 = _mm256_add_ps(res4, tmp1);             \

  454.                 tmp0 = _mm256_loadu_ps(src_dd + 10 * src_w);  \

  455.                 tmp1 = _mm256_mul_ps(tmp0, vf0);              \

  456.                 res4 = _mm256_add_ps(res4, tmp1);             \

  457.             }                                                 \

  458.             _mm256_storeu_ps(dst_dd + 0 * dst_w, res0);       \

  459.             _mm256_storeu_ps(dst_dd + 1 * dst_w, res1);       \

  460.             _mm256_storeu_ps(dst_dd + 2 * dst_w, res2);       \

  461.             _mm256_storeu_ps(dst_dd + 3 * dst_w, res3);       \

  462.             _mm256_storeu_ps(dst_dd + 4 * dst_w, res4);       \

  463.         }                                                     \

  464.     } while (0)

  465. 

  466. #define SIMD_H6                                               \

  467.     do {                                                      \

  468.         const size_t sh = dh;                                 \

  469.         const float* src_d = src + sh * src_w;                \

  470.         float* dst_d = dst + dh * dst_w;                      \

  471.         size_t dw = dst_w_beg;                                \

  472.         for (; dw < dst_w_end; dw += 8) {                     \

  473.             const size_t sw = dw;                             \

  474.             float* dst_dd = dst_d + dw;                       \

  475.             __m256 tmp0;                                      \

  476.             __m256 tmp1;                                      \

  477.             __m256 res0;                                      \

  478.             res0 = _mm256_loadu_ps(dst_dd + 0 * dst_w);       \

  479.             __m256 res1;                                      \

  480.             res1 = _mm256_loadu_ps(dst_dd + 1 * dst_w);       \

  481.             __m256 res2;                                      \

  482.             res2 = _mm256_loadu_ps(dst_dd + 2 * dst_w);       \

  483.             __m256 res3;                                      \

  484.             res3 = _mm256_loadu_ps(dst_dd + 3 * dst_w);       \

  485.             __m256 res4;                                      \

  486.             res4 = _mm256_loadu_ps(dst_dd + 4 * dst_w);       \

  487.             __m256 res5;                                      \

  488.             res5 = _mm256_loadu_ps(dst_dd + 5 * dst_w);       \

  489.             for (size_t fw = 0; fw < flt_w; ++fw) {           \

  490.                 const float* src_dd = src_d + sw + fw;        \

  491.                 __m256 vf0 = _mm256_broadcast_ss(             \

  492.                         &filter[0 * flt_w + flt_w - fw - 1]); \

  493.                 __m256 vf1 = _mm256_broadcast_ss(             \

  494.                         &filter[1 * flt_w + flt_w - fw - 1]); \

  495.                 __m256 vf2 = _mm256_broadcast_ss(             \

  496.                         &filter[2 * flt_w + flt_w - fw - 1]); \

  497.                 __m256 vf3 = _mm256_broadcast_ss(             \

  498.                         &filter[3 * flt_w + flt_w - fw - 1]); \

  499.                 __m256 vf4 = _mm256_broadcast_ss(             \

  500.                         &filter[4 * flt_w + flt_w - fw - 1]); \

  501.                 __m256 vf5 = _mm256_broadcast_ss(             \

  502.                         &filter[5 * flt_w + flt_w - fw - 1]); \

  503.                 __m256 vf6 = _mm256_broadcast_ss(             \

  504.                         &filter[6 * flt_w + flt_w - fw - 1]); \

  505.                 tmp0 = _mm256_loadu_ps(src_dd + 0 * src_w);   \

  506.                 tmp1 = _mm256_mul_ps(tmp0, vf6);              \

  507.                 res0 = _mm256_add_ps(res0, tmp1);             \

  508.                 tmp0 = _mm256_loadu_ps(src_dd + 1 * src_w);   \

  509.                 tmp1 = _mm256_mul_ps(tmp0, vf5);              \

  510.                 res0 = _mm256_add_ps(res0, tmp1);             \

  511.                 tmp1 = _mm256_mul_ps(tmp0, vf6);              \

  512.                 res1 = _mm256_add_ps(res1, tmp1);             \

  513.                 tmp0 = _mm256_loadu_ps(src_dd + 2 * src_w);   \

  514.                 tmp1 = _mm256_mul_ps(tmp0, vf4);              \

  515.                 res0 = _mm256_add_ps(res0, tmp1);             \

  516.                 tmp1 = _mm256_mul_ps(tmp0, vf5);              \

  517.                 res1 = _mm256_add_ps(res1, tmp1);             \

  518.                 tmp1 = _mm256_mul_ps(tmp0, vf6);              \

  519.                 res2 = _mm256_add_ps(res2, tmp1);             \

  520.                 tmp0 = _mm256_loadu_ps(src_dd + 3 * src_w);   \

  521.                 tmp1 = _mm256_mul_ps(tmp0, vf3);              \

  522.                 res0 = _mm256_add_ps(res0, tmp1);             \

  523.                 tmp1 = _mm256_mul_ps(tmp0, vf4);              \

  524.                 res1 = _mm256_add_ps(res1, tmp1);             \

  525.                 tmp1 = _mm256_mul_ps(tmp0, vf5);              \

  526.                 res2 = _mm256_add_ps(res2, tmp1);             \

  527.                 tmp1 = _mm256_mul_ps(tmp0, vf6);              \

  528.                 res3 = _mm256_add_ps(res3, tmp1);             \

  529.                 tmp0 = _mm256_loadu_ps(src_dd + 4 * src_w);   \

  530.                 tmp1 = _mm256_mul_ps(tmp0, vf2);              \

  531.                 res0 = _mm256_add_ps(res0, tmp1);             \

  532.                 tmp1 = _mm256_mul_ps(tmp0, vf3);              \

  533.                 res1 = _mm256_add_ps(res1, tmp1);             \

  534.                 tmp1 = _mm256_mul_ps(tmp0, vf4);              \

  535.                 res2 = _mm256_add_ps(res2, tmp1);             \

  536.                 tmp1 = _mm256_mul_ps(tmp0, vf5);              \

  537.                 res3 = _mm256_add_ps(res3, tmp1);             \

  538.                 tmp1 = _mm256_mul_ps(tmp0, vf6);              \

  539.                 res4 = _mm256_add_ps(res4, tmp1);             \

  540.                 tmp0 = _mm256_loadu_ps(src_dd + 5 * src_w);   \

  541.                 tmp1 = _mm256_mul_ps(tmp0, vf1);              \

  542.                 res0 = _mm256_add_ps(res0, tmp1);             \

  543.                 tmp1 = _mm256_mul_ps(tmp0, vf2);              \

  544.                 res1 = _mm256_add_ps(res1, tmp1);             \

  545.                 tmp1 = _mm256_mul_ps(tmp0, vf3);              \

  546.                 res2 = _mm256_add_ps(res2, tmp1);             \

  547.                 tmp1 = _mm256_mul_ps(tmp0, vf4);              \

  548.                 res3 = _mm256_add_ps(res3, tmp1);             \

  549.                 tmp1 = _mm256_mul_ps(tmp0, vf5);              \

  550.                 res4 = _mm256_add_ps(res4, tmp1);             \

  551.                 tmp1 = _mm256_mul_ps(tmp0, vf6);              \

  552.                 res5 = _mm256_add_ps(res5, tmp1);             \

  553.                 tmp0 = _mm256_loadu_ps(src_dd + 6 * src_w);   \

  554.                 tmp1 = _mm256_mul_ps(tmp0, vf0);              \

  555.                 res0 = _mm256_add_ps(res0, tmp1);             \

  556.                 tmp1 = _mm256_mul_ps(tmp0, vf1);              \

  557.                 res1 = _mm256_add_ps(res1, tmp1);             \

  558.                 tmp1 = _mm256_mul_ps(tmp0, vf2);              \

  559.                 res2 = _mm256_add_ps(res2, tmp1);             \

  560.                 tmp1 = _mm256_mul_ps(tmp0, vf3);              \

  561.                 res3 = _mm256_add_ps(res3, tmp1);             \

  562.                 tmp1 = _mm256_mul_ps(tmp0, vf4);              \

  563.                 res4 = _mm256_add_ps(res4, tmp1);             \

  564.                 tmp1 = _mm256_mul_ps(tmp0, vf5);              \

  565.                 res5 = _mm256_add_ps(res5, tmp1);             \

  566.                 tmp0 = _mm256_loadu_ps(src_dd + 7 * src_w);   \

  567.                 tmp1 = _mm256_mul_ps(tmp0, vf0);              \

  568.                 res1 = _mm256_add_ps(res1, tmp1);             \

  569.                 tmp1 = _mm256_mul_ps(tmp0, vf1);              \

  570.                 res2 = _mm256_add_ps(res2, tmp1);             \

  571.                 tmp1 = _mm256_mul_ps(tmp0, vf2);              \

  572.                 res3 = _mm256_add_ps(res3, tmp1);             \

  573.                 tmp1 = _mm256_mul_ps(tmp0, vf3);              \

  574.                 res4 = _mm256_add_ps(res4, tmp1);             \

  575.                 tmp1 = _mm256_mul_ps(tmp0, vf4);              \

  576.                 res5 = _mm256_add_ps(res5, tmp1);             \

  577.                 tmp0 = _mm256_loadu_ps(src_dd + 8 * src_w);   \

  578.                 tmp1 = _mm256_mul_ps(tmp0, vf0);              \

  579.                 res2 = _mm256_add_ps(res2, tmp1);             \

  580.                 tmp1 = _mm256_mul_ps(tmp0, vf1);              \

  581.                 res3 = _mm256_add_ps(res3, tmp1);             \

  582.                 tmp1 = _mm256_mul_ps(tmp0, vf2);              \

  583.                 res4 = _mm256_add_ps(res4, tmp1);             \

  584.                 tmp1 = _mm256_mul_ps(tmp0, vf3);              \

  585.                 res5 = _mm256_add_ps(res5, tmp1);             \

  586.                 tmp0 = _mm256_loadu_ps(src_dd + 9 * src_w);   \

  587.                 tmp1 = _mm256_mul_ps(tmp0, vf0);              \

  588.                 res3 = _mm256_add_ps(res3, tmp1);             \

  589.                 tmp1 = _mm256_mul_ps(tmp0, vf1);              \

  590.                 res4 = _mm256_add_ps(res4, tmp1);             \

  591.                 tmp1 = _mm256_mul_ps(tmp0, vf2);              \

  592.                 res5 = _mm256_add_ps(res5, tmp1);             \

  593.                 tmp0 = _mm256_loadu_ps(src_dd + 10 * src_w);  \

  594.                 tmp1 = _mm256_mul_ps(tmp0, vf0);              \

  595.                 res4 = _mm256_add_ps(res4, tmp1);             \

  596.                 tmp1 = _mm256_mul_ps(tmp0, vf1);              \

  597.                 res5 = _mm256_add_ps(res5, tmp1);             \

  598.                 tmp0 = _mm256_loadu_ps(src_dd + 11 * src_w);  \

  599.                 tmp1 = _mm256_mul_ps(tmp0, vf0);              \

  600.                 res5 = _mm256_add_ps(res5, tmp1);             \

  601.             }                                                 \

  602.             _mm256_storeu_ps(dst_dd + 0 * dst_w, res0);       \

  603.             _mm256_storeu_ps(dst_dd + 1 * dst_w, res1);       \

  604.             _mm256_storeu_ps(dst_dd + 2 * dst_w, res2);       \

  605.             _mm256_storeu_ps(dst_dd + 3 * dst_w, res3);       \

  606.             _mm256_storeu_ps(dst_dd + 4 * dst_w, res4);       \

  607.             _mm256_storeu_ps(dst_dd + 5 * dst_w, res5);       \

  608.         }                                                     \

  609.     } while (0)

  610. 

  611. #define SIMD_H7                                               \

  612.     do {                                                      \

  613.         const size_t sh = dh;                                 \

  614.         const float* src_d = src + sh * src_w;                \

  615.         float* dst_d = dst + dh * dst_w;                      \

  616.         size_t dw = dst_w_beg;                                \

  617.         for (; dw < dst_w_end; dw += 8) {                     \

  618.             const size_t sw = dw;                             \

  619.             float* dst_dd = dst_d + dw;                       \

  620.             __m256 tmp0;                                      \

  621.             __m256 tmp1;                                      \

  622.             __m256 res0;                                      \

  623.             res0 = _mm256_loadu_ps(dst_dd + 0 * dst_w);       \

  624.             __m256 res1;                                      \

  625.             res1 = _mm256_loadu_ps(dst_dd + 1 * dst_w);       \

  626.             __m256 res2;                                      \

  627.             res2 = _mm256_loadu_ps(dst_dd + 2 * dst_w);       \

  628.             __m256 res3;                                      \

  629.             res3 = _mm256_loadu_ps(dst_dd + 3 * dst_w);       \

  630.             __m256 res4;                                      \

  631.             res4 = _mm256_loadu_ps(dst_dd + 4 * dst_w);       \

  632.             __m256 res5;                                      \

  633.             res5 = _mm256_loadu_ps(dst_dd + 5 * dst_w);       \

  634.             __m256 res6;                                      \

  635.             res6 = _mm256_loadu_ps(dst_dd + 6 * dst_w);       \

  636.             for (size_t fw = 0; fw < flt_w; ++fw) {           \

  637.                 const float* src_dd = src_d + sw + fw;        \

  638.                 __m256 vf0 = _mm256_broadcast_ss(             \

  639.                         &filter[0 * flt_w + flt_w - fw - 1]); \

  640.                 __m256 vf1 = _mm256_broadcast_ss(             \

  641.                         &filter[1 * flt_w + flt_w - fw - 1]); \

  642.                 __m256 vf2 = _mm256_broadcast_ss(             \

  643.                         &filter[2 * flt_w + flt_w - fw - 1]); \

  644.                 __m256 vf3 = _mm256_broadcast_ss(             \

  645.                         &filter[3 * flt_w + flt_w - fw - 1]); \

  646.                 __m256 vf4 = _mm256_broadcast_ss(             \

  647.                         &filter[4 * flt_w + flt_w - fw - 1]); \

  648.                 __m256 vf5 = _mm256_broadcast_ss(             \

  649.                         &filter[5 * flt_w + flt_w - fw - 1]); \

  650.                 __m256 vf6 = _mm256_broadcast_ss(             \

  651.                         &filter[6 * flt_w + flt_w - fw - 1]); \

  652.                 tmp0 = _mm256_loadu_ps(src_dd + 0 * src_w);   \

  653.                 tmp1 = _mm256_mul_ps(tmp0, vf6);              \

  654.                 res0 = _mm256_add_ps(res0, tmp1);             \

  655.                 tmp0 = _mm256_loadu_ps(src_dd + 1 * src_w);   \

  656.                 tmp1 = _mm256_mul_ps(tmp0, vf5);              \

  657.                 res0 = _mm256_add_ps(res0, tmp1);             \

  658.                 tmp1 = _mm256_mul_ps(tmp0, vf6);              \

  659.                 res1 = _mm256_add_ps(res1, tmp1);             \

  660.                 tmp0 = _mm256_loadu_ps(src_dd + 2 * src_w);   \

  661.                 tmp1 = _mm256_mul_ps(tmp0, vf4);              \

  662.                 res0 = _mm256_add_ps(res0, tmp1);             \

  663.                 tmp1 = _mm256_mul_ps(tmp0, vf5);              \

  664.                 res1 = _mm256_add_ps(res1, tmp1);             \

  665.                 tmp1 = _mm256_mul_ps(tmp0, vf6);              \

  666.                 res2 = _mm256_add_ps(res2, tmp1);             \

  667.                 tmp0 = _mm256_loadu_ps(src_dd + 3 * src_w);   \

  668.                 tmp1 = _mm256_mul_ps(tmp0, vf3);              \

  669.                 res0 = _mm256_add_ps(res0, tmp1);             \

  670.                 tmp1 = _mm256_mul_ps(tmp0, vf4);              \

  671.                 res1 = _mm256_add_ps(res1, tmp1);             \

  672.                 tmp1 = _mm256_mul_ps(tmp0, vf5);              \

  673.                 res2 = _mm256_add_ps(res2, tmp1);             \

  674.                 tmp1 = _mm256_mul_ps(tmp0, vf6);              \

  675.                 res3 = _mm256_add_ps(res3, tmp1);             \

  676.                 tmp0 = _mm256_loadu_ps(src_dd + 4 * src_w);   \

  677.                 tmp1 = _mm256_mul_ps(tmp0, vf2);              \

  678.                 res0 = _mm256_add_ps(res0, tmp1);             \

  679.                 tmp1 = _mm256_mul_ps(tmp0, vf3);              \

  680.                 res1 = _mm256_add_ps(res1, tmp1);             \

  681.                 tmp1 = _mm256_mul_ps(tmp0, vf4);              \

  682.                 res2 = _mm256_add_ps(res2, tmp1);             \

  683.                 tmp1 = _mm256_mul_ps(tmp0, vf5);              \

  684.                 res3 = _mm256_add_ps(res3, tmp1);             \

  685.                 tmp1 = _mm256_mul_ps(tmp0, vf6);              \

  686.                 res4 = _mm256_add_ps(res4, tmp1);             \

  687.                 tmp0 = _mm256_loadu_ps(src_dd + 5 * src_w);   \

  688.                 tmp1 = _mm256_mul_ps(tmp0, vf1);              \

  689.                 res0 = _mm256_add_ps(res0, tmp1);             \

  690.                 tmp1 = _mm256_mul_ps(tmp0, vf2);              \

  691.                 res1 = _mm256_add_ps(res1, tmp1);             \

  692.                 tmp1 = _mm256_mul_ps(tmp0, vf3);              \

  693.                 res2 = _mm256_add_ps(res2, tmp1);             \

  694.                 tmp1 = _mm256_mul_ps(tmp0, vf4);              \

  695.                 res3 = _mm256_add_ps(res3, tmp1);             \

  696.                 tmp1 = _mm256_mul_ps(tmp0, vf5);              \

  697.                 res4 = _mm256_add_ps(res4, tmp1);             \

  698.                 tmp1 = _mm256_mul_ps(tmp0, vf6);              \

  699.                 res5 = _mm256_add_ps(res5, tmp1);             \

  700.                 tmp0 = _mm256_loadu_ps(src_dd + 6 * src_w);   \

  701.                 tmp1 = _mm256_mul_ps(tmp0, vf0);              \

  702.                 res0 = _mm256_add_ps(res0, tmp1);             \

  703.                 tmp1 = _mm256_mul_ps(tmp0, vf1);              \

  704.                 res1 = _mm256_add_ps(res1, tmp1);             \

  705.                 tmp1 = _mm256_mul_ps(tmp0, vf2);              \

  706.                 res2 = _mm256_add_ps(res2, tmp1);             \

  707.                 tmp1 = _mm256_mul_ps(tmp0, vf3);              \

  708.                 res3 = _mm256_add_ps(res3, tmp1);             \

  709.                 tmp1 = _mm256_mul_ps(tmp0, vf4);              \

  710.                 res4 = _mm256_add_ps(res4, tmp1);             \

  711.                 tmp1 = _mm256_mul_ps(tmp0, vf5);              \

  712.                 res5 = _mm256_add_ps(res5, tmp1);             \

  713.                 tmp1 = _mm256_mul_ps(tmp0, vf6);              \

  714.                 res6 = _mm256_add_ps(res6, tmp1);             \

  715.                 tmp0 = _mm256_loadu_ps(src_dd + 7 * src_w);   \

  716.                 tmp1 = _mm256_mul_ps(tmp0, vf0);              \

  717.                 res1 = _mm256_add_ps(res1, tmp1);             \

  718.                 tmp1 = _mm256_mul_ps(tmp0, vf1);              \

  719.                 res2 = _mm256_add_ps(res2, tmp1);             \

  720.                 tmp1 = _mm256_mul_ps(tmp0, vf2);              \

  721.                 res3 = _mm256_add_ps(res3, tmp1);             \

  722.                 tmp1 = _mm256_mul_ps(tmp0, vf3);              \

  723.                 res4 = _mm256_add_ps(res4, tmp1);             \

  724.                 tmp1 = _mm256_mul_ps(tmp0, vf4);              \

  725.                 res5 = _mm256_add_ps(res5, tmp1);             \

  726.                 tmp1 = _mm256_mul_ps(tmp0, vf5);              \

  727.                 res6 = _mm256_add_ps(res6, tmp1);             \

  728.                 tmp0 = _mm256_loadu_ps(src_dd + 8 * src_w);   \

  729.                 tmp1 = _mm256_mul_ps(tmp0, vf0);              \

  730.                 res2 = _mm256_add_ps(res2, tmp1);             \

  731.                 tmp1 = _mm256_mul_ps(tmp0, vf1);              \

  732.                 res3 = _mm256_add_ps(res3, tmp1);             \

  733.                 tmp1 = _mm256_mul_ps(tmp0, vf2);              \

  734.                 res4 = _mm256_add_ps(res4, tmp1);             \

  735.                 tmp1 = _mm256_mul_ps(tmp0, vf3);              \

  736.                 res5 = _mm256_add_ps(res5, tmp1);             \

  737.                 tmp1 = _mm256_mul_ps(tmp0, vf4);              \

  738.                 res6 = _mm256_add_ps(res6, tmp1);             \

  739.                 tmp0 = _mm256_loadu_ps(src_dd + 9 * src_w);   \

  740.                 tmp1 = _mm256_mul_ps(tmp0, vf0);              \

  741.                 res3 = _mm256_add_ps(res3, tmp1);             \

  742.                 tmp1 = _mm256_mul_ps(tmp0, vf1);              \

  743.                 res4 = _mm256_add_ps(res4, tmp1);             \

  744.                 tmp1 = _mm256_mul_ps(tmp0, vf2);              \

  745.                 res5 = _mm256_add_ps(res5, tmp1);             \

  746.                 tmp1 = _mm256_mul_ps(tmp0, vf3);              \

  747.                 res6 = _mm256_add_ps(res6, tmp1);             \

  748.                 tmp0 = _mm256_loadu_ps(src_dd + 10 * src_w);  \

  749.                 tmp1 = _mm256_mul_ps(tmp0, vf0);              \

  750.                 res4 = _mm256_add_ps(res4, tmp1);             \

  751.                 tmp1 = _mm256_mul_ps(tmp0, vf1);              \

  752.                 res5 = _mm256_add_ps(res5, tmp1);             \

  753.                 tmp1 = _mm256_mul_ps(tmp0, vf2);              \

  754.                 res6 = _mm256_add_ps(res6, tmp1);             \

  755.                 tmp0 = _mm256_loadu_ps(src_dd + 11 * src_w);  \

  756.                 tmp1 = _mm256_mul_ps(tmp0, vf0);              \

  757.                 res5 = _mm256_add_ps(res5, tmp1);             \

  758.                 tmp1 = _mm256_mul_ps(tmp0, vf1);              \

  759.                 res6 = _mm256_add_ps(res6, tmp1);             \

  760.                 tmp0 = _mm256_loadu_ps(src_dd + 12 * src_w);  \

  761.                 tmp1 = _mm256_mul_ps(tmp0, vf0);              \

  762.                 res6 = _mm256_add_ps(res6, tmp1);             \

  763.             }                                                 \

  764.             _mm256_storeu_ps(dst_dd + 0 * dst_w, res0);       \

  765.             _mm256_storeu_ps(dst_dd + 1 * dst_w, res1);       \

  766.             _mm256_storeu_ps(dst_dd + 2 * dst_w, res2);       \

  767.             _mm256_storeu_ps(dst_dd + 3 * dst_w, res3);       \

  768.             _mm256_storeu_ps(dst_dd + 4 * dst_w, res4);       \

  769.             _mm256_storeu_ps(dst_dd + 5 * dst_w, res5);       \

  770.             _mm256_storeu_ps(dst_dd + 6 * dst_w, res6);       \

  771.         }                                                     \

  772.     } while (0)

  773. 

  774. #include "src/x86/avx_helper.h"

  775. #include <algorithm>

  776. 

  777. #include "../convolution_direct_special_cases.h"

  778. 

  779. namespace megdnn {

  780. namespace x86 {

  781. namespace detail {

  782. 

  783. void convolution_conv_fh7_avx(const float *src, const float *filter, float *dst,

  784.         const size_t src_h, const size_t src_w, const size_t dst_h, const size_t dst_w,

  785.         const size_t flt_w)

  786. {

  787.     (void)src_h;

  788.     const size_t dst_h_beg = 0;

  789.     const size_t dst_h_end = dst_h;

  790.     const size_t dst_w_beg = 0;

  791.     const size_t dst_w_end = dst_w;

  792. 

  793.     size_t dh = dst_h_beg;

  794.     for (; dh + 7 <= dst_h_end; dh += 7) {

  795.         SIMD_H7;

  796.     }

  797.     switch (dst_h_end - dh) {

  798.         case 1:

  799.             SIMD_H1;

  800.             break;

  801.         case 2:

  802.             SIMD_H2;

  803.             break;

  804.         case 3:

  805.             SIMD_H3;

  806.             break;

  807.         case 4:

  808.             SIMD_H4;

  809.             break;

  810.         case 5:

  811.             SIMD_H5;

  812.             break;

  813.         case 6:

  814.             SIMD_H6;

  815.             break;

  816.     }

  817. }

  818. 

  819. } // namespace detail

  820. } // namespace x86

  821. } // namespace megdnn

  822. #undef SIMD_H1

  823. #undef SIMD_H2

  824. #undef SIMD_H3

  825. #undef SIMD_H4

  826. #undef SIMD_H5

  827. #undef SIMD_H6

  828. #undef SIMD_H7