OpenI
/
MegEngine
Not watched
1
0
Fork 0
Code
Releases 31 Wiki Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 369c2ccc5a
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '369c2ccc5a'
${ noResults }
MegEngine/dnn/src/aarch64/matrix_mul/asm/common.h

2184 lines
98 kB
Raw Blame History 

  1. /**

  2.  * \file dnn/src/aarch64/matrix_mul/asm/common.h

  3.  * MegEngine is Licensed under the Apache License, Version 2.0 (the "License")

  4.  *

  5.  * Copyright (c) 2014-2021 Megvii Inc. All rights reserved.

  6.  *

  7.  * Unless required by applicable law or agreed to in writing,

  8.  * software distributed under the License is distributed on an

  9.  * "AS IS" BASIS, WITHOUT ARRANTIES OR CONDITIONS OF ANY KIND, either express or

  10.  * implied.

  11.  */

  12. #pragma once

  13. #include <cmath>

  14. #include <cstdint>

  15. #include <type_traits>

  16. #include "src/arm_common/simd_macro/marm_neon.h"

  17. #include "src/common/utils.h"

  18. #include "src/fallback/conv_bias/common.h"

  19. 

  20. namespace megdnn {

  21. namespace aarch64 {

  22. 

  23. /* ======================== Prefetch ======================== */

  24. #define ASM_PREFETCH(address)    "PRFM PLDL1KEEP, " address "\n"

  25. #define ASM_PREFETCHL2(address)  "PRFM PLDL2KEEP, " address "\n"

  26. #define ASM_PREFETCHW(address)   "PRFM PSTL1KEEP, " address "\n"

  27. #define ASM_PREFETCHWL2(address) "PRFM PSTL2KEEP, " address "\n"

  28. 

  29. static inline void prefetch_6x(const void* pfp) {

  30.     // clang-format off

  31.     asm volatile(ASM_PREFETCH("[%[pfp]]")

  32.                  ASM_PREFETCH("[%[pfp], #64]")

  33.                  ASM_PREFETCH("[%[pfp], #128]")

  34.                  ASM_PREFETCH("[%[pfp], #192]")

  35.                  ASM_PREFETCH("[%[pfp], #256]")

  36.                  ASM_PREFETCH("[%[pfp], #320]")

  37.                  :

  38.                  : [pfp] "r"(pfp)

  39.                  : "memory");

  40.     // clang-format on

  41. }

  42. 

  43. static inline void prefetch_5x(const void* pfp) {

  44.     // clang-format off

  45.     asm volatile(ASM_PREFETCH("[%[pfp]]")

  46.                  ASM_PREFETCH("[%[pfp], #64]")

  47.                  ASM_PREFETCH("[%[pfp], #128]")

  48.                  ASM_PREFETCH("[%[pfp], #192]")

  49.                  ASM_PREFETCH("[%[pfp], #256]")

  50.                  :

  51.                  : [pfp] "r"(pfp)

  52.                  : "memory");

  53.     // clang-format on

  54. }

  55. 

  56. static inline void prefetch_4x(const void* pfp) {

  57.     // clang-format off

  58.     asm volatile(ASM_PREFETCH("[%[pfp]]")

  59.                  ASM_PREFETCH("[%[pfp], #64]")

  60.                  ASM_PREFETCH("[%[pfp], #128]")

  61.                  ASM_PREFETCH("[%[pfp], #192]")

  62.                  :

  63.                  : [pfp] "r"(pfp)

  64.                  : "memory");

  65.     // clang-format on

  66. }

  67. 

  68. static inline void prefetch_3x(const void* pfp) {

  69.     // clang-format off

  70.     asm volatile(ASM_PREFETCH("[%[pfp]]")

  71.                  ASM_PREFETCH("[%[pfp], #64]")

  72.                  ASM_PREFETCH("[%[pfp], #128]")

  73.                  :

  74.                  : [pfp] "r"(pfp)

  75.                  : "memory");

  76.     // clang-format on

  77. }

  78. 

  79. static inline void prefetch_2x(const void* pfp) {

  80.     // clang-format off

  81.     asm volatile(ASM_PREFETCH("[%[pfp]]")

  82.                  ASM_PREFETCH("[%[pfp], #64]")

  83.                  :

  84.                  : [pfp] "r"(pfp)

  85.                  : "memory");

  86.     // clang-format on

  87. }

  88. 

  89. static inline void prefetch_1x(const void* pfp) {

  90.     // clang-format off

  91.     asm volatile(ASM_PREFETCH("[%[pfp]]") : : [pfp] "r"(pfp) : "memory");

  92.     // clang-format on

  93. }

  94. 

  95. /* ======================== interleave pack A ======================== */

  96. 

  97. /**

  98.  * interleave_INTERLEAVE_UNROLLK_BATCH_type

  99.  *

  100.  * BATCH means process BATCH * UNROLL_K cols once, BATCH * sizeof(TYPE) *

  101.  * UNROLL_K = 16bytes(128bits, a vector size).

  102.  *

  103.  * the elements traverse order:

  104.  * rep(j, 0, INTERLEAVE) rep(i, 0, UNROLL_K) *ouptr++ = inptr[j, i]

  105.  */

  106. 

  107. template <typename T>

  108. static inline void interleave_24x1_8_h_helper(

  109.         const T*& inptr0, const T*& inptr1, const T*& inptr2, const T*& inptr3,

  110.         const T*& inptr4, const T*& inptr5, const T*& inptr6, const T*& inptr7,

  111.         T*& outptr, int skippf = 0) {

  112.     static_assert(sizeof(T) == 2, "only support size == 2");

  113.     asm volatile(

  114.             // Load up 8 elements (1 vector) from each of 8 sources.

  115.             "cbnz    %w[skippf], 1f\n"

  116.             ASM_PREFETCH("[%[inptr0], #128]")

  117.             ASM_PREFETCH("[%[inptr1], #128]")

  118.             ASM_PREFETCH("[%[inptr2], #128]")

  119.             ASM_PREFETCH("[%[inptr3], #128]")

  120.             "1:\n"

  121. 

  122.             "ldr    q0, [%[inptr0]], #16\n" // q0=A0A1A2A3A4A5A6A7

  123.             "ldr    q4, [%[inptr4]], #16\n" // q8=E0E1E2E3E4E5E6E7

  124.             "ldr    q2, [%[inptr2]], #16\n" // q4=C0C1C2C3...

  125.             "ldr    q6, [%[inptr6]], #16\n"

  126.             "zip1    v8.8h, v0.8h, v4.8h\n"  // q8=A0E0A1E1A2E2A3E3

  127.             "zip2    v16.8h, v0.8h, v4.8h\n" // q16=A4E4A5E5A6E6A7E7

  128.             "zip1    v9.8h, v2.8h, v6.8h\n"  // q9=C0G0C1G1C2G2C3G3

  129.             "zip2    v17.8h, v2.8h, v6.8h\n" // q17=C4G4C5G5C6G6C7G7

  130.             "ldr    q1, [%[inptr1]], #16\n"  // q1=B0B1B2B3B4B5B6B7

  131.             "ldr    q5, [%[inptr5]], #16\n"

  132.             "ldr    q3, [%[inptr3]], #16\n" // q3=D0D1D2D3....

  133.             "ldr    q7, [%[inptr7]], #16\n"

  134.             "zip1    v10.8h, v1.8h, v5.8h\n" // q18=B0F0B1F1B2F2B3F3

  135.             "zip2    v18.8h, v1.8h, v5.8h\n" // q18=B4F4B5F5B6F6B7F7

  136.             "zip1    v11.8h, v3.8h, v7.8h\n" // q19=D0H0D1H1D2H2D3H3

  137.             "zip2    v19.8h, v3.8h, v7.8h\n" // q19=D4H4D5H5D6H6D7H7

  138. 

  139.             "zip1    v12.8h,  v8.8h,  v9.8h\n" // q20=A0C0E0G0A1C1E1G1

  140.             "zip2    v20.8h,  v8.8h,  v9.8h\n"

  141.             "zip1    v13.8h, v10.8h, v11.8h\n" // q21=B0D0F0H0B1I1F1H1

  142.             "zip2    v21.8h, v10.8h, v11.8h\n"

  143. 

  144.             "cbnz    %w[skippf], 2f\n"

  145.             ASM_PREFETCH("[%[inptr4], #112]")

  146.             ASM_PREFETCH("[%[inptr5], #112]")

  147.             ASM_PREFETCH("[%[inptr6], #112]")

  148.             ASM_PREFETCH("[%[inptr7], #112]")

  149.             "2:\n"

  150. 

  151.             "zip1    v22.8h, v16.8h, v17.8h\n"

  152.             "zip2    v30.8h, v16.8h, v17.8h\n"

  153.             "zip1    v23.8h, v18.8h, v19.8h\n"

  154.             "zip2    v31.8h, v18.8h, v19.8h\n"

  155. 

  156.             "zip1    v14.8h, v12.8h, v13.8h\n"    // q22=A0B0C0D0E0F0G0H0

  157.             "zip2    v15.8h, v12.8h, v13.8h\n"    // q23=A1B1C1D1E1F1G1H1

  158.             "str    q14, [%[outptr]], #48\n"

  159.             "str    q15, [%[outptr]], #48\n"

  160. 

  161.             "zip1    v0.8h, v20.8h, v21.8h\n"

  162.             "zip2    v1.8h, v20.8h, v21.8h\n"

  163.             "str    q0, [%[outptr]], #48\n"

  164.             "str    q1, [%[outptr]], #48\n"

  165. 

  166.             "zip1    v2.8h, v22.8h, v23.8h\n"

  167.             "zip2    v3.8h, v22.8h, v23.8h\n"

  168.             "str    q2, [%[outptr]], #48\n"

  169.             "str    q3, [%[outptr]], #48\n"

  170. 

  171.             "zip1    v4.8h, v30.8h, v31.8h\n"

  172.             "zip2    v5.8h, v30.8h, v31.8h\n"

  173.             "str    q4, [%[outptr]], #48\n"

  174.             "str    q5, [%[outptr]], #48\n"

  175. 

  176.             : [inptr0] "+r"(inptr0), [inptr1] "+r"(inptr1),

  177.               [inptr2] "+r"(inptr2), [inptr3] "+r"(inptr3),

  178.               [inptr4] "+r"(inptr4), [inptr5] "+r"(inptr5),

  179.               [inptr6] "+r"(inptr6), [inptr7] "+r"(inptr7),

  180.               [outptr] "+r"(outptr)

  181.             : [skippf] "r"(skippf)

  182.             : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10",

  183.               "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20",

  184.               "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30",

  185.               "v31", "cc", "memory");

  186. }

  187. 

  188. template <typename T>

  189. static inline void interleave_16x1_8_h_helper(

  190.         const T*& inptr0, const T*& inptr1, const T*& inptr2, const T*& inptr3,

  191.         const T*& inptr4, const T*& inptr5, const T*& inptr6, const T*& inptr7,

  192.         T*& outptr, int skippf = 0) {

  193.     static_assert(sizeof(T) == 2, "only support size == 2");

  194.     asm volatile(

  195.             // Load up 8 elements (1 vector) from each of 8 sources.

  196.             "cbnz    %w[skippf], 1f\n"

  197.             ASM_PREFETCH("[%[inptr0], #128]")

  198.             ASM_PREFETCH("[%[inptr1], #128]")

  199.             ASM_PREFETCH("[%[inptr2], #128]")

  200.             ASM_PREFETCH("[%[inptr3], #128]")

  201.             "1:\n"

  202. 

  203.             "ldr    q0, [%[inptr0]], #16\n" // q0=A0A1A2A3A4A5A6A7

  204.             "ldr    q4, [%[inptr4]], #16\n" // q8=E0E1E2E3E4E5E6E7

  205.             "ldr    q2, [%[inptr2]], #16\n" // q4=C0C1C2C3...

  206.             "ldr    q6, [%[inptr6]], #16\n"

  207.             "zip1    v8.8h, v0.8h, v4.8h\n"  // q8=A0E0A1E1A2E2A3E3

  208.             "zip2    v16.8h, v0.8h, v4.8h\n" // q16=A4E4A5E5A6E6A7E7

  209.             "zip1    v9.8h, v2.8h, v6.8h\n"  // q9=C0G0C1G1C2G2C3G3

  210.             "zip2    v17.8h, v2.8h, v6.8h\n" // q17=C4G4C5G5C6G6C7G7

  211.             "ldr    q1, [%[inptr1]], #16\n"  // q1=B0B1B2B3B4B5B6B7

  212.             "ldr    q5, [%[inptr5]], #16\n"

  213.             "ldr    q3, [%[inptr3]], #16\n" // q3=D0D1D2D3....

  214.             "ldr    q7, [%[inptr7]], #16\n"

  215.             "zip1    v10.8h, v1.8h, v5.8h\n" // q18=B0F0B1F1B2F2B3F3

  216.             "zip2    v18.8h, v1.8h, v5.8h\n" // q18=B4F4B5F5B6F6B7F7

  217.             "zip1    v11.8h, v3.8h, v7.8h\n" // q19=D0H0D1H1D2H2D3H3

  218.             "zip2    v19.8h, v3.8h, v7.8h\n" // q19=D4H4D5H5D6H6D7H7

  219. 

  220.             "zip1    v12.8h,  v8.8h,  v9.8h\n" // q20=A0C0E0G0A1C1E1G1

  221.             "zip2    v20.8h,  v8.8h,  v9.8h\n"

  222.             "zip1    v13.8h, v10.8h, v11.8h\n" // q21=B0D0F0H0B1I1F1H1

  223.             "zip2    v21.8h, v10.8h, v11.8h\n"

  224. 

  225.             "cbnz    %w[skippf], 2f\n"

  226.             ASM_PREFETCH("[%[inptr4], #112]")

  227.             ASM_PREFETCH("[%[inptr5], #112]")

  228.             ASM_PREFETCH("[%[inptr6], #112]")

  229.             ASM_PREFETCH("[%[inptr7], #112]")

  230.             "2:\n"

  231. 

  232.             "zip1    v22.8h, v16.8h, v17.8h\n"

  233.             "zip2    v30.8h, v16.8h, v17.8h\n"

  234.             "zip1    v23.8h, v18.8h, v19.8h\n"

  235.             "zip2    v31.8h, v18.8h, v19.8h\n"

  236. 

  237.             "zip1    v14.8h, v12.8h, v13.8h\n"    // q22=A0B0C0D0E0F0G0H0

  238.             "zip2    v15.8h, v12.8h, v13.8h\n"    // q23=A1B1C1D1E1F1G1H1

  239.             "str    q14, [%[outptr]], #32\n"

  240.             "str    q15, [%[outptr]], #32\n"

  241. 

  242.             "zip1    v0.8h, v20.8h, v21.8h\n"

  243.             "zip2    v1.8h, v20.8h, v21.8h\n"

  244.             "str    q0, [%[outptr]], #32\n"

  245.             "str    q1, [%[outptr]], #32\n"

  246. 

  247.             "zip1    v2.8h, v22.8h, v23.8h\n"

  248.             "zip2    v3.8h, v22.8h, v23.8h\n"

  249.             "str    q2, [%[outptr]], #32\n"

  250.             "str    q3, [%[outptr]], #32\n"

  251. 

  252.             "zip1    v4.8h, v30.8h, v31.8h\n"

  253.             "zip2    v5.8h, v30.8h, v31.8h\n"

  254.             "str    q4, [%[outptr]], #32\n"

  255.             "str    q5, [%[outptr]], #32\n"

  256. 

  257.             : [inptr0] "+r"(inptr0), [inptr1] "+r"(inptr1),

  258.               [inptr2] "+r"(inptr2), [inptr3] "+r"(inptr3),

  259.               [inptr4] "+r"(inptr4), [inptr5] "+r"(inptr5),

  260.               [inptr6] "+r"(inptr6), [inptr7] "+r"(inptr7),

  261.               [outptr] "+r"(outptr)

  262.             : [skippf] "r"(skippf)

  263.             : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10",

  264.               "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20",

  265.               "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30",

  266.               "v31", "cc", "memory");

  267. }

  268. 

  269. template <typename T>

  270. static inline void interleave_8x1_8_h(

  271.         const T*& inptr0, const T*& inptr1, const T*& inptr2, const T*& inptr3,

  272.         const T*& inptr4, const T*& inptr5, const T*& inptr6, const T*& inptr7,

  273.         T*& outptr, int skippf = 0) {

  274.     static_assert(sizeof(T) == 2, "only support size == 2");

  275.     asm volatile(

  276.        // Load up 8 elements (1 vector) from each of 8 sources.

  277.        "cbnz    %w[skippf], 1f\n"

  278.        ASM_PREFETCH("[%[inptr0], #128]")

  279.        ASM_PREFETCH("[%[inptr1], #128]")

  280.        ASM_PREFETCH("[%[inptr2], #128]")

  281.        ASM_PREFETCH("[%[inptr3], #128]")

  282.        "1:\n"

  283. 

  284. 

  285.        "ldr    q0, [%[inptr0]], #16\n" // q0=A0A1A2A3A4A5A6A7

  286.        "ldr    q4, [%[inptr4]], #16\n" // q8=E0E1E2E3E4E5E6E7

  287.        "ldr    q2, [%[inptr2]], #16\n" // q4=C0C1C2C3...

  288.        "ldr    q6, [%[inptr6]], #16\n"

  289.        "zip1    v8.8h, v0.8h, v4.8h\n"  // q8=A0E0A1E1A2E2A3E3

  290.        "zip2    v16.8h, v0.8h, v4.8h\n" // q16=A4E4A5E5A6E6A7E7

  291.        "zip1    v9.8h, v2.8h, v6.8h\n"  // q9=C0G0C1G1C2G2C3G3

  292.        "zip2    v17.8h, v2.8h, v6.8h\n" // q17=C4G4C5G5C6G6C7G7

  293.        "ldr    q1, [%[inptr1]], #16\n"  // q1=B0B1B2B3B4B5B6B7

  294.        "ldr    q5, [%[inptr5]], #16\n"

  295.        "ldr    q3, [%[inptr3]], #16\n" // q3=D0D1D2D3....

  296.        "ldr    q7, [%[inptr7]], #16\n"

  297.        "zip1    v10.8h, v1.8h, v5.8h\n" // q18=B0F0B1F1B2F2B3F3

  298.        "zip2    v18.8h, v1.8h, v5.8h\n" // q18=B4F4B5F5B6F6B7F7

  299.        "zip1    v11.8h, v3.8h, v7.8h\n" // q19=D0H0D1H1D2H2D3H3

  300.        "zip2    v19.8h, v3.8h, v7.8h\n" // q19=D4H4D5H5D6H6D7H7

  301. 

  302.        "zip1    v12.8h,  v8.8h,  v9.8h\n" // q20=A0C0E0G0A1C1E1G1

  303.        "zip2    v20.8h,  v8.8h,  v9.8h\n"

  304.        "zip1    v13.8h, v10.8h, v11.8h\n" // q21=B0D0F0H0B1I1F1H1

  305.        "zip2    v21.8h, v10.8h, v11.8h\n"

  306. 

  307.        "cbnz    %w[skippf], 2f\n"

  308.        ASM_PREFETCH("[%[inptr4], #112]")

  309.        ASM_PREFETCH("[%[inptr5], #112]")

  310.        ASM_PREFETCH("[%[inptr6], #112]")

  311.        ASM_PREFETCH("[%[inptr7], #112]")

  312.        "2:\n"

  313. 

  314.        "zip1    v22.8h, v16.8h, v17.8h\n"

  315.        "zip2    v30.8h, v16.8h, v17.8h\n"

  316.        "zip1    v23.8h, v18.8h, v19.8h\n"

  317.        "zip2    v31.8h, v18.8h, v19.8h\n"

  318. 

  319.        "zip1    v14.8h, v12.8h, v13.8h\n"    // q22=A0B0C0D0E0F0G0H0

  320.        "zip2    v15.8h, v12.8h, v13.8h\n"    // q23=A1B1C1D1E1F1G1H1

  321.        "stp    q14, q15, [%[outptr]], #32\n" // Write back first two elements

  322. 

  323.        "zip1    v0.8h, v20.8h, v21.8h\n"

  324.        "zip2    v1.8h, v20.8h, v21.8h\n"

  325.        "stp    q0, q1, [%[outptr]], #32\n" // Write back next two elements

  326. 

  327.        "zip1    v2.8h, v22.8h, v23.8h\n"

  328.        "zip2    v3.8h, v22.8h, v23.8h\n"

  329.        "stp    q2, q3, [%[outptr]], #32\n" // Write back next two elements

  330. 

  331.        "zip1    v4.8h, v30.8h, v31.8h\n"

  332.        "zip2    v5.8h, v30.8h, v31.8h\n"

  333.        "stp    q4, q5, [%[outptr]], #32\n" // Write back last two elements

  334. 

  335.        : [inptr0] "+r"(inptr0), [inptr1] "+r"(inptr1),

  336.          [inptr2] "+r"(inptr2), [inptr3] "+r"(inptr3),

  337.          [inptr4] "+r"(inptr4), [inptr5] "+r"(inptr5),

  338.          [inptr6] "+r"(inptr6), [inptr7] "+r"(inptr7),

  339.          [outptr] "+r"(outptr)

  340. 

  341.        : [skippf] "r"(skippf)

  342.        : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10",

  343.          "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20",

  344.          "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30",

  345.          "v31", "cc", "memory");

  346. }

  347. 

  348. template <typename T>

  349. static inline void interleave_4x1_4_h(

  350.         const T*& inptr0, const T*& inptr1, const T*& inptr2, const T*& inptr3,

  351.         T*& outptr) {

  352.     static_assert(sizeof(T) == 2, "only support size == 2");

  353.     asm volatile(

  354.             "ldr d0, [%[inptr0]], #8\n"   // d0 = A0A1A2A3

  355.             "ldr d1, [%[inptr1]], #8\n"   // d1 = B0B1B2B3

  356.             "ldr d2, [%[inptr2]], #8\n"   // d2 = C0C1C2C3

  357.             "ldr d3, [%[inptr3]], #8\n"   // d3 = D0D1D2D3

  358.             "zip1 v4.4h, v0.4h, v2.4h\n"  // d4 = A0C0A1C1

  359.             "zip2 v8.4h, v0.4h, v2.4h\n"  // d8 = A2C2A3C3

  360.             "zip1 v5.4h, v1.4h, v3.4h\n"  // d5 = B0D0B1D1

  361.             "zip2 v9.4h, v1.4h, v3.4h\n"  // d9 = B2D2B3D3

  362. 

  363.             "zip1 v6.4h, v4.4h, v5.4h\n"  // d6 = A0B0C0D0

  364.             "zip2 v7.4h, v4.4h, v5.4h\n"  // d7 = A1B1C1D1

  365.             "stp d6, d7, [%[outptr]], #16\n"

  366. 

  367.             "zip1 v10.4h, v8.4h, v9.4h\n"  // d10 = A2B2C2D2

  368.             "zip2 v11.4h, v8.4h, v9.4h\n"  // d11 = A3B3C3D3

  369.             "stp d10, d11, [%[outptr]], #16\n"

  370.             : [inptr0] "+r"(inptr0), [inptr1] "+r"(inptr1), [inptr2] "+r"(inptr2),

  371.               [inptr3] "+r"(inptr3), [outptr] "+r"(outptr)

  372.             :

  373.             : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11",

  374.               "memory");

  375. }

  376. 

  377. static inline void interleave_4x1_2_d(

  378.         const int64_t*& inptr0, const int64_t*& inptr1, const int64_t*& inptr2,

  379.         const int64_t*& inptr3, int64_t*& outptr) {

  380.     asm volatile(

  381.             "ld1 {v0.2d}, [%[inptr0]], #16\n"  // d0 = A0A1

  382.             "ld1 {v1.2d}, [%[inptr1]], #16\n"  // d1 = B0B1

  383.             "ld1 {v2.2d}, [%[inptr2]], #16\n"  // d2 = C0C1

  384.             "ld1 {v3.2d}, [%[inptr3]], #16\n"  // d3 = D0D1

  385. 

  386.             "zip1 v4.2d, v0.2d, v1.2d\n"  // d8 = A0B0

  387.             "zip2 v5.2d, v0.2d, v1.2d\n"  // d9 = A1B1

  388.             "zip1 v6.2d, v2.2d, v3.2d\n"  // d10 = C0D0

  389.             "zip2 v7.2d, v2.2d, v3.2d\n"  // d11 = C1D1

  390. 

  391.             "st1 {v4.2d}, [%[outptr]], #16\n"

  392.             "st1 {v6.2d}, [%[outptr]], #16\n"

  393.             "st1 {v5.2d}, [%[outptr]], #16\n"

  394.             "st1 {v7.2d}, [%[outptr]], #16\n"

  395.             : [inptr0] "+r"(inptr0), [inptr1] "+r"(inptr1), [inptr2] "+r"(inptr2),

  396.               [inptr3] "+r"(inptr3), [outptr] "+r"(outptr)

  397.             :

  398.             : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "cc", "memory");

  399. }

  400. 

  401. static inline void interleave_4x2_2_d(

  402.         const int64_t*& inptr0, const int64_t*& inptr1, const int64_t*& inptr2,

  403.         const int64_t*& inptr3, int64_t*& outptr) {

  404.     asm volatile(

  405.             "ld1 {v0.2d}, [%[inptr0]], #16\n"  // d0 = A0

  406.             "ld1 {v1.2d}, [%[inptr0]], #16\n"  // d1 = A1

  407.             "ld1 {v2.2d}, [%[inptr1]], #16\n"  // d2 = B0

  408.             "ld1 {v3.2d}, [%[inptr1]], #16\n"  // d3 = B1

  409.             "ld1 {v4.2d}, [%[inptr2]], #16\n"  // d4 = C0

  410.             "ld1 {v5.2d}, [%[inptr2]], #16\n"  // d5 = C1

  411.             "ld1 {v6.2d}, [%[inptr3]], #16\n"  // d6 = D0

  412.             "ld1 {v7.2d}, [%[inptr3]], #16\n"  // d7 = D1

  413. 

  414.             "st1 {v0.2d}, [%[outptr]], #16\n"

  415.             "st1 {v2.2d}, [%[outptr]], #16\n"

  416.             "st1 {v4.2d}, [%[outptr]], #16\n"

  417.             "st1 {v6.2d}, [%[outptr]], #16\n"

  418.             "st1 {v1.2d}, [%[outptr]], #16\n"

  419.             "st1 {v3.2d}, [%[outptr]], #16\n"

  420.             "st1 {v5.2d}, [%[outptr]], #16\n"

  421.             "st1 {v7.2d}, [%[outptr]], #16\n"

  422.             : [inptr0] "+r"(inptr0), [inptr1] "+r"(inptr1), [inptr2] "+r"(inptr2),

  423.               [inptr3] "+r"(inptr3), [outptr] "+r"(outptr)

  424.             :

  425.             : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "cc", "memory");

  426. }

  427. 

  428. static inline void interleave_12x1_4_s(

  429.         const int32_t*& inptr0, const int32_t*& inptr1, const int32_t*& inptr2,

  430.         const int32_t*& inptr3, const int32_t*& inptr4, const int32_t*& inptr5,

  431.         const int32_t*& inptr6, const int32_t*& inptr7, const int32_t*& inptr8,

  432.         const int32_t*& inptr9, const int32_t*& inptr10, const int32_t*& inptr11,

  433.         int32_t*& outptr) {

  434.     asm volatile(

  435.             "ld1 {v0.4s}, [%[inptr0]], #16\n"  // d0 = A0A1A2A3

  436.             "ld1 {v1.4s}, [%[inptr1]], #16\n"  // d1 = B0B1B2B3

  437.             "ld1 {v2.4s}, [%[inptr2]], #16\n"  // d2 = C0C1C2C3

  438.             "ld1 {v3.4s}, [%[inptr3]], #16\n"  // d3 = D0D1D2D3

  439.             "zip1 v12.4s, v0.4s, v2.4s\n"      // d12 = A0C0A1C1

  440.             "zip2 v13.4s, v0.4s, v2.4s\n"      // d13 = A2C2A3C3

  441.             "zip1 v14.4s, v1.4s, v3.4s\n"      // d14 = B0D0B1D1

  442.             "zip2 v15.4s, v1.4s, v3.4s\n"      // d15 = B2D2B3D3

  443.             "zip1 v0.4s, v12.4s, v14.4s\n"     // d0 = A0B0C0D0

  444.             "zip2 v1.4s, v12.4s, v14.4s\n"     // d1 = A1B1C1D1

  445.             "zip1 v2.4s, v13.4s, v15.4s\n"     // d2 = A2B2C2D2

  446.             "zip2 v3.4s, v13.4s, v15.4s\n"     // d3 = A3B3C3D3

  447. 

  448.             "ld1 {v4.4s}, [%[inptr4]], #16\n"  // d4 = E0E1E2E3

  449.             "ld1 {v5.4s}, [%[inptr5]], #16\n"  // d5 = F0F1F2F3

  450.             "ld1 {v6.4s}, [%[inptr6]], #16\n"  // d6 = G0G1G2G3

  451.             "ld1 {v7.4s}, [%[inptr7]], #16\n"  // d7 = H0H1H2H3

  452.             "zip1 v16.4s, v4.4s, v6.4s\n"      // d16 = E0G0E1G1

  453.             "zip2 v17.4s, v4.4s, v6.4s\n"      // d17 = E2G2E3G3

  454.             "zip1 v18.4s, v5.4s, v7.4s\n"      // d18 = F0H0F1H1

  455.             "zip2 v19.4s, v5.4s, v7.4s\n"      // d19 = F2H2F3H3

  456.             "zip1 v4.4s, v16.4s, v18.4s\n"     // d4 = E0F0G0H0

  457.             "zip2 v5.4s, v16.4s, v18.4s\n"     // d5 = E1F1G1H1

  458.             "zip1 v6.4s, v17.4s, v19.4s\n"     // d6 = E2F2G2H2

  459.             "zip2 v7.4s, v17.4s, v19.4s\n"     // d7 = E3F3G3H3

  460. 

  461.             "ld1 {v8.4s}, [%[inptr8]], #16\n"    // d8 = I0I1I2I3

  462.             "ld1 {v9.4s}, [%[inptr9]], #16\n"    // d9 = J0J1J2J3

  463.             "ld1 {v10.4s}, [%[inptr10]], #16\n"  // d10 = K0K1K2K3

  464.             "ld1 {v11.4s}, [%[inptr11]], #16\n"  // d11 = L0L1L2L3

  465.             "zip1 v20.4s, v8.4s, v10.4s\n"       // d20 = I0K0I1K1

  466.             "zip2 v21.4s, v8.4s, v10.4s\n"       // d21 = I2K2I3K3

  467.             "zip1 v22.4s, v9.4s, v11.4s\n"       // d22 = J0L0J1L1

  468.             "zip2 v23.4s, v9.4s, v11.4s\n"       // d23 = J2L2J3L3

  469.             "zip1 v8.4s, v20.4s, v22.4s\n"       // d8 = I0J0K0L0

  470.             "zip2 v9.4s, v20.4s, v22.4s\n"       // d9 = I1J1K1L1

  471.             "zip1 v10.4s, v21.4s, v23.4s\n"      // d10 = I2J2K2L2

  472.             "zip2 v11.4s, v21.4s, v23.4s\n"      // d11 = I3J3K3L3

  473. 

  474.             "st1 {v0.4s}, [%[outptr]], #16\n"

  475.             "st1 {v4.4s}, [%[outptr]], #16\n"

  476.             "st1 {v8.4s}, [%[outptr]], #16\n"

  477.             "st1 {v1.4s}, [%[outptr]], #16\n"

  478.             "st1 {v5.4s}, [%[outptr]], #16\n"

  479.             "st1 {v9.4s}, [%[outptr]], #16\n"

  480.             "st1 {v2.4s}, [%[outptr]], #16\n"

  481.             "st1 {v6.4s}, [%[outptr]], #16\n"

  482.             "st1 {v10.4s}, [%[outptr]], #16\n"

  483.             "st1 {v3.4s}, [%[outptr]], #16\n"

  484.             "st1 {v7.4s}, [%[outptr]], #16\n"

  485.             "st1 {v11.4s}, [%[outptr]], #16\n"

  486. 

  487.             : [inptr0] "+r"(inptr0), [inptr1] "+r"(inptr1), [inptr2] "+r"(inptr2),

  488.               [inptr3] "+r"(inptr3), [inptr4] "+r"(inptr4), [inptr5] "+r"(inptr5),

  489.               [inptr6] "+r"(inptr6), [inptr7] "+r"(inptr7), [inptr8] "+r"(inptr8),

  490.               [inptr9] "+r"(inptr9), [inptr10] "+r"(inptr10), [inptr11] "+r"(inptr11),

  491.               [outptr] "+r"(outptr)

  492.             :

  493.             : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11",

  494.               "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21",

  495.               "v22", "v23", "cc", "memory");

  496. }

  497. 

  498. template <typename T>

  499. static inline void interleave_12x1_4_h(

  500.         const T*& in0, const T*& in1, const T*& in2, const T*& in3, const T*& in4,

  501.         const T*& in5, const T*& in6, const T*& in7, const T*& in8, const T*& in9,

  502.         const T*& in10, const T*& in11, T*& out) {

  503.     static_assert(

  504.             std::is_same<T, int16_t>::value || std::is_same<T, uint16_t>::value,

  505.             "interleave_12x1_4_h only support uint16_t and int16_t");

  506.     const int16_t*& inptr0 = reinterpret_cast<const int16_t*&>(in0);

  507.     const int16_t*& inptr1 = reinterpret_cast<const int16_t*&>(in1);

  508.     const int16_t*& inptr2 = reinterpret_cast<const int16_t*&>(in2);

  509.     const int16_t*& inptr3 = reinterpret_cast<const int16_t*&>(in3);

  510.     const int16_t*& inptr4 = reinterpret_cast<const int16_t*&>(in4);

  511.     const int16_t*& inptr5 = reinterpret_cast<const int16_t*&>(in5);

  512.     const int16_t*& inptr6 = reinterpret_cast<const int16_t*&>(in6);

  513.     const int16_t*& inptr7 = reinterpret_cast<const int16_t*&>(in7);

  514.     const int16_t*& inptr8 = reinterpret_cast<const int16_t*&>(in8);

  515.     const int16_t*& inptr9 = reinterpret_cast<const int16_t*&>(in9);

  516.     const int16_t*& inptr10 = reinterpret_cast<const int16_t*&>(in10);

  517.     const int16_t*& inptr11 = reinterpret_cast<const int16_t*&>(in11);

  518.     int16_t*& outptr = reinterpret_cast<int16_t*&>(out);

  519.     asm volatile(

  520.             "ld1 {v0.4h}, [%[inptr0]], #8\n"  // d0 = A0A1A2A3

  521.             "ld1 {v1.4h}, [%[inptr1]], #8\n"  // d1 = B0B1B2B3

  522.             "ld1 {v2.4h}, [%[inptr2]], #8\n"  // d2 = C0C1C2C3

  523.             "ld1 {v3.4h}, [%[inptr3]], #8\n"  // d3 = D0D1D2D3

  524.             "zip1 v12.4h, v0.4h, v2.4h\n"     // d12 = A0C0A1C1

  525.             "zip2 v13.4h, v0.4h, v2.4h\n"     // d13 = A2C2A3C3

  526.             "zip1 v14.4h, v1.4h, v3.4h\n"     // d14 = B0D0B1D1

  527.             "zip2 v15.4h, v1.4h, v3.4h\n"     // d15 = B2D2B3D3

  528.             "zip1 v0.4h, v12.4h, v14.4h\n"    // d0 = A0B0C0D0

  529.             "zip2 v1.4h, v12.4h, v14.4h\n"    // d1 = A1B1C1D1

  530.             "zip1 v2.4h, v13.4h, v15.4h\n"    // d2 = A2B2C2D2

  531.             "zip2 v3.4h, v13.4h, v15.4h\n"    // d3 = A3B3C3D3

  532. 

  533.             "ld1 {v4.4h}, [%[inptr4]], #8\n"  // d4 = E0E1E2E3

  534.             "ld1 {v5.4h}, [%[inptr5]], #8\n"  // d5 = F0F1F2F3

  535.             "ld1 {v6.4h}, [%[inptr6]], #8\n"  // d6 = G0G1G2G3

  536.             "ld1 {v7.4h}, [%[inptr7]], #8\n"  // d7 = H0H1H2H3

  537.             "zip1 v16.4h, v4.4h, v6.4h\n"     // d16 = E0G0E1G1

  538.             "zip2 v17.4h, v4.4h, v6.4h\n"     // d17 = E2G2E3G3

  539.             "zip1 v18.4h, v5.4h, v7.4h\n"     // d18 = F0H0F1H1

  540.             "zip2 v19.4h, v5.4h, v7.4h\n"     // d19 = F2H2F3H3

  541.             "zip1 v4.4h, v16.4h, v18.4h\n"    // d4 = E0F0G0H0

  542.             "zip2 v5.4h, v16.4h, v18.4h\n"    // d5 = E1F1G1H1

  543.             "zip1 v6.4h, v17.4h, v19.4h\n"    // d6 = E2F2G2H2

  544.             "zip2 v7.4h, v17.4h, v19.4h\n"    // d7 = E3F3G3H3

  545. 

  546.             "ld1 {v8.4h}, [%[inptr8]], #8\n"    // d8 = I0I1I2I3

  547.             "ld1 {v9.4h}, [%[inptr9]], #8\n"    // d9 = J0J1J2J3

  548.             "ld1 {v10.4h}, [%[inptr10]], #8\n"  // d10 = K0K1K2K3

  549.             "ld1 {v11.4h}, [%[inptr11]], #8\n"  // d11 = L0L1L2L3

  550.             "zip1 v20.4h, v8.4h, v10.4h\n"      // d20 = I0K0I1K1

  551.             "zip2 v21.4h, v8.4h, v10.4h\n"      // d21 = I2K2I3K3

  552.             "zip1 v22.4h, v9.4h, v11.4h\n"      // d22 = J0L0J1L1

  553.             "zip2 v23.4h, v9.4h, v11.4h\n"      // d23 = J2L2J3L3

  554.             "zip1 v8.4h, v20.4h, v22.4h\n"      // d8 = I0J0K0L0

  555.             "zip2 v9.4h, v20.4h, v22.4h\n"      // d9 = I1J1K1L1

  556.             "zip1 v10.4h, v21.4h, v23.4h\n"     // d10 = I2J2K2L2

  557.             "zip2 v11.4h, v21.4h, v23.4h\n"     // d11 = I3J3K3L3

  558. 

  559.             "st1 {v0.4h}, [%[outptr]], #8\n"   // d0 = A0B0C0D0

  560.             "st1 {v4.4h}, [%[outptr]], #8\n"   // d4 = E0F0G0H0

  561.             "st1 {v8.4h}, [%[outptr]], #8\n"   // d8 = I0J0K0L0

  562.             "st1 {v1.4h}, [%[outptr]], #8\n"   // d1 = A1B1C1D1

  563.             "st1 {v5.4h}, [%[outptr]], #8\n"   // d5 = E1F1G1H1

  564.             "st1 {v9.4h}, [%[outptr]], #8\n"   // d9 = I1J1K1L1

  565.             "st1 {v2.4h}, [%[outptr]], #8\n"   // d2 = A2B2C2D2

  566.             "st1 {v6.4h}, [%[outptr]], #8\n"   // d6 = E2F2G2H2

  567.             "st1 {v10.4h}, [%[outptr]], #8\n"  // d10 = I2J2K2L2

  568.             "st1 {v3.4h}, [%[outptr]], #8\n"   // d3 = A3B3C3D3

  569.             "st1 {v7.4h}, [%[outptr]], #8\n"   // d7 = E3F3G3H3

  570.             "st1 {v11.4h}, [%[outptr]], #8\n"  // d11 = I3J3K3L3

  571. 

  572.             : [inptr0] "+r"(inptr0), [inptr1] "+r"(inptr1), [inptr2] "+r"(inptr2),

  573.               [inptr3] "+r"(inptr3), [inptr4] "+r"(inptr4), [inptr5] "+r"(inptr5),

  574.               [inptr6] "+r"(inptr6), [inptr7] "+r"(inptr7), [inptr8] "+r"(inptr8),

  575.               [inptr9] "+r"(inptr9), [inptr10] "+r"(inptr10), [inptr11] "+r"(inptr11),

  576.               [outptr] "+r"(outptr)

  577.             :

  578.             : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11",

  579.               "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21",

  580.               "v22", "v23", "cc", "memory");

  581. }

  582. 

  583. template <typename T>

  584. static inline void interleave_12x4_4_b(

  585.         const T*& inptr0, const T*& inptr1, const T*& inptr2, const T*& inptr3,

  586.         const T*& inptr4, const T*& inptr5, const T*& inptr6, const T*& inptr7,

  587.         const T*& inptr8, const T*& inptr9, const T*& inptr10, const T*& inptr11,

  588.         T*& outptr) {

  589.     static_assert(

  590.             std::is_same<T, int8_t>::value || std::is_same<T, uint8_t>::value,

  591.             "interleave_12x4_4_b only support uint8_t and int8_t");

  592.     interleave_12x1_4_s(

  593.             reinterpret_cast<const int32_t*&>(inptr0),

  594.             reinterpret_cast<const int32_t*&>(inptr1),

  595.             reinterpret_cast<const int32_t*&>(inptr2),

  596.             reinterpret_cast<const int32_t*&>(inptr3),

  597.             reinterpret_cast<const int32_t*&>(inptr4),

  598.             reinterpret_cast<const int32_t*&>(inptr5),

  599.             reinterpret_cast<const int32_t*&>(inptr6),

  600.             reinterpret_cast<const int32_t*&>(inptr7),

  601.             reinterpret_cast<const int32_t*&>(inptr8),

  602.             reinterpret_cast<const int32_t*&>(inptr9),

  603.             reinterpret_cast<const int32_t*&>(inptr10),

  604.             reinterpret_cast<const int32_t*&>(inptr11),

  605.             reinterpret_cast<int32_t*&>(outptr));

  606. }

  607. 

  608. static inline void interleave_2x1_4_s(

  609.         const int32_t*& inptr0, const int32_t*& inptr1, int32_t*& outptr) {

  610.     asm volatile(

  611.             "ld1 {v0.4s}, [%[inptr0]], #16\n"  // d0 = A0A1A2A3

  612.             "ld1 {v1.4s}, [%[inptr1]], #16\n"  // d1 = B0B1B2B3

  613.             "st1 {v0.4s}, [%[outptr]], #16\n"

  614.             "st1 {v1.4s}, [%[outptr]], #16\n"

  615.             : [inptr0] "+r"(inptr0), [inptr1] "+r"(inptr1), [outptr] "+r"(outptr)

  616.             :

  617.             : "v0", "v1", "cc", "memory");

  618. }

  619. 

  620. static inline void interleave_8x1_4_s(

  621.         const int32_t*& inptr0, const int32_t*& inptr1, const int32_t*& inptr2,

  622.         const int32_t*& inptr3, const int32_t*& inptr4, const int32_t*& inptr5,

  623.         const int32_t*& inptr6, const int32_t*& inptr7, int32_t*& outptr) {

  624.     asm volatile(

  625.             "ld1 {v0.4s}, [%[inptr0]], #16\n"  // d0 = A0A1A2A3

  626.             "ld1 {v1.4s}, [%[inptr1]], #16\n"  // d1 = B0B1B2B3

  627.             "ld1 {v2.4s}, [%[inptr2]], #16\n"  // d2 = C0C1C2C3

  628.             "ld1 {v3.4s}, [%[inptr3]], #16\n"  // d3 = D0D1D2D3

  629.             "zip1 v8.4s, v0.4s, v2.4s\n"       // d8 = A0C0A1C1

  630.             "zip2 v9.4s, v0.4s, v2.4s\n"       // d9 = A2C2A3C3

  631.             "zip1 v10.4s, v1.4s, v3.4s\n"      // d10 = B0D0B1D1

  632.             "zip2 v11.4s, v1.4s, v3.4s\n"      // d11 = B2D2B3D3

  633.             "zip1 v12.4s, v8.4s, v10.4s\n"     // d12 = A0B0C0D0

  634.             "zip2 v13.4s, v8.4s, v10.4s\n"     // d13 = A1B1C1D1

  635.             "zip1 v14.4s, v9.4s, v11.4s\n"     // d14 = A2B2C2D2

  636.             "zip2 v15.4s, v9.4s, v11.4s\n"     // d15 = A3B3C3D3

  637. 

  638.             "ld1 {v4.4s}, [%[inptr4]], #16\n"  // d4 = E0E1E2E3

  639.             "ld1 {v5.4s}, [%[inptr5]], #16\n"  // d5 = F0F1F2F3

  640.             "ld1 {v6.4s}, [%[inptr6]], #16\n"  // d6 = G0G1G2G3

  641.             "ld1 {v7.4s}, [%[inptr7]], #16\n"  // d7 = H0H1H2H3

  642.             "zip1 v16.4s, v4.4s, v6.4s\n"      // d16 = E0G0E1G1

  643.             "zip2 v17.4s, v4.4s, v6.4s\n"      // d17 = E2G2E3G3

  644.             "zip1 v18.4s, v5.4s, v7.4s\n"      // d18 = F0H0F1H1

  645.             "zip2 v19.4s, v5.4s, v7.4s\n"      // d19 = F2H2F3H3

  646.             "zip1 v20.4s, v16.4s, v18.4s\n"    // d20 = E0F0G0H0

  647.             "zip2 v21.4s, v16.4s, v18.4s\n"    // d21 = E1F1G1H1

  648.             "zip1 v22.4s, v17.4s, v19.4s\n"    // d22 = E2F2G2H2

  649.             "zip2 v23.4s, v17.4s, v19.4s\n"    // d23 = E3F3G3H3

  650. 

  651.             "st1 {v12.4s}, [%[outptr]], #16\n"

  652.             "st1 {v20.4s}, [%[outptr]], #16\n"

  653.             "st1 {v13.4s}, [%[outptr]], #16\n"

  654.             "st1 {v21.4s}, [%[outptr]], #16\n"

  655.             "st1 {v14.4s}, [%[outptr]], #16\n"

  656.             "st1 {v22.4s}, [%[outptr]], #16\n"

  657.             "st1 {v15.4s}, [%[outptr]], #16\n"

  658.             "st1 {v23.4s}, [%[outptr]], #16\n"

  659. 

  660.             : [inptr0] "+r"(inptr0), [inptr1] "+r"(inptr1), [inptr2] "+r"(inptr2),

  661.               [inptr3] "+r"(inptr3), [inptr4] "+r"(inptr4), [inptr5] "+r"(inptr5),

  662.               [inptr6] "+r"(inptr6), [inptr7] "+r"(inptr7), [outptr] "+r"(outptr)

  663.             :

  664.             : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11",

  665.               "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21",

  666.               "v22", "v23", "cc", "memory");

  667. }

  668. 

  669. static inline void interleave_8x1_2_d(

  670.         const int64_t*& inptr0, const int64_t*& inptr1, const int64_t*& inptr2,

  671.         const int64_t*& inptr3, const int64_t*& inptr4, const int64_t*& inptr5,

  672.         const int64_t*& inptr6, const int64_t*& inptr7, int64_t*& outptr) {

  673.     asm volatile(

  674.             "ld1 {v0.2d}, [%[inptr0]], #16\n"  // d0 = A0A1

  675.             "ld1 {v1.2d}, [%[inptr1]], #16\n"  // d1 = B0B1

  676.             "ld1 {v2.2d}, [%[inptr2]], #16\n"  // d2 = C0C1

  677.             "ld1 {v3.2d}, [%[inptr3]], #16\n"  // d3 = D0D1

  678.             "ld1 {v4.2d}, [%[inptr4]], #16\n"  // d4 = E0E1

  679.             "ld1 {v5.2d}, [%[inptr5]], #16\n"  // d5 = F0F1

  680.             "ld1 {v6.2d}, [%[inptr6]], #16\n"  // d6 = G0G1

  681.             "ld1 {v7.2d}, [%[inptr7]], #16\n"  // d7 = H0H1

  682. 

  683.             "zip1 v8.2d, v0.2d, v1.2d\n"   // d8 = A0B0

  684.             "zip2 v9.2d, v0.2d, v1.2d\n"   // d9 = A1B1

  685.             "zip1 v10.2d, v2.2d, v3.2d\n"  // d10 = C0D0

  686.             "zip2 v11.2d, v2.2d, v3.2d\n"  // d11 = C1D1

  687.             "zip1 v12.2d, v4.2d, v5.2d\n"  // d12 = E0F0

  688.             "zip2 v13.2d, v4.2d, v5.2d\n"  // d13 = E1F1

  689.             "zip1 v14.2d, v6.2d, v7.2d\n"  // d14 = G0H0

  690.             "zip2 v15.2d, v6.2d, v7.2d\n"  // d15 = G1H1

  691. 

  692.             "st1 {v8.2d}, [%[outptr]], #16\n"

  693.             "st1 {v10.2d}, [%[outptr]], #16\n"

  694.             "st1 {v12.2d}, [%[outptr]], #16\n"

  695.             "st1 {v14.2d}, [%[outptr]], #16\n"

  696.             "st1 {v9.2d}, [%[outptr]], #16\n"

  697.             "st1 {v11.2d}, [%[outptr]], #16\n"

  698.             "st1 {v13.2d}, [%[outptr]], #16\n"

  699.             "st1 {v15.2d}, [%[outptr]], #16\n"

  700.             : [inptr0] "+r"(inptr0), [inptr1] "+r"(inptr1), [inptr2] "+r"(inptr2),

  701.               [inptr3] "+r"(inptr3), [inptr4] "+r"(inptr4), [inptr5] "+r"(inptr5),

  702.               [inptr6] "+r"(inptr6), [inptr7] "+r"(inptr7), [outptr] "+r"(outptr)

  703.             :

  704.             : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11",

  705.               "v12", "v13", "v14", "v15", "cc", "memory");

  706. }

  707. 

  708. static inline void interleave_8x2_2_d(

  709.         const int64_t*& inptr0, const int64_t*& inptr1, const int64_t*& inptr2,

  710.         const int64_t*& inptr3, const int64_t*& inptr4, const int64_t*& inptr5,

  711.         const int64_t*& inptr6, const int64_t*& inptr7, int64_t*& outptr) {

  712.     asm volatile(

  713.             "ld1 {v0.2d}, [%[inptr0]], #16\n"   // d0 = A0

  714.             "ld1 {v1.2d}, [%[inptr0]], #16\n"   // d1 = A1

  715.             "ld1 {v2.2d}, [%[inptr1]], #16\n"   // d2 = B0

  716.             "ld1 {v3.2d}, [%[inptr1]], #16\n"   // d3 = B1

  717.             "ld1 {v4.2d}, [%[inptr2]], #16\n"   // d4 = C0

  718.             "ld1 {v5.2d}, [%[inptr2]], #16\n"   // d5 = C1

  719.             "ld1 {v6.2d}, [%[inptr3]], #16\n"   // d6 = D0

  720.             "ld1 {v7.2d}, [%[inptr3]], #16\n"   // d7 = D1

  721.             "ld1 {v8.2d}, [%[inptr4]], #16\n"   // d8 = E0

  722.             "ld1 {v9.2d}, [%[inptr4]], #16\n"   // d9 = E1

  723.             "ld1 {v10.2d}, [%[inptr5]], #16\n"  // d10 = F0

  724.             "ld1 {v11.2d}, [%[inptr5]], #16\n"  // d11 = F1

  725.             "ld1 {v12.2d}, [%[inptr6]], #16\n"  // d12 = G0

  726.             "ld1 {v13.2d}, [%[inptr6]], #16\n"  // d13 = G1

  727.             "ld1 {v14.2d}, [%[inptr7]], #16\n"  // d14 = H0

  728.             "ld1 {v15.2d}, [%[inptr7]], #16\n"  // d15 = H1

  729. 

  730.             "st1 {v0.2d}, [%[outptr]], #16\n"

  731.             "st1 {v2.2d}, [%[outptr]], #16\n"

  732.             "st1 {v4.2d}, [%[outptr]], #16\n"

  733.             "st1 {v6.2d}, [%[outptr]], #16\n"

  734.             "st1 {v8.2d}, [%[outptr]], #16\n"

  735.             "st1 {v10.2d}, [%[outptr]], #16\n"

  736.             "st1 {v12.2d}, [%[outptr]], #16\n"

  737.             "st1 {v14.2d}, [%[outptr]], #16\n"

  738.             "st1 {v1.2d}, [%[outptr]], #16\n"

  739.             "st1 {v3.2d}, [%[outptr]], #16\n"

  740.             "st1 {v5.2d}, [%[outptr]], #16\n"

  741.             "st1 {v7.2d}, [%[outptr]], #16\n"

  742.             "st1 {v9.2d}, [%[outptr]], #16\n"

  743.             "st1 {v11.2d}, [%[outptr]], #16\n"

  744.             "st1 {v13.2d}, [%[outptr]], #16\n"

  745.             "st1 {v15.2d}, [%[outptr]], #16\n"

  746.             : [inptr0] "+r"(inptr0), [inptr1] "+r"(inptr1), [inptr2] "+r"(inptr2),

  747.               [inptr3] "+r"(inptr3), [inptr4] "+r"(inptr4), [inptr5] "+r"(inptr5),

  748.               [inptr6] "+r"(inptr6), [inptr7] "+r"(inptr7), [outptr] "+r"(outptr)

  749.             :

  750.             : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11",

  751.               "v12", "v13", "v14", "v15", "cc", "memory");

  752. }

  753. 

  754. template <typename T>

  755. static inline void interleave_2x4_4_b(const T*& inptr0, const T*& inptr1, T*& outptr) {

  756.     static_assert(

  757.             std::is_same<T, int8_t>::value || std::is_same<T, uint8_t>::value,

  758.             "interleave_2x4_4_b only support uint8_t and int8_t");

  759.     interleave_2x1_4_s(

  760.             reinterpret_cast<const int32_t*&>(inptr0),

  761.             reinterpret_cast<const int32_t*&>(inptr1),

  762.             reinterpret_cast<int32_t*&>(outptr));

  763. }

  764. 

  765. template <typename T>

  766. static inline void interleave_8x4_4_b(

  767.         const T*& inptr0, const T*& inptr1, const T*& inptr2, const T*& inptr3,

  768.         const T*& inptr4, const T*& inptr5, const T*& inptr6, const T*& inptr7,

  769.         T*& outptr) {

  770.     static_assert(

  771.             std::is_same<T, int8_t>::value || std::is_same<T, uint8_t>::value,

  772.             "interleave_8x4_4_b only support uint8_t and int8_t");

  773.     interleave_8x1_4_s(

  774.             reinterpret_cast<const int32_t*&>(inptr0),

  775.             reinterpret_cast<const int32_t*&>(inptr1),

  776.             reinterpret_cast<const int32_t*&>(inptr2),

  777.             reinterpret_cast<const int32_t*&>(inptr3),

  778.             reinterpret_cast<const int32_t*&>(inptr4),

  779.             reinterpret_cast<const int32_t*&>(inptr5),

  780.             reinterpret_cast<const int32_t*&>(inptr6),

  781.             reinterpret_cast<const int32_t*&>(inptr7),

  782.             reinterpret_cast<int32_t*&>(outptr));

  783. }

  784. 

  785. template <typename T>

  786. static inline void interleave_8x4_1_h(

  787.         const T*& in0, const T*& in1, const T*& in2, const T*& in3, T* out) {

  788.     static_assert(sizeof(T) == 2, "only support size == 2");

  789.     asm volatile(

  790.             "ldr q0, [%[in0]], #16\n"  // A1A2A3A4A5A6A7A8

  791.             "ldr q1, [%[in1]], #16\n"  // B1B2B3B4B5B6B7B8

  792.             "ldr q2, [%[in2]], #16\n"  // C1C2C3C4C5C6C7C8

  793.             "ldr q3, [%[in3]], #16\n"  // D1D2D3D4D5D6D7D8

  794. 

  795.             "trn1 v4.8h, v0.8h, v1.8h\n"  // A1B1A3B3A5B5A7B7

  796.             "trn2 v5.8h, v0.8h, v1.8h\n"  // A2B2A4B4A6B6A8B8

  797.             "trn1 v6.8h, v2.8h, v3.8h\n"  // C1D1C3D3C5D5C7D7

  798.             "trn2 v7.8h, v2.8h, v3.8h\n"  // C2D2C4D4C6D6C8D8

  799. 

  800.             "zip1 v8.4s, v4.4s, v6.4s\n"   // A1B1C1D1A3B3C3D3

  801.             "zip2 v9.4s, v4.4s, v6.4s\n"   // A5B5C5D5A7B7C7D7

  802.             "zip1 v10.4s, v5.4s, v7.4s\n"  // A2B2C2D2A4B4C4D4

  803.             "zip2 v11.4s, v5.4s, v7.4s\n"  // A6B6C6D6A8B8C8D8

  804. 

  805.             "zip1 v12.2d, v8.2d, v10.2d\n"  // A1B1C1D1A2B2C2D2

  806.             "zip2 v13.2d, v8.2d, v10.2d\n"  // A3B3C3D3A4B4C4D4

  807.             "zip1 v14.2d, v9.2d, v11.2d\n"  // A5B5C5D5A6B6C6D6

  808.             "zip2 v15.2d, v9.2d, v11.2d\n"  // A7B7C7D7A8B8C8D8

  809. 

  810.             "st1 {v12.2d}, [%[out]], #16\n"

  811.             "st1 {v13.2d}, [%[out]], #16\n"

  812.             "st1 {v14.2d}, [%[out]], #16\n"

  813.             "st1 {v15.2d}, [%[out]], #16\n"

  814.             : [in0] "+r"(in0), [in1] "+r"(in1), [in2] "+r"(in2), [in3] "+r"(in3),

  815.               [out] "+r"(out)

  816.             :

  817.             : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11",

  818.               "v12", "v13", "v14", "v15", "memory");

  819. }

  820. 

  821. template <typename T>

  822. static inline void interleave_8x8_2_b(

  823.         const T*& inptr0, const T*& inptr1, const T*& inptr2, const T*& inptr3,

  824.         const T*& inptr4, const T*& inptr5, const T*& inptr6, const T*& inptr7,

  825.         T*& outptr) {

  826.     static_assert(

  827.             std::is_same<T, int8_t>::value || std::is_same<T, uint8_t>::value,

  828.             "interleave_8x8_2_b only support uint8_t and int8_t");

  829.     interleave_8x1_2_d(

  830.             reinterpret_cast<const int64_t*&>(inptr0),

  831.             reinterpret_cast<const int64_t*&>(inptr1),

  832.             reinterpret_cast<const int64_t*&>(inptr2),

  833.             reinterpret_cast<const int64_t*&>(inptr3),

  834.             reinterpret_cast<const int64_t*&>(inptr4),

  835.             reinterpret_cast<const int64_t*&>(inptr5),

  836.             reinterpret_cast<const int64_t*&>(inptr6),

  837.             reinterpret_cast<const int64_t*&>(inptr7),

  838.             reinterpret_cast<int64_t*&>(outptr));

  839. }

  840. 

  841. template <typename T>

  842. static inline void interleave_8x8_2_h(

  843.         const T*& inptr0, const T*& inptr1, const T*& inptr2, const T*& inptr3,

  844.         const T*& inptr4, const T*& inptr5, const T*& inptr6, const T*& inptr7,

  845.         T*& outptr) {

  846.     static_assert(

  847.             std::is_same<T, int16_t>::value || std::is_same<T, uint16_t>::value,

  848.             "interleave_8x8_2_h only support uint16_t and int16_t");

  849.     interleave_8x2_2_d(

  850.             reinterpret_cast<const int64_t*&>(inptr0),

  851.             reinterpret_cast<const int64_t*&>(inptr1),

  852.             reinterpret_cast<const int64_t*&>(inptr2),

  853.             reinterpret_cast<const int64_t*&>(inptr3),

  854.             reinterpret_cast<const int64_t*&>(inptr4),

  855.             reinterpret_cast<const int64_t*&>(inptr5),

  856.             reinterpret_cast<const int64_t*&>(inptr6),

  857.             reinterpret_cast<const int64_t*&>(inptr7),

  858.             reinterpret_cast<int64_t*&>(outptr));

  859. }

  860. 

  861. template <typename T>

  862. static inline void interleave_8x2_8_b(

  863.         const T*& inptr0, const T*& inptr1, const T*& inptr2, const T*& inptr3,

  864.         const T*& inptr4, const T*& inptr5, const T*& inptr6, const T*& inptr7,

  865.         T*& outptr) {

  866.     static_assert(

  867.             std::is_same<T, int8_t>::value || std::is_same<T, uint8_t>::value,

  868.             "interleave_8x2_8_b only support uint8_t and int8_t");

  869.     interleave_8x1_8_h(

  870.             reinterpret_cast<const int16_t*&>(inptr0),

  871.             reinterpret_cast<const int16_t*&>(inptr1),

  872.             reinterpret_cast<const int16_t*&>(inptr2),

  873.             reinterpret_cast<const int16_t*&>(inptr3),

  874.             reinterpret_cast<const int16_t*&>(inptr4),

  875.             reinterpret_cast<const int16_t*&>(inptr5),

  876.             reinterpret_cast<const int16_t*&>(inptr6),

  877.             reinterpret_cast<const int16_t*&>(inptr7),

  878.             reinterpret_cast<int16_t*&>(outptr));

  879. }

  880. 

  881. template <typename T>

  882. static inline void interleave_8x8_1_b(

  883.         const T*& inptr0, const T*& inptr1, const T*& inptr2, const T*& inptr3,

  884.         const T*& inptr4, const T*& inptr5, const T*& inptr6, const T*& inptr7,

  885.         T*& outptr) {

  886.     static_assert(

  887.             std::is_same<T, int8_t>::value || std::is_same<T, uint8_t>::value,

  888.             "interleave_8x8_1_b only support uint8_t and int8_t");

  889.     asm volatile(

  890.             "ld1 {v0.d}[0],  [%[inptr0]], 8\n"  // A1A2A3A4A5A6A7A8

  891.             "ld1 {v0.d}[1],  [%[inptr1]], 8\n"  // B1B2B3B4B5B6B7B8

  892.             "ld1 {v1.d}[0],  [%[inptr2]], 8\n"  // C1C2C3C4C5C6C7C8

  893.             "ld1 {v1.d}[1],  [%[inptr3]], 8\n"  // D1D2D3D4D5D6D7D8

  894.             "ld1 {v2.d}[0],  [%[inptr4]], 8\n"  // E1E2E3E4E5E6E7E8

  895.             "ld1 {v2.d}[1],  [%[inptr5]], 8\n"  // F1F2F3F4F5F6F7F8

  896.             "ld1 {v3.d}[0],  [%[inptr6]], 8\n"  // G1G2G3G4G5G6G7G8

  897.             "ld1 {v3.d}[1],  [%[inptr7]], 8\n"  // H1H2H3H4H5H6H7H8

  898. 

  899.             "st1 {v0.2d},  [%[outptr]], 16\n"  // A1A2A3A4A5A6A7A8B1B2B3B4B5B6B7B8

  900.             "st1 {v1.2d},  [%[outptr]], 16\n"  // C1C2C3C4C5C6C7C8D1D2D3D4D5D6D7D8

  901.             "st1 {v2.2d},  [%[outptr]], 16\n"  // E1E2E3E4E5E6E7E8F1F2F3F4F5F6F7F8

  902.             "st1 {v3.2d},  [%[outptr]], 16\n"  // G1G2G3G4G5G6G7G8H1H2H3H4H5H6H7H8

  903.             : [inptr0] "+r"(inptr0), [inptr1] "+r"(inptr1), [inptr2] "+r"(inptr2),

  904.               [inptr3] "+r"(inptr3), [inptr4] "+r"(inptr4), [inptr5] "+r"(inptr5),

  905.               [inptr6] "+r"(inptr6), [inptr7] "+r"(inptr7), [outptr] "+r"(outptr)

  906.             :

  907.             : "v0", "v1", "v2", "v3", "memory");

  908. }

  909. 

  910. template <typename T>

  911. static inline void interleave_8x4_1_b_with_shift(

  912.         const T*& inptr0, const T*& inptr1, const T*& inptr2, const T*& inptr3,

  913.         const T*& inptr4, const T*& inptr5, const T*& inptr6, const T*& inptr7,

  914.         T* outptr) {

  915.     static_assert(sizeof(T) == 1, "only support size == 1");

  916.     asm volatile(

  917.             "ld1 {v0.s}[0], [%[inptr0]], #4\n"

  918.             "ld1 {v0.s}[1], [%[inptr1]], #4\n"

  919.             "ld1 {v0.s}[2], [%[inptr2]], #4\n"

  920.             "ld1 {v0.s}[3], [%[inptr3]], #4\n"

  921.             "ld1 {v1.s}[0], [%[inptr4]], #4\n"

  922.             "ld1 {v1.s}[1], [%[inptr5]], #4\n"

  923.             "ld1 {v1.s}[2], [%[inptr6]], #4\n"

  924.             "ld1 {v1.s}[3], [%[inptr7]], #4\n"

  925.             "shl  v2.16b, v0.16b,        #4\n"

  926.             "shl  v5.16b, v1.16b,        #4\n"

  927.             "sshr v3.16b, v0.16b,        #4\n"  // hig

  928.             "sshr v4.16b, v2.16b,        #4\n"  // low

  929.             "sshr v6.16b, v1.16b,        #4\n"  // hig

  930.             "sshr v7.16b, v5.16b,        #4\n"  // low

  931.             "zip1 v8.16b, v4.16b,    v3.16b\n"

  932.             "zip2 v9.16b, v4.16b,    v3.16b\n"

  933.             "zip1 v10.16b, v7.16b,   v6.16b\n"

  934.             "zip2 v11.16b, v7.16b,   v6.16b\n"

  935.             "st1 {v8.16b-v11.16b},[%[outptr]],#64"

  936.             : [inptr0] "+r"(inptr0), [inptr1] "+r"(inptr1), [inptr2] "+r"(inptr2),

  937.               [inptr3] "+r"(inptr3), [inptr4] "+r"(inptr4), [inptr5] "+r"(inptr5),

  938.               [inptr6] "+r"(inptr6), [inptr7] "+r"(inptr7), [outptr] "+r"(outptr)

  939.             :

  940.             : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11",

  941.               "memory");

  942. }

  943. 

  944. template <typename T>

  945. static inline void interleave_8x8_1_h(

  946.         const T*& inptr0, const T*& inptr1, const T*& inptr2, const T*& inptr3,

  947.         const T*& inptr4, const T*& inptr5, const T*& inptr6, const T*& inptr7,

  948.         T*& outptr) {

  949.     static_assert(

  950.             std::is_same<T, int16_t>::value || std::is_same<T, uint16_t>::value,

  951.             "interleave_8x8_1_h only support uint16_t and int16_t");

  952.     asm volatile(

  953.             "ld1 {v0.8h},  [%[inptr0]], #16\n"  // A1A2A3A4A5A6A7A8

  954.             "ld1 {v1.8h},  [%[inptr1]], #16\n"  // B1B2B3B4B5B6B7B8

  955.             "ld1 {v2.8h},  [%[inptr2]], #16\n"  // C1C2C3C4C5C6C7C8

  956.             "ld1 {v3.8h},  [%[inptr3]], #16\n"  // D1D2D3D4D5D6D7D8

  957.             "ld1 {v4.8h},  [%[inptr4]], #16\n"  // E1E2E3E4E5E6E7E8

  958.             "ld1 {v5.8h},  [%[inptr5]], #16\n"  // F1F2F3F4F5F6F7F8

  959.             "ld1 {v6.8h},  [%[inptr6]], #16\n"  // G1G2G3G4G5G6G7G8

  960.             "ld1 {v7.8h},  [%[inptr7]], #16\n"  // H1H2H3H4H5H6H7H8

  961. 

  962.             "st1 {v0.8h},  [%[outptr]], #16\n"  // A1A2A3A4A5A6A7A8

  963.             "st1 {v1.8h},  [%[outptr]], #16\n"  // B1B2B3B4B5B6B7B8

  964.             "st1 {v2.8h},  [%[outptr]], #16\n"  // C1C2C3C4C5C6C7C8

  965.             "st1 {v3.8h},  [%[outptr]], #16\n"  // D1D2D3D4D5D6D7D8

  966.             "st1 {v4.8h},  [%[outptr]], #16\n"  // E1E2E3E4E5E6E7E8

  967.             "st1 {v5.8h},  [%[outptr]], #16\n"  // F1F2F3F4F5F6F7F8

  968.             "st1 {v6.8h},  [%[outptr]], #16\n"  // G1G2G3G4G5G6G7G8

  969.             "st1 {v7.8h},  [%[outptr]], #16\n"  // H1H2H3H4H5H6H7H8

  970.             : [inptr0] "+r"(inptr0), [inptr1] "+r"(inptr1), [inptr2] "+r"(inptr2),

  971.               [inptr3] "+r"(inptr3), [inptr4] "+r"(inptr4), [inptr5] "+r"(inptr5),

  972.               [inptr6] "+r"(inptr6), [inptr7] "+r"(inptr7), [outptr] "+r"(outptr)

  973.             :

  974.             : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "memory");

  975. }

  976. 

  977. static inline void interleave_4x1_4_s(

  978.         const int32_t*& inptr0, const int32_t*& inptr1, const int32_t*& inptr2,

  979.         const int32_t*& inptr3, int32_t*& outptr) {

  980.     asm volatile(

  981.             "ld1 {v0.4s}, [%[inptr0]], #16\n"  // d0 = A0A1A2A3

  982.             "ld1 {v1.4s}, [%[inptr1]], #16\n"  // d1 = B0B1B2B3

  983.             "ld1 {v2.4s}, [%[inptr2]], #16\n"  // d2 = C0C1C2C3

  984.             "ld1 {v3.4s}, [%[inptr3]], #16\n"  // d3 = D0D1D2D3

  985.             "zip1 v8.4s, v0.4s, v2.4s\n"       // d8 = A0C0A1C1

  986.             "zip2 v9.4s, v0.4s, v2.4s\n"       // d9 = A2C2A3C3

  987.             "zip1 v10.4s, v1.4s, v3.4s\n"      // d10 = B0D0B1D1

  988.             "zip2 v11.4s, v1.4s, v3.4s\n"      // d11 = B2D2B3D3

  989.             "zip1 v12.4s, v8.4s, v10.4s\n"     // d12 = A0B0C0D0

  990.             "zip2 v13.4s, v8.4s, v10.4s\n"     // d13 = A1B1C1D1

  991.             "zip1 v14.4s, v9.4s, v11.4s\n"     // d14 = A2B2C2D2

  992.             "zip2 v15.4s, v9.4s, v11.4s\n"     // d15 = A3B3C3D3

  993. 

  994.             "st1 {v12.4s}, [%[outptr]], #16\n"

  995.             "st1 {v13.4s}, [%[outptr]], #16\n"

  996.             "st1 {v14.4s}, [%[outptr]], #16\n"

  997.             "st1 {v15.4s}, [%[outptr]], #16\n"

  998. 

  999.             : [inptr0] "+r"(inptr0), [inptr1] "+r"(inptr1), [inptr2] "+r"(inptr2),

  1000.               [inptr3] "+r"(inptr3), [outptr] "+r"(outptr)

  1001.             :

  1002.             : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11",

  1003.               "v12", "v13", "v14", "v15", "cc", "memory");

  1004. }

  1005. 

  1006. template <typename T>

  1007. static inline void interleave_4x8_1_s(

  1008.         const T*& inptr0, const T*& inptr1, const T*& inptr2, const T*& inptr3,

  1009.         T*& outptr) {

  1010.     static_assert(sizeof(T) == 4, "only support size == 4");

  1011.     asm volatile(

  1012.             "ld1 {v0.4s, v1.4s}, [%[inptr0]], #32\n"

  1013.             "ld1 {v2.4s, v3.4s}, [%[inptr1]], #32\n"

  1014.             "ld1 {v4.4s, v5.4s}, [%[inptr2]], #32\n"

  1015.             "ld1 {v6.4s, v7.4s}, [%[inptr3]], #32\n"

  1016.             "st1 {v0.4s, v1.4s}, [%[outptr]], #32\n"

  1017.             "st1 {v2.4s, v3.4s}, [%[outptr]], #32\n"

  1018.             "st1 {v4.4s, v5.4s}, [%[outptr]], #32\n"

  1019.             "st1 {v6.4s, v7.4s}, [%[outptr]], #32\n"

  1020. 

  1021.             : [inptr0] "+r"(inptr0), [inptr1] "+r"(inptr1), [inptr2] "+r"(inptr2),

  1022.               [inptr3] "+r"(inptr3), [outptr] "+r"(outptr)

  1023.             :

  1024.             : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "cc", "memory");

  1025. }

  1026. 

  1027. template <typename T>

  1028. static inline void interleave_4x12_1_s(

  1029.         const T*& inptr0, const T*& inptr1, const T*& inptr2, const T*& inptr3,

  1030.         T*& outptr) {

  1031.     static_assert(sizeof(T) == 4, "only support size == 4");

  1032.     asm volatile(

  1033.             "ld1 {v0.4s, v1.4s, v2.4s}, [%[inptr0]], #48\n"

  1034.             "ld1 {v4.4s, v5.4s, v6.4s}, [%[inptr1]], #48\n"

  1035.             "ld1 {v8.4s, v9.4s, v10.4s}, [%[inptr2]], #48\n"

  1036.             "ld1 {v12.4s, v13.4s, v14.4s}, [%[inptr3]], #48\n"

  1037.             "st1 {v0.4s, v1.4s, v2.4s}, [%[outptr]], #48\n"

  1038.             "st1 {v4.4s, v5.4s, v6.4s}, [%[outptr]], #48\n"

  1039.             "st1 {v8.4s, v9.4s, v10.4s}, [%[outptr]], #48\n"

  1040.             "st1 {v12.4s, v13.4s, v14.4s}, [%[outptr]], #48\n"

  1041. 

  1042.             : [inptr0] "+r"(inptr0), [inptr1] "+r"(inptr1), [inptr2] "+r"(inptr2),

  1043.               [inptr3] "+r"(inptr3), [outptr] "+r"(outptr)

  1044.             :

  1045.             : "v0", "v1", "v2", "v4", "v5", "v6", "v8", "v9", "v10", "v12", "v13",

  1046.               "v14", "cc", "memory");

  1047. }

  1048. 

  1049. template <typename T>

  1050. static inline void interleave_4x16_1_b(

  1051.         const T*& inptr0, const T*& inptr1, const T*& inptr2, const T*& inptr3,

  1052.         T*& outptr) {

  1053.     static_assert(sizeof(T) == 1, "only support size == 1");

  1054.     asm volatile(

  1055.             "ld1 {v0.4s}, [%[inptr0]], #16\n"  // d0 = A0A1A2A3

  1056.             "ld1 {v1.4s}, [%[inptr1]], #16\n"  // d1 = B0B1B2B3

  1057.             "ld1 {v2.4s}, [%[inptr2]], #16\n"  // d2 = C0C1C2C3

  1058.             "ld1 {v3.4s}, [%[inptr3]], #16\n"  // d3 = D0D1D2D3

  1059.             "st1 {v0.4s}, [%[outptr]], #16\n"

  1060.             "st1 {v1.4s}, [%[outptr]], #16\n"

  1061.             "st1 {v2.4s}, [%[outptr]], #16\n"

  1062.             "st1 {v3.4s}, [%[outptr]], #16\n"

  1063. 

  1064.             : [inptr0] "+r"(inptr0), [inptr1] "+r"(inptr1), [inptr2] "+r"(inptr2),

  1065.               [inptr3] "+r"(inptr3), [outptr] "+r"(outptr)

  1066.             :

  1067.             : "v0", "v1", "v2", "v3", "v4", "cc", "memory");

  1068. }

  1069. 

  1070. template <typename T>

  1071. static inline void interleave_4x16_1_s(

  1072.         const T*& inptr0, const T*& inptr1, const T*& inptr2, const T*& inptr3,

  1073.         T*& outptr) {

  1074.     static_assert(sizeof(T) == 4, "only support size == 4");

  1075.     asm volatile(

  1076.             "ld1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%[inptr0]], #64\n"

  1077.             "ld1 {v4.4s, v5.4s, v6.4s, v7.4s}, [%[inptr1]], #64\n"

  1078.             "ld1 {v8.4s, v9.4s, v10.4s, v11.4s}, [%[inptr2]], #64\n"

  1079.             "ld1 {v12.4s, v13.4s, v14.4s, v15.4s}, [%[inptr3]], #64\n"

  1080.             "st1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%[outptr]], #64\n"

  1081.             "st1 {v4.4s, v5.4s, v6.4s, v7.4s}, [%[outptr]], #64\n"

  1082.             "st1 {v8.4s, v9.4s, v10.4s, v11.4s}, [%[outptr]], #64\n"

  1083.             "st1 {v12.4s, v13.4s, v14.4s, v15.4s}, [%[outptr]], #64\n"

  1084. 

  1085.             : [inptr0] "+r"(inptr0), [inptr1] "+r"(inptr1), [inptr2] "+r"(inptr2),

  1086.               [inptr3] "+r"(inptr3), [outptr] "+r"(outptr)

  1087.             :

  1088.             : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11",

  1089.               "v12", "v13", "v14", "v15", "cc", "memory");

  1090. }

  1091. 

  1092. template <typename T>

  1093. static inline void interleave_4x2_4_b(

  1094.         const T*& inptr0, const T*& inptr1, const T*& inptr2, const T*& inptr3,

  1095.         T*& outptr) {

  1096.     static_assert(

  1097.             std::is_same<T, int8_t>::value || std::is_same<T, uint8_t>::value,

  1098.             "interleave_4x2_4_b only support uint8_t and int8_t");

  1099.     interleave_4x1_4_h(

  1100.             reinterpret_cast<const int16_t*&>(inptr0),

  1101.             reinterpret_cast<const int16_t*&>(inptr1),

  1102.             reinterpret_cast<const int16_t*&>(inptr2),

  1103.             reinterpret_cast<const int16_t*&>(inptr3),

  1104.             reinterpret_cast<int16_t*&>(outptr));

  1105. }

  1106. 

  1107. template <typename T>

  1108. static inline void interleave_4x4_4_b(

  1109.         const T*& inptr0, const T*& inptr1, const T*& inptr2, const T*& inptr3,

  1110.         T*& outptr) {

  1111.     static_assert(

  1112.             std::is_same<T, int8_t>::value || std::is_same<T, uint8_t>::value,

  1113.             "interleave_4x4_4_b only support uint8_t and int8_t");

  1114.     interleave_4x1_4_s(

  1115.             reinterpret_cast<const int32_t*&>(inptr0),

  1116.             reinterpret_cast<const int32_t*&>(inptr1),

  1117.             reinterpret_cast<const int32_t*&>(inptr2),

  1118.             reinterpret_cast<const int32_t*&>(inptr3),

  1119.             reinterpret_cast<int32_t*&>(outptr));

  1120. }

  1121. 

  1122. template <typename T>

  1123. static inline void interleave_4x4_1_s(

  1124.         const T*& inptr0, const T*& inptr1, const T*& inptr2, const T*& inptr3,

  1125.         T*& outptr) {

  1126.     static_assert(sizeof(T) == 4, "interleave_4x4_1_s only support size == 4");

  1127.     asm volatile(

  1128.             "ld1 {v0.4s}, [%[inptr0]], #16\n"

  1129.             "ld1 {v1.4s}, [%[inptr1]], #16\n"

  1130.             "ld1 {v2.4s}, [%[inptr2]], #16\n"

  1131.             "ld1 {v3.4s}, [%[inptr3]], #16\n"

  1132.             "st1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%[outptr]], #64\n"

  1133. 

  1134.             : [inptr0] "+r"(inptr0), [inptr1] "+r"(inptr1), [inptr2] "+r"(inptr2),

  1135.               [inptr3] "+r"(inptr3), [outptr] "+r"(outptr)

  1136.             :

  1137.             : "v0", "v1", "v2", "v3", "cc", "memory");

  1138. }

  1139. 

  1140. template <typename T>

  1141. static inline void interleave_2x4_4_s(const T*& inptr0, const T*& inptr1, T* outptr) {

  1142.     static_assert(sizeof(T) == 4, "interleave_2x4_4_s only support size == 4");

  1143.     asm volatile(

  1144.             "ld1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%[inptr0]], #64\n"

  1145.             "ld1 {v4.4s, v5.4s, v6.4s, v7.4s}, [%[inptr1]], #64\n"

  1146.             "stp q0, q4, [%[outptr]]\n"

  1147.             "stp q1, q5, [%[outptr], #32]\n"

  1148.             "stp q2, q6, [%[outptr], #64]\n"

  1149.             "stp q3, q7, [%[outptr], #96]\n"

  1150. 

  1151.             : [inptr0] "+r"(inptr0), [inptr1] "+r"(inptr1), [outptr] "+r"(outptr)

  1152.             :

  1153.             : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "memory");

  1154. }

  1155. 

  1156. template <typename T>

  1157. static inline void interleave_1x4_4_s(const T*& inptr0, T* outptr) {

  1158.     static_assert(sizeof(T) == 4, "interleave_1x4_4_s only support size == 4");

  1159.     asm volatile(

  1160.             "ld1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%[inptr0]], #64\n"

  1161.             "st1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%[outptr]]\n"

  1162. 

  1163.             : [inptr0] "+r"(inptr0), [outptr] "+r"(outptr)

  1164.             :

  1165.             : "v0", "v1", "v2", "v3", "memory");

  1166. }

  1167. 

  1168. template <typename T>

  1169. static inline void interleave_4x8_2_b(

  1170.         const T*& inptr0, const T*& inptr1, const T*& inptr2, const T*& inptr3,

  1171.         T*& outptr) {

  1172.     static_assert(

  1173.             std::is_same<T, int8_t>::value || std::is_same<T, uint8_t>::value,

  1174.             "interleave_4x8_2_b only support uint8_t and int8_t");

  1175.     interleave_4x1_2_d(

  1176.             reinterpret_cast<const int64_t*&>(inptr0),

  1177.             reinterpret_cast<const int64_t*&>(inptr1),

  1178.             reinterpret_cast<const int64_t*&>(inptr2),

  1179.             reinterpret_cast<const int64_t*&>(inptr3),

  1180.             reinterpret_cast<int64_t*&>(outptr));

  1181. }

  1182. 

  1183. template <typename T>

  1184. static inline void interleave_4x8_2_h(

  1185.         const T*& inptr0, const T*& inptr1, const T*& inptr2, const T*& inptr3,

  1186.         T*& outptr) {

  1187.     static_assert(

  1188.             std::is_same<T, int16_t>::value || std::is_same<T, uint16_t>::value,

  1189.             "interleave_4x8_2_h only support uint16_t and int16_t");

  1190.     interleave_4x2_2_d(

  1191.             reinterpret_cast<const int64_t*&>(inptr0),

  1192.             reinterpret_cast<const int64_t*&>(inptr1),

  1193.             reinterpret_cast<const int64_t*&>(inptr2),

  1194.             reinterpret_cast<const int64_t*&>(inptr3),

  1195.             reinterpret_cast<int64_t*&>(outptr));

  1196. }

  1197. 

  1198. template <typename T>

  1199. static inline void interleave_1x16_1_s(const T*& inptr0, T*& outptr) {

  1200.     static_assert(sizeof(T) == 4, "interleave_1x16_1_s only support size == 4");

  1201.     asm volatile(

  1202.             "ld1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%[inptr0]], #64\n"

  1203.             "st1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%[outptr]], #64\n"

  1204. 

  1205.             : [inptr0] "+r"(inptr0), [outptr] "+r"(outptr)

  1206.             :

  1207.             : "v0", "v1", "v2", "v3", "cc", "memory");

  1208. }

  1209. template <typename T>

  1210. static inline void interleave_1x12_1_s(const T*& inptr0, T*& outptr) {

  1211.     static_assert(sizeof(T) == 4, "interleave_1x12_1_s only support size == 4");

  1212.     asm volatile(

  1213.             "ld1 {v0.4s, v1.4s, v2.4s}, [%[inptr0]], #48\n"

  1214.             "st1 {v0.4s, v1.4s, v2.4s}, [%[outptr]], #48\n"

  1215. 

  1216.             : [inptr0] "+r"(inptr0), [outptr] "+r"(outptr)

  1217.             :

  1218.             : "v0", "v1", "v2", "cc", "memory");

  1219. }

  1220. 

  1221. template <typename T>

  1222. static inline void interleave_1x8_1_s(const T*& inptr0, T*& outptr) {

  1223.     static_assert(sizeof(T) == 4, "interleave_1x8_1_s only support size == 4");

  1224.     asm volatile(

  1225.             "ld1 {v0.4s, v1.4s}, [%[inptr0]], #32\n"

  1226.             "st1 {v0.4s, v1.4s}, [%[outptr]], #32\n"

  1227. 

  1228.             : [inptr0] "+r"(inptr0), [outptr] "+r"(outptr)

  1229.             :

  1230.             : "v0", "v1", "cc", "memory");

  1231. }

  1232. 

  1233. template <typename T>

  1234. static inline void interleave_1x4_1_s(const T*& inptr0, T*& outptr) {

  1235.     static_assert(sizeof(T) == 4, "interleave_1x4_1_s only support size == 4");

  1236.     asm volatile(

  1237.             "ld1 {v0.4s}, [%[inptr0]], #16\n"

  1238.             "st1 {v0.4s}, [%[outptr]], #16\n"

  1239. 

  1240.             : [inptr0] "+r"(inptr0), [outptr] "+r"(outptr)

  1241.             :

  1242.             : "v0", "cc", "memory");

  1243. }

  1244. 

  1245. template <typename T>

  1246. static inline void interleave_helper(

  1247.         const T*& inptr, T*& outptr, int unroll_k, int ksize, T val = 0) {

  1248.     int k = 0;

  1249.     for (; k < ksize; k++) {

  1250.         *outptr++ = *inptr++;

  1251.     }

  1252.     for (; k < unroll_k; k++) {

  1253.         *outptr++ = val;

  1254.     }

  1255. }

  1256. 

  1257. template <typename T>

  1258. static inline void interleave_1(

  1259.         const T*& inptr0, T*& outptr, int unroll_k, int ksize, T val = 0) {

  1260.     for (int k = 0; k < ksize; k += unroll_k) {

  1261.         int size = std::min(unroll_k, ksize - k);

  1262.         interleave_helper(inptr0, outptr, unroll_k, size, val);

  1263.     }

  1264. }

  1265. 

  1266. template <typename T>

  1267. static inline void interleave_4(

  1268.         const T*& inptr0, const T*& inptr1, const T*& inptr2, const T*& inptr3,

  1269.         T*& outptr, int unroll_k, int ksize, T val = 0) {

  1270.     for (int k = 0; k < ksize; k += unroll_k) {

  1271.         int size = std::min(unroll_k, ksize - k);

  1272.         interleave_helper(inptr0, outptr, unroll_k, size, val);

  1273.         interleave_helper(inptr1, outptr, unroll_k, size, val);

  1274.         interleave_helper(inptr2, outptr, unroll_k, size, val);

  1275.         interleave_helper(inptr3, outptr, unroll_k, size, val);

  1276.     }

  1277. }

  1278. 

  1279. template <typename T>

  1280. static inline void interleave_8(

  1281.         const T*& inptr0, const T*& inptr1, const T*& inptr2, const T*& inptr3,

  1282.         const T*& inptr4, const T*& inptr5, const T*& inptr6, const T*& inptr7,

  1283.         T*& outptr, int unroll_k, int ksize, T val = 0) {

  1284.     for (int k = 0; k < ksize; k += unroll_k) {

  1285.         int size = std::min(unroll_k, ksize - k);

  1286.         interleave_helper(inptr0, outptr, unroll_k, size, val);

  1287.         interleave_helper(inptr1, outptr, unroll_k, size, val);

  1288.         interleave_helper(inptr2, outptr, unroll_k, size, val);

  1289.         interleave_helper(inptr3, outptr, unroll_k, size, val);

  1290.         interleave_helper(inptr4, outptr, unroll_k, size, val);

  1291.         interleave_helper(inptr5, outptr, unroll_k, size, val);

  1292.         interleave_helper(inptr6, outptr, unroll_k, size, val);

  1293.         interleave_helper(inptr7, outptr, unroll_k, size, val);

  1294.     }

  1295. }

  1296. 

  1297. template <typename T>

  1298. static inline void interleave_12(

  1299.         const T*& inptr0, const T*& inptr1, const T*& inptr2, const T*& inptr3,

  1300.         const T*& inptr4, const T*& inptr5, const T*& inptr6, const T*& inptr7,

  1301.         const T*& inptr8, const T*& inptr9, const T*& inptr10, const T*& inptr11,

  1302.         T*& outptr, int unroll_k, int ksize) {

  1303.     for (int k = 0; k < ksize; k += unroll_k) {

  1304.         int size = std::min(unroll_k, ksize - k);

  1305.         interleave_helper(inptr0, outptr, unroll_k, size);

  1306.         interleave_helper(inptr1, outptr, unroll_k, size);

  1307.         interleave_helper(inptr2, outptr, unroll_k, size);

  1308.         interleave_helper(inptr3, outptr, unroll_k, size);

  1309.         interleave_helper(inptr4, outptr, unroll_k, size);

  1310.         interleave_helper(inptr5, outptr, unroll_k, size);

  1311.         interleave_helper(inptr6, outptr, unroll_k, size);

  1312.         interleave_helper(inptr7, outptr, unroll_k, size);

  1313.         interleave_helper(inptr8, outptr, unroll_k, size);

  1314.         interleave_helper(inptr9, outptr, unroll_k, size);

  1315.         interleave_helper(inptr10, outptr, unroll_k, size);

  1316.         interleave_helper(inptr11, outptr, unroll_k, size);

  1317.     }

  1318. }

  1319. /* ======================== transpose pack B ======================== */

  1320. /**

  1321.  * transpose_INTERLEAVE_UNROLLK_BATCH_type

  1322.  *

  1323.  * BATCH means process BATCH * INTERLEAVE cols once, BATCH * sizeof(TYPE) *

  1324.  * INTERLEAVE = 16bytes(128bits, a vector size).

  1325.  *

  1326.  * the elements traverse order:

  1327.  * rep(j, 0, INTERLEAVE) rep(i, 0, UNROLL_K) *ouptr++ = inptr[i, j]

  1328.  */

  1329. template <typename T>

  1330. static inline void transpose_24x4_1_h(

  1331.         const T*& in0, const T*& in1, const T*& in2, const T*& in3, T* out) {

  1332.     static_assert(sizeof(T) == 2, "only support size == 2");

  1333.     asm volatile(

  1334.         "ldp q0, q1, [%[in0]], #32\n"

  1335.         "stp q0, q1, [%[out]]\n"

  1336.         "ldr q2, [%[in0]], #16\n"

  1337.         ASM_PREFETCH("[%[in0], #192]")

  1338.         "ldp q3, q4, [%[in1]], #32\n"

  1339.         "stp q2, q3, [%[out], #32]\n"

  1340.         "ldr q5, [%[in1]], #16\n"

  1341.         ASM_PREFETCH("[%[in1], #192]")

  1342.         "stp q4, q5, [%[out], #64]\n"

  1343.         "ldp q6, q7, [%[in2]], #32\n"

  1344.         "stp q6, q7, [%[out], #96]\n"

  1345.         "ldr q8, [%[in2]], #16\n"

  1346.         ASM_PREFETCH("[%[in2], #192]")

  1347.         "ldp q9, q10, [%[in3]], #32\n"

  1348.         "stp q8, q9, [%[out], #128]\n"

  1349.         "ldr q11, [%[in3]], #16\n"

  1350.         "stp q10, q11, [%[out], #160]\n"

  1351.         ASM_PREFETCH("[%[in3], #192]")

  1352. 

  1353.         : [in0] "+r"(in0), [in1] "+r"(in1), [in2] "+r"(in2),

  1354.           [in3] "+r"(in3), [out] "+r"(out)

  1355.         :

  1356.         : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10",

  1357.         "v11", "memory");

  1358. }

  1359. 

  1360. template <typename T>

  1361. static inline void transpose_16x4_1_h(

  1362.         const T*& in0, const T*& in1, const T*& in2, const T*& in3, T* out) {

  1363.     static_assert(sizeof(T) == 2, "only support size == 2");

  1364.     asm volatile(

  1365.             "ldp q0, q1, [%[in0]], #32\n"

  1366.             "stp q0, q1, [%[out]]\n"

  1367.             "ldp q2, q3, [%[in1]], #32\n"

  1368.             "stp q2, q3, [%[out], #32]\n"

  1369.             "ldp q4, q5, [%[in2]], #32\n"

  1370.             "stp q4, q5, [%[out], #64]\n"

  1371.             "ldp q6, q7, [%[in3]], #32\n"

  1372.             "stp q6, q7, [%[out], #96]\n"

  1373.             : [in0] "+r"(in0), [in1] "+r"(in1), [in2] "+r"(in2), [in3] "+r"(in3),

  1374.               [out] "+r"(out)

  1375.             :

  1376.             : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "memory");

  1377. }

  1378. 

  1379. template <typename T>

  1380. static inline void transpose_8x4_1_h(

  1381.         const T*& in0, const T*& in1, const T*& in2, const T*& in3, T* out) {

  1382.     static_assert(sizeof(T) == 2, "only support size == 2");

  1383.     asm volatile(

  1384.             "ldr q0, [%[in0]], #16\n"

  1385.             "str q0, [%[out]]\n"

  1386.             "ldr q1, [%[in1]], #16\n"

  1387.             "str q1, [%[out], #16]\n"

  1388.             "ldr q2, [%[in2]], #16\n"

  1389.             "str q2, [%[out], #32]\n"

  1390.             "ldr q3, [%[in3]], #16\n"

  1391.             "str q3, [%[out], #48]\n"

  1392.             : [in0] "+r"(in0), [in1] "+r"(in1), [in2] "+r"(in2), [in3] "+r"(in3),

  1393.               [out] "+r"(out)

  1394.             :

  1395.             : "v0", "v1", "v2", "v3", "memory");

  1396. }

  1397. 

  1398. template <typename T>

  1399. static inline void transpose_24x2_1_h(const T*& in0, const T*& in1, T* out) {

  1400.     static_assert(sizeof(T) == 2, "only support size == 2");

  1401.     asm volatile(

  1402.         "ldp q0, q1, [%[in0]], #32\n"

  1403.         "stp q0, q1, [%[out]]\n"

  1404.         "ldr q2, [%[in0]], #16\n"

  1405.         ASM_PREFETCH("[%[in0], #192]")

  1406.         "ldp q3, q4, [%[in1]], #32\n"

  1407.         "stp q2, q3, [%[out], #32]\n"

  1408.         "ldr q5, [%[in1]], #16\n"

  1409.         ASM_PREFETCH("[%[in1], #192]")

  1410.         "stp q4, q5, [%[out], #64]\n"

  1411.         : [in0] "+r"(in0), [in1] "+r"(in1), [out] "+r"(out)

  1412.         :

  1413.         : "v0", "v1", "v2", "v3", "v4", "v5", "memory");

  1414. }

  1415. 

  1416. template <typename T>

  1417. static inline void transpose_16x2_1_h(const T*& in0, const T*& in1, T* out) {

  1418.     static_assert(sizeof(T) == 2, "only support size == 2");

  1419.     asm volatile(

  1420.             "ldp q0, q1, [%[in0]], #32\n"

  1421.             "stp q0, q1, [%[out]]\n"

  1422.             "ldp q2, q3, [%[in1]], #32\n"

  1423.             "stp q2, q3, [%[out], #32]\n"

  1424.             : [in0] "+r"(in0), [in1] "+r"(in1), [out] "+r"(out)

  1425.             :

  1426.             : "v0", "v1", "v2", "v3", "memory");

  1427. }

  1428. 

  1429. template <typename T>

  1430. static inline void transpose_8x2_1_h(const T*& in0, const T*& in1, T* out) {

  1431.     static_assert(sizeof(T) == 2, "only support size == 2");

  1432.     asm volatile(

  1433.             "ldr q0, [%[in0]], #16\n"

  1434.             "str q0, [%[out]]\n"

  1435.             "ldr q1, [%[in1]], #16\n"

  1436.             "str q1, [%[out], #16]\n"

  1437.             : [in0] "+r"(in0), [in1] "+r"(in1), [out] "+r"(out)

  1438.             :

  1439.             : "v0", "v1", "memory");

  1440. }

  1441. 

  1442. template <typename T>

  1443. static inline void transpose_24x1_1_h(const T*& in0, T* out) {

  1444.     static_assert(sizeof(T) == 2, "only support size == 2");

  1445.     // clang-format off

  1446.     asm volatile(

  1447.             "ldp q0, q1, [%[in0]], #32\n"

  1448.             "stp q0, q1, [%[out]] \n"

  1449.             "ldr q2, [%[in0]], #16 \n"

  1450.             ASM_PREFETCH("[%[in0], #192]")

  1451.             "str q2, [%[out], #32] \n"

  1452.             : [in0] "+r"(in0), [out] "+r"(out)

  1453.             :

  1454.             : "v0", "v1", "v2", "memory");

  1455.     // clang-format on

  1456. }

  1457. 

  1458. template <typename T>

  1459. static inline void transpose_16x1_1_h(const T*& in0, T* out) {

  1460.     static_assert(sizeof(T) == 2, "only support size == 2");

  1461.     asm volatile(

  1462.             "ldp q0, q1, [%[in0]], #32\n"

  1463.             "stp q0, q1, [%[out]]\n"

  1464.             : [in0] "+r"(in0), [out] "+r"(out)

  1465.             :

  1466.             : "v0", "v1", "memory");

  1467. }

  1468. 

  1469. template <typename T>

  1470. static inline void transpose_12x1_1_h(const T*& in0, T* out) {

  1471.     static_assert(sizeof(T) == 2, "only support size == 2");

  1472.     // clang-format off

  1473.     asm volatile(

  1474.             "ld1 {v0.8h}, [%[in0]], #16\n"

  1475.             "ld1 {v1.4h}, [%[in0]], #8\n"

  1476.             "st1 {v0.8h}, [%[out]], #16\n"

  1477.             "st1 {v1.4h}, [%[out]], #8\n"

  1478.             : [in0] "+r"(in0), [out] "+r"(out)

  1479.             :

  1480.             : "v0", "v1", "memory");

  1481.     // clang-format on

  1482. }

  1483. 

  1484. template <typename T>

  1485. static inline void transpose_8x1_1_h(const T*& in0, T* out) {

  1486.     static_assert(sizeof(T) == 2, "only support size == 2");

  1487.     asm volatile(

  1488.             "ldr q0, [%[in0]], #16\n"

  1489.             "str q0, [%[out]]\n"

  1490.             : [in0] "+r"(in0), [out] "+r"(out)

  1491.             :

  1492.             : "v0", "memory");

  1493. }

  1494. 

  1495. template <typename T>

  1496. static inline void transpose_4x1_1_h(const T*& in0, T* out) {

  1497.     static_assert(sizeof(T) == 2, "only support size == 2");

  1498.     // clang-format off

  1499.     asm volatile(

  1500.             "ld1 {v0.4h}, [%[in0]], #8\n"

  1501.             "st1 {v0.4h}, [%[out]], #8\n"

  1502.             : [in0] "+r"(in0), [out] "+r"(out)

  1503.             :

  1504.             : "v0", "memory");

  1505.     // clang-format on

  1506. }

  1507. 

  1508. template <typename T>

  1509. static inline void transpose_4x4_1_s(

  1510.         const T*& inptr0, const T*& inptr1, const T*& inptr2, const T*& inptr3,

  1511.         T* outptr, int stride = 16) {

  1512.     static_assert(sizeof(T) == 4, "transpose_4x4_1_s only support sizeof(T) == 4");

  1513. 

  1514.     asm volatile(

  1515.             "ld1 {v0.4s},  [%[inptr0]], 16\n"  // A0A1A2A3

  1516.             "ld1 {v1.4s},  [%[inptr1]], 16\n"  // B0B1B2B3

  1517.             "ld1 {v2.4s},  [%[inptr2]], 16\n"  // C0C1C2C3

  1518.             "ld1 {v3.4s},  [%[inptr3]], 16\n"  // D0D1D2D3

  1519. 

  1520.             "zip1 v4.4s, v0.4s, v1.4s\n"

  1521.             "zip1 v5.4s, v2.4s, v3.4s\n"

  1522.             "zip2 v6.4s, v0.4s, v1.4s\n"

  1523.             "zip2 v7.4s, v2.4s, v3.4s\n"

  1524. 

  1525.             "zip1 v8.2d, v4.2d, v5.2d\n"

  1526.             "zip1 v9.2d, v6.2d, v7.2d\n"

  1527.             "zip2 v10.2d, v4.2d, v5.2d\n"

  1528.             "zip2 v11.2d, v6.2d, v7.2d\n"

  1529. 

  1530.             "st1 {v8.4s},  [%[outptr]], %x[stride]\n"

  1531.             "st1 {v10.4s},  [%[outptr]], %x[stride]\n"

  1532.             "st1 {v9.4s},  [%[outptr]], %x[stride]\n"

  1533.             "st1 {v11.4s},  [%[outptr]], %x[stride]\n"

  1534.             : [inptr0] "+r"(inptr0), [inptr1] "+r"(inptr1), [inptr2] "+r"(inptr2),

  1535.               [inptr3] "+r"(inptr3), [outptr] "+r"(outptr), [stride] "+r"(stride)

  1536.             :

  1537.             : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11",

  1538.               "memory");

  1539. }

  1540. 

  1541. template <typename T>

  1542. static inline void transpose_1x12_4_s(const T*& inptr0, T* outptr) {

  1543.     static_assert(sizeof(T) == 4, "transpose_1x12_4_s only support sizeof(T) == 4");

  1544. 

  1545.     asm volatile(

  1546.             "ld4 {v0.4s, v1.4s, v2.4s, v3.4s},  [%[inptr0]], #64\n"

  1547.             "ld4 {v4.4s, v5.4s, v6.4s, v7.4s},  [%[inptr0]], #64\n"

  1548.             "ld4 {v8.4s, v9.4s, v10.4s, v11.4s},[%[inptr0]], #64\n"

  1549. 

  1550.             "stp q0, q4, [%[outptr]] \n"

  1551.             "stp q8, q1, [%[outptr], #32] \n"

  1552.             "stp q5, q9, [%[outptr], #64] \n"

  1553.             "stp q2, q6, [%[outptr], #96] \n"

  1554.             "stp q10, q3, [%[outptr], #128] \n"

  1555.             "stp q7, q11, [%[outptr], #160] \n"

  1556.             : [inptr0] "+r"(inptr0), [outptr] "+r"(outptr)

  1557.             :

  1558.             : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11",

  1559.               "memory");

  1560. }

  1561. 

  1562. template <typename T>

  1563. static inline void transpose_1x4_4_s(const T*& inptr0, T* outptr) {

  1564.     static_assert(sizeof(T) == 4, "transpose_1x4_4_s only support sizeof(T) == 4");

  1565. 

  1566.     asm volatile(

  1567.             "ld4 {v0.4s, v1.4s, v2.4s, v3.4s},  [%[inptr0]], #64\n"

  1568.             "st1 {v0.4s, v1.4s, v2.4s, v3.4s},  [%[outptr]]\n"

  1569.             : [inptr0] "+r"(inptr0), [outptr] "+r"(outptr)

  1570.             :

  1571.             : "v0", "v1", "v2", "v3", "memory");

  1572. }

  1573. 

  1574. template <typename T>

  1575. static inline void transpose_8x4_1_s(

  1576.         const T*& inptr0, const T*& inptr1, const T*& inptr2, const T*& inptr3,

  1577.         const T*& inptr4, const T*& inptr5, const T*& inptr6, const T*& inptr7,

  1578.         T* outptr) {

  1579.     static_assert(sizeof(T) == 4, "transpose_8x4_1_s only support sizeof(T) == 4");

  1580. 

  1581.     asm volatile(

  1582.             "ld1 {v0.4s},  [%[inptr0]], 16\n"  // A0A1A2A3

  1583.             "ld1 {v1.4s},  [%[inptr1]], 16\n"  // B0B1B2B3

  1584.             "ld1 {v2.4s},  [%[inptr2]], 16\n"  // C0C1C2C3

  1585.             "ld1 {v3.4s},  [%[inptr3]], 16\n"  // D0D1D2D3

  1586.             "ld1 {v4.4s},  [%[inptr4]], 16\n"  // E0E1E2E3

  1587.             "ld1 {v5.4s},  [%[inptr5]], 16\n"  // F0F1F2F3

  1588.             "ld1 {v6.4s},  [%[inptr6]], 16\n"  // G0G1G2G3

  1589.             "ld1 {v7.4s},  [%[inptr7]], 16\n"  // H0H1H2H3

  1590. 

  1591.             "zip1 v8.4s, v0.4s, v1.4s\n"   // A0B0A1B1

  1592.             "zip2 v9.4s, v0.4s, v1.4s\n"   // A2B2A3B3

  1593.             "zip1 v10.4s, v2.4s, v3.4s\n"  // C0D0C1D1

  1594.             "zip2 v11.4s, v2.4s, v3.4s\n"  // C2D2C3D3

  1595.             "zip1 v12.4s, v4.4s, v5.4s\n"  // E0F0E1F1

  1596.             "zip2 v13.4s, v4.4s, v5.4s\n"  // E2F2E3F3

  1597.             "zip1 v14.4s, v6.4s, v7.4s\n"  // G0H0G1H1

  1598.             "zip2 v15.4s, v6.4s, v7.4s\n"  // G2H2G3H3

  1599. 

  1600.             "zip1 v0.2d, v8.2d, v10.2d\n"  // A0B0C0D0

  1601.             "zip2 v2.2d, v8.2d, v10.2d\n"  // A1B1C1D1

  1602. 

  1603.             "zip1 v4.2d, v9.2d, v11.2d\n"  // A2B2C2D2

  1604.             "zip2 v6.2d, v9.2d, v11.2d\n"  // A3B3C3D3

  1605. 

  1606.             "zip1 v1.2d, v12.2d, v14.2d\n"  // E0F0G0H0

  1607.             "zip2 v3.2d, v12.2d, v14.2d\n"  // E1F1G1H1

  1608. 

  1609.             "zip1 v5.2d, v13.2d, v15.2d\n"  // E2F2G2H2

  1610.             "zip2 v7.2d, v13.2d, v15.2d\n"  // E3F3G3H3

  1611. 

  1612.             "st1 {v0.4s,v1.4s,v2.4s,v3.4s},  [%[outptr]], #64\n"

  1613.             "st1 {v4.4s,v5.4s,v6.4s,v7.4s},  [%[outptr]], #64\n"

  1614.             : [inptr0] "+r"(inptr0), [inptr1] "+r"(inptr1), [inptr2] "+r"(inptr2),

  1615.               [inptr3] "+r"(inptr3), [inptr4] "+r"(inptr4), [inptr5] "+r"(inptr5),

  1616.               [inptr6] "+r"(inptr6), [inptr7] "+r"(inptr7), [outptr] "+r"(outptr)

  1617.             :

  1618.             : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11",

  1619.               "v12", "v13", "v14", "v15", "memory");

  1620. }

  1621. 

  1622. template <typename T>

  1623. static inline void transpose_12x4_1_s(

  1624.         const T*& inptr0, const T*& inptr1, const T*& inptr2, const T*& inptr3,

  1625.         const T*& inptr4, const T*& inptr5, const T*& inptr6, const T*& inptr7,

  1626.         const T*& inptr8, const T*& inptr9, const T*& inptr10, const T*& inptr11,

  1627.         T* outptr) {

  1628.     static_assert(sizeof(T) == 4, "transpose_12x4_1_s only support sizeof(T) == 4");

  1629.     asm volatile(

  1630.             "ld1 {v0.4s},  [%[inptr0]], 16\n"    // A0A1A2A3

  1631.             "ld1 {v1.4s},  [%[inptr1]], 16\n"    // B0B1B2B3

  1632.             "ld1 {v2.4s},  [%[inptr2]], 16\n"    // C0C1C2C3

  1633.             "ld1 {v3.4s},  [%[inptr3]], 16\n"    // D0D1D2D3

  1634.             "ld1 {v4.4s},  [%[inptr4]], 16\n"    // E0E1E2E3

  1635.             "ld1 {v5.4s},  [%[inptr5]], 16\n"    // F0F1F2F3

  1636.             "ld1 {v6.4s},  [%[inptr6]], 16\n"    // G0G1G2G3

  1637.             "ld1 {v7.4s},  [%[inptr7]], 16\n"    // H0H1H2H3

  1638.             "ld1 {v16.4s},  [%[inptr8]], 16\n"   // I0I1I2I3

  1639.             "ld1 {v17.4s},  [%[inptr9]], 16\n"   // J0J1J2J3

  1640.             "ld1 {v18.4s},  [%[inptr10]], 16\n"  // K0K1K2K3

  1641.             "ld1 {v19.4s},  [%[inptr11]], 16\n"  // L0L1L2L3

  1642. 

  1643.             "zip1 v8.4s, v0.4s, v1.4s\n"   // A0B0A1B1

  1644.             "zip2 v9.4s, v0.4s, v1.4s\n"   // A2B2A3B3

  1645.             "zip1 v10.4s, v2.4s, v3.4s\n"  // C0D0C1D1

  1646.             "zip2 v11.4s, v2.4s, v3.4s\n"  // C2D2C3D3

  1647. 

  1648.             "zip1 v12.4s, v4.4s, v5.4s\n"  // E0F0E1F1

  1649.             "zip2 v13.4s, v4.4s, v5.4s\n"  // E2F2E3F3

  1650.             "zip1 v14.4s, v6.4s, v7.4s\n"  // G0H0G1H1

  1651.             "zip2 v15.4s, v6.4s, v7.4s\n"  // G2H2G3H3

  1652. 

  1653.             "zip1 v20.4s, v16.4s, v17.4s\n"  // I0J0I1J1

  1654.             "zip2 v21.4s, v16.4s, v17.4s\n"  // I2J2I3J3

  1655.             "zip1 v22.4s, v18.4s, v19.4s\n"  // K0L0K1L1

  1656.             "zip2 v23.4s, v18.4s, v19.4s\n"  // K2L2K3L3

  1657. 

  1658.             "zip1 v0.2d, v8.2d, v10.2d\n"  // A0B0C0D0

  1659.             "zip2 v3.2d, v8.2d, v10.2d\n"  // A1B1C1D1

  1660. 

  1661.             "zip1 v6.2d, v9.2d, v11.2d\n"   // A2B2C2D2

  1662.             "zip2 v24.2d, v9.2d, v11.2d\n"  // A3B3C3D3

  1663. 

  1664.             "zip1 v1.2d, v12.2d, v14.2d\n"  // E0F0G0H0

  1665.             "zip2 v4.2d, v12.2d, v14.2d\n"  // E1F1G1H1

  1666. 

  1667.             "zip1 v7.2d, v13.2d, v15.2d\n"   // E2F2G2H2

  1668.             "zip2 v25.2d, v13.2d, v15.2d\n"  // E3F3G3H3

  1669. 

  1670.             "zip1 v2.2d, v20.2d, v22.2d\n"  // I0J0K0L0

  1671.             "zip2 v5.2d, v20.2d, v22.2d\n"  // I1J1K1L1

  1672. 

  1673.             "zip1 v8.2d, v21.2d, v23.2d\n"   // I2J2K2L2

  1674.             "zip2 v26.2d, v21.2d, v23.2d\n"  // I3J3K3L3

  1675. 

  1676.             "st1 {v0.4s,v1.4s,v2.4s},  [%[outptr]], #48\n"

  1677.             "st1 {v3.4s,v4.4s,v5.4s},  [%[outptr]], #48\n"

  1678.             "st1 {v6.4s,v7.4s,v8.4s},  [%[outptr]], #48\n"

  1679.             "st1 {v24.4s,v25.4s,v26.4s},  [%[outptr]], #48\n"

  1680.             : [inptr0] "+r"(inptr0), [inptr1] "+r"(inptr1), [inptr2] "+r"(inptr2),

  1681.               [inptr3] "+r"(inptr3), [inptr4] "+r"(inptr4), [inptr5] "+r"(inptr5),

  1682.               [inptr6] "+r"(inptr6), [inptr7] "+r"(inptr7), [inptr8] "+r"(inptr8),

  1683.               [inptr9] "+r"(inptr9), [inptr10] "+r"(inptr10), [inptr11] "+r"(inptr11),

  1684.               [outptr] "+r"(outptr)

  1685.             :

  1686.             : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11",

  1687.               "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21",

  1688.               "v22", "v23", "v24", "v25", "v26", "memory");

  1689. }

  1690. 

  1691. template <typename T>

  1692. static inline void transpose_12x4_1_b(

  1693.         const T*& inptr0, const T*& inptr1, const T*& inptr2, const T*& inptr3,

  1694.         T* outptr) {

  1695.     static_assert(

  1696.             std::is_same<T, int8_t>::value || std::is_same<T, uint8_t>::value,

  1697.             "transpose_12x4_1_b only support uint8_t and int8_t");

  1698.     asm volatile(

  1699.             "ldr q0,  [%[inptr0]], #12\n"  // A1A2A3A4A5A6A7A8A9A10A11A12A13A14A15A16

  1700.             "ldr q1,  [%[inptr1]], #12\n"  // B1B2B3B4B5B6B7B8B9B10B11B12B13B14B15B16

  1701.             "ldr q2,  [%[inptr2]], #12\n"  // C1C2C3C4C5C6C7C8C9C10C11C12C13C14C15C16

  1702.             //! \warning the last inptr3 may less than 16bytes, so we should

  1703.             //! split read it

  1704.             "ldr d3,  [%[inptr3]], #8\n"  // D1D2D3D4D5D6D7D8D9D10D11D12D13D14D15D16

  1705.             "ldr w1, [%[inptr3]], #4\n"

  1706.             "ins v3.s[2], w1\n"

  1707. 

  1708.             "trn1 v4.16b, v0.16b, v1.16b\n"  // v4: A1B1A3B3....

  1709.             "trn2 v5.16b, v0.16b, v1.16b\n"  // v5: A2B2A4B4....

  1710.             "trn1 v6.16b, v2.16b, v3.16b\n"  // v6: C1D1C3D3....

  1711.             "trn2 v7.16b, v2.16b, v3.16b\n"  // v7: C2D2C4D4....

  1712. 

  1713.             "trn1 v8.8h, v4.8h, v6.8h\n"   // v8: A1B1C1D1A5B5C5D5...

  1714.             "trn2 v9.8h, v4.8h, v6.8h\n"   // v9: A3B3C3D3A7B7C7D7...

  1715.             "trn1 v10.8h, v5.8h, v7.8h\n"  // v10: A2B2C2D2A6B6C6D6...

  1716.             "trn2 v11.8h, v5.8h, v7.8h\n"  // v11: A4B4C4D4A8B8C8D8...

  1717. 

  1718.             //! ABCD=E then

  1719.             //! v8: E1E5E9E13 v10: E2E6E10E14 v9: E3E7E11E15 v11:

  1720.             //! E4E8E12E16

  1721.             "zip1 v12.4s, v8.4s, v10.4s\n"   // v12: E1E2E5E6

  1722.             "zip2 v13.4s, v8.4s, v10.4s\n"   // v13: E9E10E13E14

  1723.             "zip1 v14.4s, v9.4s, v11.4s\n"   // v14: E3E4E7E8

  1724.             "zip2 v15.4s, v9.4s, v11.4s\n"   // v15: E11E12E15E16

  1725.             "zip1 v17.2d, v12.2d, v14.2d\n"  // v17: E1E2E3E4

  1726.             "zip2 v18.2d, v12.2d, v14.2d\n"  // v18: E5E6E7E8

  1727.             "zip1 v19.2d, v13.2d, v15.2d\n"  // v19: E8E10E11E12

  1728.             "zip2 v20.2d, v13.2d, v15.2d\n"  // v19: E13E14E15E16

  1729. 

  1730.             "stp q17, q18, [%[outptr]], #32\n"

  1731.             "str q19, [%[outptr]], #16\n"

  1732.             : [inptr0] "+r"(inptr0), [inptr1] "+r"(inptr1), [inptr2] "+r"(inptr2),

  1733.               [inptr3] "+r"(inptr3), [outptr] "+r"(outptr)

  1734.             :

  1735.             : "w1", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10",

  1736.               "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "memory");

  1737. }

  1738. 

  1739. template <typename T>

  1740. static inline void transpose_8x4_1_b(

  1741.         const T*& inptr0, const T*& inptr1, const T*& inptr2, const T*& inptr3,

  1742.         T* outptr) {

  1743.     static_assert(

  1744.             std::is_same<T, int8_t>::value || std::is_same<T, uint8_t>::value,

  1745.             "transpose_8x4_1_b only support uint8_t and int8_t");

  1746.     asm volatile(

  1747.             "ld1 {v0.d}[0],  [%[inptr0]], #8\n"  // A1A2A3A4A5A6A7A8

  1748.             "ld1 {v1.d}[0],  [%[inptr1]], #8\n"  // B1B2B3B4B5B6B7B8

  1749.             "ld1 {v0.d}[1],  [%[inptr2]], #8\n"  // C1C2C3C4C5C6C7C8

  1750.             "ld1 {v1.d}[1],  [%[inptr3]], #8\n"  // D1D2D3D4D5D6D7D8

  1751. 

  1752.             "zip1 v2.16b, v0.16b, v1.16b\n"  // A1B1A2B2A3B3A4B4A5B5A6B6A7B7A8B8

  1753.             "zip2 v3.16b, v0.16b, v1.16b\n"  // C1D1C2D2C3D3C4D4C5D5C6D6C7D7C8D8

  1754. 

  1755.             "zip1 v4.8h, v2.8h, v3.8h\n"  // A1B1C1D1A2B2C2D2A3B3C3D3A4B4C4D4

  1756.             "zip2 v5.8h, v2.8h, v3.8h\n"  // A5B5C5D5A6B6C6D6A7B7C7D7A8B8C8D8

  1757. 

  1758.             "st1 {v4.2d}, [%[outptr]], #16\n"

  1759.             "st1 {v5.2d}, [%[outptr]], #16\n"

  1760.             : [inptr0] "+r"(inptr0), [inptr1] "+r"(inptr1), [inptr2] "+r"(inptr2),

  1761.               [inptr3] "+r"(inptr3), [outptr] "+r"(outptr)

  1762.             :

  1763.             : "v0", "v1", "v2", "v3", "v4", "v5", "memory");

  1764. }

  1765. 

  1766. template <typename T>

  1767. static inline void transpose_4x8_1_b_with_shift(

  1768.         const T*& inptr0, const T*& inptr1, const T*& inptr2, const T*& inptr3,

  1769.         const T*& inptr4, const T*& inptr5, const T*& inptr6, const T*& inptr7,

  1770.         T*& outptr) {

  1771.     static int8x16_t shuffle_idx = {0, 4, 8,  12, 1, 5, 9,  13,

  1772.                                     2, 6, 10, 14, 3, 7, 11, 15};

  1773.     static_assert(

  1774.             std::is_same<T, int8_t>::value || std::is_same<T, uint8_t>::value,

  1775.             "transpose_8x4_1_b only support uint8_t and int8_t");

  1776.     asm volatile(

  1777.             "ld1 {v0.s}[0],  [%[inptr0]], #4\n"  // A1A2A3A4

  1778.             "ld1 {v0.s}[1],  [%[inptr1]], #4\n"  // B1B2B3B4

  1779.             "ld1 {v0.s}[2],  [%[inptr2]], #4\n"  // C1C2C3C4

  1780.             "ld1 {v0.s}[3],  [%[inptr3]], #4\n"  // D1D2D3D4

  1781.             "ld1 {v1.s}[0],  [%[inptr4]], #4\n"  // E1E2E3E4

  1782.             "ld1 {v1.s}[1],  [%[inptr5]], #4\n"  // F1F2F3F4

  1783.             "ld1 {v1.s}[2],  [%[inptr6]], #4\n"  // G1G2G3G4

  1784.             "ld1 {v1.s}[3],  [%[inptr7]], #4\n"  // H1H2H3H4

  1785. 

  1786.             "tbl v2.16b, {v0.16b}, %[shuffle_idx].16b \n"  // A1B1C1D1A2B2C2D2A3B3C3D3A4B4C4D4

  1787.             "tbl v3.16b, {v1.16b}, %[shuffle_idx].16b \n"  // E1F1G1H1E2F2G2H2E3F3G3H3E4F4G4H4

  1788. 

  1789.             "zip1 v4.4s, v2.4s, v3.4s\n"  // A1B1C1D1E1F1G1H1 A2B2C2D2E2F2G2H2

  1790.             "zip2 v5.4s, v2.4s, v3.4s\n"  // A3B3C3D3E3F3G3H3 A4B4C4D4E4F4G4H4

  1791. 

  1792.             "shl  v6.16b, v4.16b,        #4\n"

  1793.             "sshr v7.16b, v4.16b,        #4\n"  // hig

  1794.             "sshr v8.16b, v6.16b,        #4\n"  // low

  1795.             "shl  v9.16b, v5.16b,        #4\n"

  1796.             "sshr v10.16b, v5.16b,        #4\n"  // hig

  1797.             "sshr v11.16b, v9.16b,        #4\n"  // low

  1798.             "zip1 v0.2d,v8.2d,v7.2d\n"

  1799.             "zip2 v1.2d,v8.2d,v7.2d\n"

  1800.             "zip1 v2.2d,v11.2d,v10.2d\n"

  1801.             "zip2 v3.2d,v11.2d,v10.2d\n"

  1802.             "st1 {v0.2d-v3.2d},[%[outptr]],#64\n"

  1803. 

  1804.             : [inptr0] "+r"(inptr0), [inptr1] "+r"(inptr1), [inptr2] "+r"(inptr2),

  1805.               [inptr3] "+r"(inptr3), [inptr4] "+r"(inptr4), [inptr5] "+r"(inptr5),

  1806.               [inptr6] "+r"(inptr6), [inptr7] "+r"(inptr7),

  1807.               [shuffle_idx] "+w"(shuffle_idx), [outptr] "+r"(outptr)

  1808.             :

  1809.             : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11",

  1810.               "memory");

  1811. }

  1812. template <typename T>

  1813. static inline void transpose_8x8_1_b(

  1814.         const T*& inptr0, const T*& inptr1, const T*& inptr2, const T*& inptr3,

  1815.         const T*& inptr4, const T*& inptr5, const T*& inptr6, const T*& inptr7,

  1816.         T* outptr) {

  1817.     static_assert(

  1818.             std::is_same<T, int8_t>::value || std::is_same<T, uint8_t>::value,

  1819.             "transpose_8x8_1_b only support uint8_t and int8_t");

  1820.     asm volatile(

  1821.             "ld1 {v0.8b},  [%[inptr0]], #8\n"  // A1A2A3A4A5A6A7A8

  1822.             "ld1 {v1.8b},  [%[inptr1]], #8\n"  // B1B2B3B4B5B6B7B8

  1823.             "ld1 {v2.8b},  [%[inptr2]], #8\n"  // C1C2C3C4C5C6C7C8

  1824.             "ld1 {v3.8b},  [%[inptr3]], #8\n"  // D1D2D3D4D5D6D7D8

  1825.             "ld1 {v4.8b},  [%[inptr4]], #8\n"  // E1E2E3E4E5E6E7E8

  1826.             "ld1 {v5.8b},  [%[inptr5]], #8\n"  // F1F2F3F4F5F6F7F8

  1827.             "ld1 {v6.8b},  [%[inptr6]], #8\n"  // G1G2G3G4G5G6G7G8

  1828.             "ld1 {v7.8b},  [%[inptr7]], #8\n"  // H1H2H3H4H5H6H7H8

  1829. 

  1830.             "zip1 v8.16b, v0.16b, v1.16b\n"   // A1B1A2B2A3B3A4B4

  1831.                                               // A5B5A6B6A7B7A8B8

  1832.             "zip1 v9.16b, v2.16b, v3.16b\n"   // C1D1C2D2C3D3C4D4

  1833.                                               // C5D5C6D6C7D7C8D8

  1834.             "zip1 v10.16b, v4.16b, v5.16b\n"  // E1F1E2F2E3F3E4F4

  1835.                                               // E5F5E6F6E7F7E8F8

  1836.             "zip1 v11.16b, v6.16b, v7.16b\n"  // G1H1G2H2G3H3G4H4

  1837.                                               // G5H5G6H6G7H7G8H8

  1838. 

  1839.             "zip1 v12.8h, v8.8h, v9.8h\n"    // A1B1C1D1A2B2C2D2

  1840.                                              // A3B3C3D3A4B4C4D4

  1841.             "zip1 v13.8h, v10.8h, v11.8h\n"  // E1F1G1H1E2F2G2H2

  1842.                                              // E3F3G3H3E4F4G4H4

  1843.             "zip2 v14.8h, v8.8h, v9.8h\n"    // A5B5C5D5A6B6C6D6

  1844.                                              // A7B7C7D7A8B8C8D8

  1845.             "zip2 v15.8h, v10.8h, v11.8h\n"  // E5F5G5H5E6F6G6H6

  1846.                                              // E7F7G7H7E8F8G8H8

  1847. 

  1848.             "zip1 v16.4s, v12.4s, v13.4s\n"  // A1B1C1D1E1F1G1H1

  1849.                                              // A2B2C2D2E2F2G2H2

  1850.             "zip1 v18.4s, v14.4s, v15.4s\n"  // A5B5C5D5E5F5G5H5

  1851.                                              // A6B6C6D6E6F6G6H6

  1852.             "zip2 v17.4s, v12.4s, v13.4s\n"  // A3B3C3D3E3F3G3H3

  1853.                                              // A4B4C4D4E4F4G4H4

  1854.             "zip2 v19.4s, v14.4s, v15.4s\n"  // A7B7C7D7E7F7G7H7

  1855.                                              // A8B8C8D8E8F8G8H8

  1856. 

  1857.             "st1 {v16.16b},  [%[outptr]], #16\n"  // A1B1C1D1E1F1G1H1

  1858.                                                   // A2B2C2D2E2F2G2H2

  1859.             "st1 {v17.16b},  [%[outptr]], #16\n"  // A3B3C3D3E3F3G3H3

  1860.                                                   // A4B4C4D4E4F4G4H4

  1861.             "st1 {v18.16b},  [%[outptr]], #16\n"  // A5B5C5D5E5F5G5H5

  1862.                                                   // A6B6C6D6E6F6G6H6

  1863.             "st1 {v19.16b},  [%[outptr]], #16\n"  // A7B7C7D7E7F7G7H7

  1864.                                                   // A8B8C8D8E8F8G8H8

  1865.             : [inptr0] "+r"(inptr0), [inptr1] "+r"(inptr1), [inptr2] "+r"(inptr2),

  1866.               [inptr3] "+r"(inptr3), [inptr4] "+r"(inptr4), [inptr5] "+r"(inptr5),

  1867.               [inptr6] "+r"(inptr6), [inptr7] "+r"(inptr7), [outptr] "+r"(outptr)

  1868.             :

  1869.             : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11",

  1870.               "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "memory");

  1871. }

  1872. 

  1873. template <typename T>

  1874. static inline void transpose_4x16_1_b_helper(

  1875.         const T*& inptr0, const T*& inptr1, const T*& inptr2, const T*& inptr3,

  1876.         const T*& inptr4, const T*& inptr5, const T*& inptr6, const T*& inptr7,

  1877.         T* outptr) {

  1878.     static_assert(sizeof(T) == 1, "only support size == 1");

  1879.     static int8x16_t shuffle_idx = {0, 4, 8,  12, 1, 5, 9,  13,

  1880.                                     2, 6, 10, 14, 3, 7, 11, 15};

  1881.     asm volatile(

  1882.             "ld1 {v0.s}[0], [%[inptr0]], #4\n"

  1883.             "ld1 {v0.s}[1], [%[inptr1]], #4\n"

  1884.             "ld1 {v0.s}[2], [%[inptr2]], #4\n"

  1885.             "ld1 {v0.s}[3], [%[inptr3]], #4\n"

  1886.             "ld1 {v1.s}[0], [%[inptr4]], #4\n"

  1887.             "ld1 {v1.s}[1], [%[inptr5]], #4\n"

  1888.             "ld1 {v1.s}[2], [%[inptr6]], #4\n"

  1889.             "ld1 {v1.s}[3], [%[inptr7]], #4\n"

  1890. 

  1891.             "tbl v2.16b, {v0.16b}, %[shuffle_idx].16b\n"

  1892.             "tbl v3.16b, {v1.16b}, %[shuffle_idx].16b\n"

  1893. 

  1894.             "zip1 v4.4s, v2.4s, v3.4s\n"

  1895.             "zip2 v5.4s, v2.4s, v3.4s\n"

  1896. 

  1897.             "dup v6.2d, v4.d[1]\n"

  1898.             "dup v7.2d, v5.d[1]\n"

  1899. 

  1900.             "str d4, [%[outptr]], #16\n"

  1901.             "str d6, [%[outptr]], #16\n"

  1902.             "str d5, [%[outptr]], #16\n"

  1903.             "str d7, [%[outptr]], #16\n"

  1904. 

  1905.             : [inptr0] "+r"(inptr0), [inptr1] "+r"(inptr1), [inptr2] "+r"(inptr2),

  1906.               [inptr3] "+r"(inptr3), [inptr4] "+r"(inptr4), [inptr5] "+r"(inptr5),

  1907.               [inptr6] "+r"(inptr6), [inptr7] "+r"(inptr7), [outptr] "+r"(outptr),

  1908.               [shuffle_idx] "+w"(shuffle_idx)

  1909.             :

  1910.             : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "memory");

  1911. }

  1912. 

  1913. template <typename T>

  1914. static inline void transpose_4(

  1915.         const T*& inptr0, const T*& inptr1, const T*& inptr2, const T*& inptr3,

  1916.         T* outptr, int interleave, int size, T val = 0) {

  1917.     megdnn_assert(size <= interleave);

  1918.     int i = 0;

  1919.     for (; i < size; i++) {

  1920.         *outptr++ = *inptr0++;

  1921.         *outptr++ = *inptr1++;

  1922.         *outptr++ = *inptr2++;

  1923.         *outptr++ = *inptr3++;

  1924.     }

  1925.     for (; i < interleave; i++) {

  1926.         *outptr++ = val;

  1927.         *outptr++ = val;

  1928.         *outptr++ = val;

  1929.         *outptr++ = val;

  1930.     }

  1931. }

  1932. 

  1933. template <typename T>

  1934. static inline void transpose_8(

  1935.         const T*& inptr0, const T*& inptr1, const T*& inptr2, const T*& inptr3,

  1936.         const T*& inptr4, const T*& inptr5, const T*& inptr6, const T*& inptr7,

  1937.         T* outptr, int interleave, int size, T val = 0) {

  1938.     megdnn_assert(size <= interleave);

  1939.     int i = 0;

  1940.     for (; i < size; i++) {

  1941.         *outptr++ = *inptr0++;

  1942.         *outptr++ = *inptr1++;

  1943.         *outptr++ = *inptr2++;

  1944.         *outptr++ = *inptr3++;

  1945.         *outptr++ = *inptr4++;

  1946.         *outptr++ = *inptr5++;

  1947.         *outptr++ = *inptr6++;

  1948.         *outptr++ = *inptr7++;

  1949.     }

  1950.     for (; i < interleave; i++) {

  1951.         *outptr++ = val;

  1952.         *outptr++ = val;

  1953.         *outptr++ = val;

  1954.         *outptr++ = val;

  1955.         *outptr++ = val;

  1956.         *outptr++ = val;

  1957.         *outptr++ = val;

  1958.         *outptr++ = val;

  1959.     }

  1960. }

  1961. /***************************** Transpose then interleave ********************/

  1962. 

  1963. //! pack form {1, 4(icb), 4(ic), 4(oc)} to {1, 1, 4(oc), 16(ic)}

  1964. template <typename T>

  1965. static inline void transpose_interleave_4x4_4_b(

  1966.         const T*& inptr0, const T*& inptr1, const T*& inptr2, const T*& inptr3,

  1967.         T* outptr, int stride = 64) {

  1968.     static_assert(

  1969.             sizeof(T) == 1, "transpose_interleave_4x4_4_b only support sizeof(T) == 1");

  1970. 

  1971.     asm volatile(

  1972.             "ld4 {v0.16b, v1.16b, v2.16b, v3.16b},[%[inptr0]], 64\n"

  1973.             "ld4 {v4.16b, v5.16b, v6.16b, v7.16b},[%[inptr1]], 64\n"

  1974.             "ld4 {v8.16b, v9.16b, v10.16b, v11.16b},[%[inptr2]], 64\n"

  1975.             "ld4 {v12.16b, v13.16b, v14.16b, v15.16b},[%[inptr3]], 64\n"

  1976. 

  1977.             "st1 {v0.16b, v1.16b, v2.16b, v3.16b},[%[outptr]], %x[stride]\n"

  1978.             "st1 {v4.16b, v5.16b, v6.16b, v7.16b},[%[outptr]], %x[stride]\n"

  1979.             "st1 {v8.16b, v9.16b, v10.16b, v11.16b},[%[outptr]], %x[stride]\n"

  1980.             "st1 {v12.16b, v13.16b, v14.16b, v15.16b},[%[outptr]], %x[stride]\n"

  1981.             : [inptr0] "+r"(inptr0), [inptr1] "+r"(inptr1), [inptr2] "+r"(inptr2),

  1982.               [inptr3] "+r"(inptr3), [outptr] "+r"(outptr), [stride] "+r"(stride)

  1983.             :

  1984.             : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11",

  1985.               "v12", "v14", "v15", "memory");

  1986. }

  1987. 

  1988. template <typename T>

  1989. static inline void transpose_interleave_1x4_4_b(

  1990.         const T*& inptr0, T* outptr, int stride = 64) {

  1991.     static_assert(

  1992.             sizeof(T) == 1, "transpose_interleave_1x4_4_b only support sizeof(T) == 1");

  1993. 

  1994.     asm volatile(

  1995.             "ld4 {v0.16b, v1.16b, v2.16b, v3.16b},[%[inptr0]], 64\n"

  1996.             "st1 {v0.16b, v1.16b, v2.16b, v3.16b},[%[outptr]], %x[stride]\n"

  1997.             : [inptr0] "+r"(inptr0), [outptr] "+r"(outptr), [stride] "+r"(stride)

  1998.             :

  1999.             : "v0", "v1", "v2", "v3", "v4", "memory");

  2000. }

  2001. 

  2002. static inline void interleave_4x4_16x4_s8_s16(

  2003.         const int8_t* inptr0, const int8_t* inptr1, const int8_t* inptr2,

  2004.         const int8_t* inptr3, int16_t* outptr) {

  2005.     int8x16_t row0 = vld1q_s8(inptr0);

  2006.     int16x8_t row0_01 = vmovl_low_s8(row0);

  2007.     int16x8_t row0_23 = vmovl_high_s8(row0);

  2008.     int16x4_t row0_0 = vget_low_s16(row0_01);

  2009.     int16x4_t row0_1 = vget_high_s16(row0_01);

  2010.     int16x4_t row0_2 = vget_low_s16(row0_23);

  2011.     int16x4_t row0_3 = vget_high_s16(row0_23);

  2012. 

  2013.     int8x16_t row1 = vld1q_s8(inptr1);

  2014.     int16x8_t row1_01 = vmovl_low_s8(row1);

  2015.     int16x8_t row1_23 = vmovl_high_s8(row1);

  2016.     int16x4_t row1_0 = vget_low_s16(row1_01);

  2017.     int16x4_t row1_1 = vget_high_s16(row1_01);

  2018.     int16x4_t row1_2 = vget_low_s16(row1_23);

  2019.     int16x4_t row1_3 = vget_high_s16(row1_23);

  2020. 

  2021.     int8x16_t row2 = vld1q_s8(inptr2);

  2022.     int16x8_t row2_01 = vmovl_low_s8(row2);

  2023.     int16x8_t row2_23 = vmovl_high_s8(row2);

  2024.     int16x4_t row2_0 = vget_low_s16(row2_01);

  2025.     int16x4_t row2_1 = vget_high_s16(row2_01);

  2026.     int16x4_t row2_2 = vget_low_s16(row2_23);

  2027.     int16x4_t row2_3 = vget_high_s16(row2_23);

  2028. 

  2029.     int8x16_t row3 = vld1q_s8(inptr3);

  2030.     int16x8_t row3_01 = vmovl_low_s8(row3);

  2031.     int16x8_t row3_23 = vmovl_high_s8(row3);

  2032.     int16x4_t row3_0 = vget_low_s16(row3_01);

  2033.     int16x4_t row3_1 = vget_high_s16(row3_01);

  2034.     int16x4_t row3_2 = vget_low_s16(row3_23);

  2035.     int16x4_t row3_3 = vget_high_s16(row3_23);

  2036. 

  2037.     vst1_s16(outptr, row0_0);

  2038.     vst1_s16(outptr + 1 * 4, row1_0);

  2039.     vst1_s16(outptr + 2 * 4, row2_0);

  2040.     vst1_s16(outptr + 3 * 4, row3_0);

  2041.     vst1_s16(outptr + 4 * 4, row0_1);

  2042.     vst1_s16(outptr + 5 * 4, row1_1);

  2043.     vst1_s16(outptr + 6 * 4, row2_1);

  2044.     vst1_s16(outptr + 7 * 4, row3_1);

  2045.     vst1_s16(outptr + 8 * 4, row0_2);

  2046.     vst1_s16(outptr + 9 * 4, row1_2);

  2047.     vst1_s16(outptr + 10 * 4, row2_2);

  2048.     vst1_s16(outptr + 11 * 4, row3_2);

  2049.     vst1_s16(outptr + 12 * 4, row0_3);

  2050.     vst1_s16(outptr + 13 * 4, row1_3);

  2051.     vst1_s16(outptr + 14 * 4, row2_3);

  2052.     vst1_s16(outptr + 15 * 4, row3_3);

  2053. };

  2054. static inline void interleave_4x4_8x4_s8_s16(

  2055.         const int8_t* inptr0, const int8_t* inptr1, int16_t* outptr) {

  2056.     int8x16_t row0 = vld1q_s8(inptr0);

  2057.     int16x8_t row0_01 = vmovl_low_s8(row0);

  2058.     int16x8_t row0_23 = vmovl_high_s8(row0);

  2059.     int16x4_t row0_0 = vget_low_s16(row0_01);

  2060.     int16x4_t row0_1 = vget_high_s16(row0_01);

  2061.     int16x4_t row0_2 = vget_low_s16(row0_23);

  2062.     int16x4_t row0_3 = vget_high_s16(row0_23);

  2063. 

  2064.     int8x16_t row1 = vld1q_s8(inptr1);

  2065.     int16x8_t row1_01 = vmovl_low_s8(row1);

  2066.     int16x8_t row1_23 = vmovl_high_s8(row1);

  2067.     int16x4_t row1_0 = vget_low_s16(row1_01);

  2068.     int16x4_t row1_1 = vget_high_s16(row1_01);

  2069.     int16x4_t row1_2 = vget_low_s16(row1_23);

  2070.     int16x4_t row1_3 = vget_high_s16(row1_23);

  2071. 

  2072.     vst1_s16(outptr, row0_0);

  2073.     vst1_s16(outptr + 1 * 4, row1_0);

  2074.     vst1_s16(outptr + 2 * 4, row0_1);

  2075.     vst1_s16(outptr + 3 * 4, row1_1);

  2076.     vst1_s16(outptr + 4 * 4, row0_2);

  2077.     vst1_s16(outptr + 5 * 4, row1_2);

  2078.     vst1_s16(outptr + 6 * 4, row0_3);

  2079.     vst1_s16(outptr + 7 * 4, row1_3);

  2080. };

  2081. 

  2082. static inline void memcpy_s8_s16(const int8_t* inptr, int16_t* outptr, int count) {

  2083.     for (; count >= 32; count -= 32) {

  2084.         int8x8_t in0 = vld1_s8(inptr);

  2085.         int8x8_t in1 = vld1_s8(inptr + 1 * 8);

  2086.         int8x8_t in2 = vld1_s8(inptr + 2 * 8);

  2087.         int8x8_t in3 = vld1_s8(inptr + 3 * 8);

  2088.         vst1q_s16(outptr, vmovl_s8(in0));

  2089.         vst1q_s16(outptr + 1 * 8, vmovl_s8(in1));

  2090.         vst1q_s16(outptr + 2 * 8, vmovl_s8(in2));

  2091.         vst1q_s16(outptr + 3 * 8, vmovl_s8(in3));

  2092.         inptr += 32;

  2093.         outptr += 32;

  2094.     }

  2095.     for (; count >= 8; count -= 8) {

  2096.         int8x8_t in0 = vld1_s8(inptr);

  2097.         vst1q_s16(outptr, vmovl_s8(in0));

  2098.         inptr += 8;

  2099.         outptr += 8;

  2100.     }

  2101.     for (; count > 0; --count) {

  2102.         *outptr++ = (int16_t)(*inptr++);

  2103.     }

  2104. }

  2105. 

  2106. static inline void transpos_12x4_s8(const int8_t* inptr0, int8_t* outptr) {

  2107.     static const uint8_t src_idx_buffer[16] = {0, 4, 8,  12, 1, 5, 9,  13,

  2108.                                                2, 6, 10, 14, 3, 7, 11, 15};

  2109.     static const uint8x16_t vtbl = vld1q_u8(&src_idx_buffer[0]);

  2110.     int8x8x4_t input = vld4_s8(inptr0);

  2111.     int8x16_t input2 = vqtbl1q_s8(vld1q_s8(inptr0 + 4 * 8), vtbl);

  2112. 

  2113.     vst1_s8(outptr, input.val[0]);

  2114.     vst1q_lane_s32(

  2115.             reinterpret_cast<int32_t*>(outptr + 8), vreinterpretq_s32_s8(input2), 0);

  2116.     vst1_s8(outptr + 1 * 12, input.val[1]);

  2117.     vst1q_lane_s32(

  2118.             reinterpret_cast<int32_t*>(outptr + 1 * 12 + 8),

  2119.             vreinterpretq_s32_s8(input2), 1);

  2120.     vst1_s8(outptr + 2 * 12, input.val[2]);

  2121.     vst1q_lane_s32(

  2122.             reinterpret_cast<int32_t*>(outptr + 2 * 12 + 8),

  2123.             vreinterpretq_s32_s8(input2), 2);

  2124.     vst1_s8(outptr + 3 * 12, input.val[3]);

  2125.     vst1q_lane_s32(

  2126.             reinterpret_cast<int32_t*>(outptr + 3 * 12 + 8),

  2127.             vreinterpretq_s32_s8(input2), 3);

  2128. }

  2129. 

  2130. template <typename T>

  2131. static inline void interleave_8x8_mk4_b(

  2132.         const T*& inptr0, const T*& inptr1, T*& outptr) {

  2133.     static_assert(

  2134.             std::is_same<T, int8_t>::value || std::is_same<T, uint8_t>::value,

  2135.             "transpose_8x4_1_b only support uint8_t and int8_t");

  2136.     asm volatile(

  2137.             "ld1 {v0.4s},  [%[inptr0]], #16\n"

  2138.             "ld1 {v1.4s},  [%[inptr1]], #16\n"

  2139.             "ld1 {v2.4s},  [%[inptr0]], #16\n"

  2140.             "ld1 {v3.4s},  [%[inptr1]], #16\n"

  2141. 

  2142.             "zip1 v4.4s, v0.4s, v1.4s \n"

  2143.             "zip2 v5.4s, v0.4s, v1.4s \n"

  2144. 

  2145.             "zip1 v6.4s, v2.4s, v3.4s\n"

  2146.             "zip2 v7.4s, v2.4s, v3.4s\n"

  2147. 

  2148.             "st1 {v4.4s},[%[outptr]],#16\n"

  2149.             "st1 {v5.4s},[%[outptr]],#16\n"

  2150.             "st1 {v6.4s},[%[outptr]],#16\n"

  2151.             "st1 {v7.4s},[%[outptr]],#16\n"

  2152. 

  2153.             : [inptr0] "+r"(inptr0), [inptr1] "+r"(inptr1), [outptr] "+r"(outptr)

  2154.             :

  2155.             : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "memory");

  2156. }

  2157. 

  2158. template <typename T>

  2159. static inline void transpose_8x8_mk4_b(const T*& inptr0, const T*& inptr1, T* outptr) {

  2160.     static_assert(

  2161.             std::is_same<T, int8_t>::value || std::is_same<T, uint8_t>::value,

  2162.             "transpose_8x4_1_b only support uint8_t and int8_t");

  2163.     asm volatile(

  2164.             "ld4 {v0.8b-v3.8b},  [%[inptr0]], #32\n"

  2165.             "ld4 {v4.8b-v7.8b},  [%[inptr1]], #32\n"

  2166.             "st1 {v0.2s},[%[outptr]],#8\n"

  2167.             "st1 {v1.2s},[%[outptr]],#8\n"

  2168.             "st1 {v2.2s},[%[outptr]],#8\n"

  2169.             "st1 {v3.2s},[%[outptr]],#8\n"

  2170.             "st1 {v4.2s},[%[outptr]],#8\n"

  2171.             "st1 {v5.2s},[%[outptr]],#8\n"

  2172.             "st1 {v6.2s},[%[outptr]],#8\n"

  2173.             "st1 {v7.2s},[%[outptr]],#8\n"

  2174. 

  2175.             : [inptr0] "+r"(inptr0), [inptr1] "+r"(inptr1), [outptr] "+r"(outptr)

  2176.             :

  2177.             : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "memory");

  2178. }

  2179. 

  2180. }  // namespace aarch64

  2181. }  // namespace megdnn

  2182. 

  2183. // vim: syntax=cpp.doxygen