OpenI
/
MegEngine
Not watched
1
0
Fork 0
Code
Releases 31 Wiki Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 364afec033
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '364afec033'
${ noResults }
MegEngine/dnn/src/aarch64/matrix_mul/int8/kernel_8x8x8.h

1376 lines
50 kB
Raw Blame History 

  1. /**

  2.  * \file dnn/src/aarch64/matrix_mul/int8/kernel_8x8x8.h

  3.  * MegEngine is Licensed under the Apache License, Version 2.0 (the "License")

  4.  *

  5.  * Copyright (c) 2014-2021 Megvii Inc. All rights reserved.

  6.  *

  7.  * Unless required by applicable law or agreed to in writing,

  8.  * software distributed under the License is distributed on an

  9.  * "AS IS" BASIS, WITHOUT ARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

  10.  */

  11. 

  12. #if !(__ARM_FEATURE_DOTPROD)

  13. #include "src/aarch64/matrix_mul/asm/common.h"

  14. #include "src/arm_common/simd_macro/marm_neon.h"

  15. 

  16. namespace megdnn {

  17. namespace aarch64 {

  18. namespace matmul_8x8x8 {

  19. 

  20. /**

  21.  * Overview of register layout:

  22.  *

  23.  * A 8x8x8 cell of Rhs is stored in 8bit in q26-q27

  24.  * A 8x8x8 cell of Lhs is stored in 8bit in q0-q7

  25.  * A 8x8 block of accumulators is stored in 32bit in q8-q23

  26.  *

  27.  *                     +--------+--------+

  28.  *                     |v26[0-8]|v27[0-8]|

  29.  *                Rhs  +--------+--------+

  30.  *    Lhs              |        |        |

  31.  *

  32.  *  +--------+ - - - - +-----------------+

  33.  *  |v0[0-8]|          | v8[0-4]| v9[0-4]|

  34.  *  |v1[0-8]|          |v10[0-4]|v11[0-4]|

  35.  *  |v2[0-8]|          |v12[0-4]|v13[0-4]|

  36.  *  |v3[0-8]|          |v14[0-4]|v15[0-4]|

  37.  *  |v4[0-8]|          |v16[0-4]|v17[0-4]|

  38.  *  |v5[0-8]|          |v18[0-4]|v19[0-4]|

  39.  *  |v6[0-8]|          |v20[0-4]|v21[0-4]|

  40.  *  |v7[0-8]|          |v22[0-4]|v23[0-4]|

  41.  *  +--------+ - - - - +-----------------+

  42.  *

  43.  *                            Accumulator

  44.  */

  45. 

  46. static void kern_8x8(const int8_t* packA, const int8_t* packB, int K,

  47.                      int32_t* output, int LDC, bool is_first_k) {

  48.     K /= 8;

  49.     const int8_t* a_ptr = packA;

  50.     const int8_t* b_ptr = packB;

  51. 

  52.     LDC = LDC * sizeof(int32_t);

  53. 

  54.     asm volatile(

  55.             // load accumulator C

  56.             "add x1, %[output], %x[LDC]\n"

  57.             "add x2, x1, %x[LDC]\n"

  58.             "add x3, x2, %x[LDC]\n"

  59.             "add x4, x3, %x[LDC]\n"

  60.             "add x5, x4, %x[LDC]\n"

  61.             "add x6, x5, %x[LDC]\n"

  62.             "add x7, x6, %x[LDC]\n"

  63.             "cmp %w[is_first_k], #1\n"

  64.             "beq 1f\n"

  65. 

  66.             "ldp q8, q9, [%[output]]\n"

  67.             "ldp q10, q11, [x1]\n"

  68.             "ldp q12, q13, [x2]\n"

  69.             "ldp q14, q15, [x3]\n"

  70.             "ldp q16, q17, [x4]\n"

  71.             "ldp q18, q19, [x5]\n"

  72.             "ldp q20, q21, [x6]\n"

  73.             "ldp q22, q23, [x7]\n"

  74.             "b 2f\n"

  75. 

  76.             "1:\n"

  77.             "eor v8.16b, v8.16b, v8.16b\n"

  78.             "eor v9.16b, v9.16b, v9.16b\n"

  79.             "eor v10.16b, v10.16b, v10.16b\n"

  80.             "eor v11.16b, v11.16b, v11.16b\n"

  81.             "eor v12.16b, v12.16b, v12.16b\n"

  82.             "eor v13.16b, v13.16b, v13.16b\n"

  83.             "eor v14.16b, v14.16b, v14.16b\n"

  84.             "eor v15.16b, v15.16b, v15.16b\n"

  85.             "eor v16.16b, v16.16b, v16.16b\n"

  86.             "eor v17.16b, v17.16b, v17.16b\n"

  87.             "eor v18.16b, v18.16b, v18.16b\n"

  88.             "eor v19.16b, v19.16b, v19.16b\n"

  89.             "eor v20.16b, v20.16b, v20.16b\n"

  90.             "eor v21.16b, v21.16b, v21.16b\n"

  91.             "eor v22.16b, v22.16b, v22.16b\n"

  92.             "eor v23.16b, v23.16b, v23.16b\n"

  93. 

  94.             "2: \n"

  95.             "ld1 {v26.8b}, [%[b_ptr]], 8\n"

  96.             "ld1 {v0.8b}, [%[a_ptr]], 8\n"

  97.             "ld1 {v1.8b}, [%[a_ptr]], 8\n"

  98.             "ld1 {v2.8b}, [%[a_ptr]], 8\n"

  99.             "ld1 {v3.8b}, [%[a_ptr]], 8\n"

  100.             "ld1 {v4.8b}, [%[a_ptr]], 8\n"

  101.             "ld1 {v5.8b}, [%[a_ptr]], 8\n"

  102.             "ld1 {v6.8b}, [%[a_ptr]], 8\n"

  103.             "ld1 {v7.8b}, [%[a_ptr]], 8\n"

  104.             "sshll v26.8h, v26.8b, #0\n"

  105.             "sshll v0.8h, v0.8b, #0\n"

  106.             "sshll v1.8h, v1.8b, #0\n"

  107.             "sshll v2.8h, v2.8b, #0\n"

  108.             "sshll v3.8h, v3.8b, #0\n"

  109.             "sshll v4.8h, v4.8b, #0\n"

  110.             "sshll v5.8h, v5.8b, #0\n"

  111.             "sshll v6.8h, v6.8b, #0\n"

  112.             "sshll v7.8h, v7.8b, #0\n"

  113. 

  114.             "ld1 {v27.8b}, [%[b_ptr]], 8\n"

  115.             "smlal v8.4s, v26.4h, v0.h[0]\n"

  116.             "smlal v10.4s, v26.4h, v1.h[0]\n"

  117.             "smlal v12.4s, v26.4h, v2.h[0]\n"

  118.             "smlal v14.4s, v26.4h, v3.h[0]\n"

  119.             "smlal v16.4s, v26.4h, v4.h[0]\n"

  120.             "smlal v18.4s, v26.4h, v5.h[0]\n"

  121.             "smlal v20.4s, v26.4h, v6.h[0]\n"

  122.             "smlal v22.4s, v26.4h, v7.h[0]\n"

  123.             "sshll v27.8h, v27.8b, #0\n"

  124.             "smlal2 v9.4s, v26.8h, v0.h[0]\n"

  125.             "smlal2 v11.4s, v26.8h, v1.h[0]\n"

  126.             "smlal2 v13.4s, v26.8h, v2.h[0]\n"

  127.             "smlal2 v15.4s, v26.8h, v3.h[0]\n"

  128.             "smlal2 v17.4s, v26.8h, v4.h[0]\n"

  129.             "smlal2 v19.4s, v26.8h, v5.h[0]\n"

  130.             "smlal2 v21.4s, v26.8h, v6.h[0]\n"

  131.             "smlal2 v23.4s, v26.8h, v7.h[0]\n"

  132. 

  133.             "ld1 {v26.8b}, [%[b_ptr]], 8\n"

  134.             "smlal v8.4s, v27.4h, v0.h[1]\n"

  135.             "smlal v10.4s, v27.4h, v1.h[1]\n"

  136.             "smlal v12.4s, v27.4h, v2.h[1]\n"

  137.             "smlal v14.4s, v27.4h, v3.h[1]\n"

  138.             "smlal v16.4s, v27.4h, v4.h[1]\n"

  139.             "smlal v18.4s, v27.4h, v5.h[1]\n"

  140.             "smlal v20.4s, v27.4h, v6.h[1]\n"

  141.             "smlal v22.4s, v27.4h, v7.h[1]\n"

  142.             "sshll v26.8h, v26.8b, #0\n"

  143.             "smlal2 v9.4s, v27.8h, v0.h[1]\n"

  144.             "smlal2 v11.4s, v27.8h, v1.h[1]\n"

  145.             "smlal2 v13.4s, v27.8h, v2.h[1]\n"

  146.             "smlal2 v15.4s, v27.8h, v3.h[1]\n"

  147.             "smlal2 v17.4s, v27.8h, v4.h[1]\n"

  148.             "smlal2 v19.4s, v27.8h, v5.h[1]\n"

  149.             "smlal2 v21.4s, v27.8h, v6.h[1]\n"

  150.             "smlal2 v23.4s, v27.8h, v7.h[1]\n"

  151. 

  152.             "ld1 {v27.8b}, [%[b_ptr]], 8\n"

  153.             "smlal v8.4s, v26.4h, v0.h[2]\n"

  154.             "smlal v10.4s, v26.4h, v1.h[2]\n"

  155.             "smlal v12.4s, v26.4h, v2.h[2]\n"

  156.             "smlal v14.4s, v26.4h, v3.h[2]\n"

  157.             "smlal v16.4s, v26.4h, v4.h[2]\n"

  158.             "smlal v18.4s, v26.4h, v5.h[2]\n"

  159.             "smlal v20.4s, v26.4h, v6.h[2]\n"

  160.             "smlal v22.4s, v26.4h, v7.h[2]\n"

  161.             "sshll v27.8h, v27.8b, #0\n"

  162.             "smlal2 v9.4s, v26.8h, v0.h[2]\n"

  163.             "smlal2 v11.4s, v26.8h, v1.h[2]\n"

  164.             "smlal2 v13.4s, v26.8h, v2.h[2]\n"

  165.             "smlal2 v15.4s, v26.8h, v3.h[2]\n"

  166.             "smlal2 v17.4s, v26.8h, v4.h[2]\n"

  167.             "smlal2 v19.4s, v26.8h, v5.h[2]\n"

  168.             "smlal2 v21.4s, v26.8h, v6.h[2]\n"

  169.             "smlal2 v23.4s, v26.8h, v7.h[2]\n"

  170. 

  171.             "ld1 {v26.8b}, [%[b_ptr]], 8\n"

  172.             "smlal v8.4s, v27.4h, v0.h[3]\n"

  173.             "smlal v10.4s, v27.4h, v1.h[3]\n"

  174.             "smlal v12.4s, v27.4h, v2.h[3]\n"

  175.             "smlal v14.4s, v27.4h, v3.h[3]\n"

  176.             "smlal v16.4s, v27.4h, v4.h[3]\n"

  177.             "smlal v18.4s, v27.4h, v5.h[3]\n"

  178.             "smlal v20.4s, v27.4h, v6.h[3]\n"

  179.             "smlal v22.4s, v27.4h, v7.h[3]\n"

  180.             "sshll v26.8h, v26.8b, #0\n"

  181.             "smlal2 v9.4s, v27.8h, v0.h[3]\n"

  182.             "smlal2 v11.4s, v27.8h, v1.h[3]\n"

  183.             "smlal2 v13.4s, v27.8h, v2.h[3]\n"

  184.             "smlal2 v15.4s, v27.8h, v3.h[3]\n"

  185.             "smlal2 v17.4s, v27.8h, v4.h[3]\n"

  186.             "smlal2 v19.4s, v27.8h, v5.h[3]\n"

  187.             "smlal2 v21.4s, v27.8h, v6.h[3]\n"

  188.             "smlal2 v23.4s, v27.8h, v7.h[3]\n"

  189. 

  190.             "ld1 {v27.8b}, [%[b_ptr]], 8\n"

  191.             "smlal v8.4s, v26.4h, v0.h[4]\n"

  192.             "smlal v10.4s, v26.4h, v1.h[4]\n"

  193.             "smlal v12.4s, v26.4h, v2.h[4]\n"

  194.             "smlal v14.4s, v26.4h, v3.h[4]\n"

  195.             "smlal v16.4s, v26.4h, v4.h[4]\n"

  196.             "smlal v18.4s, v26.4h, v5.h[4]\n"

  197.             "smlal v20.4s, v26.4h, v6.h[4]\n"

  198.             "smlal v22.4s, v26.4h, v7.h[4]\n"

  199.             "sshll v27.8h, v27.8b, #0\n"

  200.             "smlal2 v9.4s, v26.8h, v0.h[4]\n"

  201.             "smlal2 v11.4s, v26.8h, v1.h[4]\n"

  202.             "smlal2 v13.4s, v26.8h, v2.h[4]\n"

  203.             "smlal2 v15.4s, v26.8h, v3.h[4]\n"

  204.             "smlal2 v17.4s, v26.8h, v4.h[4]\n"

  205.             "smlal2 v19.4s, v26.8h, v5.h[4]\n"

  206.             "smlal2 v21.4s, v26.8h, v6.h[4]\n"

  207.             "smlal2 v23.4s, v26.8h, v7.h[4]\n"

  208. 

  209.             "ld1 {v26.8b}, [%[b_ptr]], 8\n"

  210.             "smlal v8.4s, v27.4h, v0.h[5]\n"

  211.             "smlal v10.4s, v27.4h, v1.h[5]\n"

  212.             "smlal v12.4s, v27.4h, v2.h[5]\n"

  213.             "smlal v14.4s, v27.4h, v3.h[5]\n"

  214.             "smlal v16.4s, v27.4h, v4.h[5]\n"

  215.             "smlal v18.4s, v27.4h, v5.h[5]\n"

  216.             "smlal v20.4s, v27.4h, v6.h[5]\n"

  217.             "smlal v22.4s, v27.4h, v7.h[5]\n"

  218.             "sshll v26.8h, v26.8b, #0\n"

  219.             "smlal2 v9.4s, v27.8h, v0.h[5]\n"

  220.             "smlal2 v11.4s, v27.8h, v1.h[5]\n"

  221.             "smlal2 v13.4s, v27.8h, v2.h[5]\n"

  222.             "smlal2 v15.4s, v27.8h, v3.h[5]\n"

  223.             "smlal2 v17.4s, v27.8h, v4.h[5]\n"

  224.             "smlal2 v19.4s, v27.8h, v5.h[5]\n"

  225.             "smlal2 v21.4s, v27.8h, v6.h[5]\n"

  226.             "smlal2 v23.4s, v27.8h, v7.h[5]\n"

  227. 

  228.             "ld1 {v27.8b}, [%[b_ptr]], 8\n"

  229.             "smlal v8.4s, v26.4h, v0.h[6]\n"

  230.             "smlal v10.4s, v26.4h, v1.h[6]\n"

  231.             "smlal v12.4s, v26.4h, v2.h[6]\n"

  232.             "smlal v14.4s, v26.4h, v3.h[6]\n"

  233.             "smlal v16.4s, v26.4h, v4.h[6]\n"

  234.             "smlal v18.4s, v26.4h, v5.h[6]\n"

  235.             "smlal v20.4s, v26.4h, v6.h[6]\n"

  236.             "smlal v22.4s, v26.4h, v7.h[6]\n"

  237.             "sshll v27.8h, v27.8b, #0\n"

  238.             "smlal2 v9.4s, v26.8h, v0.h[6]\n"

  239.             "smlal2 v11.4s, v26.8h, v1.h[6]\n"

  240.             "smlal2 v13.4s, v26.8h, v2.h[6]\n"

  241.             "smlal2 v15.4s, v26.8h, v3.h[6]\n"

  242.             "smlal2 v17.4s, v26.8h, v4.h[6]\n"

  243.             "smlal2 v19.4s, v26.8h, v5.h[6]\n"

  244.             "smlal2 v21.4s, v26.8h, v6.h[6]\n"

  245.             "smlal2 v23.4s, v26.8h, v7.h[6]\n"

  246. 

  247.             "smlal v8.4s, v27.4h, v0.h[7]\n"

  248.             "smlal v10.4s, v27.4h, v1.h[7]\n"

  249.             "smlal v12.4s, v27.4h, v2.h[7]\n"

  250.             "smlal v14.4s, v27.4h, v3.h[7]\n"

  251.             "smlal v16.4s, v27.4h, v4.h[7]\n"

  252.             "smlal v18.4s, v27.4h, v5.h[7]\n"

  253.             "smlal v20.4s, v27.4h, v6.h[7]\n"

  254.             "smlal v22.4s, v27.4h, v7.h[7]\n"

  255.             "smlal2 v9.4s, v27.8h, v0.h[7]\n"

  256.             "smlal2 v11.4s, v27.8h, v1.h[7]\n"

  257.             "smlal2 v13.4s, v27.8h, v2.h[7]\n"

  258.             "smlal2 v15.4s, v27.8h, v3.h[7]\n"

  259.             "smlal2 v17.4s, v27.8h, v4.h[7]\n"

  260.             "smlal2 v19.4s, v27.8h, v5.h[7]\n"

  261.             "smlal2 v21.4s, v27.8h, v6.h[7]\n"

  262.             "smlal2 v23.4s, v27.8h, v7.h[7]\n"

  263. 

  264.             "subs %w[K], %w[K], #1\n"

  265.             "cbnz %w[K], 2b\n"

  266. 

  267.             "3:\n"

  268.             "stp q8, q9, [%[output]]\n"

  269.             "stp q10, q11, [x1]\n"

  270.             "stp q12, q13, [x2]\n"

  271.             "stp q14, q15, [x3]\n"

  272.             "stp q16, q17, [x4]\n"

  273.             "stp q18, q19, [x5]\n"

  274.             "stp q20, q21, [x6]\n"

  275.             "stp q22, q23, [x7]\n"

  276.             : [a_ptr] "+r"(a_ptr), [b_ptr] "+r"(b_ptr),

  277.               [is_first_k] "+r"(is_first_k), [K] "+r"(K), [LDC] "+r"(LDC),

  278.               [output] "+r"(output)

  279.             :

  280.             : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10",

  281.               "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19",

  282.               "v20", "v21", "v22", "v23", "v26", "v27", "x1",

  283.               "x2", "x3", "x4", "x5", "x6", "x7", "cc", "memory");

  284. }

  285. 

  286. /**

  287.  * Overview of register layout:

  288.  *

  289.  * A 8x4x8 cell of Rhs is stored in 8bit in q16-q17

  290.  * A 8x8x8 cell of Lhs is stored in 8bit in q0-q7

  291.  * A 8x4 block of accumulators is stored in 32bit in q8-q15

  292.  *

  293.  *                     +--------+

  294.  *                     |v16[0-4]|

  295.  *                Rhs  +--------+

  296.  *                     |v17[0-4]|

  297.  *     Lhs             +--------+

  298.  *

  299.  *  +--------+ - - - - +--------+

  300.  *  |v0[0-8]|          | v8[0-4]|

  301.  *  |v1[0-8]|          | v9[0-4]|

  302.  *  |v2[0-8]|          |v10[0-4]|

  303.  *  |v3[0-8]|          |v11[0-4]|

  304.  *  |v4[0-8]|          |v12[0-4]|

  305.  *  |v5[0-8]|          |v13[0-4]|

  306.  *  |v6[0-8]|          |v14[0-4]|

  307.  *  |v7[0-8]|          |v15[0-4]|

  308.  *  +--------+ - - - - +--------+

  309.  *

  310.  *                            Accumulator

  311.  */

  312. 

  313. static void kern_8x4(const int8_t* packA, const int8_t* packB, int K,

  314.                      int32_t* output, int LDC, bool is_first_k,

  315.                      size_t n_remain) {

  316.     K /= 8;

  317.     const int8_t* a_ptr = packA;

  318.     const int8_t* b_ptr = packB;

  319. 

  320.     LDC = LDC * sizeof(int32_t);

  321.     int32_t* outptr0 = output;

  322.     int32_t* outptr1;

  323.     int32_t* outptr2;

  324.     int32_t* outptr3;

  325.     int32_t* outptr4;

  326.     int32_t* outptr5;

  327.     int32_t* outptr6;

  328.     int32_t* outptr7;

  329.     size_t x0 = 0;

  330. 

  331. // clang-format off

  332. #define LOAD_LINE(reg_index, n)                \

  333.     "mov %[x0], %[outptr" n "]\n"              \

  334.     "cmp %w[n_remain], #4\n"                   \

  335.     "blt 100" n "f\n"                          \

  336.     "ldr q" reg_index ", [%[x0]] \n"           \

  337.     "b 101" n "f\n"                            \

  338.     "100" n ":\n"                              \

  339.     "cmp %w[n_remain], #0\n"                   \

  340.     "beq 101" n "f\n"                          \

  341.     "ld1 {v" reg_index ".s}[0], [%[x0]], #4\n" \

  342.     "cmp %w[n_remain], #1\n"                   \

  343.     "beq 101" n "f\n"                          \

  344.     "ld1 {v" reg_index ".s}[1], [%[x0]], #4\n" \

  345.     "cmp %w[n_remain], #2\n"                   \

  346.     "beq 101" n "f\n"                          \

  347.     "ld1 {v" reg_index ".s}[2], [%[x0]], #4\n" \

  348.     "101" n ":\n"

  349. 

  350. #define LOAD_C            \

  351.     LOAD_LINE("8", "0")   \

  352.     LOAD_LINE("9", "1")   \

  353.     LOAD_LINE("10", "2")  \

  354.     LOAD_LINE("11", "3")  \

  355.     LOAD_LINE("12", "4")  \

  356.     LOAD_LINE("13", "5")  \

  357.     LOAD_LINE("14", "6")  \

  358.     LOAD_LINE("15", "7")

  359. 

  360. #define STORE_LINE(reg_index, n)               \

  361.     "mov %[x0], %[outptr" n "]\n"              \

  362.     "cmp %w[n_remain], #4\n"                   \

  363.     "blt 102" n "f\n"                          \

  364.     "str q" reg_index ", [%[x0]]\n"            \

  365.     "b 103" n "f\n"                            \

  366.     "102" n ":\n"                              \

  367.     "cmp %w[n_remain], #0\n"                   \

  368.     "beq 103" n "f\n"                          \

  369.     "st1 {v" reg_index ".s}[0], [%[x0]], #4\n" \

  370.     "cmp %w[n_remain], #1\n"                   \

  371.     "beq 103" n "f\n"                          \

  372.     "st1 {v" reg_index ".s}[1], [%[x0]], #4\n" \

  373.     "cmp %w[n_remain], #2\n"                   \

  374.     "beq 103" n "f\n"                          \

  375.     "st1 {v" reg_index ".s}[2], [%[x0]], #4\n" \

  376.     "103" n ":\n"

  377. 

  378. #define STORE_C            \

  379.     STORE_LINE("8", "0")   \

  380.     STORE_LINE("9", "1")   \

  381.     STORE_LINE("10", "2")  \

  382.     STORE_LINE("11", "3")  \

  383.     STORE_LINE("12", "4")  \

  384.     STORE_LINE("13", "5")  \

  385.     STORE_LINE("14", "6")  \

  386.     STORE_LINE("15", "7")

  387. 

  388.     // clang-format on

  389. 

  390.     asm volatile(

  391.             // load accumulator C

  392.             "add %[outptr1], %[outptr0], %x[LDC]\n"

  393.             "add %[outptr2], %[outptr1], %x[LDC]\n"

  394.             "add %[outptr3], %[outptr2], %x[LDC]\n"

  395.             "add %[outptr4], %[outptr3], %x[LDC]\n"

  396.             "add %[outptr5], %[outptr4], %x[LDC]\n"

  397.             "add %[outptr6], %[outptr5], %x[LDC]\n"

  398.             "add %[outptr7], %[outptr6], %x[LDC]\n"

  399.             "cmp %w[is_first_k], #1\n"

  400.             "beq 1f\n" LOAD_C

  401. 

  402.             "b 2f\n"

  403. 

  404.             "1:\n"

  405.             "eor v8.16b, v8.16b, v8.16b\n"

  406.             "eor v9.16b, v9.16b, v9.16b\n"

  407.             "eor v10.16b, v10.16b, v10.16b\n"

  408.             "eor v11.16b, v11.16b, v11.16b\n"

  409.             "eor v12.16b, v12.16b, v12.16b\n"

  410.             "eor v13.16b, v13.16b, v13.16b\n"

  411.             "eor v14.16b, v14.16b, v14.16b\n"

  412.             "eor v15.16b, v15.16b, v15.16b\n"

  413. 

  414.             "2: \n"

  415.             "ld1 {v16.s}[0], [%[b_ptr]], 4\n"

  416.             "ld1 {v0.8b}, [%[a_ptr]], 8\n"

  417.             "ld1 {v1.8b}, [%[a_ptr]], 8\n"

  418.             "ld1 {v2.8b}, [%[a_ptr]], 8\n"

  419.             "ld1 {v3.8b}, [%[a_ptr]], 8\n"

  420.             "ld1 {v4.8b}, [%[a_ptr]], 8\n"

  421.             "ld1 {v5.8b}, [%[a_ptr]], 8\n"

  422.             "ld1 {v6.8b}, [%[a_ptr]], 8\n"

  423.             "ld1 {v7.8b}, [%[a_ptr]], 8\n"

  424.             "sshll v16.8h, v16.8b, #0\n"

  425.             "sshll v0.8h, v0.8b, #0\n"

  426.             "sshll v1.8h, v1.8b, #0\n"

  427.             "sshll v2.8h, v2.8b, #0\n"

  428.             "sshll v3.8h, v3.8b, #0\n"

  429.             "sshll v4.8h, v4.8b, #0\n"

  430.             "sshll v5.8h, v5.8b, #0\n"

  431.             "sshll v6.8h, v6.8b, #0\n"

  432.             "sshll v7.8h, v7.8b, #0\n"

  433. 

  434.             "ld1 {v17.s}[0], [%[b_ptr]], 4\n"

  435.             "smlal v8.4s, v16.4h, v0.h[0]\n"

  436.             "smlal v9.4s, v16.4h, v1.h[0]\n"

  437.             "smlal v10.4s, v16.4h, v2.h[0]\n"

  438.             "smlal v11.4s, v16.4h, v3.h[0]\n"

  439.             "sshll v17.8h, v17.8b, #0\n"

  440.             "smlal v12.4s, v16.4h, v4.h[0]\n"

  441.             "smlal v13.4s, v16.4h, v5.h[0]\n"

  442.             "smlal v14.4s, v16.4h, v6.h[0]\n"

  443.             "smlal v15.4s, v16.4h, v7.h[0]\n"

  444. 

  445.             "ld1 {v16.s}[0], [%[b_ptr]], 4\n"

  446.             "smlal v8.4s, v17.4h, v0.h[1]\n"

  447.             "smlal v9.4s, v17.4h, v1.h[1]\n"

  448.             "smlal v10.4s, v17.4h, v2.h[1]\n"

  449.             "smlal v11.4s, v17.4h, v3.h[1]\n"

  450.             "sshll v16.8h, v16.8b, #0\n"

  451.             "smlal v12.4s, v17.4h, v4.h[1]\n"

  452.             "smlal v13.4s, v17.4h, v5.h[1]\n"

  453.             "smlal v14.4s, v17.4h, v6.h[1]\n"

  454.             "smlal v15.4s, v17.4h, v7.h[1]\n"

  455. 

  456.             "ld1 {v17.s}[0], [%[b_ptr]], 4\n"

  457.             "smlal v8.4s, v16.4h, v0.h[2]\n"

  458.             "smlal v9.4s, v16.4h, v1.h[2]\n"

  459.             "smlal v10.4s, v16.4h, v2.h[2]\n"

  460.             "smlal v11.4s, v16.4h, v3.h[2]\n"

  461.             "sshll v17.8h, v17.8b, #0\n"

  462.             "smlal v12.4s, v16.4h, v4.h[2]\n"

  463.             "smlal v13.4s, v16.4h, v5.h[2]\n"

  464.             "smlal v14.4s, v16.4h, v6.h[2]\n"

  465.             "smlal v15.4s, v16.4h, v7.h[2]\n"

  466. 

  467.             "ld1 {v16.s}[0], [%[b_ptr]], 4\n"

  468.             "smlal v8.4s, v17.4h, v0.h[3]\n"

  469.             "smlal v9.4s, v17.4h, v1.h[3]\n"

  470.             "smlal v10.4s, v17.4h, v2.h[3]\n"

  471.             "smlal v11.4s, v17.4h, v3.h[3]\n"

  472.             "sshll v16.8h, v16.8b, #0\n"

  473.             "smlal v12.4s, v17.4h, v4.h[3]\n"

  474.             "smlal v13.4s, v17.4h, v5.h[3]\n"

  475.             "smlal v14.4s, v17.4h, v6.h[3]\n"

  476.             "smlal v15.4s, v17.4h, v7.h[3]\n"

  477. 

  478.             "ld1 {v17.s}[0], [%[b_ptr]], 4\n"

  479.             "smlal v8.4s, v16.4h, v0.h[4]\n"

  480.             "smlal v9.4s, v16.4h, v1.h[4]\n"

  481.             "smlal v10.4s, v16.4h, v2.h[4]\n"

  482.             "smlal v11.4s, v16.4h, v3.h[4]\n"

  483.             "sshll v17.8h, v17.8b, #0\n"

  484.             "smlal v12.4s, v16.4h, v4.h[4]\n"

  485.             "smlal v13.4s, v16.4h, v5.h[4]\n"

  486.             "smlal v14.4s, v16.4h, v6.h[4]\n"

  487.             "smlal v15.4s, v16.4h, v7.h[4]\n"

  488. 

  489.             "ld1 {v16.s}[0], [%[b_ptr]], 4\n"

  490.             "smlal v8.4s, v17.4h, v0.h[5]\n"

  491.             "smlal v9.4s, v17.4h, v1.h[5]\n"

  492.             "smlal v10.4s, v17.4h, v2.h[5]\n"

  493.             "smlal v11.4s, v17.4h, v3.h[5]\n"

  494.             "sshll v16.8h, v16.8b, #0\n"

  495.             "smlal v12.4s, v17.4h, v4.h[5]\n"

  496.             "smlal v13.4s, v17.4h, v5.h[5]\n"

  497.             "smlal v14.4s, v17.4h, v6.h[5]\n"

  498.             "smlal v15.4s, v17.4h, v7.h[5]\n"

  499. 

  500.             "ld1 {v17.s}[0], [%[b_ptr]], 4\n"

  501.             "smlal v8.4s, v16.4h, v0.h[6]\n"

  502.             "smlal v9.4s, v16.4h, v1.h[6]\n"

  503.             "smlal v10.4s, v16.4h, v2.h[6]\n"

  504.             "smlal v11.4s, v16.4h, v3.h[6]\n"

  505.             "sshll v17.8h, v17.8b, #0\n"

  506.             "smlal v12.4s, v16.4h, v4.h[6]\n"

  507.             "smlal v13.4s, v16.4h, v5.h[6]\n"

  508.             "smlal v14.4s, v16.4h, v6.h[6]\n"

  509.             "smlal v15.4s, v16.4h, v7.h[6]\n"

  510. 

  511.             "smlal v8.4s, v17.4h, v0.h[7]\n"

  512.             "smlal v9.4s, v17.4h, v1.h[7]\n"

  513.             "smlal v10.4s, v17.4h, v2.h[7]\n"

  514.             "smlal v11.4s, v17.4h, v3.h[7]\n"

  515.             "smlal v12.4s, v17.4h, v4.h[7]\n"

  516.             "smlal v13.4s, v17.4h, v5.h[7]\n"

  517.             "smlal v14.4s, v17.4h, v6.h[7]\n"

  518.             "smlal v15.4s, v17.4h, v7.h[7]\n"

  519. 

  520.             "subs %w[K], %w[K], #1\n"

  521.             "cbnz %w[K], 2b\n"

  522. 

  523.             "3:\n" STORE_C

  524.             : [a_ptr] "+r"(a_ptr), [b_ptr] "+r"(b_ptr),

  525.               [is_first_k] "+r"(is_first_k), [K] "+r"(K), [LDC] "+r"(LDC),

  526.               [outptr0] "+r"(outptr0), [outptr1] "=r"(outptr1),

  527.               [outptr2] "=r"(outptr2), [outptr3] "=r"(outptr3),

  528.               [outptr4] "=r"(outptr4), [outptr5] "=r"(outptr5),

  529.               [outptr6] "=r"(outptr6), [outptr7] "=r"(outptr7), [x0] "+r"(x0),

  530.               [n_remain] "+r"(n_remain)

  531.             :

  532.             : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10",

  533.               "v11", "v12", "v13", "v14", "v15", "v16", "v17", "cc", "memory");

  534. 

  535. #undef LOAD_LINE

  536. #undef LOAD_C

  537. #undef STORE_LINE

  538. #undef STORE_C

  539. }

  540. 

  541. /**

  542.  * Overview of register layout:

  543.  *

  544.  * A 8x8x8 cell of Rhs is stored in 8bit in q12-q13

  545.  * A 8x8x4 cell of Lhs is stored in 8bit in q0-q3

  546.  * A 4x8 block of accumulators is stored in 32bit in q4-q11

  547.  *

  548.  *                     +--------+--------+

  549.  *                     |v12[0-8]|v13[0-8]|

  550.  *                Rhs  +--------+--------+

  551.  *    Lhs              |        |        |

  552.  *

  553.  *  +--------+ - - - - +-----------------+

  554.  *  |v0[0-8]|          | v4[0-4]| v5[0-4]|

  555.  *  |v1[0-8]|          | v6[0-4]| v7[0-4]|

  556.  *  |v2[0-8]|          | v8[0-4]| v9[0-4]|

  557.  *  |v3[0-8]|          |v10[0-4]|v11[0-4]|

  558.  *  +--------+ - - - - +-----------------+

  559.  *

  560.  *                            Accumulator

  561.  */

  562. 

  563. static void kern_4x8(const int8_t* packA, const int8_t* packB, int K,

  564.                      int32_t* output, int LDC, bool is_first_k,

  565.                      size_t m_remain) {

  566.     K /= 8;

  567.     const int8_t* a_ptr = packA;

  568.     const int8_t* b_ptr = packB;

  569. 

  570.     LDC = LDC * sizeof(int32_t);

  571.     int32_t* outptr0 = output;

  572.     int32_t* outptr1;

  573.     int32_t* outptr2;

  574.     int32_t* outptr3;

  575.     size_t x0 = 0;

  576. 

  577. // clang-format off

  578. #define LOAD_LINE(v1, v2, m)                \

  579.     "cbz %[x0], 100f\n"                     \

  580.     "ldp " v1 "," v2 ", [%[outptr" m "]]\n" \

  581.     "subs %[x0], %[x0], #1\n"

  582. 

  583. #define LOAD_C                      \

  584.     "mov %[x0], %x[m_remain]\n"     \

  585.     LOAD_LINE("q4", "q5", "0")      \

  586.     LOAD_LINE("q6", "q7", "1")      \

  587.     LOAD_LINE("q8", "q9", "2")      \

  588.     LOAD_LINE("q10", "q11", "3")    \

  589.     "100:\n"

  590. 

  591. #define STORE_LINE(v1, v2, m)              \

  592.     "cbz %[x0], 101f\n"                    \

  593.     "stp " v1 "," v2", [%[outptr" m "]]\n" \

  594.     "subs %[x0], %[x0], #1\n"

  595. 

  596. #define STORE_C                     \

  597.     "mov %[x0], %x[m_remain]\n"     \

  598.     STORE_LINE("q4", "q5", "0")     \

  599.     STORE_LINE("q6", "q7", "1")     \

  600.     STORE_LINE("q8", "q9", "2")     \

  601.     STORE_LINE("q10", "q11", "3")   \

  602.     "101:\n"

  603. 

  604.     // clang-format on

  605. 

  606.     asm volatile(

  607.             // load accumulator C

  608.             "add %[outptr1], %[outptr0], %x[LDC]\n"

  609.             "add %[outptr2], %[outptr1], %x[LDC]\n"

  610.             "add %[outptr3], %[outptr2], %x[LDC]\n"

  611.             "cmp %w[is_first_k], #1\n"

  612.             "beq 1f\n" LOAD_C

  613. 

  614.             "b 2f\n"

  615. 

  616.             "1:\n"

  617.             "eor v4.16b, v4.16b, v4.16b\n"

  618.             "eor v5.16b, v5.16b, v5.16b\n"

  619.             "eor v6.16b, v6.16b, v6.16b\n"

  620.             "eor v7.16b, v7.16b, v7.16b\n"

  621.             "eor v8.16b, v8.16b, v8.16b\n"

  622.             "eor v9.16b, v9.16b, v9.16b\n"

  623.             "eor v10.16b, v10.16b, v10.16b\n"

  624.             "eor v11.16b, v11.16b, v11.16b\n"

  625. 

  626.             "2: \n"

  627.             "ld1 {v12.8b}, [%[b_ptr]], 8\n"

  628.             "ld1 {v0.8b}, [%[a_ptr]], 8\n"

  629.             "ld1 {v1.8b}, [%[a_ptr]], 8\n"

  630.             "ld1 {v2.8b}, [%[a_ptr]], 8\n"

  631.             "ld1 {v3.8b}, [%[a_ptr]], 8\n"

  632.             "sshll v12.8h, v12.8b, #0\n"

  633.             "sshll v0.8h, v0.8b, #0\n"

  634.             "sshll v1.8h, v1.8b, #0\n"

  635.             "sshll v2.8h, v2.8b, #0\n"

  636.             "sshll v3.8h, v3.8b, #0\n"

  637. 

  638.             "ld1 {v13.8b}, [%[b_ptr]], 8\n"

  639.             "smlal v4.4s, v12.4h, v0.h[0]\n"

  640.             "smlal v6.4s, v12.4h, v1.h[0]\n"

  641.             "smlal v8.4s, v12.4h, v2.h[0]\n"

  642.             "smlal v10.4s, v12.4h, v3.h[0]\n"

  643.             "sshll v13.8h, v13.8b, #0\n"

  644.             "smlal2 v5.4s, v12.8h, v0.h[0]\n"

  645.             "smlal2 v7.4s, v12.8h, v1.h[0]\n"

  646.             "smlal2 v9.4s, v12.8h, v2.h[0]\n"

  647.             "smlal2 v11.4s, v12.8h, v3.h[0]\n"

  648. 

  649.             "ld1 {v12.8b}, [%[b_ptr]], 8\n"

  650.             "smlal v4.4s, v13.4h, v0.h[1]\n"

  651.             "smlal v6.4s, v13.4h, v1.h[1]\n"

  652.             "smlal v8.4s, v13.4h, v2.h[1]\n"

  653.             "smlal v10.4s, v13.4h, v3.h[1]\n"

  654.             "sshll v12.8h, v12.8b, #0\n"

  655.             "smlal2 v5.4s, v13.8h, v0.h[1]\n"

  656.             "smlal2 v7.4s, v13.8h, v1.h[1]\n"

  657.             "smlal2 v9.4s, v13.8h, v2.h[1]\n"

  658.             "smlal2 v11.4s, v13.8h, v3.h[1]\n"

  659. 

  660.             "ld1 {v13.8b}, [%[b_ptr]], 8\n"

  661.             "smlal v4.4s, v12.4h, v0.h[2]\n"

  662.             "smlal v6.4s, v12.4h, v1.h[2]\n"

  663.             "smlal v8.4s, v12.4h, v2.h[2]\n"

  664.             "smlal v10.4s, v12.4h, v3.h[2]\n"

  665.             "sshll v13.8h, v13.8b, #0\n"

  666.             "smlal2 v5.4s, v12.8h, v0.h[2]\n"

  667.             "smlal2 v7.4s, v12.8h, v1.h[2]\n"

  668.             "smlal2 v9.4s, v12.8h, v2.h[2]\n"

  669.             "smlal2 v11.4s, v12.8h, v3.h[2]\n"

  670. 

  671.             "ld1 {v12.8b}, [%[b_ptr]], 8\n"

  672.             "smlal v4.4s, v13.4h, v0.h[3]\n"

  673.             "smlal v6.4s, v13.4h, v1.h[3]\n"

  674.             "smlal v8.4s, v13.4h, v2.h[3]\n"

  675.             "smlal v10.4s, v13.4h, v3.h[3]\n"

  676.             "sshll v12.8h, v12.8b, #0\n"

  677.             "smlal2 v5.4s, v13.8h, v0.h[3]\n"

  678.             "smlal2 v7.4s, v13.8h, v1.h[3]\n"

  679.             "smlal2 v9.4s, v13.8h, v2.h[3]\n"

  680.             "smlal2 v11.4s, v13.8h, v3.h[3]\n"

  681. 

  682.             "ld1 {v13.8b}, [%[b_ptr]], 8\n"

  683.             "smlal v4.4s, v12.4h, v0.h[4]\n"

  684.             "smlal v6.4s, v12.4h, v1.h[4]\n"

  685.             "smlal v8.4s, v12.4h, v2.h[4]\n"

  686.             "smlal v10.4s, v12.4h, v3.h[4]\n"

  687.             "sshll v13.8h, v13.8b, #0\n"

  688.             "smlal2 v5.4s, v12.8h, v0.h[4]\n"

  689.             "smlal2 v7.4s, v12.8h, v1.h[4]\n"

  690.             "smlal2 v9.4s, v12.8h, v2.h[4]\n"

  691.             "smlal2 v11.4s, v12.8h, v3.h[4]\n"

  692. 

  693.             "ld1 {v12.8b}, [%[b_ptr]], 8\n"

  694.             "smlal v4.4s, v13.4h, v0.h[5]\n"

  695.             "smlal v6.4s, v13.4h, v1.h[5]\n"

  696.             "smlal v8.4s, v13.4h, v2.h[5]\n"

  697.             "smlal v10.4s, v13.4h, v3.h[5]\n"

  698.             "sshll v12.8h, v12.8b, #0\n"

  699.             "smlal2 v5.4s, v13.8h, v0.h[5]\n"

  700.             "smlal2 v7.4s, v13.8h, v1.h[5]\n"

  701.             "smlal2 v9.4s, v13.8h, v2.h[5]\n"

  702.             "smlal2 v11.4s, v13.8h, v3.h[5]\n"

  703. 

  704.             "ld1 {v13.8b}, [%[b_ptr]], 8\n"

  705.             "smlal v4.4s, v12.4h, v0.h[6]\n"

  706.             "smlal v6.4s, v12.4h, v1.h[6]\n"

  707.             "smlal v8.4s, v12.4h, v2.h[6]\n"

  708.             "smlal v10.4s, v12.4h, v3.h[6]\n"

  709.             "sshll v13.8h, v13.8b, #0\n"

  710.             "smlal2 v5.4s, v12.8h, v0.h[6]\n"

  711.             "smlal2 v7.4s, v12.8h, v1.h[6]\n"

  712.             "smlal2 v9.4s, v12.8h, v2.h[6]\n"

  713.             "smlal2 v11.4s, v12.8h, v3.h[6]\n"

  714. 

  715.             "smlal v4.4s, v13.4h, v0.h[7]\n"

  716.             "smlal v6.4s, v13.4h, v1.h[7]\n"

  717.             "smlal v8.4s, v13.4h, v2.h[7]\n"

  718.             "smlal v10.4s, v13.4h, v3.h[7]\n"

  719.             "smlal2 v5.4s, v13.8h, v0.h[7]\n"

  720.             "smlal2 v7.4s, v13.8h, v1.h[7]\n"

  721.             "smlal2 v9.4s, v13.8h, v2.h[7]\n"

  722.             "smlal2 v11.4s, v13.8h, v3.h[7]\n"

  723. 

  724.             "subs %w[K], %w[K], #1\n"

  725.             "cbnz %w[K], 2b\n"

  726. 

  727.             "3:\n" STORE_C

  728.             : [a_ptr] "+r"(a_ptr), [b_ptr] "+r"(b_ptr),

  729.               [is_first_k] "+r"(is_first_k), [K] "+r"(K), [LDC] "+r"(LDC),

  730.               [outptr0] "+r"(outptr0),

  731.               [outptr1] "=r"(outptr1), [outptr2] "=r"(outptr2),

  732.               [outptr3] "=r"(outptr3), [x0] "+r"(x0), [m_remain] "+r"(m_remain)

  733.             :

  734.             : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10",

  735.               "v11", "v12", "v13", "cc", "memory");

  736. 

  737. #undef LOAD_LINE

  738. #undef LOAD_C

  739. #undef STORE_LINE

  740. #undef STORE_C

  741. }

  742. 

  743. /**

  744.  * Overview of register layout:

  745.  *

  746.  * A 8x4x8 cell of Rhs is stored in 8bit in q8-q9

  747.  * A 8x8x4 cell of Lhs is stored in 8bit in q0-q3

  748.  * A 4x4 block of accumulators is stored in 32bit in q4-q7

  749.  *

  750.  *                     +--------+

  751.  *                     | v8[0-4]|

  752.  *                Rhs  +--------+

  753.  *                     | v9[0-4]|

  754.  *     Lhs             +--------+

  755.  *

  756.  *  +--------+ - - - - +--------+

  757.  *  |v0[0-8]|          | v4[0-4]|

  758.  *  |v1[0-8]|          | v5[0-4]|

  759.  *  |v2[0-8]|          | v6[0-4]|

  760.  *  |v3[0-8]|          | v7[0-4]|

  761.  *  +--------+ - - - - +--------+

  762.  *

  763.  *                            Accumulator

  764.  */

  765. 

  766. static void kern_4x4(const int8_t* packA, const int8_t* packB, int K,

  767.                      int32_t* output, int LDC, bool is_first_k, size_t m_remain,

  768.                      size_t n_remain) {

  769.     K /= 8;

  770.     const int8_t* a_ptr = packA;

  771.     const int8_t* b_ptr = packB;

  772. 

  773.     LDC = LDC * sizeof(int32_t);

  774.     int32_t* outptr0 = output;

  775.     int32_t* outptr1;

  776.     int32_t* outptr2;

  777.     int32_t* outptr3;

  778.     size_t x0 = 0;

  779.     size_t x1 = 0;

  780. 

  781. // clang-format off

  782. #define LOAD_LINE(reg_index, n)                 \

  783.     "cbz %[x1], 102f\n"                         \

  784.     "mov %[x0], %[outptr" n "]\n"               \

  785.     "cmp %w[n_remain], #4\n"                    \

  786.     "blt 100" n "f\n"                           \

  787.     "ldr q" reg_index ", [%[x0]]\n"             \

  788.     "b 101" n "f\n"                             \

  789.     "100" n ":\n"                               \

  790.     "cmp %w[n_remain], #0\n"                    \

  791.     "beq 101" n "f\n"                           \

  792.     "ld1 {v" reg_index ".s}[0], [%[x0]], #4\n"  \

  793.     "cmp %w[n_remain], #1\n"                    \

  794.     "beq 101" n "f\n"                           \

  795.     "ld1 {v" reg_index ".s}[1], [%[x0]], #4\n"  \

  796.     "cmp %w[n_remain], #2\n"                    \

  797.     "beq 101" n "f\n"                           \

  798.     "ld1 {v" reg_index ".s}[2], [%[x0]], #4\n"  \

  799.     "101" n ":\n" \

  800.     "subs %[x1], %[x1], #1\n"

  801. 

  802. #define LOAD_C                  \

  803.     "mov %[x1], %x[m_remain]\n" \

  804.     LOAD_LINE("4", "0")         \

  805.     LOAD_LINE("5", "1")         \

  806.     LOAD_LINE("6", "2")         \

  807.     LOAD_LINE("7", "3")         \

  808.     "102:\n"

  809. 

  810. #define STORE_LINE(reg_index, n)               \

  811.     "cbz %[x1], 105f\n"                        \

  812.     "mov %[x0], %[outptr" n "]\n"              \

  813.     "cmp %w[n_remain], #4\n"                   \

  814.     "blt 103" n "f\n"                          \

  815.     "str q" reg_index ", [%[x0]]\n"            \

  816.     "b 104" n "f\n"                            \

  817.     "103" n ":\n"                              \

  818.     "cmp %w[n_remain], #0\n"                   \

  819.     "beq 104" n "f\n"                          \

  820.     "st1 {v" reg_index ".s}[0], [%[x0]], #4\n" \

  821.     "cmp %w[n_remain], #1\n"                   \

  822.     "beq 104" n "f\n"                          \

  823.     "st1 {v" reg_index ".s}[1], [%[x0]], #4\n" \

  824.     "cmp %w[n_remain], #2\n"                   \

  825.     "beq 104" n "f\n"                          \

  826.     "st1 {v" reg_index ".s}[2], [%[x0]], #4\n" \

  827.     "104" n ":\n"                              \

  828.     "subs %[x1], %[x1], #1\n"

  829. 

  830. #define STORE_C                 \

  831.     "mov %[x1], %x[m_remain]\n" \

  832.     STORE_LINE("4", "0")        \

  833.     STORE_LINE("5", "1")        \

  834.     STORE_LINE("6", "2")        \

  835.     STORE_LINE("7", "3")        \

  836.     "105:\n"

  837. 

  838.     // clang-format on

  839. 

  840.     asm volatile(

  841.             // load accumulator C

  842.             "add %[outptr1], %[outptr0], %x[LDC]\n"

  843.             "add %[outptr2], %[outptr1], %x[LDC]\n"

  844.             "add %[outptr3], %[outptr2], %x[LDC]\n"

  845.             "cmp %w[is_first_k], #1\n"

  846.             "beq 1f\n" LOAD_C

  847. 

  848.             "b 2f\n"

  849. 

  850.             "1:\n"

  851.             "eor v4.16b, v4.16b, v4.16b\n"

  852.             "eor v5.16b, v5.16b, v5.16b\n"

  853.             "eor v6.16b, v6.16b, v6.16b\n"

  854.             "eor v7.16b, v7.16b, v7.16b\n"

  855. 

  856.             "2: \n"

  857.             "ld1 {v8.s}[0], [%[b_ptr]], 4\n"

  858.             "ld1 {v0.8b}, [%[a_ptr]], 8\n"

  859.             "ld1 {v1.8b}, [%[a_ptr]], 8\n"

  860.             "ld1 {v2.8b}, [%[a_ptr]], 8\n"

  861.             "ld1 {v3.8b}, [%[a_ptr]], 8\n"

  862.             "sshll v8.8h, v8.8b, #0\n"

  863.             "sshll v0.8h, v0.8b, #0\n"

  864.             "sshll v1.8h, v1.8b, #0\n"

  865.             "sshll v2.8h, v2.8b, #0\n"

  866.             "sshll v3.8h, v3.8b, #0\n"

  867. 

  868.             "ld1 {v9.s}[0], [%[b_ptr]], 4\n"

  869.             "smlal v4.4s, v8.4h, v0.h[0]\n"

  870.             "smlal v5.4s, v8.4h, v1.h[0]\n"

  871.             "sshll v9.8h, v9.8b, #0\n"

  872.             "smlal v6.4s, v8.4h, v2.h[0]\n"

  873.             "smlal v7.4s, v8.4h, v3.h[0]\n"

  874. 

  875.             "ld1 {v8.s}[0], [%[b_ptr]], 4\n"

  876.             "smlal v4.4s, v9.4h, v0.h[1]\n"

  877.             "smlal v5.4s, v9.4h, v1.h[1]\n"

  878.             "sshll v8.8h, v8.8b, #0\n"

  879.             "smlal v6.4s, v9.4h, v2.h[1]\n"

  880.             "smlal v7.4s, v9.4h, v3.h[1]\n"

  881. 

  882.             "ld1 {v9.s}[0], [%[b_ptr]], 4\n"

  883.             "smlal v4.4s, v8.4h, v0.h[2]\n"

  884.             "smlal v5.4s, v8.4h, v1.h[2]\n"

  885.             "sshll v9.8h, v9.8b, #0\n"

  886.             "smlal v6.4s, v8.4h, v2.h[2]\n"

  887.             "smlal v7.4s, v8.4h, v3.h[2]\n"

  888. 

  889.             "ld1 {v8.s}[0], [%[b_ptr]], 4\n"

  890.             "smlal v4.4s, v9.4h, v0.h[3]\n"

  891.             "smlal v5.4s, v9.4h, v1.h[3]\n"

  892.             "sshll v8.8h, v8.8b, #0\n"

  893.             "smlal v6.4s, v9.4h, v2.h[3]\n"

  894.             "smlal v7.4s, v9.4h, v3.h[3]\n"

  895. 

  896.             "ld1 {v9.s}[0], [%[b_ptr]], 4\n"

  897.             "smlal v4.4s, v8.4h, v0.h[4]\n"

  898.             "smlal v5.4s, v8.4h, v1.h[4]\n"

  899.             "sshll v9.8h, v9.8b, #0\n"

  900.             "smlal v6.4s, v8.4h, v2.h[4]\n"

  901.             "smlal v7.4s, v8.4h, v3.h[4]\n"

  902. 

  903.             "ld1 {v8.s}[0], [%[b_ptr]], 4\n"

  904.             "smlal v4.4s, v9.4h, v0.h[5]\n"

  905.             "smlal v5.4s, v9.4h, v1.h[5]\n"

  906.             "sshll v8.8h, v8.8b, #0\n"

  907.             "smlal v6.4s, v9.4h, v2.h[5]\n"

  908.             "smlal v7.4s, v9.4h, v3.h[5]\n"

  909. 

  910.             "ld1 {v9.s}[0], [%[b_ptr]], 4\n"

  911.             "smlal v4.4s, v8.4h, v0.h[6]\n"

  912.             "smlal v5.4s, v8.4h, v1.h[6]\n"

  913.             "sshll v9.8h, v9.8b, #0\n"

  914.             "smlal v6.4s, v8.4h, v2.h[6]\n"

  915.             "smlal v7.4s, v8.4h, v3.h[6]\n"

  916. 

  917.             "smlal v4.4s, v9.4h, v0.h[7]\n"

  918.             "smlal v5.4s, v9.4h, v1.h[7]\n"

  919.             "smlal v6.4s, v9.4h, v2.h[7]\n"

  920.             "smlal v7.4s, v9.4h, v3.h[7]\n"

  921. 

  922.             "subs %w[K], %w[K], #1\n"

  923.             "cbnz %w[K], 2b\n"

  924. 

  925.             "3:\n" STORE_C

  926.             : [a_ptr] "+r"(a_ptr), [b_ptr] "+r"(b_ptr),

  927.               [is_first_k] "+r"(is_first_k), [K] "+r"(K), [LDC] "+r"(LDC),

  928.               [outptr0] "+r"(outptr0), [outptr1] "=r"(outptr1),

  929.               [outptr2] "=r"(outptr2), [outptr3] "=r"(outptr3), [x0] "+r"(x0),

  930.               [x1] "+r"(x1), [m_remain] "+r"(m_remain),

  931.               [n_remain] "+r"(n_remain)

  932.             :

  933.             : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v11", "cc",

  934.               "memory");

  935. 

  936. #undef LOAD_LINE

  937. #undef LOAD_C

  938. #undef STORE_LINE

  939. #undef STORE_C

  940. }

  941. 

  942. static void gemm_s8_8x8_pack_A_n(int8_t* outptr, const int8_t* inptr, int ldin,

  943.                                  int y0, int ymax, int k0, int kmax) {

  944.     int8_t zerobuff[16];

  945.     std::memset(zerobuff, 0, sizeof(int8_t) * 16);

  946. 

  947.     int y = y0;

  948.     for (; y + 7 < ymax; y += 8) {

  949.         const int8_t* inptr0 = inptr + y * ldin + k0;

  950.         const int8_t* inptr1 = inptr0 + ldin;

  951.         const int8_t* inptr2 = inptr1 + ldin;

  952.         const int8_t* inptr3 = inptr2 + ldin;

  953.         const int8_t* inptr4 = inptr3 + ldin;

  954.         const int8_t* inptr5 = inptr4 + ldin;

  955.         const int8_t* inptr6 = inptr5 + ldin;

  956.         const int8_t* inptr7 = inptr6 + ldin;

  957. 

  958.         prefetch_2x(inptr0);

  959.         prefetch_2x(inptr1);

  960.         prefetch_2x(inptr2);

  961.         prefetch_2x(inptr3);

  962.         prefetch_2x(inptr4);

  963.         prefetch_2x(inptr5);

  964.         prefetch_2x(inptr6);

  965.         prefetch_2x(inptr7);

  966. 

  967.         int K = kmax - k0;

  968.         for (; K > 15; K -= 16) {

  969.             interleave_8x8_2_b(inptr0, inptr1, inptr2, inptr3, inptr4, inptr5,

  970.                                inptr6, inptr7, outptr);

  971.         }

  972. 

  973.         if (K > 0) {

  974.             interleave_8(inptr0, inptr1, inptr2, inptr3, inptr4, inptr5, inptr6,

  975.                          inptr7, outptr, 8, K);

  976.         }

  977.     }

  978. 

  979.     for (; y < ymax; y += 4) {

  980.         const int8_t* inptr0 = inptr + y * ldin + k0;

  981.         const int8_t* inptr1 = inptr0 + ldin;

  982.         const int8_t* inptr2 = inptr1 + ldin;

  983.         const int8_t* inptr3 = inptr2 + ldin;

  984. 

  985.         prefetch_2x(inptr0);

  986.         prefetch_2x(inptr1);

  987.         prefetch_2x(inptr2);

  988.         prefetch_2x(inptr3);

  989. 

  990.         int K = kmax - k0;

  991.         for (; K > 15; K -= 16) {

  992.             if (y + 3 >= ymax) {

  993.                 switch (y + 3 - ymax) {

  994.                     case 2:

  995.                         inptr1 = zerobuff; MEGDNN_FALLTHRU

  996.                     case 1:

  997.                         inptr2 = zerobuff; MEGDNN_FALLTHRU

  998.                     case 0:

  999.                         inptr3 = zerobuff;

  1000.                         break;

  1001.                     default:

  1002.                         megdnn_assert(0);

  1003.                 }

  1004.             }

  1005. 

  1006.             interleave_4x8_2_b(inptr0, inptr1, inptr2, inptr3, outptr);

  1007.         }

  1008. 

  1009.         if (K > 0) {

  1010.             if (y + 3 >= ymax) {

  1011.                 switch (y + 3 - ymax) {

  1012.                     case 2:

  1013.                         inptr1 = zerobuff; MEGDNN_FALLTHRU

  1014.                     case 1:

  1015.                         inptr2 = zerobuff; MEGDNN_FALLTHRU

  1016.                     case 0:

  1017.                         inptr3 = zerobuff;

  1018.                         break;

  1019.                     default:

  1020.                         megdnn_assert(0);

  1021.                 }

  1022.             }

  1023.             interleave_4(inptr0, inptr1, inptr2, inptr3, outptr, 8, K);

  1024.         }

  1025.     }

  1026. }

  1027. 

  1028. static void gemm_s8_8x8_transpose_pack_A_n(int8_t* out, const int8_t* in,

  1029.                                            int ldin, int x0, int xmax, int k0,

  1030.                                            int kmax) {

  1031.     int8_t zerobuff[16];

  1032.     std::memset(zerobuff, 0, sizeof(int8_t) * 16);

  1033.     const int ksize = kmax - k0;

  1034.     const int ksize4 = round_up(ksize, 8) * 4;

  1035.     const int ksize8 = ksize4 * 2;

  1036.     int8_t* outptr = out;

  1037.     int8_t* outptr_base = out;

  1038.     //! 4x4 block output start pos

  1039.     int8_t* outptr_base4 = out + ((xmax - x0) / 8) * ksize8;

  1040. 

  1041.     int k = k0;

  1042.     for (; k < kmax; k += 8) {

  1043.         const int8_t* inptr0 = in + k * ldin + x0;

  1044.         const int8_t* inptr1 = inptr0 + ldin;

  1045.         const int8_t* inptr2 = inptr1 + ldin;

  1046.         const int8_t* inptr3 = inptr2 + ldin;

  1047.         const int8_t* inptr4 = inptr3 + ldin;

  1048.         const int8_t* inptr5 = inptr4 + ldin;

  1049.         const int8_t* inptr6 = inptr5 + ldin;

  1050.         const int8_t* inptr7 = inptr6 + ldin;

  1051.         prefetch_2x(inptr0);

  1052.         prefetch_2x(inptr1);

  1053.         prefetch_2x(inptr2);

  1054.         prefetch_2x(inptr3);

  1055.         prefetch_2x(inptr4);

  1056.         prefetch_2x(inptr5);

  1057.         prefetch_2x(inptr6);

  1058.         prefetch_2x(inptr7);

  1059. 

  1060.         int x = x0;

  1061.         outptr = outptr_base;

  1062. 

  1063.         for (; x + 7 < xmax; x += 8) {

  1064.             if (k + 7 >= kmax) {

  1065.                 switch (k + 7 - kmax) {

  1066.                     case 6:

  1067.                         inptr1 = zerobuff; MEGDNN_FALLTHRU

  1068.                     case 5:

  1069.                         inptr2 = zerobuff; MEGDNN_FALLTHRU

  1070.                     case 4:

  1071.                         inptr3 = zerobuff; MEGDNN_FALLTHRU

  1072.                     case 3:

  1073.                         inptr4 = zerobuff; MEGDNN_FALLTHRU

  1074.                     case 2:

  1075.                         inptr5 = zerobuff; MEGDNN_FALLTHRU

  1076.                     case 1:

  1077.                         inptr6 = zerobuff; MEGDNN_FALLTHRU

  1078.                     case 0:

  1079.                         inptr7 = zerobuff;

  1080.                         break;

  1081.                     default:

  1082.                         megdnn_assert(0);

  1083.                 }

  1084.             }

  1085.             transpose_8x8_1_b(inptr0, inptr1, inptr2, inptr3, inptr4, inptr5,

  1086.                               inptr6, inptr7, outptr);

  1087.             outptr += ksize8;

  1088.         }

  1089. 

  1090.         outptr = outptr_base4;

  1091.         for (; x + 3 < xmax; x += 4) {

  1092.             if (k + 7 >= kmax) {

  1093.                 switch (k + 7 - kmax) {

  1094.                     case 6:

  1095.                         inptr1 = zerobuff; MEGDNN_FALLTHRU

  1096.                     case 5:

  1097.                         inptr2 = zerobuff; MEGDNN_FALLTHRU

  1098.                     case 4:

  1099.                         inptr3 = zerobuff; MEGDNN_FALLTHRU

  1100.                     case 3:

  1101.                         inptr4 = zerobuff; MEGDNN_FALLTHRU

  1102.                     case 2:

  1103.                         inptr5 = zerobuff; MEGDNN_FALLTHRU

  1104.                     case 1:

  1105.                         inptr6 = zerobuff; MEGDNN_FALLTHRU

  1106.                     case 0:

  1107.                         inptr7 = zerobuff;

  1108.                         break;

  1109.                     default:

  1110.                         megdnn_assert(0);

  1111.                 }

  1112.             }

  1113. 

  1114.             transpose_8(inptr0, inptr1, inptr2, inptr3, inptr4, inptr5, inptr6,

  1115.                         inptr7, outptr, 4, 4);

  1116.             outptr += ksize4;

  1117.         }

  1118. 

  1119.         if (x < xmax) {

  1120.             if (k + 7 >= kmax) {

  1121.                 switch (k + 7 - kmax) {

  1122.                     case 6:

  1123.                         inptr1 = zerobuff; MEGDNN_FALLTHRU

  1124.                     case 5:

  1125.                         inptr2 = zerobuff; MEGDNN_FALLTHRU

  1126.                     case 4:

  1127.                         inptr3 = zerobuff; MEGDNN_FALLTHRU

  1128.                     case 3:

  1129.                         inptr4 = zerobuff; MEGDNN_FALLTHRU

  1130.                     case 2:

  1131.                         inptr5 = zerobuff; MEGDNN_FALLTHRU

  1132.                     case 1:

  1133.                         inptr6 = zerobuff; MEGDNN_FALLTHRU

  1134.                     case 0:

  1135.                         inptr7 = zerobuff;

  1136.                         break;

  1137.                     default:

  1138.                         megdnn_assert(0);

  1139.                 }

  1140.             }

  1141. 

  1142.             transpose_8(inptr0, inptr1, inptr2, inptr3, inptr4, inptr5, inptr6,

  1143.                         inptr7, outptr, 4, xmax - x);

  1144.         }

  1145. 

  1146.         outptr_base += 8 * 8;

  1147.         outptr_base4 += 4 * 8;

  1148.     }

  1149. }

  1150. 

  1151. static void gemm_s8_8x8_pack_B_n(int8_t* out, const int8_t* in, int ldin,

  1152.                                  int x0, int xmax, int k0, int kmax) {

  1153.     int8_t zerobuff[16];

  1154.     std::memset(zerobuff, 0, sizeof(int8_t) * 16);

  1155.     const int ksize = kmax - k0;

  1156.     const int ksize4 = round_up(ksize, 8) * 4;

  1157.     const int ksize8 = ksize4 * 2;

  1158.     int8_t* outptr = out;

  1159.     int8_t* outptr_base = out;

  1160.     int8_t* outptr_interleave = nullptr;

  1161.     //! 4x4 block output start pos

  1162.     int8_t* outptr_base4 = out + ((xmax - x0) / 8) * ksize8;

  1163. 

  1164.     int k = k0;

  1165.     for (; k < kmax; k += 8) {

  1166.         const int8_t* inptr0 = in + k * ldin + x0;

  1167.         const int8_t* inptr1 = inptr0 + ldin;

  1168.         const int8_t* inptr2 = inptr1 + ldin;

  1169.         const int8_t* inptr3 = inptr2 + ldin;

  1170.         const int8_t* inptr4 = inptr3 + ldin;

  1171.         const int8_t* inptr5 = inptr4 + ldin;

  1172.         const int8_t* inptr6 = inptr5 + ldin;

  1173.         const int8_t* inptr7 = inptr6 + ldin;

  1174.         prefetch_2x(inptr0);

  1175.         prefetch_2x(inptr1);

  1176.         prefetch_2x(inptr2);

  1177.         prefetch_2x(inptr3);

  1178.         prefetch_2x(inptr4);

  1179.         prefetch_2x(inptr5);

  1180.         prefetch_2x(inptr6);

  1181.         prefetch_2x(inptr7);

  1182. 

  1183.         int x = x0;

  1184.         outptr = outptr_base;

  1185. 

  1186.         for (; x + 7 < xmax; x += 8) {

  1187.             if (k + 7 >= kmax) {

  1188.                 switch (k + 7 - kmax) {

  1189.                     case 6:

  1190.                         inptr1 = zerobuff; MEGDNN_FALLTHRU

  1191.                     case 5:

  1192.                         inptr2 = zerobuff; MEGDNN_FALLTHRU

  1193.                     case 4:

  1194.                         inptr3 = zerobuff; MEGDNN_FALLTHRU

  1195.                     case 3:

  1196.                         inptr4 = zerobuff; MEGDNN_FALLTHRU

  1197.                     case 2:

  1198.                         inptr5 = zerobuff; MEGDNN_FALLTHRU

  1199.                     case 1:

  1200.                         inptr6 = zerobuff; MEGDNN_FALLTHRU

  1201.                     case 0:

  1202.                         inptr7 = zerobuff;

  1203.                         break;

  1204.                     default:

  1205.                         megdnn_assert(0);

  1206.                 }

  1207.             }

  1208.             outptr_interleave = outptr;

  1209.             interleave_8x8_1_b(inptr0, inptr1, inptr2, inptr3, inptr4, inptr5,

  1210.                                inptr6, inptr7, outptr_interleave);

  1211.             outptr += ksize8;

  1212.         }

  1213. 

  1214.         outptr = outptr_base4;

  1215.         for (; x + 3 < xmax; x += 4) {

  1216.             if (k + 7 >= kmax) {

  1217.                 switch (k + 7 - kmax) {

  1218.                     case 6:

  1219.                         inptr1 = zerobuff; MEGDNN_FALLTHRU

  1220.                     case 5:

  1221.                         inptr2 = zerobuff; MEGDNN_FALLTHRU

  1222.                     case 4:

  1223.                         inptr3 = zerobuff; MEGDNN_FALLTHRU

  1224.                     case 3:

  1225.                         inptr4 = zerobuff; MEGDNN_FALLTHRU

  1226.                     case 2:

  1227.                         inptr5 = zerobuff; MEGDNN_FALLTHRU

  1228.                     case 1:

  1229.                         inptr6 = zerobuff; MEGDNN_FALLTHRU

  1230.                     case 0:

  1231.                         inptr7 = zerobuff;

  1232.                         break;

  1233.                     default:

  1234.                         megdnn_assert(0);

  1235.                 }

  1236.             }

  1237. 

  1238.             outptr_interleave = outptr;

  1239.             interleave_8(inptr0, inptr1, inptr2, inptr3, inptr4, inptr5, inptr6,

  1240.                          inptr7, outptr_interleave, 4, 4);

  1241.             outptr += ksize4;

  1242.         }

  1243. 

  1244.         if (x < xmax) {

  1245.             if (k + 7 >= kmax) {

  1246.                 switch (k + 7 - kmax) {

  1247.                     case 6:

  1248.                         inptr1 = zerobuff; MEGDNN_FALLTHRU

  1249.                     case 5:

  1250.                         inptr2 = zerobuff; MEGDNN_FALLTHRU

  1251.                     case 4:

  1252.                         inptr3 = zerobuff; MEGDNN_FALLTHRU

  1253.                     case 3:

  1254.                         inptr4 = zerobuff; MEGDNN_FALLTHRU

  1255.                     case 2:

  1256.                         inptr5 = zerobuff; MEGDNN_FALLTHRU

  1257.                     case 1:

  1258.                         inptr6 = zerobuff; MEGDNN_FALLTHRU

  1259.                     case 0:

  1260.                         inptr7 = zerobuff;

  1261.                         break;

  1262.                     default:

  1263.                         megdnn_assert(0);

  1264.                 }

  1265.             }

  1266. 

  1267.             outptr_interleave = outptr;

  1268.             interleave_8(inptr0, inptr1, inptr2, inptr3, inptr4, inptr5, inptr6,

  1269.                          inptr7, outptr_interleave, 4, xmax - x);

  1270.         }

  1271. 

  1272.         outptr_base += 8 * 8;

  1273.         outptr_base4 += 4 * 8;

  1274.     }

  1275. }

  1276. 

  1277. static void gemm_s8_8x8_transpose_pack_B_n(int8_t* outptr, const int8_t* inptr,

  1278.                                            int ldin, int y0, int ymax, int k0,

  1279.                                            int kmax) {

  1280.     int8_t zerobuff[16];

  1281.     std::memset(zerobuff, 0, sizeof(int8_t) * 16);

  1282.     constexpr int interleave4 = 32;

  1283.     constexpr int interleave8 = 64;

  1284. 

  1285.     int y = y0;

  1286.     for (; y + 7 < ymax; y += 8) {

  1287.         const int8_t* inptr0 = inptr + y * ldin + k0;

  1288.         const int8_t* inptr1 = inptr0 + ldin;

  1289.         const int8_t* inptr2 = inptr1 + ldin;

  1290.         const int8_t* inptr3 = inptr2 + ldin;

  1291.         const int8_t* inptr4 = inptr3 + ldin;

  1292.         const int8_t* inptr5 = inptr4 + ldin;

  1293.         const int8_t* inptr6 = inptr5 + ldin;

  1294.         const int8_t* inptr7 = inptr6 + ldin;

  1295. 

  1296.         prefetch_2x(inptr0);

  1297.         prefetch_2x(inptr1);

  1298.         prefetch_2x(inptr2);

  1299.         prefetch_2x(inptr3);

  1300.         prefetch_2x(inptr4);

  1301.         prefetch_2x(inptr5);

  1302.         prefetch_2x(inptr6);

  1303.         prefetch_2x(inptr7);

  1304. 

  1305.         int K = kmax - k0;

  1306.         for (; K > 7; K -= 8) {

  1307.             transpose_8x8_1_b(inptr0, inptr1, inptr2, inptr3, inptr4, inptr5,

  1308.                               inptr6, inptr7, outptr);

  1309.             outptr += interleave8;

  1310.         }

  1311. 

  1312.         if (K > 0) {

  1313.             transpose_8(inptr0, inptr1, inptr2, inptr3, inptr4, inptr5, inptr6,

  1314.                         inptr7, outptr, 8, K);

  1315.             outptr += interleave8;

  1316.         }

  1317.     }

  1318. 

  1319.     for (; y < ymax; y += 4) {

  1320.         const int8_t* inptr0 = inptr + y * ldin + k0;

  1321.         const int8_t* inptr1 = inptr0 + ldin;

  1322.         const int8_t* inptr2 = inptr1 + ldin;

  1323.         const int8_t* inptr3 = inptr2 + ldin;

  1324. 

  1325.         prefetch_2x(inptr0);

  1326.         prefetch_2x(inptr1);

  1327.         prefetch_2x(inptr2);

  1328.         prefetch_2x(inptr3);

  1329. 

  1330.         int K = kmax - k0;

  1331.         for (; K > 7; K -= 8) {

  1332.             if (y + 3 >= ymax) {

  1333.                 switch (y + 3 - ymax) {

  1334.                     case 2:

  1335.                         inptr1 = zerobuff; MEGDNN_FALLTHRU

  1336.                     case 1:

  1337.                         inptr2 = zerobuff; MEGDNN_FALLTHRU

  1338.                     case 0:

  1339.                         inptr3 = zerobuff;

  1340.                         break;

  1341.                     default:

  1342.                         megdnn_assert(0);

  1343.                 }

  1344.             }

  1345. 

  1346.             transpose_8x4_1_b(inptr0, inptr1, inptr2, inptr3, outptr);

  1347.             outptr += interleave4;

  1348.         }

  1349. 

  1350.         if (K > 0) {

  1351.             if (y + 3 >= ymax) {

  1352.                 switch (y + 3 - ymax) {

  1353.                     case 2:

  1354.                         inptr1 = zerobuff; MEGDNN_FALLTHRU

  1355.                     case 1:

  1356.                         inptr2 = zerobuff; MEGDNN_FALLTHRU

  1357.                     case 0:

  1358.                         inptr3 = zerobuff;

  1359.                         break;

  1360.                     default:

  1361.                         megdnn_assert(0);

  1362.                 }

  1363.             }

  1364.             transpose_4(inptr0, inptr1, inptr2, inptr3, outptr, 8, K);

  1365.             outptr += interleave4;

  1366.         }

  1367.     }

  1368. }

  1369. 

  1370. }  // namespace matmul_8x8x8

  1371. }  // namespace aarch64

  1372. }  // namespace megdnn

  1373. 

  1374. // vim: syntax=cpp.doxygen

  1375. #endif