OpenI
/
MegEngine
Not watched
1
0
Fork 0
Code
Releases 31 Wiki Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 69fe5ab3b3
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '69fe5ab3b3'
${ noResults }
MegEngine/dnn/test/cuda/conv_bias_int8.cpp

1245 lines
55 kB
Raw Blame History 

  1. /**

  2.  * \file dnn/test/cuda/conv_bias_int8.cpp

  3.  * MegEngine is Licensed under the Apache License, Version 2.0 (the "License")

  4.  *

  5.  * Copyright (c) 2014-2020 Megvii Inc. All rights reserved.

  6.  *

  7.  * Unless required by applicable law or agreed to in writing,

  8.  * software distributed under the License is distributed on an

  9.  * "AS IS" BASIS, WITHOUT ARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

  10.  */

  11. #include "megdnn/oprs/nn.h"

  12. 

  13. #include "src/common/utils.h"

  14. #include "src/cuda/cudnn_with_check.h"

  15. #include "test/common/checker.h"

  16. #include "test/common/conv_bias.h"

  17. #include "test/cuda/benchmark.h"

  18. #include "test/cuda/fixture.h"

  19. #include "test/cuda/utils.h"

  20. 

  21. #define V1(x) #x

  22. #define V(x) V1(x)

  23. 

  24. namespace megdnn {

  25. namespace test {

  26. namespace {

  27. 

  28. #if MEGDNN_WITH_BENCHMARK

  29. struct BenchArgs {

  30.     size_t n, ci, hi, wi, co, f, s;

  31. };

  32. 

  33. std::vector<BenchArgs> get_resnet50_bench_args(size_t batch = 64) {

  34.     std::vector<BenchArgs> args;

  35.     args.emplace_back(BenchArgs{batch, 64, 56, 56, 256, 1, 1});

  36. 

  37.     args.emplace_back(BenchArgs{batch, 256, 56, 56, 32, 3, 1});

  38.     args.emplace_back(BenchArgs{batch, 256, 56, 56, 32, 3, 2});

  39.     args.emplace_back(BenchArgs{batch, 4, 256, 256, 32, 7, 2});

  40.  

  41.     args.emplace_back(BenchArgs{batch, 256, 56, 56, 64, 1, 1});

  42.     args.emplace_back(BenchArgs{batch, 64, 56, 56, 64, 1, 1});

  43.     args.emplace_back(BenchArgs{batch, 64, 56, 56, 64, 3, 1});

  44.     args.emplace_back(BenchArgs{batch, 64, 56, 56, 64, 3, 2});

  45.     args.emplace_back(BenchArgs{batch, 256, 56, 56, 64, 3, 2});

  46.     args.emplace_back(BenchArgs{batch, 64, 56, 56, 256, 1, 1});

  47. 

  48.     args.emplace_back(BenchArgs{batch, 256, 56, 56, 512, 1, 2});

  49.     args.emplace_back(BenchArgs{batch, 256, 56, 56, 128, 1, 2});

  50.     args.emplace_back(BenchArgs{batch, 512, 28, 28, 128, 1, 1});

  51.     args.emplace_back(BenchArgs{batch, 128, 28, 28, 128, 3, 1});

  52.     args.emplace_back(BenchArgs{batch, 128, 28, 28, 512, 1, 1});

  53. 

  54.     args.emplace_back(BenchArgs{batch, 512, 28, 28, 1024, 1, 2});

  55.     args.emplace_back(BenchArgs{batch, 512, 28, 28, 256, 1, 2});

  56.     args.emplace_back(BenchArgs{batch, 1024, 14, 14, 256, 1, 1});

  57.     args.emplace_back(BenchArgs{batch, 256, 14, 14, 256, 3, 1});

  58.     args.emplace_back(BenchArgs{batch, 256, 14, 14, 1024, 1, 1});

  59. 

  60.     args.emplace_back(BenchArgs{batch, 1024, 14, 14, 2048, 1, 2});

  61.     args.emplace_back(BenchArgs{batch, 1024, 14, 14, 512, 1, 2});

  62.     args.emplace_back(BenchArgs{batch, 2048, 7, 7, 512, 1, 1});

  63.     args.emplace_back(BenchArgs{batch, 512, 7, 7, 512, 3, 1});

  64.     args.emplace_back(BenchArgs{batch, 512, 7, 7, 2048, 1, 1});

  65.     return args;

  66. }

  67. 

  68. std::vector<BenchArgs> get_detection_bench_args(size_t batch = 16) {

  69.     std::vector<BenchArgs> args;

  70.     args.emplace_back(BenchArgs{batch, 4, 736, 1280, 8, 3, 2});

  71.     args.emplace_back(BenchArgs{batch, 32, 184, 320, 16, 3, 1});

  72.     args.emplace_back(BenchArgs{batch, 16, 184, 320, 32, 3, 1});

  73.     args.emplace_back(BenchArgs{batch, 8, 184, 320, 16, 3, 1});

  74.     args.emplace_back(BenchArgs{batch, 8, 184, 320, 32, 3, 1});

  75.     args.emplace_back(BenchArgs{batch, 64, 92, 160, 32, 3, 1});

  76.     args.emplace_back(BenchArgs{batch, 32, 184, 320, 64, 3, 2});

  77.     args.emplace_back(BenchArgs{batch, 32, 184, 320, 32, 3, 2});

  78.     args.emplace_back(BenchArgs{batch, 32, 92, 160, 64, 3, 1});

  79.     args.emplace_back(BenchArgs{batch, 64, 92, 160, 8, 3, 1});

  80.     args.emplace_back(BenchArgs{batch, 64, 92, 160, 128, 3, 2});

  81.     args.emplace_back(BenchArgs{batch, 128, 46, 80, 32, 3, 1});

  82.     args.emplace_back(BenchArgs{batch, 128, 46, 80, 256, 3, 2});

  83.     args.emplace_back(BenchArgs{batch, 128, 46, 80, 8, 3, 1});

  84.     args.emplace_back(BenchArgs{batch, 64, 92, 160, 32, 3, 2});

  85.     args.emplace_back(BenchArgs{batch, 32, 46, 80, 128, 3, 1});

  86.     args.emplace_back(BenchArgs{batch, 8, 46, 80, 32, 3, 1});

  87.     args.emplace_back(BenchArgs{batch, 64, 23, 40, 256, 3, 1});

  88.     args.emplace_back(BenchArgs{batch, 256, 23, 40, 64, 3, 1});

  89.     args.emplace_back(BenchArgs{batch, 128, 46, 80, 64, 3, 2});

  90.     args.emplace_back(BenchArgs{batch, 256, 23, 40, 8, 3, 1});

  91.     args.emplace_back(BenchArgs{batch, 8, 23, 40, 32, 3, 2});

  92.     args.emplace_back(BenchArgs{batch, 8, 12, 20, 8, 3, 1});

  93.     args.emplace_back(BenchArgs{batch, 8, 12, 20, 8, 3, 2});

  94.     args.emplace_back(BenchArgs{batch, 8, 6, 10, 8, 3, 1});

  95.     return args;

  96. }

  97. 

  98. void benchmark_target_algo(

  99.         Handle* handle, const std::vector<BenchArgs>& args, DType src_dtype,

  100.         DType filter_dtype, DType bias_dtype, DType dst_dtype,

  101.         const char* algo = nullptr,

  102.         param::ConvBias::Format format = param::ConvBias::Format::NCHW4) {

  103.     megdnn_assert(src_dtype.enumv() == filter_dtype.enumv());

  104.     CUBenchmarker<ConvBiasForward> benchmarker(handle);

  105.     CUBenchmarker<ConvBiasForward> benchmarker_cudnn(handle);

  106.     size_t RUNS = 1000;

  107.     benchmarker.set_display(false).set_times(RUNS);

  108.     benchmarker_cudnn.set_display(false).set_times(RUNS);

  109. 

  110.     if (algo) {

  111.         benchmarker.set_before_exec_callback(

  112.                 conv_bias::ConvBiasAlgoChecker<ConvBiasForward>(algo));

  113.     }

  114. 

  115. #define CUDNN_VERSION_STRING \

  116.     "v" V(CUDNN_MAJOR) "." V(CUDNN_MINOR) "." V(CUDNN_PATCHLEVEL)

  117.     benchmarker_cudnn.set_before_exec_callback(

  118.             conv_bias::ConvBiasAlgoChecker<ConvBiasForward>(

  119.                     "DEFAULT:CUDNN:ConvBiasActivation:CUDNN_CONVOLUTION_FWD_"

  120.                     "ALGO_IMPLICIT_PRECOMP_"

  121.                     "GEMM" CUDNN_VERSION_STRING));

  122. 

  123.     benchmarker.set_dtype(0, src_dtype)

  124.             .set_dtype(1, filter_dtype)

  125.             .set_dtype(2, bias_dtype)

  126.             .set_dtype(3, dst_dtype)

  127.             .set_dtype(4, dst_dtype);

  128.     benchmarker_cudnn.set_dtype(0, src_dtype)

  129.             .set_dtype(1, filter_dtype)

  130.             .set_dtype(2, bias_dtype)

  131.             .set_dtype(3, dst_dtype)

  132.             .set_dtype(4, dst_dtype);

  133. 

  134.     using Param = ConvBias::Param;

  135.     using Format = Param::Format;

  136.     if (format == Format::NCHW4) {

  137.         for (auto&& arg : args) {

  138.             Param param;

  139.             param.pad_h = param.pad_w = arg.f / 2;

  140.             param.stride_h = param.stride_w = arg.s;

  141.             param.format = Format::NCHW4;

  142. 

  143.             size_t ho = infer_conv_shape(arg.hi, arg.f, arg.s, arg.f / 2);

  144.             size_t wo = infer_conv_shape(arg.wi, arg.f, arg.s, arg.f / 2);

  145. 

  146.             benchmarker.set_param(param);

  147.             auto time_in_ms =

  148.                     benchmarker.execs({{arg.n, arg.ci / 4, arg.hi, arg.wi, 4},

  149.                                        {arg.co, arg.ci / 4, arg.f, arg.f, 4},

  150.                                        {1, arg.co / 4, 1, 1, 4},

  151.                                        {},

  152.                                        {}}) /

  153.                     RUNS;

  154.             param.nonlineMode = Param::NonlineMode::IDENTITY;

  155.             benchmarker_cudnn.set_param(param);

  156.             auto time_in_ms_cudnn =

  157.                     benchmarker_cudnn.execs(

  158.                             {{arg.n, arg.ci / 4, arg.hi, arg.wi, 4},

  159.                              {arg.co, arg.ci / 4, arg.f, arg.f, 4},

  160.                              {1, arg.co / 4, 1, 1, 4},

  161.                              {},

  162.                              {}}) /

  163.                     RUNS;

  164.             float flo = 2.0 * arg.n * arg.co * ho * wo * arg.ci * arg.f *

  165.                         arg.f / (1e12);

  166.             TensorShape src{arg.n, arg.ci, arg.hi, arg.wi},

  167.                     filter{arg.co, arg.ci, arg.f, arg.f};

  168.             printf("src=%s, filter=%s, time(algo=%s)=%.2f %.2fTops, "

  169.                    "time(cudnn)=%.2f %.2fTops, "

  170.                    "perf(algo=%s)/perf(cudnn)=%.2f\n",

  171.                    src.to_string().c_str(), filter.to_string().c_str(), algo,

  172.                    time_in_ms, (flo / (time_in_ms * 1e-3)), time_in_ms_cudnn,

  173.                    (flo / (time_in_ms_cudnn * 1e-3)), algo,

  174.                    time_in_ms_cudnn / time_in_ms);

  175.         }

  176.         printf("bench with z tensor\n");

  177.         for (auto&& arg : args) {

  178.             Param param;

  179.             param.pad_h = param.pad_w = arg.f / 2;

  180.             param.stride_h = param.stride_w = arg.s;

  181.             param.format = Format::NCHW4;

  182. 

  183.             size_t ho = infer_conv_shape(arg.hi, arg.f, arg.s, arg.f / 2);

  184.             size_t wo = infer_conv_shape(arg.wi, arg.f, arg.s, arg.f / 2);

  185. 

  186.             benchmarker.set_param(param);

  187.             auto time_in_ms =

  188.                     benchmarker.execs({{arg.n, arg.ci / 4, arg.hi, arg.wi, 4},

  189.                                        {arg.co, arg.ci / 4, arg.f, arg.f, 4},

  190.                                        {1, arg.co / 4, 1, 1, 4},

  191.                                        {arg.n, arg.co / 4, ho, wo, 4},

  192.                                        {}}) /

  193.                     RUNS;

  194.             param.format = Format::NCHW4;

  195.             param.nonlineMode = Param::NonlineMode::IDENTITY;

  196.             benchmarker_cudnn.set_param(param);

  197.             auto time_in_ms_cudnn =

  198.                     benchmarker_cudnn.execs(

  199.                             {{arg.n, arg.ci / 4, arg.hi, arg.wi, 4},

  200.                              {arg.co, arg.ci / 4, arg.f, arg.f, 4},

  201.                              {1, arg.co / 4, 1, 1, 4},

  202.                              {arg.n, arg.co / 4, ho, wo, 4},

  203.                              {}}) /

  204.                     RUNS;

  205.             float flo = 2.0 * arg.n * arg.co * ho * wo * arg.ci * arg.f *

  206.                         arg.f / (1e12);

  207.             TensorShape src{arg.n, arg.ci, arg.hi, arg.wi},

  208.                     filter{arg.co, arg.ci, arg.f, arg.f};

  209.             printf("src=%s, filter=%s, time(algo=%s)=%.2f %.2fTops, "

  210.                    "time(cudnn)=%.2f %.2fTops, "

  211.                    "perf(algo=%s)/perf(cudnn)=%.2f\n",

  212.                    src.to_string().c_str(), filter.to_string().c_str(), algo,

  213.                    time_in_ms, (flo / (time_in_ms * 1e-3)), time_in_ms_cudnn,

  214.                    (flo / (time_in_ms_cudnn * 1e-3)), algo,

  215.                    time_in_ms_cudnn / time_in_ms);

  216.         }

  217.     } else if (format == Format::CHWN4) {

  218.         for (auto&& arg : args) {

  219.             Param param;

  220.             param.pad_h = param.pad_w = arg.f / 2;

  221.             param.stride_h = param.stride_w = arg.s;

  222.             param.format = Format::CHWN4;

  223. 

  224.             size_t ho = infer_conv_shape(arg.hi, arg.f, arg.s, arg.f / 2);

  225.             size_t wo = infer_conv_shape(arg.wi, arg.f, arg.s, arg.f / 2);

  226. 

  227.             benchmarker.set_param(param);

  228.             auto time_in_ms =

  229.                     benchmarker.execs({{arg.ci / 4, arg.hi, arg.wi, arg.n, 4},

  230.                                        {arg.ci / 4, arg.f, arg.f, arg.co, 4},

  231.                                        {arg.co / 4, 1, 1, 1, 4},

  232.                                        {},

  233.                                        {}}) /

  234.                     RUNS;

  235.             param.format = Format::NCHW4;

  236.             benchmarker_cudnn.set_param(param);

  237.             auto time_in_ms_cudnn =

  238.                     benchmarker_cudnn.execs(

  239.                             {{arg.n, arg.ci / 4, arg.hi, arg.wi, 4},

  240.                              {arg.co, arg.ci / 4, arg.f, arg.f, 4},

  241.                              {1, arg.co / 4, 1, 1, 4},

  242.                              {},

  243.                              {}}) /

  244.                     RUNS;

  245.             float flo = 2.0 * arg.n * arg.co * ho * wo * arg.ci * arg.f *

  246.                         arg.f / (1e12);

  247.             TensorShape src{arg.n, arg.ci, arg.hi, arg.wi},

  248.                     filter{arg.co, arg.ci, arg.f, arg.f};

  249.             printf("src=%s, filter=%s, time(algo=%s)=%.2f %.2fTops, "

  250.                    "time(cudnn)=%.2f %.2fTops, "

  251.                    "perf(algo=%s)/perf(cudnn)=%.2f\n",

  252.                    src.to_string().c_str(), filter.to_string().c_str(), algo,

  253.                    time_in_ms, (flo / (time_in_ms * 1e-3)), time_in_ms_cudnn,

  254.                    (flo / (time_in_ms_cudnn * 1e-3)), algo,

  255.                    time_in_ms_cudnn / time_in_ms);

  256.         }

  257.         printf("bench with z tensor\n");

  258.         for (auto&& arg : args) {

  259.             Param param;

  260.             param.pad_h = param.pad_w = arg.f / 2;

  261.             param.stride_h = param.stride_w = arg.s;

  262.             param.format = Format::CHWN4;

  263. 

  264.             size_t ho = infer_conv_shape(arg.hi, arg.f, arg.s, arg.f / 2);

  265.             size_t wo = infer_conv_shape(arg.wi, arg.f, arg.s, arg.f / 2);

  266. 

  267.             benchmarker.set_param(param);

  268.             auto time_in_ms =

  269.                     benchmarker.execs({{arg.ci / 4, arg.hi, arg.wi, arg.n, 4},

  270.                                        {arg.ci / 4, arg.f, arg.f, arg.co, 4},

  271.                                        {arg.co / 4, 1, 1, 1, 4},

  272.                                        {arg.co / 4, ho, wo, arg.n, 4},

  273.                                        {}}) /

  274.                     RUNS;

  275.             param.format = Format::NCHW4;

  276.             benchmarker_cudnn.set_param(param);

  277.             param.nonlineMode = Param::NonlineMode::IDENTITY;

  278.             auto time_in_ms_cudnn =

  279.                     benchmarker_cudnn.execs(

  280.                             {{arg.n, arg.ci / 4, arg.hi, arg.wi, 4},

  281.                              {arg.co, arg.ci / 4, arg.f, arg.f, 4},

  282.                              {1, arg.co / 4, 1, 1, 4},

  283.                              {arg.n, arg.co / 4, ho, wo, 4},

  284.                              {}}) /

  285.                     RUNS;

  286.             float flo = 2.0 * arg.n * arg.co * ho * wo * arg.ci * arg.f *

  287.                         arg.f / (1e12);

  288.             TensorShape src{arg.n, arg.ci, arg.hi, arg.wi},

  289.                     filter{arg.co, arg.ci, arg.f, arg.f};

  290.             printf("src=%s, filter=%s, time(algo=%s)=%.2f %.2fTops, "

  291.                    "time(cudnn)=%.2f %.2fTops, "

  292.                    "perf(algo=%s)/perf(cudnn)=%.2f\n",

  293.                    src.to_string().c_str(), filter.to_string().c_str(), algo,

  294.                    time_in_ms, (flo / (time_in_ms * 1e-3)), time_in_ms_cudnn,

  295.                    (flo / (time_in_ms_cudnn * 1e-3)), algo,

  296.                    time_in_ms_cudnn / time_in_ms);

  297.         }

  298.     }

  299. }

  300. 

  301. void benchmark_target_algo_with_cudnn_tsc(

  302.         Handle* handle, const std::vector<BenchArgs>& args, DType src_dtype,

  303.         DType filter_dtype, DType bias_dtype, DType dst_dtype,

  304.         const char* algo = nullptr,

  305.         param::ConvBias::Format format = param::ConvBias::Format::NCHW4) {

  306.     megdnn_assert(src_dtype.enumv() == filter_dtype.enumv());

  307.     CUBenchmarker<ConvBiasForward> benchmarker(handle);

  308.     CUBenchmarker<ConvBiasForward> benchmarker_cudnn(handle);

  309.     size_t RUNS = 1000;

  310.     benchmarker.set_display(false).set_times(RUNS);

  311.     benchmarker_cudnn.set_display(false).set_times(RUNS);

  312. 

  313.     std::unique_ptr<OprProxy<ConvBiasForward>> proxy{

  314.             new OprProxy<ConvBiasForward>{true}};

  315. 

  316.     if (algo) {

  317.         benchmarker.set_before_exec_callback(

  318.                 conv_bias::ConvBiasAlgoChecker<ConvBiasForward>(algo));

  319.     } else {

  320.         benchmarker.set_proxy(proxy);

  321.     }

  322. 

  323.     benchmarker_cudnn.set_before_exec_callback(

  324.             conv_bias::ConvBiasAlgoChecker<ConvBiasForward>(

  325.                     "DEFAULT:CUDNN:ConvBiasActivation:CUDNN_CONVOLUTION_FWD_"

  326.                     "ALGO_IMPLICIT_PRECOMP_"

  327.                     "GEMM" CUDNN_VERSION_STRING));

  328. #undef CUDNN_VERSION_STRING

  329. 

  330.     benchmarker.set_dtype(0, src_dtype)

  331.             .set_dtype(1, filter_dtype)

  332.             .set_dtype(2, bias_dtype)

  333.             .set_dtype(3, dst_dtype)

  334.             .set_dtype(4, dst_dtype);

  335.     benchmarker_cudnn.set_dtype(0, src_dtype)

  336.             .set_dtype(1, filter_dtype)

  337.             .set_dtype(2, bias_dtype)

  338.             .set_dtype(3, dst_dtype)

  339.             .set_dtype(4, dst_dtype);

  340. 

  341.     using Param = ConvBias::Param;

  342.     using Format = Param::Format;

  343.     if (format == Format::NCHW4) {

  344.         for (auto&& arg : args) {

  345.             Param param;

  346.             param.pad_h = param.pad_w = arg.f / 2;

  347.             param.stride_h = param.stride_w = arg.s;

  348.             param.format = Format::NCHW4;

  349. 

  350.             size_t ho = infer_conv_shape(arg.hi, arg.f, arg.s, arg.f / 2);

  351.             size_t wo = infer_conv_shape(arg.wi, arg.f, arg.s, arg.f / 2);

  352. 

  353.             benchmarker.set_param(param);

  354.             if (!algo) {

  355.                 benchmarker.proxy()->target_algo = nullptr;

  356.             }

  357.             auto time_in_ms =

  358.                     benchmarker.execs({{arg.n, arg.ci / 4, arg.hi, arg.wi, 4},

  359.                                        {arg.co, arg.ci / 4, arg.f, arg.f, 4},

  360.                                        {1, arg.co / 4, 1, 1, 4},

  361.                                        {},

  362.                                        {}}) /

  363.                     RUNS;

  364.             param.format = Format::NCHW32;

  365.             benchmarker_cudnn.set_param(param);

  366.             auto time_in_ms_cudnn =

  367.                     benchmarker_cudnn.execs(

  368.                             {{arg.n, arg.ci / 32, arg.hi, arg.wi, 32},

  369.                              {arg.co, arg.ci / 32, arg.f, arg.f, 32},

  370.                              {1, arg.co / 32, 1, 1, 32},

  371.                              {},

  372.                              {}}) /

  373.                     RUNS;

  374.             float flo = 2.0 * arg.n * arg.co * ho * wo * arg.ci * arg.f *

  375.                         arg.f / (1e12);

  376.             TensorShape src{arg.n, arg.ci, arg.hi, arg.wi},

  377.                     filter{arg.co, arg.ci, arg.f, arg.f};

  378.             printf("src=%s, filter=%s, time(algo=%s)=%.2f %.2fTops, "

  379.                    "time(cudnn)=%.2f %.2fTops, "

  380.                    "perf(algo=%s)/perf(cudnn)=%.2f\n",

  381.                    src.to_string().c_str(), filter.to_string().c_str(), algo,

  382.                    time_in_ms, (flo / (time_in_ms * 1e-3)), time_in_ms_cudnn,

  383.                    (flo / (time_in_ms_cudnn * 1e-3)), algo,

  384.                    time_in_ms_cudnn / time_in_ms);

  385.         }

  386.     } else if (format == Format::CHWN4) {

  387.         for (auto&& arg : args) {

  388.             Param param;

  389.             param.pad_h = param.pad_w = arg.f / 2;

  390.             param.stride_h = param.stride_w = arg.s;

  391.             param.format = Format::CHWN4;

  392. 

  393.             size_t ho = infer_conv_shape(arg.hi, arg.f, arg.s, arg.f / 2);

  394.             size_t wo = infer_conv_shape(arg.wi, arg.f, arg.s, arg.f / 2);

  395. 

  396.             benchmarker.set_param(param);

  397.             if (!algo) {

  398.                 benchmarker.proxy()->target_algo = nullptr;

  399.             }

  400.             auto time_in_ms =

  401.                     benchmarker.execs({{arg.ci / 4, arg.hi, arg.wi, arg.n, 4},

  402.                                        {arg.ci / 4, arg.f, arg.f, arg.co, 4},

  403.                                        {arg.co / 4, 1, 1, 1, 4},

  404.                                        {},

  405.                                        {}}) /

  406.                     RUNS;

  407.             float time_in_ms_cudnn = 0.f;

  408.             if (arg.ci % 32 == 0 && arg.co % 32 == 0) {

  409.                 param.format = Format::NCHW32;

  410.                 benchmarker_cudnn.set_param(param);

  411.                 time_in_ms_cudnn =

  412.                         benchmarker_cudnn.execs(

  413.                                 {{arg.n, arg.ci / 32, arg.hi, arg.wi, 32},

  414.                                  {arg.co, arg.ci / 32, arg.f, arg.f, 32},

  415.                                  {1, arg.co / 32, 1, 1, 32},

  416.                                  {},

  417.                                  {}}) /

  418.                         RUNS;

  419.             } else {

  420.                 param.format = Format::NCHW4;

  421.                 benchmarker_cudnn.set_param(param);

  422.                 time_in_ms_cudnn =

  423.                         benchmarker_cudnn.execs(

  424.                                 {{arg.n, arg.ci / 4, arg.hi, arg.wi, 4},

  425.                                  {arg.co, arg.ci / 4, arg.f, arg.f, 4},

  426.                                  {1, arg.co / 4, 1, 1, 4},

  427.                                  {},

  428.                                  {}}) /

  429.                         RUNS;

  430.             }

  431.             float flo = 2.0 * arg.n * arg.co * ho * wo * arg.ci * arg.f *

  432.                         arg.f / (1e12);

  433.             TensorShape src{arg.n, arg.ci, arg.hi, arg.wi},

  434.                     filter{arg.co, arg.ci, arg.f, arg.f};

  435.             printf("src=%s, filter=%s, time(algo=%s)=%.2f %.2fTops, "

  436.                    "time(cudnn)=%.2f %.2fTops, "

  437.                    "perf(algo=%s)/perf(cudnn)=%.2f\n",

  438.                    src.to_string().c_str(), filter.to_string().c_str(), algo,

  439.                    time_in_ms, (flo / (time_in_ms * 1e-3)), time_in_ms_cudnn,

  440.                    (flo / (time_in_ms_cudnn * 1e-3)), algo,

  441.                    time_in_ms_cudnn / time_in_ms);

  442.         }

  443.         printf("bench with z tensor\n");

  444.         for (auto&& arg : args) {

  445.             Param param;

  446.             param.pad_h = param.pad_w = arg.f / 2;

  447.             param.stride_h = param.stride_w = arg.s;

  448.             param.format = Format::CHWN4;

  449. 

  450.             size_t ho = infer_conv_shape(arg.hi, arg.f, arg.s, arg.f / 2);

  451.             size_t wo = infer_conv_shape(arg.wi, arg.f, arg.s, arg.f / 2);

  452. 

  453.             benchmarker.set_param(param);

  454.             if (!algo) {

  455.                 benchmarker.proxy()->target_algo = nullptr;

  456.             }

  457.             auto time_in_ms =

  458.                     benchmarker.execs({{arg.ci / 4, arg.hi, arg.wi, arg.n, 4},

  459.                                        {arg.ci / 4, arg.f, arg.f, arg.co, 4},

  460.                                        {arg.co / 4, 1, 1, 1, 4},

  461.                                        {arg.co / 4, ho, wo, arg.n, 4},

  462.                                        {}}) /

  463.                     RUNS;

  464.             float time_in_ms_cudnn = 0.f;

  465.             if (arg.ci % 32 == 0 && arg.co % 32 == 0) {

  466.                 param.format = Format::NCHW32;

  467.                 benchmarker_cudnn.set_param(param);

  468.                 time_in_ms_cudnn =

  469.                         benchmarker_cudnn.execs(

  470.                                 {{arg.n, arg.ci / 32, arg.hi, arg.wi, 32},

  471.                                  {arg.co, arg.ci / 32, arg.f, arg.f, 32},

  472.                                  {1, arg.co / 32, 1, 1, 32},

  473.                                  {arg.n, arg.co / 32, ho, wo, 32},

  474.                                  {}}) /

  475.                         RUNS;

  476.             } else {

  477.                 param.format = Format::NCHW4;

  478.                 benchmarker_cudnn.set_param(param);

  479.                 time_in_ms_cudnn =

  480.                         benchmarker_cudnn.execs(

  481.                                 {{arg.n, arg.ci / 4, arg.hi, arg.wi, 4},

  482.                                  {arg.co, arg.ci / 4, arg.f, arg.f, 4},

  483.                                  {1, arg.co / 4, 1, 1, 4},

  484.                                  {arg.n, arg.co / 4, ho, wo, 4},

  485.                                  {}}) /

  486.                         RUNS;

  487.             }

  488.             float flo = 2.0 * arg.n * arg.co * ho * wo * arg.ci * arg.f *

  489.                         arg.f / (1e12);

  490.             TensorShape src{arg.n, arg.ci, arg.hi, arg.wi},

  491.                     filter{arg.co, arg.ci, arg.f, arg.f};

  492.             printf("src=%s, filter=%s, time(algo=%s)=%.2f %.2fTops, "

  493.                    "time(cudnn)=%.2f %.2fTops, "

  494.                    "perf(algo=%s)/perf(cudnn)=%.2f\n",

  495.                    src.to_string().c_str(), filter.to_string().c_str(), algo,

  496.                    time_in_ms, (flo / (time_in_ms * 1e-3)), time_in_ms_cudnn,

  497.                    (flo / (time_in_ms_cudnn * 1e-3)), algo,

  498.                    time_in_ms_cudnn / time_in_ms);

  499.         }

  500.     }

  501. }

  502. #endif

  503. }  // namespace

  504. 

  505. TEST_F(CUDA, CONV_BIAS_INT8_NCHW4_1x1) {

  506.     require_compute_capability(6, 1);

  507.     conv_bias::check_conv_bias(

  508.             dtype::QuantizedS8{1.2f}, dtype::QuantizedS8{1.3f},

  509.             dtype::QuantizedS32{1.2f * 1.3f}, dtype::QuantizedS8{1.3f},

  510.             handle_cuda(), "INT8_NCHW4_DOTPROD_IMPLICIT_GEMM",

  511.             param::ConvBias::Format::NCHW4, conv_bias::get_int8_nchw4_args(1));

  512. }

  513. 

  514. TEST_F(CUDA, CONV_BIAS_INT8_NCHW4_3x3) {

  515.     require_compute_capability(6, 1);

  516.     conv_bias::check_conv_bias(

  517.             dtype::QuantizedS8{1.2f}, dtype::QuantizedS8{1.3f},

  518.             dtype::QuantizedS32{1.2f * 1.3f}, dtype::QuantizedS8{1.3f},

  519.             handle_cuda(), "INT8_NCHW4_DOTPROD_IMPLICIT_GEMM",

  520.             param::ConvBias::Format::NCHW4);

  521. }

  522. 

  523. TEST_F(CUDA, CONV_BIAS_INT8_NCHW4_5x5) {

  524.     require_compute_capability(6, 1);

  525.     conv_bias::check_conv_bias(

  526.             dtype::QuantizedS8{1.2f}, dtype::QuantizedS8{1.3f},

  527.             dtype::QuantizedS32{1.2f * 1.3f}, dtype::QuantizedS8{1.3f},

  528.             handle_cuda(), "INT8_NCHW4_DOTPROD_IMPLICIT_GEMM",

  529.             param::ConvBias::Format::NCHW4, conv_bias::get_int8_nchw4_args(5));

  530. }

  531. 

  532. TEST_F(CUDA, CONV_BIAS_INT8_NCHW4_7x7) {

  533.     require_compute_capability(6, 1);

  534.     conv_bias::check_conv_bias(

  535.             dtype::QuantizedS8{1.2f}, dtype::QuantizedS8{1.3f},

  536.             dtype::QuantizedS32{1.2f * 1.3f}, dtype::QuantizedS8{1.3f},

  537.             handle_cuda(), "INT8_NCHW4_DOTPROD_IMPLICIT_GEMM",

  538.             param::ConvBias::Format::NCHW4, conv_bias::get_int8_nchw4_args(7));

  539. }

  540. 

  541. TEST_F(CUDA, CONV_BIAS_INT8_NCHW4_WITH_Z) {

  542.     require_compute_capability(6, 1);

  543.     Checker<ConvBiasForward> checker(handle_cuda());

  544.     checker.set_before_exec_callback(

  545.             conv_bias::ConvBiasAlgoChecker<ConvBiasForward>(

  546.                     "INT8_NCHW4_DOTPROD_IMPLICIT_GEMM"));

  547.     UniformIntRNG rng{-3, 3};

  548.     UniformIntRNG bias_rng{-50, 50};

  549.     checker.set_rng(0, &rng)

  550.             .set_rng(1, &rng)

  551.             .set_rng(2, &bias_rng)

  552.             .set_rng(3, &rng)

  553.             .set_dtype(0, dtype::QuantizedS8{1.2f})

  554.             .set_dtype(1, dtype::QuantizedS8{1.3f})

  555.             .set_dtype(2, dtype::QuantizedS32{1.2f * 1.3f})

  556.             .set_dtype(3, dtype::QuantizedS8{1.1f})

  557.             .set_dtype(4, dtype::QuantizedS8{1.0f})

  558.             .set_epsilon(1 + 1e-3)

  559.             .set_max_avg_error(1e-1)

  560.             .set_max_avg_biased_error(1e-1);

  561.     param::ConvBias param;

  562.     param.pad_h = param.pad_w = 1;

  563.     param.stride_h = param.stride_w = 1;

  564.     param.format = param::ConvBias::Format::NCHW4;

  565.     checker.set_param(param).execs({{32, 4, 12, 12, 4},

  566.                                     {16, 4, 3, 3, 4},

  567.                                     {1, 4, 1, 1, 4},

  568.                                     {32, 4, 12, 12, 4},

  569.                                     {}});

  570. }

  571. 

  572. TEST_F(CUDA, CONV_BIAS_INT8_NCHW4_STRIDE2_WITH_Z) {

  573.     require_compute_capability(6, 1);

  574.     Checker<ConvBiasForward> checker(handle_cuda());

  575.     checker.set_before_exec_callback(

  576.             conv_bias::ConvBiasAlgoChecker<ConvBiasForward>(

  577.                     "INT8_NCHW4_DOTPROD_IMPLICIT_GEMM"));

  578.     UniformIntRNG rng{-3, 3};

  579.     UniformIntRNG bias_rng{-50, 50};

  580.     checker.set_rng(0, &rng)

  581.             .set_rng(1, &rng)

  582.             .set_rng(2, &bias_rng)

  583.             .set_rng(3, &rng)

  584.             .set_dtype(0, dtype::QuantizedS8{1.2f})

  585.             .set_dtype(1, dtype::QuantizedS8{1.3f})

  586.             .set_dtype(2, dtype::QuantizedS32{1.2f * 1.3f})

  587.             .set_dtype(3, dtype::QuantizedS8{1.1f})

  588.             .set_dtype(4, dtype::QuantizedS8{1.0f})

  589.             .set_epsilon(1 + 1e-3)

  590.             .set_max_avg_error(1e-1)

  591.             .set_max_avg_biased_error(1e-1);

  592.     param::ConvBias param;

  593.     param.pad_h = param.pad_w = 1;

  594.     param.stride_h = param.stride_w = 2;

  595.     param.format = param::ConvBias::Format::NCHW4;

  596.     checker.set_param(param).execs({{32, 4, 12, 12, 4},

  597.                                     {16, 4, 3, 3, 4},

  598.                                     {1, 4, 1, 1, 4},

  599.                                     {32, 4, 6, 6, 4},

  600.                                     {}});

  601. }

  602. 

  603. TEST_F(CUDA, CONV_BIAS_INT8_NCHW4_CHECK_BOUNDS_1x1) {

  604.     require_compute_capability(6, 1);

  605.     conv_bias::check_conv_bias(

  606.             dtype::QuantizedS8{1.2f}, dtype::QuantizedS8{1.3f},

  607.             dtype::QuantizedS32{1.2f * 1.3f}, dtype::QuantizedS8{1.3f},

  608.             handle_cuda(), "INT8_NCHW4_DOTPROD_IMPLICIT_GEMM",

  609.             param::ConvBias::Format::NCHW4,

  610.             conv_bias::get_int8_nchw4_args_check_bounds(1));

  611. }

  612. 

  613. TEST_F(CUDA, CONV_BIAS_INT8_NCHW4_CHECK_BOUNDS_3x3) {

  614.     require_compute_capability(6, 1);

  615.     conv_bias::check_conv_bias(

  616.             dtype::QuantizedS8{1.2f}, dtype::QuantizedS8{1.3f},

  617.             dtype::QuantizedS32{1.2f * 1.3f}, dtype::QuantizedS8{1.3f},

  618.             handle_cuda(), "INT8_NCHW4_DOTPROD_IMPLICIT_GEMM",

  619.             param::ConvBias::Format::NCHW4,

  620.             conv_bias::get_int8_nchw4_args_check_bounds(3));

  621. }

  622. 

  623. TEST_F(CUDA, CONV_BIAS_INT8_NCHW4_CHECK_BOUNDS_5x5) {

  624.     require_compute_capability(6, 1);

  625.     conv_bias::check_conv_bias(

  626.             dtype::QuantizedS8{1.2f}, dtype::QuantizedS8{1.3f},

  627.             dtype::QuantizedS32{1.2f * 1.3f}, dtype::QuantizedS8{1.3f},

  628.             handle_cuda(), "INT8_NCHW4_DOTPROD_IMPLICIT_GEMM",

  629.             param::ConvBias::Format::NCHW4,

  630.             conv_bias::get_int8_nchw4_args_check_bounds(5));

  631. }

  632. 

  633. TEST_F(CUDA, CONV_BIAS_INT8_NCHW4_CHECK_BOUNDS_7x7) {

  634.     require_compute_capability(6, 1);

  635.     conv_bias::check_conv_bias(

  636.             dtype::QuantizedS8{1.2f}, dtype::QuantizedS8{1.3f},

  637.             dtype::QuantizedS32{1.2f * 1.3f}, dtype::QuantizedS8{1.3f},

  638.             handle_cuda(), "INT8_NCHW4_DOTPROD_IMPLICIT_GEMM",

  639.             param::ConvBias::Format::NCHW4,

  640.             conv_bias::get_int8_nchw4_args_check_bounds(7));

  641. }

  642. 

  643. TEST_F(CUDA, CONV_BIAS_INT8_CHWN4) {

  644.     require_compute_capability(6, 1);

  645.     conv_bias::check_conv_bias(

  646.             dtype::QuantizedS8{1.2f}, dtype::QuantizedS8{1.3f},

  647.             dtype::QuantizedS32{1.2f * 1.3f}, dtype::QuantizedS8{1.3f},

  648.             handle_cuda(), "INT8_CHWN4_DOTPROD_IMPLICIT_GEMM",

  649.             param::ConvBias::Format::CHWN4);

  650. }

  651. 

  652. TEST_F(CUDA, CONV_BIAS_INT8_CHWN4_WITH_Z) {

  653.     require_compute_capability(6, 1);

  654.     Checker<ConvBiasForward> checker(handle_cuda());

  655.     checker.set_before_exec_callback(

  656.             conv_bias::ConvBiasAlgoChecker<ConvBiasForward>(

  657.                     "INT8_CHWN4_DOTPROD_IMPLICIT_GEMM"));

  658.     UniformIntRNG rng{-3, 3};

  659.     UniformIntRNG bias_rng{-50, 50};

  660.     checker.set_rng(0, &rng)

  661.             .set_rng(1, &rng)

  662.             .set_rng(2, &bias_rng)

  663.             .set_rng(3, &rng)

  664.             .set_dtype(0, dtype::QuantizedS8{1.2f})

  665.             .set_dtype(1, dtype::QuantizedS8{1.3f})

  666.             .set_dtype(2, dtype::QuantizedS32{1.2f * 1.3f})

  667.             .set_dtype(3, dtype::QuantizedS8{1.1f})

  668.             .set_dtype(4, dtype::QuantizedS8{1.1f})

  669.             .set_epsilon(1 + 1e-3)

  670.             .set_max_avg_error(1e-1)

  671.             .set_max_avg_biased_error(1e-1);

  672.     param::ConvBias param;

  673.     param.pad_h = param.pad_w = 1;

  674.     param.stride_h = param.stride_w = 1;

  675.     param.format = param::ConvBias::Format::CHWN4;

  676.     checker.set_param(param).execs({{4, 12, 12, 32, 4},

  677.                                     {4, 3, 3, 16, 4},

  678.                                     {4, 1, 1, 1, 4},

  679.                                     {4, 12, 12, 32, 4},

  680.                                     {}});

  681. }

  682. 

  683. TEST_F(CUDA, CONV_BIAS_INT8_CHWN4_HSWISH) {

  684.     require_compute_capability(6, 1);

  685.     Checker<ConvBiasForward> checker(handle_cuda());

  686.     checker.set_before_exec_callback(

  687.             conv_bias::ConvBiasAlgoChecker<ConvBiasForward>(

  688.                     "INT8_CHWN4_DOTPROD_IMPLICIT_GEMM"));

  689.     UniformIntRNG rng{-3, 3};

  690.     UniformIntRNG bias_rng{-50, 50};

  691.     checker.set_rng(0, &rng)

  692.             .set_rng(1, &rng)

  693.             .set_rng(2, &bias_rng)

  694.             .set_rng(3, &rng)

  695.             .set_dtype(0, dtype::QuantizedS8{1.2f})

  696.             .set_dtype(1, dtype::QuantizedS8{1.3f})

  697.             .set_dtype(2, dtype::QuantizedS32{1.2f * 1.3f})

  698.             .set_dtype(4, dtype::QuantizedS8{0.001f})

  699.             .set_epsilon(1 + 1e-3)

  700.             .set_max_avg_error(1e-1)

  701.             .set_max_avg_biased_error(1e-1);

  702.     param::ConvBias param;

  703.     param.pad_h = param.pad_w = 1;

  704.     param.stride_h = param.stride_w = 1;

  705.     param.format = param::ConvBias::Format::CHWN4;

  706.     param.nonlineMode = param::ConvBias::NonlineMode::H_SWISH;

  707.     checker.set_param(param).execs({{4, 12, 12, 32, 4},

  708.                                     {4, 3, 3, 16, 4},

  709.                                     {4, 1, 1, 1, 4},

  710.                                     {},

  711.                                     {}});

  712. }

  713. 

  714. TEST_F(CUDA, CONV_BIAS_INT8_CHWN4_CHECK_BOUNDS) {

  715.     require_compute_capability(6, 1);

  716.     conv_bias::check_conv_bias(

  717.             dtype::QuantizedS8{1.2f}, dtype::QuantizedS8{1.3f},

  718.             dtype::QuantizedS32{1.2f * 1.3f}, dtype::QuantizedS8{1.3f},

  719.             handle_cuda(), "INT8_CHWN4_DOTPROD_IMPLICIT_GEMM",

  720.             param::ConvBias::Format::CHWN4,

  721.             conv_bias::get_int8_chwn4_args_check_bounds(3));

  722. }

  723. 

  724. TEST_F(CUDA, CONV_BIAS_INT8_CHWN4_SMALL_CHANNEL_1x1) {

  725.     require_compute_capability(6, 1);

  726.     conv_bias::check_conv_bias(

  727.             dtype::QuantizedS8{1.2f}, dtype::QuantizedS8{1.3f},

  728.             dtype::QuantizedS32{1.2f * 1.3f}, dtype::QuantizedS8{1.3f},

  729.             handle_cuda(), "INT8_CHWN4_DOTPROD_IMPLICIT_GEMM",

  730.             param::ConvBias::Format::CHWN4,

  731.             conv_bias::get_int8_chwn4_small_channel_args(1));

  732. }

  733. 

  734. TEST_F(CUDA, CONV_BIAS_INT8_CHWN4_SMALL_CHANNEL_3x3) {

  735.     require_compute_capability(6, 1);

  736.     conv_bias::check_conv_bias(

  737.             dtype::QuantizedS8{1.2f}, dtype::QuantizedS8{1.3f},

  738.             dtype::QuantizedS32{1.2f * 1.3f}, dtype::QuantizedS8{1.3f},

  739.             handle_cuda(), "INT8_CHWN4_DOTPROD_IMPLICIT_GEMM",

  740.             param::ConvBias::Format::CHWN4,

  741.             conv_bias::get_int8_chwn4_small_channel_args(3));

  742. }

  743. 

  744. TEST_F(CUDA, CONV_BIAS_INT8_CHWN4_SMALL_CHANNEL_5x5) {

  745.     require_compute_capability(6, 1);

  746.     conv_bias::check_conv_bias(

  747.             dtype::QuantizedS8{1.2f}, dtype::QuantizedS8{1.3f},

  748.             dtype::QuantizedS32{1.2f * 1.3f}, dtype::QuantizedS8{1.3f},

  749.             handle_cuda(), "INT8_CHWN4_DOTPROD_IMPLICIT_GEMM",

  750.             param::ConvBias::Format::CHWN4,

  751.             conv_bias::get_int8_chwn4_small_channel_args(5));

  752. }

  753. 

  754. TEST_F(CUDA, CONV_BIAS_INT8_CHWN4_SMALL_CHANNEL_7x7) {

  755.     require_compute_capability(6, 1);

  756.     conv_bias::check_conv_bias(

  757.             dtype::QuantizedS8{1.2f}, dtype::QuantizedS8{1.3f},

  758.             dtype::QuantizedS32{1.2f * 1.3f}, dtype::QuantizedS8{1.3f},

  759.             handle_cuda(), "INT8_CHWN4_DOTPROD_IMPLICIT_GEMM",

  760.             param::ConvBias::Format::CHWN4,

  761.             conv_bias::get_int8_chwn4_small_channel_args(7));

  762. }

  763. 

  764. TEST_F(CUDA, CONV_BIAS_INT8_NCHW4_SMALL_CHANNEL_CHECK_BOUNDS) {

  765.     require_compute_capability(6, 1);

  766.     conv_bias::check_conv_bias(

  767.             dtype::QuantizedS8{1.2f}, dtype::QuantizedS8{1.3f},

  768.             dtype::QuantizedS32{1.2f * 1.3f}, dtype::QuantizedS8{1.3f},

  769.             handle_cuda(), "INT8_NCHW4_DOTPROD_IMPLICIT_GEMM",

  770.             param::ConvBias::Format::NCHW4,

  771.             conv_bias::get_int8_nchw4_small_channel_args_check_bounds(3));

  772. }

  773. 

  774. TEST_F(CUDA, CONV_BIAS_INT8_CHWN4_SMALL_CHANNEL_1x1_CHECK_BOUNDS) {

  775.     require_compute_capability(6, 1);

  776.     conv_bias::check_conv_bias(

  777.             dtype::QuantizedS8{1.2f}, dtype::QuantizedS8{1.3f},

  778.             dtype::QuantizedS32{1.2f * 1.3f}, dtype::QuantizedS8{1.3f},

  779.             handle_cuda(), "INT8_CHWN4_DOTPROD_IMPLICIT_GEMM",

  780.             param::ConvBias::Format::CHWN4,

  781.             conv_bias::get_int8_chwn4_small_channel_args_check_bounds(1));

  782. }

  783. 

  784. TEST_F(CUDA, CONV_BIAS_INT8_CHWN4_SMALL_CHANNEL_5x5_CHECK_BOUNDS) {

  785.     require_compute_capability(6, 1);

  786.     conv_bias::check_conv_bias(

  787.             dtype::QuantizedS8{1.2f}, dtype::QuantizedS8{1.3f},

  788.             dtype::QuantizedS32{1.2f * 1.3f}, dtype::QuantizedS8{1.3f},

  789.             handle_cuda(), "INT8_CHWN4_DOTPROD_IMPLICIT_GEMM",

  790.             param::ConvBias::Format::CHWN4,

  791.             conv_bias::get_int8_chwn4_small_channel_args_check_bounds(5));

  792. }

  793. 

  794. TEST_F(CUDA, CONV_BIAS_INT8_CHWN4_SMALL_CHANNEL_7x7_CHECK_BOUNDS) {

  795.     require_compute_capability(6, 1);

  796.     conv_bias::check_conv_bias(

  797.             dtype::QuantizedS8{1.2f}, dtype::QuantizedS8{1.3f},

  798.             dtype::QuantizedS32{1.2f * 1.3f}, dtype::QuantizedS8{1.3f},

  799.             handle_cuda(), "INT8_CHWN4_DOTPROD_IMPLICIT_GEMM",

  800.             param::ConvBias::Format::CHWN4,

  801.             conv_bias::get_int8_chwn4_small_channel_args_check_bounds(7));

  802. }

  803. 

  804. TEST_F(CUDA, CONV_BIAS_INT8_NCHW4_TENSORCORE_1x1) {

  805.     require_compute_capability(7, 5);

  806.     conv_bias::check_conv_bias(

  807.             dtype::QuantizedS8{1.2f}, dtype::QuantizedS8{1.3f},

  808.             dtype::QuantizedS32{1.2f * 1.3f}, dtype::QuantizedS8{1.3f},

  809.             handle_cuda(), "INT8_NCHW4_IMMA_IMPLICIT_GEMM_mma16x16x16",

  810.             param::ConvBias::Format::NCHW4,

  811.             conv_bias::get_int8_nchw4_tensorcore_args(1));

  812. }

  813. 

  814. TEST_F(CUDA, CONV_BIAS_INT8_NCHW4_TENSORCORE_3x3) {

  815.     require_compute_capability(7, 5);

  816.     conv_bias::check_conv_bias(

  817.             dtype::QuantizedS8{1.2f}, dtype::QuantizedS8{1.3f},

  818.             dtype::QuantizedS32{1.2f * 1.3f}, dtype::QuantizedS8{1.3f},

  819.             handle_cuda(), "INT8_NCHW4_IMMA_IMPLICIT_GEMM_mma16x16x16",

  820.             param::ConvBias::Format::NCHW4,

  821.             conv_bias::get_int8_nchw4_tensorcore_args(3));

  822. }

  823. 

  824. TEST_F(CUDA, CONV_BIAS_INT8_NCHW4_TENSORCORE_5x5) {

  825.     require_compute_capability(7, 5);

  826.     conv_bias::check_conv_bias(

  827.             dtype::QuantizedS8{1.2f}, dtype::QuantizedS8{1.3f},

  828.             dtype::QuantizedS32{1.2f * 1.3f}, dtype::QuantizedS8{1.3f},

  829.             handle_cuda(), "INT8_NCHW4_IMMA_IMPLICIT_GEMM_mma16x16x16",

  830.             param::ConvBias::Format::NCHW4,

  831.             conv_bias::get_int8_nchw4_tensorcore_args(5));

  832. }

  833. 

  834. TEST_F(CUDA, CONV_BIAS_INT8_NCHW4_TENSORCORE_7x7) {

  835.     require_compute_capability(7, 5);

  836.     conv_bias::check_conv_bias(

  837.             dtype::QuantizedS8{1.2f}, dtype::QuantizedS8{1.3f},

  838.             dtype::QuantizedS32{1.2f * 1.3f}, dtype::QuantizedS8{1.3f},

  839.             handle_cuda(), "INT8_NCHW4_IMMA_IMPLICIT_GEMM_mma16x16x16",

  840.             param::ConvBias::Format::NCHW4,

  841.             conv_bias::get_int8_nchw4_tensorcore_args(7));

  842. }

  843. 

  844. TEST_F(CUDA, CONV_BIAS_INT8_NCHW4_TENSORCORE_CHECK_BOUNDS_ALGO_0) {

  845.     require_compute_capability(7, 5);

  846.     conv_bias::check_conv_bias(

  847.             dtype::QuantizedS8{1.2f}, dtype::QuantizedS8{1.3f},

  848.             dtype::QuantizedS32{1.2f * 1.3f}, dtype::QuantizedS8{1.3f},

  849.             handle_cuda(), "INT8_NCHW4_IMMA_IMPLICIT_GEMM_mma16x16x16",

  850.             param::ConvBias::Format::NCHW4,

  851.             conv_bias::get_int8_nchw4_args_check_bounds(3));

  852. }

  853. 

  854. TEST_F(CUDA, CONV_BIAS_INT8_NCHW4_TENSORCORE_CHECK_BOUNDS_ALGO_1) {

  855.     require_compute_capability(7, 5);

  856.     conv_bias::check_conv_bias(

  857.             dtype::QuantizedS8{1.2f}, dtype::QuantizedS8{1.3f},

  858.             dtype::QuantizedS32{1.2f * 1.3f}, dtype::QuantizedS8{1.3f},

  859.             handle_cuda(), "INT8_NCHW4_IMMA_IMPLICIT_GEMM_mma8x32x16",

  860.             param::ConvBias::Format::NCHW4,

  861.             conv_bias::get_int8_nchw4_args_check_bounds(3));

  862. }

  863. 

  864. TEST_F(CUDA, CONV_BIAS_INT8_NCHW4_TENSORCORE_CHECK_BOUNDS_ALGO_2) {

  865.     require_compute_capability(7, 5);

  866.     conv_bias::check_conv_bias(

  867.             dtype::QuantizedS8{1.2f}, dtype::QuantizedS8{1.3f},

  868.             dtype::QuantizedS32{1.2f * 1.3f}, dtype::QuantizedS8{1.3f},

  869.             handle_cuda(), "INT8_NCHW4_IMMA_IMPLICIT_GEMM_mma32x8x16",

  870.             param::ConvBias::Format::NCHW4,

  871.             conv_bias::get_int8_nchw4_args_check_bounds(3));

  872. }

  873. 

  874. TEST_F(CUDA, CONV_BIAS_INT8_CHWN4_TENSORCORE_ALGO_0) {

  875.     require_compute_capability(7, 5);

  876.     conv_bias::check_conv_bias(

  877.             dtype::QuantizedS8{1.2f}, dtype::QuantizedS8{1.3f},

  878.             dtype::QuantizedS32{1.2f * 1.3f}, dtype::QuantizedS8{1.1f},

  879.             handle_cuda(), "INT8_CHWN4_IMMA_IMPLICIT_GEMM_mma16x16x16",

  880.             param::ConvBias::Format::CHWN4,

  881.             conv_bias::get_int8_chwn4_tensorcore_args(3));

  882. }

  883. 

  884. TEST_F(CUDA, CONV_BIAS_INT8_CHWN4_TENSORCORE_ALGO_1) {

  885.     require_compute_capability(7, 5);

  886.     conv_bias::check_conv_bias(

  887.             dtype::QuantizedS8{1.2f}, dtype::QuantizedS8{1.3f},

  888.             dtype::QuantizedS32{1.2f * 1.3f}, dtype::QuantizedS8{1.1f},

  889.             handle_cuda(), "INT8_CHWN4_IMMA_IMPLICIT_GEMM_mma32x8x16",

  890.             param::ConvBias::Format::CHWN4,

  891.             conv_bias::get_int8_chwn4_tensorcore_args(3));

  892. }

  893. 

  894. TEST_F(CUDA, CONV_BIAS_INT8_CHWN4_TENSORCORE_ALGO_2) {

  895.     require_compute_capability(7, 5);

  896.     conv_bias::check_conv_bias(

  897.             dtype::QuantizedS8{1.2f}, dtype::QuantizedS8{1.3f},

  898.             dtype::QuantizedS32{1.2f * 1.3f}, dtype::QuantizedS8{1.1f},

  899.             handle_cuda(), "INT8_CHWN4_IMMA_IMPLICIT_GEMM_mma8x32x16",

  900.             param::ConvBias::Format::CHWN4,

  901.             conv_bias::get_int8_chwn4_tensorcore_args(3));

  902. }

  903. 

  904. TEST_F(CUDA, CONV_BIAS_INT8_CHWN4_TENSORCORE_CHECK_BOUNDS_1x1) {

  905.     require_compute_capability(7, 5);

  906.     conv_bias::check_conv_bias(

  907.             dtype::QuantizedS8{1.2f}, dtype::QuantizedS8{1.3f},

  908.             dtype::QuantizedS32{1.2f * 1.3f}, dtype::QuantizedS8{1.3f},

  909.             handle_cuda(), "INT8_CHWN4_IMMA_IMPLICIT_GEMM_mma16x16x16",

  910.             param::ConvBias::Format::CHWN4,

  911.             conv_bias::get_int8_chwn4_args_check_bounds(1));

  912. }

  913. 

  914. TEST_F(CUDA, CONV_BIAS_INT8_CHWN4_TENSORCORE_CHECK_BOUNDS_5x5) {

  915.     require_compute_capability(7, 5);

  916.     conv_bias::check_conv_bias(

  917.             dtype::QuantizedS8{1.2f}, dtype::QuantizedS8{1.3f},

  918.             dtype::QuantizedS32{1.2f * 1.3f}, dtype::QuantizedS8{1.3f},

  919.             handle_cuda(), "INT8_CHWN4_IMMA_IMPLICIT_GEMM_mma16x16x16",

  920.             param::ConvBias::Format::CHWN4,

  921.             conv_bias::get_int8_chwn4_args_check_bounds(5));

  922. }

  923. 

  924. TEST_F(CUDA, CONV_BIAS_INT8_CHWN4_TENSORCORE_CHECK_BOUNDS_7x7) {

  925.     require_compute_capability(7, 5);

  926.     conv_bias::check_conv_bias(

  927.             dtype::QuantizedS8{1.2f}, dtype::QuantizedS8{1.3f},

  928.             dtype::QuantizedS32{1.2f * 1.3f}, dtype::QuantizedS8{1.3f},

  929.             handle_cuda(), "INT8_CHWN4_IMMA_IMPLICIT_GEMM_mma16x16x16",

  930.             param::ConvBias::Format::CHWN4,

  931.             conv_bias::get_int8_chwn4_args_check_bounds(7));

  932. }

  933. 

  934. TEST_F(CUDA, CONV_BIAS_INT8_NCHW4_TENSORCORE_WITH_Z) {

  935.     require_compute_capability(7, 5);

  936.     Checker<ConvBiasForward> checker(handle_cuda());

  937.     checker.set_before_exec_callback(

  938.             conv_bias::ConvBiasAlgoChecker<ConvBiasForward>(

  939.                     "INT8_NCHW4_IMMA_IMPLICIT_GEMM_mma16x16x16"));

  940.     UniformIntRNG rng{-3, 3};

  941.     UniformIntRNG bias_rng{-50, 50};

  942.     checker.set_rng(0, &rng)

  943.             .set_rng(1, &rng)

  944.             .set_rng(2, &bias_rng)

  945.             .set_rng(3, &rng)

  946.             .set_dtype(0, dtype::QuantizedS8{1.2f})

  947.             .set_dtype(1, dtype::QuantizedS8{1.3f})

  948.             .set_dtype(2, dtype::QuantizedS32{1.2f * 1.3f})

  949.             .set_dtype(3, dtype::QuantizedS8{1.1f})

  950.             .set_dtype(4, dtype::QuantizedS8{1.0f})

  951.             .set_epsilon(1 + 1e-3)

  952.             .set_max_avg_error(1e-1)

  953.             .set_max_avg_biased_error(1e-1);

  954.     param::ConvBias param;

  955.     param.pad_h = param.pad_w = 1;

  956.     param.stride_h = param.stride_w = 1;

  957.     param.format = param::ConvBias::Format::NCHW4;

  958.     checker.set_param(param).execs({{64, 8, 12, 12, 4},

  959.                                     {64, 8, 3, 3, 4},

  960.                                     {1, 16, 1, 1, 4},

  961.                                     {64, 16, 12, 12, 4},

  962.                                     {}});

  963. }

  964. 

  965. TEST_F(CUDA, CONV_BIAS_INT8_CHWN4_TENSORCORE_WITH_Z) {

  966.     require_compute_capability(7, 5);

  967.     Checker<ConvBiasForward> checker(handle_cuda());

  968.     checker.set_before_exec_callback(

  969.             conv_bias::ConvBiasAlgoChecker<ConvBiasForward>(

  970.                     "INT8_CHWN4_IMMA_IMPLICIT_GEMM_mma16x16x16"));

  971.     UniformIntRNG rng{-3, 3};

  972.     UniformIntRNG bias_rng{-50, 50};

  973.     checker.set_rng(0, &rng)

  974.             .set_rng(1, &rng)

  975.             .set_rng(2, &bias_rng)

  976.             .set_rng(3, &rng)

  977.             .set_dtype(0, dtype::QuantizedS8{1.2f})

  978.             .set_dtype(1, dtype::QuantizedS8{1.3f})

  979.             .set_dtype(2, dtype::QuantizedS32{1.2f * 1.3f})

  980.             .set_dtype(3, dtype::QuantizedS8{1.1f})

  981.             .set_dtype(4, dtype::QuantizedS8{1.0f})

  982.             .set_epsilon(1 + 1e-3)

  983.             .set_max_avg_error(1e-1)

  984.             .set_max_avg_biased_error(1e-1);

  985.     param::ConvBias param;

  986.     param.pad_h = param.pad_w = 1;

  987.     param.stride_h = param.stride_w = 1;

  988.     param.format = param::ConvBias::Format::CHWN4;

  989.     checker.set_param(param).execs({{8, 12, 12, 64, 4},

  990.                                     {8, 3, 3, 64, 4},

  991.                                     {16, 1, 1, 1, 4},

  992.                                     {16, 12, 12, 64, 4},

  993.                                     {}});

  994. }

  995. 

  996. TEST_F(CUDA,

  997.        CONV_BIAS_INT8_CHWN4_REFORMAT_FILTER_TENSORCORE_CHECK_BOUNDS_ALGO_0) {

  998.     require_compute_capability(7, 5);

  999.     conv_bias::check_conv_bias(

  1000.             dtype::QuantizedS8{1.2f}, dtype::QuantizedS8{1.3f},

  1001.             dtype::QuantizedS32{1.2f * 1.3f}, dtype::QuantizedS8{1.3f},

  1002.             handle_cuda(),

  1003.             "INT8_CHWN4_IMMA_IMPLICIT_GEMM_REORDER_FILTER_mma16x16x16",

  1004.             param::ConvBias::Format::CHWN4,

  1005.             conv_bias::get_int8_chwn4_args_check_bounds(3));

  1006. }

  1007. 

  1008. TEST_F(CUDA,

  1009.        CONV_BIAS_INT8_CHWN4_REFORMAT_FILTER_TENSORCORE_CHECK_BOUNDS_ALGO_1) {

  1010.     require_compute_capability(7, 5);

  1011.     conv_bias::check_conv_bias(

  1012.             dtype::QuantizedS8{1.2f}, dtype::QuantizedS8{1.3f},

  1013.             dtype::QuantizedS32{1.2f * 1.3f}, dtype::QuantizedS8{1.3f},

  1014.             handle_cuda(),

  1015.             "INT8_CHWN4_IMMA_IMPLICIT_GEMM_REORDER_FILTER_mma8x32x16",

  1016.             param::ConvBias::Format::CHWN4,

  1017.             conv_bias::get_int8_chwn4_args_check_bounds(3));

  1018. }

  1019. 

  1020. TEST_F(CUDA,

  1021.        CONV_BIAS_INT8_CHWN4_REFORMAT_FILTER_TENSORCORE_CHECK_BOUNDS_ALGO_2) {

  1022.     require_compute_capability(7, 5);

  1023.     conv_bias::check_conv_bias(

  1024.             dtype::QuantizedS8{1.2f}, dtype::QuantizedS8{1.3f},

  1025.             dtype::QuantizedS32{1.2f * 1.3f}, dtype::QuantizedS8{1.3f},

  1026.             handle_cuda(),

  1027.             "INT8_CHWN4_IMMA_IMPLICIT_GEMM_REORDER_FILTER_mma32x8x16",

  1028.             param::ConvBias::Format::CHWN4,

  1029.             conv_bias::get_int8_chwn4_args_check_bounds(3));

  1030. }

  1031. 

  1032. TEST_F(CUDA, CONV_BIAS_INT8_CHWN4_REFORMAT_FILTER_TENSORCORE_ALGO_0) {

  1033.     require_compute_capability(7, 5);

  1034.     conv_bias::check_conv_bias(

  1035.             dtype::QuantizedS8{1.2f}, dtype::QuantizedS8{1.3f},

  1036.             dtype::QuantizedS32{1.2f * 1.3f}, dtype::QuantizedS8{1.3f},

  1037.             handle_cuda(),

  1038.             "INT8_CHWN4_IMMA_IMPLICIT_GEMM_REORDER_FILTER_mma16x16x16",

  1039.             param::ConvBias::Format::CHWN4, conv_bias::get_int8_chwn4_args(3));

  1040. }

  1041. 

  1042. TEST_F(CUDA, CONV_BIAS_INT8_CHWN4_REFORMAT_FILTER_TENSORCORE_ALGO_1) {

  1043.     require_compute_capability(7, 5);

  1044.     conv_bias::check_conv_bias(

  1045.             dtype::QuantizedS8{1.2f}, dtype::QuantizedS8{1.3f},

  1046.             dtype::QuantizedS32{1.2f * 1.3f}, dtype::QuantizedS8{1.3f},

  1047.             handle_cuda(),

  1048.             "INT8_CHWN4_IMMA_IMPLICIT_GEMM_REORDER_FILTER_mma8x32x16",

  1049.             param::ConvBias::Format::CHWN4, conv_bias::get_int8_chwn4_args(3));

  1050. }

  1051. 

  1052. TEST_F(CUDA, CONV_BIAS_INT8_CHWN4_REFORMAT_FILTER_TENSORCORE_ALGO_2) {

  1053.     require_compute_capability(7, 5);

  1054.     conv_bias::check_conv_bias(

  1055.             dtype::QuantizedS8{1.2f}, dtype::QuantizedS8{1.3f},

  1056.             dtype::QuantizedS32{1.2f * 1.3f}, dtype::QuantizedS8{1.3f},

  1057.             handle_cuda(),

  1058.             "INT8_CHWN4_IMMA_IMPLICIT_GEMM_REORDER_FILTER_mma32x8x16",

  1059.             param::ConvBias::Format::CHWN4, conv_bias::get_int8_chwn4_args(3));

  1060. }

  1061. 

  1062. TEST_F(CUDA, CONV_BIAS_INT8_CHWN4_UNROLL_WIDTH_TENSORCORE_ALGO_0) {

  1063.     require_compute_capability(7, 5);

  1064.     conv_bias::check_conv_bias(

  1065.             dtype::QuantizedS8{1.2f}, dtype::QuantizedS8{1.3f},

  1066.             dtype::QuantizedS32{1.2f * 1.3f}, dtype::QuantizedS8{1.1f},

  1067.             handle_cuda(),

  1068.             "INT8_CHWN4_IMMA_IMPLICIT_GEMM_UNROLL_WIDTH_mma16x16x16",

  1069.             param::ConvBias::Format::CHWN4, conv_bias::get_int8_chwn4_args(3));

  1070. }

  1071. 

  1072. TEST_F(CUDA, CONV_BIAS_INT8_CHWN4_UNROLL_WIDTH_TENSORCORE_ALGO_1) {

  1073.     require_compute_capability(7, 5);

  1074.     conv_bias::check_conv_bias(

  1075.             dtype::QuantizedS8{1.2f}, dtype::QuantizedS8{1.3f},

  1076.             dtype::QuantizedS32{1.2f * 1.3f}, dtype::QuantizedS8{1.3f},

  1077.             handle_cuda(),

  1078.             "INT8_CHWN4_IMMA_IMPLICIT_GEMM_UNROLL_WIDTH_mma8x32x16",

  1079.             param::ConvBias::Format::CHWN4, conv_bias::get_int8_chwn4_args(3));

  1080. }

  1081. 

  1082. TEST_F(CUDA, CONV_BIAS_INT8_CHWN4_UNROLL_WIDTH_TENSORCORE_ALGO_2) {

  1083.     require_compute_capability(7, 5);

  1084.     conv_bias::check_conv_bias(

  1085.             dtype::QuantizedS8{1.2f}, dtype::QuantizedS8{1.3f},

  1086.             dtype::QuantizedS32{1.2f * 1.3f}, dtype::QuantizedS8{1.3f},

  1087.             handle_cuda(),

  1088.             "INT8_CHWN4_IMMA_IMPLICIT_GEMM_UNROLL_WIDTH_mma32x8x16",

  1089.             param::ConvBias::Format::CHWN4, conv_bias::get_int8_chwn4_args(3));

  1090. }

  1091. 

  1092. TEST_F(CUDA, CONV_BIAS_INT8_CHWN4_UNROLL_WIDTH_TENSORCORE_1x1) {

  1093.     require_compute_capability(7, 5);

  1094.     conv_bias::check_conv_bias(

  1095.             dtype::QuantizedS8{1.2f}, dtype::QuantizedS8{1.3f},

  1096.             dtype::QuantizedS32{1.2f * 1.3f}, dtype::QuantizedS8{1.1f},

  1097.             handle_cuda(),

  1098.             "INT8_CHWN4_IMMA_IMPLICIT_GEMM_UNROLL_WIDTH_mma16x16x16",

  1099.             param::ConvBias::Format::CHWN4, conv_bias::get_int8_chwn4_args(1));

  1100. }

  1101. 

  1102. TEST_F(CUDA, CONV_BIAS_INT8_CHWN4_UNROLL_WIDTH_TENSORCORE_5x5) {

  1103.     require_compute_capability(7, 5);

  1104.     conv_bias::check_conv_bias(

  1105.             dtype::QuantizedS8{1.2f}, dtype::QuantizedS8{1.3f},

  1106.             dtype::QuantizedS32{1.2f * 1.3f}, dtype::QuantizedS8{1.1f},

  1107.             handle_cuda(),

  1108.             "INT8_CHWN4_IMMA_IMPLICIT_GEMM_UNROLL_WIDTH_mma16x16x16",

  1109.             param::ConvBias::Format::CHWN4,

  1110.             conv_bias::get_int8_chwn4_args_small_batch(5));

  1111. }

  1112. 

  1113. TEST_F(CUDA, CONV_BIAS_INT8_CHWN4_UNROLL_WIDTH_TENSORCORE_7x7) {

  1114.     require_compute_capability(7, 5);

  1115.     conv_bias::check_conv_bias(

  1116.             dtype::QuantizedS8{1.2f}, dtype::QuantizedS8{1.3f},

  1117.             dtype::QuantizedS32{1.2f * 1.3f}, dtype::QuantizedS8{1.1f},

  1118.             handle_cuda(),

  1119.             "INT8_CHWN4_IMMA_IMPLICIT_GEMM_UNROLL_WIDTH_mma16x16x16",

  1120.             param::ConvBias::Format::CHWN4,

  1121.             conv_bias::get_int8_chwn4_args_small_batch(7));

  1122. }

  1123. 

  1124. TEST_F(CUDA, CONV_BIAS_INT8_CHWN4_UNROLL_WIDTH_TENSORCORE_5x5_ALGO_1) {

  1125.     require_compute_capability(7, 5);

  1126.     conv_bias::check_conv_bias(

  1127.             dtype::QuantizedS8{1.2f}, dtype::QuantizedS8{1.3f},

  1128.             dtype::QuantizedS32{1.2f * 1.3f}, dtype::QuantizedS8{1.1f},

  1129.             handle_cuda(),

  1130.             "INT8_CHWN4_IMMA_IMPLICIT_GEMM_UNROLL_WIDTH_mma32x8x16",

  1131.             param::ConvBias::Format::CHWN4,

  1132.             conv_bias::get_int8_chwn4_args_small_batch(5));

  1133. }

  1134. 

  1135. TEST_F(CUDA, CONV_BIAS_INT8_CHWN4_UNROLL_WIDTH_TENSORCORE_5x5_ALGO_2) {

  1136.     require_compute_capability(7, 5);

  1137.     conv_bias::check_conv_bias(

  1138.             dtype::QuantizedS8{1.2f}, dtype::QuantizedS8{1.3f},

  1139.             dtype::QuantizedS32{1.2f * 1.3f}, dtype::QuantizedS8{1.1f},

  1140.             handle_cuda(),

  1141.             "INT8_CHWN4_IMMA_IMPLICIT_GEMM_UNROLL_WIDTH_mma8x32x16",

  1142.             param::ConvBias::Format::CHWN4,

  1143.             conv_bias::get_int8_chwn4_args_small_batch(5));

  1144. }

  1145. 

  1146. TEST_F(CUDA, CONV_BIAS_INT8_CHWN4_UNROLL_WIDTH_TENSORCORE_1x1_ALGO_1) {

  1147.     require_compute_capability(7, 5);

  1148.     conv_bias::check_conv_bias(

  1149.             dtype::QuantizedS8{1.2f}, dtype::QuantizedS8{1.3f},

  1150.             dtype::QuantizedS32{1.2f * 1.3f}, dtype::QuantizedS8{1.1f},

  1151.             handle_cuda(),

  1152.             "INT8_CHWN4_IMMA_IMPLICIT_GEMM_UNROLL_WIDTH_mma32x8x16",

  1153.             param::ConvBias::Format::CHWN4,

  1154.             conv_bias::get_int8_chwn4_args_small_batch(1));

  1155. }

  1156. 

  1157. TEST_F(CUDA, CONV_BIAS_INT8_CHWN4_UNROLL_WIDTH_TENSORCORE_1x1_ALGO_2) {

  1158.     require_compute_capability(7, 5);

  1159.     conv_bias::check_conv_bias(

  1160.             dtype::QuantizedS8{1.2f}, dtype::QuantizedS8{1.3f},

  1161.             dtype::QuantizedS32{1.2f * 1.3f}, dtype::QuantizedS8{1.1f},

  1162.             handle_cuda(),

  1163.             "INT8_CHWN4_IMMA_IMPLICIT_GEMM_UNROLL_WIDTH_mma8x32x16",

  1164.             param::ConvBias::Format::CHWN4,

  1165.             conv_bias::get_int8_chwn4_args_small_batch(1));

  1166. }

  1167. 

  1168. 

  1169. #if MEGDNN_WITH_BENCHMARK

  1170. TEST_F(CUDA, BENCHMARK_CONV_BIAS_INT8_CHWN4) {

  1171.     require_compute_capability(6, 1);

  1172.     benchmark_target_algo(

  1173.             handle_cuda(), get_resnet50_bench_args(), dtype::QuantizedS8{1.2f},

  1174.             dtype::QuantizedS8{1.3f}, dtype::QuantizedS32{1.2f * 1.3f},

  1175.             dtype::QuantizedS8{1.0f}, "INT8_CHWN4_DOTPROD_IMPLICIT_GEMM",

  1176.             param::ConvBias::Format::CHWN4);

  1177. }

  1178. 

  1179. TEST_F(CUDA, BENCHMARK_CONV_BIAS_INT8_NCHW4) {

  1180.     require_compute_capability(6, 1);

  1181.     benchmark_target_algo(

  1182.             handle_cuda(), get_resnet50_bench_args(), dtype::QuantizedS8{1.2f},

  1183.             dtype::QuantizedS8{1.3f}, dtype::QuantizedS32{1.2f * 1.3f},

  1184.             dtype::QuantizedS8{1.0f}, "INT8_NCHW4_DOTPROD_IMPLICIT_GEMM",

  1185.             param::ConvBias::Format::NCHW4);

  1186. }

  1187. 

  1188. TEST_F(CUDA, BENCHMARK_CONV_BIAS_INT8_CHWN4_TENSORCORE) {

  1189.     require_compute_capability(7, 5);

  1190.     benchmark_target_algo_with_cudnn_tsc(

  1191.             handle_cuda(), get_resnet50_bench_args(256),

  1192.             dtype::QuantizedS8{1.2f}, dtype::QuantizedS8{1.3f},

  1193.             dtype::QuantizedS32{1.2f * 1.3f}, dtype::QuantizedS8{1.0f},

  1194.             "INT8_CHWN4_IMMA_IMPLICIT_GEMM_mma16x16x16",

  1195.             param::ConvBias::Format::CHWN4);

  1196. }

  1197. 

  1198. TEST_F(CUDA, BENCHMARK_CONV_BIAS_INT8_CHWN4_TENSORCORE_ALL_ALGO) {

  1199.     require_compute_capability(7, 5);

  1200.     benchmark_target_algo_with_cudnn_tsc(

  1201.             handle_cuda(), get_resnet50_bench_args(256),

  1202.             dtype::QuantizedS8{1.2f}, dtype::QuantizedS8{1.3f},

  1203.             dtype::QuantizedS32{1.2f * 1.3f}, dtype::QuantizedS8{1.0f}, nullptr,

  1204.             param::ConvBias::Format::CHWN4);

  1205. }

  1206. 

  1207. TEST_F(CUDA, BENCHMARK_CONV_BIAS_INT8_CHWN4_DET_ALL_ALGO) {

  1208.     require_compute_capability(7, 5);

  1209.     benchmark_target_algo_with_cudnn_tsc(

  1210.             handle_cuda(), get_detection_bench_args(), dtype::QuantizedS8{1.2f},

  1211.             dtype::QuantizedS8{1.3f}, dtype::QuantizedS32{1.2f * 1.3f},

  1212.             dtype::QuantizedS8{1.0f}, nullptr, param::ConvBias::Format::CHWN4);

  1213. }

  1214. 

  1215. TEST_F(CUDA, BENCHMARK_CONV_BIAS_INT8_NCHW4_TENSORCORE) {

  1216.     require_compute_capability(7, 5);

  1217.     benchmark_target_algo_with_cudnn_tsc(

  1218.             handle_cuda(), get_resnet50_bench_args(256),

  1219.             dtype::QuantizedS8{1.2f}, dtype::QuantizedS8{1.3f},

  1220.             dtype::QuantizedS32{1.2f * 1.3f}, dtype::QuantizedS8{1.0f},

  1221.             "INT8_NCHW4_IMMA_IMPLICIT_GEMM_mma16x16x16",

  1222.             param::ConvBias::Format::NCHW4);

  1223. }

  1224. 

  1225. TEST_F(CUDA, BENCHMARK_CONV_BIAS_INT8_CHWN4_SMALL_CHANNEL) {

  1226.     require_compute_capability(6, 1);

  1227.     std::vector<BenchArgs> args;

  1228.     args.push_back(BenchArgs{64, 4, 224, 224, 64, 7, 2});

  1229.     benchmark_target_algo(

  1230.             handle_cuda(), args, dtype::QuantizedS8{1.2f},

  1231.             dtype::QuantizedS8{1.3f}, dtype::QuantizedS32{1.2f * 1.3f},

  1232.             dtype::QuantizedS8{1.0f}, "INT8_CHWN4_DOTPROD_IMPLICIT_GEMM",

  1233.             param::ConvBias::Format::CHWN4);

  1234. }

  1235. 

  1236. #endif

  1237. 

  1238. }  // namespace test

  1239. }  // namespace megdnn

  1240. 

  1241. #undef V1

  1242. #undef V

  1243. 

  1244. // vim: syntax=cpp.doxygen