OpenI
/
MegEngine
Not watched
1
0
Fork 0
Code
Releases 31 Wiki Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: c42ce93705
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'c42ce93705'
${ noResults }
MegEngine/dnn/test/cuda/chanwise_convolution.cpp

1557 lines
63 kB
Raw Blame History 

  1. /**

  2.  * \file dnn/test/cuda/chanwise_convolution.cpp

  3.  * MegEngine is Licensed under the Apache License, Version 2.0 (the "License")

  4.  *

  5.  * Copyright (c) 2014-2021 Megvii Inc. All rights reserved.

  6.  *

  7.  * Unless required by applicable law or agreed to in writing,

  8.  * software distributed under the License is distributed on an

  9.  * "AS IS" BASIS, WITHOUT ARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

  10.  */

  11. 

  12. #include "megdnn/oprs/nn.h"

  13. 

  14. #include "cuda.h"

  15. #include "megcore_cuda.h"

  16. #include "test/common/benchmarker.h"

  17. #include "test/common/checker.h"

  18. #include "test/common/convolution.h"

  19. #include "test/common/tensor.h"

  20. #include "test/common/workspace_wrapper.h"

  21. #include "test/cuda/benchmark.h"

  22. #include "test/cuda/fixture.h"

  23. #include "test/cuda/utils.h"

  24. 

  25. #include <cuda_profiler_api.h>

  26. #include <cuda_runtime_api.h>

  27. 

  28. using namespace megdnn;

  29. using namespace test;

  30. 

  31. namespace {

  32. 

  33. #if MEGDNN_WITH_BENCHMARK

  34. bool check_need_full_bench() {

  35.     if (getenv("MEGDNN_CHANWISE_CONV_FULLBENCH"))

  36.         return true;

  37.     printf("set MEGDNN_CHANWISE_CONV_FULLBENCH to run full benchmark\n");

  38.     return false;

  39. }

  40. #endif

  41. 

  42. Convolution::Param gconv_param(Convolution::Param p, bool io16xc32 = false) {

  43.     p.sparse = Convolution::Param::Sparse::GROUP;

  44.     if (io16xc32)

  45.         p.compute_mode = Convolution::Param::ComputeMode::FLOAT32;

  46.     return p;

  47. }

  48. 

  49. template <int P0, int P1, int P2>

  50. class BenchmarkEnv {

  51.     Handle *handle, *handle_cpu;

  52.     std::unique_ptr<GaussianRNG> rng;

  53.     TensorLayout lsrc, lflt0, lflt1, ldst;

  54.     std::unique_ptr<Tensor<>> src0, src1, flt0, flt0_cpu, flt1, flt1_cpu, dst0, dst1;

  55.     cudaEvent_t cuda_ev[3];

  56.     cudaStream_t cuda_stream;

  57.     size_t pad_h, pad_w;

  58. 

  59.     template <typename T>

  60.     static std::tuple<T, T, T> shuffle(std::tuple<T, T, T> data) {

  61.         return std::make_tuple(

  62.                 std::get<P0>(data), std::get<P1>(data), std::get<P2>(data));

  63.     }

  64. 

  65. public:

  66.     BenchmarkEnv(Handle* handle, Handle* handle_cpu) {

  67.         this->handle = handle;

  68.         this->handle_cpu = handle_cpu;

  69.         rng = handle->create_operator<GaussianRNG>();

  70.         // make cpu handle used

  71.         handle_cpu->create_operator<Sleep>()->exec();

  72. 

  73.         for (int i = 0; i < 3; ++i)

  74.             cudaEventCreate(&cuda_ev[i]);

  75.         megcoreGetCUDAStream(handle->megcore_computing_handle(), &cuda_stream);

  76.     }

  77. 

  78.     ~BenchmarkEnv() {

  79.         for (int i = 0; i < 3; ++i)

  80.             cudaEventDestroy(cuda_ev[i]);

  81.     }

  82. 

  83.     void alloc(

  84.             size_t N, size_t IC, size_t IH, size_t IW, size_t CHL_MUL, size_t FH,

  85.             size_t FW, size_t PH, size_t PW) {

  86.         pad_h = PH;

  87.         pad_w = PW;

  88.         auto mkly = [](const TensorShape& s) {

  89.             return TensorLayout{s, dtype::Float32()};

  90.         };

  91.         lsrc = mkly({N, IC, IH, IW});

  92.         lflt0 = mkly({CHL_MUL * IC, IC, FH, FW});

  93.         lflt1 = mkly({IC, CHL_MUL, 1, FH, FW});

  94.         ldst = mkly({N, IC * CHL_MUL, IH - FH + 1 + PH * 2, IW - FW + 1 + PW * 2});

  95.         src0.reset(new Tensor<>(handle, lsrc));

  96.         src1.reset(new Tensor<>(handle, lsrc));

  97.         flt0.reset(new Tensor<>(handle, lflt0));

  98.         flt0_cpu.reset(new Tensor<>(handle_cpu, lflt0));

  99.         flt1.reset(new Tensor<>(handle, lflt1));

  100.         flt1_cpu.reset(new Tensor<>(handle_cpu, lflt1));

  101.         dst0.reset(new Tensor<>(handle, ldst));

  102.         dst1.reset(new Tensor<>(handle, ldst));

  103.     }

  104. 

  105.     void fill_src() {

  106.         rng->exec(src0->tensornd(), {});

  107.         megdnn_memcpy_D2D(handle, src1->ptr(), src0->ptr(), lsrc.span().dist_byte());

  108.     }

  109. 

  110.     void fill_flt() {

  111.         rng->exec(flt1->tensornd(), {});

  112.         megdnn_memcpy_D2H(

  113.                 handle, flt1_cpu->ptr(), flt1->ptr(), lflt1.span().dist_byte());

  114. 

  115.         const size_t IC = lflt1[0], CHL_MUL = lflt1[1], FSIZE = lflt1[3] * lflt1[4];

  116. 

  117.         // fill flt0 from flt1

  118.         float* src = flt1_cpu->ptr();

  119.         float* dst = flt0_cpu->ptr();

  120.         memset(dst, 0, lflt0.span().dist_byte());

  121.         for (size_t i = 0; i < IC; ++i) {

  122.             for (size_t j = 0; j < CHL_MUL; ++j) {

  123.                 memcpy(dst + ((i * CHL_MUL + j) * IC + i) * FSIZE,

  124.                        src + (i * CHL_MUL + j) * FSIZE, FSIZE * sizeof(float));

  125.             }

  126.         }

  127. 

  128.         megdnn_memcpy_H2D(handle, flt0->ptr(), dst, lflt0.span().dist_byte());

  129.     }

  130. 

  131.     void fill_dst() {

  132.         rng->exec(dst0->tensornd(), {});

  133.         megdnn_memcpy_D2D(handle, dst1->ptr(), dst0->ptr(), ldst.span().dist_byte());

  134.     }

  135. 

  136.     template <class Opr>

  137.     void exec(Opr* opr0, Opr* opr1) {

  138.         opr0->param().pad_h = pad_h;

  139.         opr0->param().pad_w = pad_w;

  140.         opr1->param() = opr0->param();

  141.         opr1->param().sparse = param::Convolution::Sparse::GROUP;

  142. 

  143.         TensorND a0, b0, c0, a1, b1, c1;

  144.         std::tie(a0, b0, c0) = shuffle(

  145.                 std::make_tuple(src0->tensornd(), flt0->tensornd(), dst0->tensornd()));

  146.         std::tie(a1, b1, c1) = shuffle(

  147.                 std::make_tuple(src1->tensornd(), flt1->tensornd(), dst1->tensornd()));

  148.         WorkspaceWrapper wk(

  149.                 handle,

  150.                 std::max(

  151.                         opr0->get_workspace_in_bytes(a0.layout, b0.layout, c0.layout),

  152.                         opr1->get_workspace_in_bytes(a1.layout, b1.layout, c1.layout)));

  153.         cudaProfilerStart();

  154.         cudaEventRecord(cuda_ev[0], cuda_stream);

  155.         opr0->exec(a0, b0, c0, wk.workspace());

  156.         cudaEventRecord(cuda_ev[1], cuda_stream);

  157.         opr1->exec(a1, b1, c1, wk.workspace());

  158.         cudaEventRecord(cuda_ev[2], cuda_stream);

  159.         cudaProfilerStop();

  160. 

  161.         if (getenv("MEGDNN_CHANWISE_CONV_VERBOSE") ||

  162.             getenv("MEGDNN_CHANWISE_CONV_FULLBENCH")) {

  163.             cudaStreamSynchronize(cuda_stream);

  164.             float t0 = -1, t1 = -1;

  165.             cudaEventElapsedTime(&t0, cuda_ev[0], cuda_ev[1]);

  166.             cudaEventElapsedTime(&t1, cuda_ev[1], cuda_ev[2]);

  167.             printf("%s;%s;%s: cudnn/megdnn: %.3fms/%.3fms=%.3f\n",

  168.                    lsrc.TensorShape::to_string().c_str(),

  169.                    lflt1.TensorShape::to_string().c_str(),

  170.                    ldst.TensorShape::to_string().c_str(), t0, t1, t0 / t1);

  171.         }

  172.     }

  173. 

  174.     //! special for weight preprocess

  175.     void exec_convolution(ConvolutionForward* opr0, ConvolutionForward* opr1) {

  176.         opr0->param().pad_h = pad_h;

  177.         opr0->param().pad_w = pad_w;

  178.         opr1->param() = opr0->param();

  179.         opr1->param().sparse = param::Convolution::Sparse::GROUP;

  180. 

  181.         TensorND a0, b0, c0, a1, b1, c1;

  182.         std::tie(a0, b0, c0) = shuffle(

  183.                 std::make_tuple(src0->tensornd(), flt0->tensornd(), dst0->tensornd()));

  184.         std::tie(a1, b1, c1) = shuffle(

  185.                 std::make_tuple(src1->tensornd(), flt1->tensornd(), dst1->tensornd()));

  186.         WorkspaceWrapper wk(

  187.                 handle, std::max(

  188.                                 opr0->get_workspace_in_bytes(

  189.                                         a0.layout, b0.layout, c0.layout, nullptr),

  190.                                 opr1->get_workspace_in_bytes(

  191.                                         a1.layout, b1.layout, c1.layout, nullptr)));

  192.         cudaProfilerStart();

  193.         cudaEventRecord(cuda_ev[0], cuda_stream);

  194.         opr0->exec(a0, b0, c0, nullptr, wk.workspace());

  195.         cudaEventRecord(cuda_ev[1], cuda_stream);

  196.         opr1->exec(a1, b1, c1, nullptr, wk.workspace());

  197.         cudaEventRecord(cuda_ev[2], cuda_stream);

  198.         cudaProfilerStop();

  199. 

  200.         if (getenv("MEGDNN_CHANWISE_CONV_VERBOSE") ||

  201.             getenv("MEGDNN_CHANWISE_CONV_FULLBENCH")) {

  202.             cudaStreamSynchronize(cuda_stream);

  203.             float t0 = -1, t1 = -1;

  204.             cudaEventElapsedTime(&t0, cuda_ev[0], cuda_ev[1]);

  205.             cudaEventElapsedTime(&t1, cuda_ev[1], cuda_ev[2]);

  206.             printf("%s;%s;%s: cudnn/megdnn: %.3fms/%.3fms=%.3f\n",

  207.                    lsrc.TensorShape::to_string().c_str(),

  208.                    lflt1.TensorShape::to_string().c_str(),

  209.                    ldst.TensorShape::to_string().c_str(), t0, t1, t0 / t1);

  210.         }

  211.     }

  212. 

  213.     void cmp_dst() {

  214.         Tensor<> dst0_cpu(handle_cpu, ldst), dst1_cpu(handle_cpu, ldst);

  215.         megdnn_memcpy_D2H(handle, dst0_cpu.ptr(), dst0->ptr(), ldst.span().dist_byte());

  216.         megdnn_memcpy_D2H(handle, dst1_cpu.ptr(), dst1->ptr(), ldst.span().dist_byte());

  217.         dst0_cpu.check_with(dst1_cpu);

  218.     }

  219. 

  220.     void cmp_src() {

  221.         Tensor<> src0_cpu(handle_cpu, lsrc), src1_cpu(handle_cpu, lsrc);

  222.         megdnn_memcpy_D2H(handle, src0_cpu.ptr(), src0->ptr(), lsrc.span().dist_byte());

  223.         megdnn_memcpy_D2H(handle, src1_cpu.ptr(), src1->ptr(), lsrc.span().dist_byte());

  224.         src0_cpu.check_with(src1_cpu);

  225.     }

  226. 

  227.     void cmp_flt() {

  228.         Tensor<> flt0_cpu(handle_cpu, lflt0), flt1_cpu(handle_cpu, lflt1);

  229.         float* p0 = flt0_cpu.ptr();

  230.         float* p1 = flt1_cpu.ptr();

  231.         megdnn_memcpy_D2H(handle, p0, flt0->ptr(), lflt0.span().dist_byte());

  232.         megdnn_memcpy_D2H(handle, p1, flt1->ptr(), lflt1.span().dist_byte());

  233. 

  234.         size_t IC = lflt1[0], CHL_MUL = lflt1[1], FSIZE = lflt1[3] * lflt1[4];

  235. 

  236.         double tot_err = 0, tot_err_num = 0;

  237.         for (size_t i = 0; i < IC; ++i) {

  238.             for (size_t j = 0; j < CHL_MUL; ++j) {

  239.                 auto t0 = p0 + ((i * CHL_MUL + j) * IC + i) * FSIZE,

  240.                      t1 = p1 + (i * CHL_MUL + j) * FSIZE;

  241.                 for (size_t k = 0; k < FSIZE; ++k) {

  242.                     auto err = std::abs(diff(t0[k], t1[k]));

  243.                     tot_err += err;

  244.                     tot_err_num += 1;

  245.                     ASSERT_LT(err, 1e-2) << "failed at " << i << " " << j << " " << k

  246.                                          << " vals=" << t0[k] << "," << t1[k];

  247.                 }

  248.             }

  249.         }

  250.         auto avg_err = tot_err / tot_err_num;

  251.         ASSERT_LT(avg_err, 1e-4);

  252.     }

  253. };

  254. 

  255. }  // anonymous namespace

  256. 

  257. constexpr auto M = Convolution::Mode::CROSS_CORRELATION;

  258. 

  259. TEST_F(CUDA, CHANWISE_CONVOLUTION_FORWARD) {

  260.     Checker<Convolution> checker(handle_cuda());

  261.     bool require_algo = false;

  262.     checker.set_before_exec_callback(AlgoChecker<ConvolutionForward>(

  263.             ExecutionPolicyAlgoName{

  264.                     "DEFAULT",

  265.                     {{ConvBiasForward::algo_name<ConvBiasForward::DirectParam>(

  266.                               "CHANNEL_WISE", {})

  267.                               .c_str(),

  268.                       {}}}},

  269.             &require_algo));

  270. 

  271.     for (auto dtype : std::vector<DType>{dtype::Float32(), dtype::Float16()}) {

  272.         checker.set_dtype(0, dtype).set_dtype(1, dtype).set_dtype(2, dtype);

  273.         if (dtype.enumv() == DTypeEnum::Float16)

  274.             checker.set_epsilon(2e-2);

  275. 

  276.         // simple case

  277.         // clang-format off

  278.         for (uint32_t s : {1, 2})

  279.         for (uint32_t p : {0, 1, 2, 3})

  280.         for (size_t f : {2, 3, 5, 7})

  281.         for (size_t ocpg : {1, 3}) {

  282.             checker.set_param(gconv_param({M, p, p, s, s}))

  283.                     .execs({{2, 3, 16, 16}, {3, ocpg, 1, f, f}, {}});

  284.         }

  285.         // clang-format on

  286. 

  287.         checker.set_param(gconv_param({M, 2, 3, 2, 1}))

  288.                 .execs({{32, 12, 20, 10}, {12, 2, 1, 4, 5}, {}});

  289. 

  290.         // padding larger than kern

  291.         checker.set_param(gconv_param({M, 20, 30, 4, 5}))

  292.                 .execs({{32, 12, 20, 10}, {12, 2, 1, 4, 5}, {}});

  293.     }

  294. }

  295. 

  296. TEST_F(CUDA, CHANWISE_CONVOLUTION_FORWARD_SMALL) {

  297.     Checker<Convolution> checker(handle_cuda());

  298.     bool require_algo = false;

  299.     checker.set_before_exec_callback(AlgoChecker<ConvolutionForward>(

  300.             ExecutionPolicyAlgoName{

  301.                     "DEFAULT",

  302.                     {{ConvBiasForward::algo_name<ConvBiasForward::DirectParam>(

  303.                               "CHANNEL_WISE_SMALL", {})

  304.                               .c_str(),

  305.                       {}}}},

  306.             &require_algo));

  307.     for (auto dtype : std::vector<DType> {

  308.              dtype::Float32(),

  309. #if CUDA_VERSION >= 9000

  310.                      dtype::Float16()

  311. #endif

  312.          }) {

  313.         checker.set_dtype(0, dtype).set_dtype(1, dtype).set_dtype(2, dtype);

  314.         if (dtype.enumv() == DTypeEnum::Float16)

  315.             checker.set_epsilon(2e-2);

  316. 

  317.         // clang-format off

  318.         for (uint32_t s : {1})

  319.         for (uint32_t f : {1, 3, 5, 7}) {

  320.             checker.set_param(gconv_param({M, f / 2, f / 2, s, s}))

  321.                     .execs({{2, 3, 16, 16}, {3, 1, 1, f, f}, {}});

  322.         }

  323.         // clang-format on

  324.         checker.set_param(gconv_param({M, 1, 1, 1, 1}))

  325.                 .execs({{2, 3, 3, 16}, {3, 1, 1, 3, 3}, {}})

  326.                 .execs({{2, 3, 8, 3}, {3, 1, 1, 3, 3}, {}});

  327.     }

  328. }

  329. 

  330. TEST_F(CUDA, CHANWISE_CONVOLUTION_BACKWARD_DATA) {

  331.     Checker<ConvolutionBackwardData> checker(handle_cuda());

  332.     bool require_algo = false;

  333.     checker.set_before_exec_callback(

  334.             AlgoChecker<ConvolutionBackwardData>("CHANNEL_WISE", &require_algo));

  335. 

  336.     for (auto dtype : std::vector<DType>{dtype::Float32(), dtype::Float16()}) {

  337.         checker.set_dtype(0, dtype).set_dtype(1, dtype).set_dtype(2, dtype);

  338.         if (dtype.enumv() == DTypeEnum::Float16)

  339.             checker.set_epsilon(1e-1);

  340.         // simple case

  341.         // clang-format off

  342.         for (uint32_t s : {1, 2})

  343.         for (uint32_t p : {0, 1, 2, 3})

  344.         for (size_t f : {1, 2, 3, 5, 7})

  345.         for (size_t ocpg : {1, 3}) {

  346.             size_t ii = infer_conv_shape(16, f, s, p, true);

  347.             checker.set_param(gconv_param({M, p, p, s, s}))

  348.                     .execs({{3, ocpg, 1, f, f},

  349.                             {2, 3 * ocpg, ii, ii},

  350.                             {2, 3, 16, 16}});

  351.         }

  352.         // clang-format on

  353. 

  354.         checker.set_param(gconv_param({M, 2, 3, 2, 1}))

  355.                 .execs({{12, 3, 1, 4, 5}, {32, 36, 20, 10}, {32, 12, 39, 8}});

  356.         checker.set_param(gconv_param({M, 30, 20, 5, 4}))

  357.                 .execs({{6, 2, 1, 5, 4}, {32, 12, 12, 10}, {32, 6, 3, 2}});

  358.         checker.set_param(gconv_param({M, 20, 30, 4, 5}))

  359.                 .execs({{6, 2, 1, 4, 5}, {32, 12, 10, 12}, {32, 6, 2, 3}});

  360.     }

  361. }

  362. 

  363. TEST_F(CUDA, CHANWISE_CONVOLUTION_BACKWARD_DATA_SMALL) {

  364.     Checker<ConvolutionBackwardData> checker(handle_cuda());

  365.     bool require_algo = false;

  366.     checker.set_before_exec_callback(

  367.             AlgoChecker<ConvolutionBackwardData>("CHANNEL_WISE_SMALL", &require_algo));

  368.     for (auto dtype : std::vector<DType> {

  369.              dtype::Float32(),

  370. #if CUDA_VERSION >= 9000

  371.                      dtype::Float16()

  372. #endif

  373.          }) {

  374.         checker.set_dtype(0, dtype).set_dtype(1, dtype).set_dtype(2, dtype);

  375. 

  376.         if (dtype.enumv() == DTypeEnum::Float16)

  377.             checker.set_epsilon(2e-2);

  378. 

  379.         for (uint32_t f : {1, 3, 5, 7}) {

  380.             checker.set_param(gconv_param({M, f / 2, f / 2, 1, 1}))

  381.                     .execs({{3, 1, 1, f, f}, {2, 3, 16, 16}, {2, 3, 16, 16}});

  382.         }

  383.         checker.set_param(gconv_param({M, 1, 1, 1, 1}))

  384.                 .execs({{3, 1, 1, 3, 3}, {2, 3, 3, 16}, {2, 3, 3, 16}})

  385.                 .execs({{3, 1, 1, 3, 3}, {2, 3, 8, 3}, {2, 3, 8, 3}});

  386.     }

  387. }

  388. 

  389. TEST_F(CUDA, CHANWISE_CONVOLUTION_BACKWARD_FILTER) {

  390.     Checker<ConvolutionBackwardFilter> checker(handle_cuda());

  391.     bool require_algo = false;

  392.     checker.set_before_exec_callback(

  393.             AlgoChecker<ConvolutionBackwardFilter>("CHANNEL_WISE", &require_algo));

  394.     UniformFloatRNG rng(-0.1, 0.1);

  395.     for (auto dtype : std::vector<DType>{dtype::Float32(), dtype::Float16()}) {

  396.         checker.set_dtype(0, dtype)

  397.                 .set_dtype(1, dtype)

  398.                 .set_dtype(2, dtype)

  399.                 .set_rng(0, &rng)

  400.                 .set_rng(1, &rng);

  401.         if (dtype.enumv() == DTypeEnum::Float16)

  402.             checker.set_epsilon(2e-1);

  403.         // simple case

  404.         // clang-format off

  405.         for (uint32_t s : {1, 2})

  406.         for (uint32_t p : {0, 1, 2, 3})

  407.         for (uint32_t f : {1, 2, 3, 5, 7})

  408.         for (uint32_t ocpg : {1, 3})

  409.         for (uint32_t i : {8, 16, 32, 64}){

  410.             size_t ii = infer_conv_shape(i, f, s, p, true);

  411.             checker.set_param(gconv_param({M, p, p, s, s}))

  412.                     .execs({{2, 3, i, i},

  413.                             {2, 3 * ocpg, ii, ii},

  414.                             {3, ocpg, 1, f, f}});

  415.         }

  416.         // clang-format on

  417. 

  418.         // padding larger than kern

  419.         checker.set_param(gconv_param({M, 20, 30, 4, 5}))

  420.                 .execs({{32, 6, 2, 3}, {32, 12, 10, 12}, {6, 2, 1, 4, 5}});

  421.         // unused filter items

  422.         checker.set_param(gconv_param({M, 2, 3, 2, 3}))

  423.                 .execs({{32, 6, 1, 1}, {32, 12, 1, 1}, {6, 2, 1, 5, 7}});

  424.     }

  425. }

  426. 

  427. namespace {

  428. template <typename Op>

  429. struct AlgoCheckerMaker {

  430.     static auto make(const char* name, bool* require_algo) {

  431.         return AlgoChecker<Op>(name, require_algo);

  432.     }

  433. };

  434. 

  435. template <>

  436. struct AlgoCheckerMaker<ConvolutionForward> {

  437.     static auto make(const char* name, bool* require_algo) {

  438.         return AlgoChecker<ConvolutionForward>(

  439.                 ExecutionPolicyAlgoName{

  440.                         "DEFAULT",

  441.                         {{ConvBiasForward::algo_name<ConvBiasForward::DirectParam>(

  442.                                   name, {})

  443.                                   .c_str(),

  444.                           {}}}},

  445.                 require_algo);

  446.     }

  447. };

  448. 

  449. template <typename Op>

  450. void check_chanwise(DType io_type, DType comp_type, Handle* handle, const char* name) {

  451.     Checker<Op> checker(handle);

  452.     bool require_algo = false;

  453.     checker.set_before_exec_callback(AlgoCheckerMaker<Op>::make(name, &require_algo));

  454.     checker.set_dtype(0, io_type).set_dtype(1, io_type).set_dtype(2, io_type);

  455.     bool io16xc32 = false;

  456.     if (io_type == dtype::Float16()) {

  457.         if (comp_type == dtype::Float16()) {

  458.             checker.set_epsilon(1e-1);

  459.         } else {

  460.             io16xc32 = true;

  461.         }

  462.     }

  463.     // dispatch testcase by operation

  464.     if (std::is_same<Op, ConvolutionForward>::value) {

  465.         // align 8

  466.         checker.set_param(gconv_param({M, 7, 7, 1, 1}, io16xc32))

  467.                 .execs({{8, 2, 16, 16}, {2, 1, 1, 15, 15}, {}});

  468.         // align 1

  469.         checker.set_param(gconv_param({M, 7, 7, 1, 1}, io16xc32))

  470.                 .execs({{8, 2, 15, 15}, {2, 1, 1, 15, 15}, {}});

  471.         // align 2

  472.         checker.set_param(gconv_param({M, 7, 7, 1, 1}, io16xc32))

  473.                 .execs({{8, 2, 14, 14}, {2, 1, 1, 15, 15}, {}});

  474.         // custom padding

  475.         checker.set_param(gconv_param({M, 3, 3, 1, 1}, io16xc32))

  476.                 .execs({{8, 2, 16, 16}, {2, 1, 1, 15, 15}, {}});

  477.         // custom stride

  478.         checker.set_param(gconv_param({M, 7, 7, 2, 2}, io16xc32))

  479.                 .execs({{8, 2, 16, 16}, {2, 1, 1, 15, 15}, {}});

  480.     } else if (std::is_same<Op, ConvolutionBackwardData>::value) {

  481.         // align 8

  482.         checker.set_param(gconv_param({M, 7, 7, 1, 1}, io16xc32))

  483.                 .execs({{2, 1, 1, 15, 15}, {8, 2, 16, 16}, {8, 2, 16, 16}});

  484.         // align 1

  485.         checker.set_param(gconv_param({M, 7, 7, 1, 1}, io16xc32))

  486.                 .execs({{2, 1, 1, 15, 15}, {8, 2, 15, 15}, {8, 2, 15, 15}});

  487.         // align 2

  488.         checker.set_param(gconv_param({M, 7, 7, 1, 1}, io16xc32))

  489.                 .execs({{2, 1, 1, 15, 15}, {8, 2, 14, 14}, {8, 2, 14, 14}});

  490.         // custom padding

  491.         checker.set_param(gconv_param({M, 3, 3, 1, 1}, io16xc32))

  492.                 .execs({{2, 1, 1, 15, 15}, {8, 2, 8, 8}, {8, 2, 16, 16}});

  493.         // custom stride

  494.         checker.set_param(gconv_param({M, 7, 7, 2, 2}, io16xc32))

  495.                 .execs({{2, 1, 1, 15, 15}, {8, 2, 7, 7}, {8, 2, 14, 14}});

  496.     } else if (std::is_same<Op, ConvolutionBackwardFilter>::value) {

  497.         // align 8

  498.         checker.set_param(gconv_param({M, 7, 7, 1, 1}, io16xc32))

  499.                 .execs({{8, 2, 16, 16}, {8, 2, 16, 16}, {2, 1, 1, 15, 15}});

  500.         // align 1

  501.         checker.set_param(gconv_param({M, 7, 7, 1, 1}, io16xc32))

  502.                 .execs({{8, 2, 15, 15}, {8, 2, 15, 15}, {2, 1, 1, 15, 15}});

  503.         // align 2

  504.         checker.set_param(gconv_param({M, 7, 7, 1, 1}, io16xc32))

  505.                 .execs({{8, 2, 14, 14}, {8, 2, 14, 14}, {2, 1, 1, 15, 15}});

  506.         // custom padding

  507.         checker.set_param(gconv_param({M, 3, 3, 1, 1}, io16xc32))

  508.                 .execs({{8, 2, 16, 16}, {8, 2, 8, 8}, {2, 1, 1, 15, 15}});

  509.         // custom stride

  510.         checker.set_param(gconv_param({M, 7, 7, 2, 2}, io16xc32))

  511.                 .execs({{8, 2, 14, 14}, {8, 2, 7, 7}, {2, 1, 1, 15, 15}});

  512.     }

  513. }

  514. }  // namespace

  515. 

  516. #define MEGDNN_FOREACH_CUTLASS_CHANWISE_CONV_FMA_KERNEL(cb) \

  517.     cb(1, 128, 128, 8, 32, 64, 8);                          \

  518.     cb(2, 128, 64, 8, 64, 32, 8);                           \

  519.     cb(3, 128, 32, 8, 64, 32, 8);                           \

  520.     cb(4, 64, 128, 8, 32, 64, 8);                           \

  521.     cb(5, 32, 128, 8, 32, 64, 8);                           \

  522.     cb(6, 64, 64, 8, 32, 64, 8);                            \

  523.     cb(7, 32, 64, 8, 32, 64, 8);                            \

  524.     cb(8, 32, 32, 8, 32, 32, 8);                            \

  525.     cb(9, 64, 32, 8, 64, 32, 8);

  526. 

  527. #define cb(tag, tbm, tbn, tbk, wm, wn, wk)                                       \

  528.     TEST_F(CUDA, CHANWISE_CONVOLUTION_FORWARD_CUTLASS_FMA_##tag) {               \

  529.         require_compute_capability(6, 1);                                        \

  530.         check_chanwise<ConvolutionForward>(                                      \

  531.                 dtype::Float32(), dtype::Float32(), handle_cuda(),               \

  532.                 "FLOAT32_NCHW_FMA_IMPLICIT_BATCHED_GEMM_" #tbm "X" #tbn "X" #tbk \

  533.                 "_" #wm "X" #wn "X" #wk "_2stage");                              \

  534.     }

  535. 

  536. MEGDNN_FOREACH_CUTLASS_CHANWISE_CONV_FMA_KERNEL(cb)

  537. 

  538. #undef cb

  539. 

  540. #define cb(tag, tbm, tbn, tbk, wm, wn, wk)                                       \

  541.     TEST_F(CUDA, CHANWISE_CONVOLUTION_BACKWARD_DATA_CUTLASS_FMA_##tag) {         \

  542.         require_compute_capability(6, 1);                                        \

  543.         check_chanwise<ConvolutionBackwardData>(                                 \

  544.                 dtype::Float32(), dtype::Float32(), handle_cuda(),               \

  545.                 "FLOAT32_NCHW_FMA_IMPLICIT_BATCHED_GEMM_" #tbm "X" #tbn "X" #tbk \

  546.                 "_" #wm "X" #wn "X" #wk "_2stage");                              \

  547.     }

  548. 

  549. MEGDNN_FOREACH_CUTLASS_CHANWISE_CONV_FMA_KERNEL(cb)

  550. 

  551. #undef cb

  552. 

  553. #define cb(tag, tbm, tbn, tbk, wm, wn, wk)                                       \

  554.     TEST_F(CUDA, CHANWISE_CONVOLUTION_BACKWARD_FILTER_CUTLASS_FMA_##tag) {       \

  555.         require_compute_capability(6, 1);                                        \

  556.         check_chanwise<ConvolutionBackwardFilter>(                               \

  557.                 dtype::Float32(), dtype::Float32(), handle_cuda(),               \

  558.                 "FLOAT32_NCHW_FMA_IMPLICIT_BATCHED_GEMM_" #tbm "X" #tbn "X" #tbk \

  559.                 "_" #wm "X" #wn "X" #wk "_2stage");                              \

  560.     }

  561. 

  562. MEGDNN_FOREACH_CUTLASS_CHANWISE_CONV_FMA_KERNEL(cb)

  563. 

  564. #undef cb

  565. 

  566. #undef MEGDNN_FOREACH_CUTLASS_CHANWISE_CONV_FMA_KERNEL

  567. 

  568. #if CUDA_VERSION >= 10010

  569. #define MEGDNN_FOREACH_CUTLASS_CHANWISE_CONV_HMMA_KERNEL(cb) \

  570.     cb(1, 128, 128, 32, 32, 32, 32);                         \

  571.     cb(2, 128, 256, 32, 64, 64, 32);                         \

  572.     cb(3, 128, 64, 32, 32, 32, 32);                          \

  573.     cb(4, 64, 128, 32, 32, 32, 32);                          \

  574.     cb(5, 64, 64, 32, 32, 32, 32);

  575. #else

  576. // hmma instruction need cuda version >= 10.2, disable hmma testcases in this path

  577. #define MEGDNN_FOREACH_CUTLASS_CHANWISE_CONV_HMMA_KERNEL(cb)

  578. #endif

  579. 

  580. // check both ioc16 and io16xc32

  581. #define cb(tag, tbm, tbn, tbk, wm, wn, wk)                                        \

  582.     TEST_F(CUDA, CHANWISE_CONVOLUTION_FORWARD_CUTLASS_HMMA_##tag) {               \

  583.         require_compute_capability(7, 0);                                         \

  584.         check_chanwise<ConvolutionForward>(                                       \

  585.                 dtype::Float16(), dtype::Float16(), handle_cuda(),                \

  586.                 "FLOAT16_NCHW_HMMA_IMPLICIT_BATCHED_GEMM_" #tbm "X" #tbn "X" #tbk \

  587.                 "_" #wm "X" #wn "X" #wk "_2stage");                               \

  588.         check_chanwise<ConvolutionForward>(                                       \

  589.                 dtype::Float16(), dtype::Float32(), handle_cuda(),                \

  590.                 "FLOAT16_NCHW_HMMA_IMPLICIT_BATCHED_GEMM_" #tbm "X" #tbn "X" #tbk \

  591.                 "_" #wm "X" #wn "X" #wk "_2stage");                               \

  592.     }

  593. 

  594. MEGDNN_FOREACH_CUTLASS_CHANWISE_CONV_HMMA_KERNEL(cb)

  595. 

  596. #undef cb

  597. 

  598. #define cb(tag, tbm, tbn, tbk, wm, wn, wk)                                        \

  599.     TEST_F(CUDA, CHANWISE_CONVOLUTION_BACKWARD_DATA_CUTLASS_HMMA_##tag) {         \

  600.         require_compute_capability(7, 0);                                         \

  601.         check_chanwise<ConvolutionBackwardData>(                                  \

  602.                 dtype::Float16(), dtype::Float16(), handle_cuda(),                \

  603.                 "FLOAT16_NCHW_HMMA_IMPLICIT_BATCHED_GEMM_" #tbm "X" #tbn "X" #tbk \

  604.                 "_" #wm "X" #wn "X" #wk "_mma8X8X4_2stage");                      \

  605.         check_chanwise<ConvolutionBackwardData>(                                  \

  606.                 dtype::Float16(), dtype::Float32(), handle_cuda(),                \

  607.                 "FLOAT16_NCHW_HMMA_IMPLICIT_BATCHED_GEMM_" #tbm "X" #tbn "X" #tbk \

  608.                 "_" #wm "X" #wn "X" #wk "_mma8X8X4_2stage");                      \

  609.     }

  610. 

  611. MEGDNN_FOREACH_CUTLASS_CHANWISE_CONV_HMMA_KERNEL(cb)

  612. 

  613. #undef cb

  614. 

  615. #define cb(tag, tbm, tbn, tbk, wm, wn, wk)                                        \

  616.     TEST_F(CUDA, CHANWISE_CONVOLUTION_BACKWARD_FILTER_CUTLASS_HMMA_##tag) {       \

  617.         require_compute_capability(7, 0);                                         \

  618.         check_chanwise<ConvolutionBackwardData>(                                  \

  619.                 dtype::Float16(), dtype::Float32(), handle_cuda(),                \

  620.                 "FLOAT16_NCHW_HMMA_IMPLICIT_BATCHED_GEMM_" #tbm "X" #tbn "X" #tbk \

  621.                 "_" #wm "X" #wn "X" #wk "_mma8X8X4_2stage");                      \

  622.     }

  623. 

  624. MEGDNN_FOREACH_CUTLASS_CHANWISE_CONV_HMMA_KERNEL(cb)

  625. 

  626. #undef cb

  627. 

  628. #undef MEGDNN_FOREACH_CUTLASS_CHANWISE_CONV_FWD_HMMA_KERNEL

  629. 

  630. #if MEGDNN_WITH_BENCHMARK

  631. TEST_F(CUDA, CHANWISE_CONVOLUTION_FORWARD_BENCH_CHECK) {

  632.     auto handle = handle_cuda();

  633.     auto handle_cpu = handle_naive();

  634.     auto conv0 = handle->create_operator<ConvolutionForward>();

  635.     auto conv1 = handle->create_operator<ConvolutionForward>();

  636.     BenchmarkEnv<0, 1, 2> benv(handle, handle_cpu);

  637. 

  638.     auto run = [&](size_t N, size_t IC, size_t IH, size_t IW, size_t CHL_MUL, size_t FH,

  639.                    size_t FW, size_t PH, size_t PW) {

  640.         benv.alloc(N, IC, IH, IW, CHL_MUL, FH, FW, PH, PW);

  641.         benv.fill_src();

  642.         benv.fill_flt();

  643.         benv.exec_convolution(conv0.get(), conv1.get());

  644.         benv.cmp_dst();

  645.     };

  646. 

  647.     run(64, 60, 50, 50, 1, 3, 3, 1, 1);

  648.     if (check_need_full_bench()) {

  649.         run(64, 728, 18, 18, 2, 5, 5, 2, 2);

  650.         run(64, 64, 150, 150, 2, 3, 3, 1, 1);

  651.         run(1, 2048, 4, 4, 2, 3, 3, 1, 1);

  652.     }

  653. }

  654. 

  655. TEST_F(CUDA, CHANWISE_CONVOLUTION_BWD_DATA_BENCH_CHECK) {

  656.     auto handle = handle_cuda();

  657.     auto handle_cpu = handle_naive();

  658.     auto conv0 = handle->create_operator<ConvolutionBackwardData>();

  659.     auto conv1 = handle->create_operator<ConvolutionBackwardData>();

  660.     BenchmarkEnv<1, 2, 0> benv(handle, handle_cpu);

  661. 

  662.     auto run = [&](size_t N, size_t IC, size_t IH, size_t IW, size_t CHL_MUL, size_t FH,

  663.                    size_t FW, size_t PH, size_t PW) {

  664.         benv.alloc(N, IC, IH, IW, CHL_MUL, FH, FW, PH, PW);

  665.         benv.fill_dst();

  666.         benv.fill_flt();

  667.         benv.exec(conv0.get(), conv1.get());

  668.         benv.cmp_src();

  669.     };

  670. 

  671.     run(64, 60, 50, 50, 1, 3, 3, 1, 1);

  672.     if (check_need_full_bench()) {

  673.         run(64, 728, 18, 18, 2, 5, 5, 2, 2);

  674.         run(64, 64, 150, 150, 2, 3, 3, 1, 1);

  675.         run(1, 2048, 4, 4, 2, 3, 3, 1, 1);

  676.     }

  677. }

  678. 

  679. TEST_F(CUDA, CHANWISE_CONVOLUTION_BWD_FILTER_BENCH_CHECK) {

  680.     auto handle = handle_cuda();

  681.     auto handle_cpu = handle_naive();

  682.     auto conv0 = handle->create_operator<ConvolutionBackwardFilter>();

  683.     auto conv1 = handle->create_operator<ConvolutionBackwardFilter>();

  684.     BenchmarkEnv<0, 2, 1> benv(handle, handle_cpu);

  685. 

  686.     auto run = [&](size_t N, size_t IC, size_t IH, size_t IW, size_t CHL_MUL, size_t FH,

  687.                    size_t FW, size_t PH, size_t PW) {

  688.         benv.alloc(N, IC, IH, IW, CHL_MUL, FH, FW, PH, PW);

  689.         benv.fill_src();

  690.         benv.fill_dst();

  691.         benv.exec(conv0.get(), conv1.get());

  692.         benv.cmp_flt();

  693.     };

  694. 

  695.     run(64, 60, 50, 50, 1, 3, 3, 1, 1);

  696.     if (check_need_full_bench()) {

  697.         run(64, 728, 18, 18, 2, 5, 5, 2, 2);

  698.         run(64, 64, 150, 150, 2, 3, 3, 1, 1);

  699.         run(1, 2048, 4, 4, 2, 3, 3, 1, 1);

  700.     }

  701. }

  702. 

  703. TEST_F(CUDA, CHANWISE_CONVOLUTION_BENCH_ALL_ALGO_FWD) {

  704.     // enable profiling

  705.     std::unique_ptr<OprProxy<ConvolutionForward>> proxy{

  706.             new OprProxy<ConvolutionForward>{true}};

  707.     proxy->warmup_times = 1;

  708.     proxy->exec_times = 10;

  709.     Benchmarker<ConvolutionForward> checker(handle_cuda());

  710.     checker.set_times(1);

  711.     ConvolutionForward::Param param;

  712.     param.sparse = ConvolutionForward::Param::Sparse::GROUP;

  713.     checker.set_param(param);

  714.     checker.set_proxy(proxy);

  715. 

  716.     auto run = [&](size_t N, size_t C, size_t IH, size_t IW, size_t FH, size_t FW) {

  717.         checker.proxy()->target_execution_policy = {};

  718.         checker.execs({{N, C, IH, IW}, {C, 1, 1, FH, FW}, {}});

  719.     };

  720. 

  721.     run(128, 64, 90, 80, 3, 3);

  722.     run(128, 90, 100, 100, 3, 5);

  723.     run(128, 32, 62, 62, 5, 5);

  724. }

  725. 

  726. TEST_F(CUDA, CHANWISE_CONVOLUTION_BENCH_ALL_ALGO_BWD_DATA) {

  727.     // enable profiling

  728.     std::unique_ptr<OprProxy<ConvolutionBackwardData>> proxy{

  729.             new OprProxy<ConvolutionBackwardData>{true}};

  730.     proxy->warmup_times = 1;

  731.     proxy->exec_times = 10;

  732.     Benchmarker<ConvolutionBackwardData> checker(handle_cuda());

  733.     checker.set_times(1);

  734.     ConvolutionBackwardData::Param param;

  735.     param.sparse = ConvolutionForward::Param::Sparse::GROUP;

  736.     checker.set_param(param);

  737.     checker.set_proxy(proxy);

  738. 

  739.     auto run = [&](size_t N, size_t C, size_t IH, size_t IW, size_t FH, size_t FW) {

  740.         checker.proxy()->target_execution_policy.algo.reset();

  741.         checker.execs(

  742.                 {{C, 1, 1, FH, FW}, {N, C, IH - FH + 1, IW - FW + 1}, {N, C, IH, IW}});

  743.     };

  744. 

  745.     run(128, 64, 90, 80, 3, 3);

  746.     run(128, 90, 100, 100, 3, 5);

  747.     run(128, 32, 62, 62, 5, 5);

  748. }

  749. 

  750. TEST_F(CUDA, CHANWISE_CONVOLUTION_BENCH_ALL_ALGO_BWD_FILTER) {

  751.     // enable profiling

  752.     std::unique_ptr<OprProxy<ConvolutionBackwardFilter>> proxy{

  753.             new OprProxy<ConvolutionBackwardFilter>{true}};

  754.     proxy->warmup_times = 1;

  755.     proxy->exec_times = 10;

  756.     Benchmarker<ConvolutionBackwardFilter> checker(handle_cuda());

  757.     checker.set_times(1);

  758.     ConvolutionBackwardFilter::Param param;

  759.     param.sparse = ConvolutionForward::Param::Sparse::GROUP;

  760.     checker.set_param(param);

  761.     checker.set_proxy(proxy);

  762. 

  763.     auto run = [&](size_t N, size_t C, size_t IH, size_t IW, size_t FH, size_t FW) {

  764.         checker.proxy()->target_execution_policy.algo.reset();

  765.         checker.execs(

  766.                 {{N, C, IH, IW}, {N, C, IH - FH + 1, IW - FW + 1}, {C, 1, 1, FH, FW}});

  767.     };

  768. 

  769.     run(128, 64, 90, 80, 3, 3);

  770.     run(128, 90, 100, 100, 3, 5);

  771.     run(128, 32, 62, 62, 5, 5);

  772. }

  773. 

  774. TEST_F(CUDA, BENCHMARK_CHANWISE_CONV_ALL_ALGO_FORWARD) {

  775.     CUBenchmarker<ConvolutionForward> bencher(handle_cuda());

  776.     size_t RUNS = 10;

  777.     bencher.set_display(false).set_times(RUNS);

  778.     std::unique_ptr<OprProxy<ConvolutionForward>> proxy{

  779.             new OprProxy<ConvolutionForward>{true}};

  780.     bencher.set_proxy(proxy);

  781. 

  782.     Convolution::Param param;

  783.     param.format = ConvBias::Param::Format::NCHW;

  784.     param.sparse = Convolution::Param::Sparse::GROUP;

  785.     NormalRNG rng;

  786. 

  787.     auto run = [&](size_t batch, size_t c, size_t ih, size_t iw, size_t f, size_t s) {

  788.         param.pad_h = f / 2;

  789.         param.pad_w = f / 2;

  790.         param.stride_h = s;

  791.         param.stride_w = s;

  792.         param.compute_mode = param::Convolution::ComputeMode::DEFAULT;

  793. 

  794.         TensorShape src = {batch, c, ih, iw}, filter = {c, 1, 1, f, f};

  795. 

  796.         TensorLayout dst_layout;

  797.         auto opr = handle_cuda()->create_operator<Convolution>();

  798.         opr->param() = param;

  799.         opr->deduce_layout(

  800.                 {src, dtype::Float32()}, {filter, dtype::Float32()}, dst_layout);

  801.         float bandwith = static_cast<float>(

  802.                                  src.total_nr_elems() + filter.total_nr_elems() +

  803.                                  dst_layout.total_nr_elems()) /

  804.                          (1024 * 1024 * 1024) * 1e3;

  805. 

  806.         bencher.set_param(param)

  807.                 .set_dtype(0, dtype::Float32())

  808.                 .set_dtype(1, dtype::Float32())

  809.                 .set_dtype(2, dtype::Float32())

  810.                 .set_rng(0, &rng)

  811.                 .set_rng(1, &rng);

  812.         bencher.proxy()->target_execution_policy = {};

  813.         auto time_in_ms_fp32 = bencher.execs({src, filter, {}}) / RUNS;

  814. 

  815.         bencher.set_param(param)

  816.                 .set_dtype(0, dtype::Float16())

  817.                 .set_dtype(1, dtype::Float16())

  818.                 .set_dtype(2, dtype::Float16())

  819.                 .set_rng(0, &rng)

  820.                 .set_rng(1, &rng);

  821.         bencher.proxy()->target_execution_policy = {};

  822.         auto time_in_ms_fp16 = bencher.execs({src, filter, {}}) / RUNS;

  823. 

  824.         bencher.proxy()->target_execution_policy.algo.reset();

  825.         param.compute_mode = param::Convolution::ComputeMode::FLOAT32;

  826.         bencher.set_param(param);

  827.         auto time_in_ms_pseudo_fp16 = bencher.execs({src, filter, {}}) / RUNS;

  828. 

  829.         printf("stride=%zu src=%s, filter=%s, float32: %.2fms %.2fGB/s "

  830.                "float16: %.2fms %.2fGB/s "

  831.                "pseudo float16: %.2fms %.2fGB/s "

  832.                "speedup: "

  833.                "%0.2f (fp16/fp32) %.2f (fp16/pseudo fp16)\n",

  834.                s, src.to_string().c_str(), filter.to_string().c_str(), time_in_ms_fp32,

  835.                bandwith * 4 / time_in_ms_fp32, time_in_ms_fp16,

  836.                bandwith * 2 / time_in_ms_fp16, time_in_ms_pseudo_fp16,

  837.                bandwith * 2 / time_in_ms_pseudo_fp16, time_in_ms_fp32 / time_in_ms_fp16,

  838.                time_in_ms_pseudo_fp16 / time_in_ms_fp16);

  839.     };

  840. 

  841.     // clang-format off

  842.     for (size_t s : {1, 2})

  843.     for (size_t f : {3, 5, 7})

  844.     for (size_t batch : {64})

  845.     for (size_t c : {16, 32, 64, 128})

  846.     for (size_t ih: {128, 256})

  847.     for (size_t iw : {128, 256})

  848.         run(batch, c, ih, iw, f, s);

  849.     // clang-format on

  850. 

  851.     run(128, 192, 28, 28, 3, 1);

  852.     run(128, 192, 28, 28, 3, 2);

  853.     run(128, 576, 14, 14, 3, 1);

  854.     run(128, 384, 14, 14, 3, 1);

  855.     run(128, 32, 112, 112, 3, 1);

  856.     run(128, 960, 7, 7, 3, 1);

  857.     run(128, 384, 14, 14, 3, 1);

  858.     run(128, 144, 56, 56, 3, 2);

  859.     run(128, 384, 14, 14, 3, 1);

  860.     run(128, 144, 56, 56, 3, 1);

  861.     run(128, 96, 112, 112, 3, 2);

  862.     run(128, 384, 14, 14, 3, 1);

  863.     run(128, 192, 28, 28, 3, 1);

  864.     run(128, 576, 14, 14, 3, 1);

  865.     run(128, 576, 14, 14, 3, 2);

  866. }

  867. 

  868. TEST_F(CUDA, BENCHMARK_CHANWISE_CONV_FORWARD_FLOAT) {

  869.     CUBenchmarker<ConvolutionForward> bencher(handle_cuda());

  870.     size_t RUNS = 1;

  871.     bencher.set_display(false).set_times(RUNS);

  872.     bencher.set_before_exec_callback(

  873.             AlgoChecker<ConvolutionForward>(ExecutionPolicyAlgoName{

  874.                     "DEFAULT",

  875.                     {{ConvBiasForward::algo_name<ConvBiasForward::DirectParam>(

  876.                               "CHANNEL_WISE", {})

  877.                               .c_str(),

  878.                       {}}}}));

  879. 

  880.     Convolution::Param param;

  881.     param.format = ConvBias::Param::Format::NCHW;

  882.     param.sparse = Convolution::Param::Sparse::GROUP;

  883.     NormalRNG rng;

  884. 

  885.     auto run = [&](size_t batch, size_t c, size_t ih, size_t iw, size_t f, size_t s) {

  886.         param.pad_h = f / 2;

  887.         param.pad_w = f / 2;

  888.         param.stride_h = s;

  889.         param.stride_w = s;

  890.         param.compute_mode = param::Convolution::ComputeMode::DEFAULT;

  891. 

  892.         TensorShape src = {batch, c, ih, iw}, filter = {c, 1, 1, f, f};

  893. 

  894.         TensorLayout dst_layout;

  895.         auto opr = handle_cuda()->create_operator<Convolution>();

  896.         opr->param() = param;

  897.         opr->deduce_layout(

  898.                 {src, dtype::Float32()}, {filter, dtype::Float32()}, dst_layout);

  899.         float bandwith = static_cast<float>(

  900.                                  src.total_nr_elems() + filter.total_nr_elems() +

  901.                                  dst_layout.total_nr_elems()) /

  902.                          (1024 * 1024 * 1024) * 1e3;

  903. 

  904.         bencher.set_param(param)

  905.                 .set_dtype(0, dtype::Float32())

  906.                 .set_dtype(1, dtype::Float32())

  907.                 .set_dtype(2, dtype::Float32())

  908.                 .set_rng(0, &rng)

  909.                 .set_rng(1, &rng);

  910.         auto time_in_ms_fp32 = bencher.execs({src, filter, {}}) / RUNS;

  911. 

  912.         bencher.set_param(param)

  913.                 .set_dtype(0, dtype::Float16())

  914.                 .set_dtype(1, dtype::Float16())

  915.                 .set_dtype(2, dtype::Float16())

  916.                 .set_rng(0, &rng)

  917.                 .set_rng(1, &rng);

  918.         auto time_in_ms_fp16 = bencher.execs({src, filter, {}}) / RUNS;

  919. 

  920.         printf("stride=%zu src=%s, filter=%s, float32: %.2fms %.2fGB/s "

  921.                "float16: %.2fms %.2fGB/s "

  922.                "speedup: "

  923.                "%0.2f (fp16/fp32)\n",

  924.                s, src.to_string().c_str(), filter.to_string().c_str(), time_in_ms_fp32,

  925.                bandwith * 4 / time_in_ms_fp32, time_in_ms_fp16,

  926.                bandwith * 2 / time_in_ms_fp16, time_in_ms_fp32 / time_in_ms_fp16);

  927.     };

  928. 

  929.     // clang-format off

  930.     for (size_t s : {1})

  931.     for (size_t f : {3, 5, 7})

  932.     for (size_t batch : {64})

  933.     for (size_t c : {16, 32, 64, 128})

  934.     for (size_t ih: {8, 16, 32, 128, 256})

  935.     for (size_t iw : {8, 16, 32, 128, 256})

  936.         run(batch, c, ih, iw, f, s);

  937.     // clang-format on

  938. }

  939. 

  940. TEST_F(CUDA, BENCHMARK_CHANWISE_CONV_FORWARD_FLOAT_SMALL) {

  941.     CUBenchmarker<ConvolutionForward> bencher(handle_cuda());

  942.     size_t RUNS = 1;

  943.     bencher.set_display(false).set_times(RUNS);

  944. 

  945.     Convolution::Param param;

  946.     param.format = ConvBias::Param::Format::NCHW;

  947.     param.sparse = Convolution::Param::Sparse::GROUP;

  948.     NormalRNG rng;

  949. 

  950.     auto run = [&](size_t batch, size_t c, size_t ih, size_t iw, size_t f, size_t s) {

  951.         param.pad_h = f / 2;

  952.         param.pad_w = f / 2;

  953.         param.stride_h = s;

  954.         param.stride_w = s;

  955.         param.compute_mode = param::Convolution::ComputeMode::DEFAULT;

  956. 

  957.         TensorShape src = {batch, c, ih, iw}, filter = {c, 1, 1, f, f};

  958. 

  959.         TensorLayout dst_layout;

  960.         auto opr = handle_cuda()->create_operator<Convolution>();

  961.         opr->param() = param;

  962.         opr->deduce_layout(

  963.                 {src, dtype::Float32()}, {filter, dtype::Float32()}, dst_layout);

  964.         float bandwith = static_cast<float>(

  965.                                  src.total_nr_elems() + filter.total_nr_elems() +

  966.                                  dst_layout.total_nr_elems()) /

  967.                          (1024 * 1024 * 1024) * 1e3;

  968. 

  969.         bencher.set_param(param)

  970.                 .set_dtype(0, dtype::Float32())

  971.                 .set_dtype(1, dtype::Float32())

  972.                 .set_dtype(2, dtype::Float32())

  973.                 .set_rng(0, &rng)

  974.                 .set_rng(1, &rng)

  975.                 .set_before_exec_callback(AlgoChecker<

  976.                                           ConvolutionForward>(ExecutionPolicyAlgoName{

  977.                         "DEFAULT",

  978.                         {{ConvBiasForward::algo_name<ConvBiasForward::DirectParam>(

  979.                                   "CHANNEL_WISE", {})

  980.                                   .c_str(),

  981.                           {}}}}));

  982.         auto time_in_ms_fp32_normal = bencher.execs({src, filter, {}}) / RUNS;

  983. 

  984.         bencher.set_before_exec_callback(

  985.                 AlgoChecker<ConvolutionForward>(ExecutionPolicyAlgoName{

  986.                         "DEFAULT",

  987.                         {{ConvBiasForward::algo_name<ConvBiasForward::DirectParam>(

  988.                                   "CHANNEL_WISE", {})

  989.                                   .c_str(),

  990.                           {}}}}));

  991. 

  992.         auto time_in_ms_fp32_small = bencher.execs({src, filter, {}}) / RUNS;

  993. 

  994.         bencher.set_param(param)

  995.                 .set_dtype(0, dtype::Float16())

  996.                 .set_dtype(1, dtype::Float16())

  997.                 .set_dtype(2, dtype::Float16())

  998.                 .set_rng(0, &rng)

  999.                 .set_rng(1, &rng);

  1000.         auto time_in_ms_fp16_small = bencher.execs({src, filter, {}}) / RUNS;

  1001. 

  1002.         printf("stride=%zu src=%s, filter=%s, fp32 normal: %.2fms %.2fGB/s "

  1003.                "small: %.2fms %.2fGB/s, fp16 small: %.2fms %.2fGB/s, "

  1004.                "speedup: "

  1005.                "%0.2f (fp32 small/normal) %0.2f (small fp16/fp32)\n",

  1006.                s, src.to_string().c_str(), filter.to_string().c_str(),

  1007.                time_in_ms_fp32_normal, bandwith * 4 / time_in_ms_fp32_normal,

  1008.                time_in_ms_fp32_small, bandwith * 4 / time_in_ms_fp32_small,

  1009.                time_in_ms_fp16_small, bandwith * 2 / time_in_ms_fp16_small,

  1010.                time_in_ms_fp32_normal / time_in_ms_fp32_small,

  1011.                time_in_ms_fp32_small / time_in_ms_fp16_small);

  1012.     };

  1013. 

  1014.     // clang-format off

  1015.     for (size_t s : {1})

  1016.     for (size_t f : {3, 5})

  1017.     for (size_t batch : {64})

  1018.     for (size_t c : {16, 32, 64, 128})

  1019.     for (size_t ih: {8, 16, 32})

  1020.     for (size_t iw : {8, 16, 32})

  1021.         run(batch, c, ih, iw, f, s);

  1022.     // clang-format on

  1023. 

  1024.     run(128, 192, 28, 28, 3, 1);

  1025.     run(128, 576, 14, 14, 3, 1);

  1026.     run(128, 384, 14, 14, 3, 1);

  1027.     run(128, 960, 7, 7, 3, 1);

  1028.     run(128, 384, 14, 14, 3, 1);

  1029.     run(128, 384, 14, 14, 3, 1);

  1030.     run(128, 384, 14, 14, 3, 1);

  1031.     run(128, 192, 28, 28, 3, 1);

  1032.     run(128, 576, 14, 14, 3, 1);

  1033. }

  1034. 

  1035. TEST_F(CUDA, BENCHMARK_CHANWISE_CONV_FORWARD_CUDNN_DNN) {

  1036.     CUBenchmarker<ConvBiasForward> bencher(handle_cuda());

  1037.     size_t RUNS = 1;

  1038.     bencher.set_display(false).set_times(RUNS);

  1039. 

  1040.     ConvBias::Param param;

  1041.     param.format = ConvBias::Param::Format::NCHW;

  1042.     param.sparse = ConvBias::Param::Sparse::GROUP;

  1043.     NormalRNG rng;

  1044. 

  1045.     auto run = [&](size_t batch, size_t c, size_t ih, size_t iw, size_t f, size_t s) {

  1046.         param.pad_h = f / 2;

  1047.         param.pad_w = f / 2;

  1048.         param.stride_h = s;

  1049.         param.stride_w = s;

  1050.         param.compute_mode = param::ConvBias::ComputeMode::DEFAULT;

  1051. 

  1052.         TensorShape src = {batch, c, ih, iw}, filter = {c, 1, 1, f, f},

  1053.                     bias = {1, c, 1, 1};

  1054. 

  1055.         TensorLayout dst_layout;

  1056.         auto opr = handle_cuda()->create_operator<ConvBias>();

  1057.         opr->param() = param;

  1058.         opr->deduce_layout(

  1059.                 {src, dtype::Float32()}, {filter, dtype::Float32()},

  1060.                 {bias, dtype::Float32()}, {}, dst_layout);

  1061.         float computation_mops =

  1062.                 static_cast<float>(dst_layout.total_nr_elems() * f * f * 2) * 1e-6;

  1063. 

  1064.         bencher.set_param(param)

  1065.                 .set_dtype(0, dtype::Float32())

  1066.                 .set_dtype(1, dtype::Float32())

  1067.                 .set_dtype(2, dtype::Float32())

  1068.                 .set_rng(0, &rng)

  1069.                 .set_rng(1, &rng);

  1070.         bencher.set_before_exec_callback(

  1071.                 AlgoChecker<ConvBiasForward>(".+CHANNEL_WISE.+"));

  1072.         auto time_in_ms_dnn = bencher.execs({src, filter, bias, {}, {}}) / RUNS;

  1073. 

  1074.         bencher.set_param(param)

  1075.                 .set_dtype(0, dtype::Float32())

  1076.                 .set_dtype(1, dtype::Float32())

  1077.                 .set_dtype(2, dtype::Float32())

  1078.                 .set_rng(0, &rng)

  1079.                 .set_rng(1, &rng);

  1080.         bencher.set_before_exec_callback(AlgoChecker<ConvBiasForward>(

  1081.                 ".+CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_GEMM.+"));

  1082.         auto time_in_ms_cudnn = bencher.execs({src, filter, bias, {}, {}}) / RUNS;

  1083. 

  1084.         printf("stride=%zu src=%s, filter=%s, dst=%s, dnn: %.2fms %.2fGB/s "

  1085.                "cudnn: %.2fms %.2fGB/s "

  1086.                "speedup: "

  1087.                "%0.2f (dnn/cudnn)\n",

  1088.                s, src.to_string().c_str(), filter.to_string().c_str(),

  1089.                dst_layout.to_string().c_str(), time_in_ms_dnn,

  1090.                computation_mops / time_in_ms_dnn, time_in_ms_cudnn,

  1091.                computation_mops / time_in_ms_cudnn, time_in_ms_cudnn / time_in_ms_dnn);

  1092.     };

  1093. 

  1094.     // clang-format off

  1095.     for(size_t batch:{1, 16, 32, 64, 128}){

  1096.         run(batch, 32, 112, 112, 3, 1);

  1097.         run(batch, 96, 112, 112, 3, 2);

  1098.         run(batch, 96, 112, 112, 3, 1);

  1099.         run(batch, 144, 56, 56, 3, 2);

  1100.         run(batch, 144, 56, 56, 3, 1);

  1101.         run(batch, 192, 28, 28, 3, 1);

  1102.         run(batch, 384, 14, 14, 3, 1);

  1103.         run(batch, 576, 14, 14, 3, 1);

  1104.         run(batch, 960, 7, 7, 3, 1);

  1105.         //! calibrate heu algo policy hw_size param

  1106.         run(batch, 144, 24, 24, 3, 1);

  1107.         run(batch, 144, 22, 22, 3, 1);

  1108.         run(batch, 144, 20, 20, 3, 1);

  1109.         run(batch, 144, 18, 18, 3, 1);

  1110.     }

  1111.     // clang-format on

  1112. }

  1113. 

  1114. TEST_F(CUDA, BENCHMARK_CHANWISE_CONV_BACKWARD_DATA_FLOAT_SMALL) {

  1115.     CUBenchmarker<ConvolutionBackwardData> bencher(handle_cuda());

  1116.     size_t RUNS = 1;

  1117.     bencher.set_display(false).set_times(RUNS);

  1118. 

  1119.     ConvolutionBackwardData::Param param;

  1120.     param.format = Convolution::Param::Format::NCHW;

  1121.     param.sparse = Convolution::Param::Sparse::GROUP;

  1122.     NormalRNG rng;

  1123. 

  1124.     auto run = [&](size_t batch, size_t c, size_t ih, size_t iw, size_t f, size_t s) {

  1125.         param.pad_h = f / 2;

  1126.         param.pad_w = f / 2;

  1127.         param.stride_h = s;

  1128.         param.stride_w = s;

  1129.         param.compute_mode = param::Convolution::ComputeMode::DEFAULT;

  1130. 

  1131.         TensorShape src = {batch, c, ih, iw}, filter = {c, 1, 1, f, f};

  1132.         float bandwith = static_cast<float>(

  1133.                                  src.total_nr_elems() + filter.total_nr_elems() +

  1134.                                  src.total_nr_elems()) /

  1135.                          (1024 * 1024 * 1024) * 1e3;

  1136. 

  1137.         bencher.set_param(param)

  1138.                 .set_dtype(0, dtype::Float32())

  1139.                 .set_dtype(1, dtype::Float32())

  1140.                 .set_dtype(2, dtype::Float32())

  1141.                 .set_rng(0, &rng)

  1142.                 .set_rng(1, &rng)

  1143.                 .set_before_exec_callback(

  1144.                         AlgoChecker<ConvolutionBackwardData>("CHANNEL_WISE"));

  1145.         auto time_in_ms_fp32_normal = bencher.execs({filter, src, src}) / RUNS;

  1146. 

  1147.         bencher.set_before_exec_callback(

  1148.                 AlgoChecker<ConvolutionBackwardData>("CHANNEL_WISE_SMALL"));

  1149.         auto time_in_ms_fp32_small = bencher.execs({filter, src, src}) / RUNS;

  1150. 

  1151.         bencher.set_param(param)

  1152.                 .set_dtype(0, dtype::Float16())

  1153.                 .set_dtype(1, dtype::Float16())

  1154.                 .set_dtype(2, dtype::Float16())

  1155.                 .set_rng(0, &rng)

  1156.                 .set_rng(1, &rng);

  1157.         auto time_in_ms_fp16_small = bencher.execs({filter, src, src}) / RUNS;

  1158. 

  1159.         printf("stride=%zu src=%s, filter=%s, fp32 normal: %.2fms %.2fGB/s "

  1160.                "small: %.2fms %.2fGB/s, fp16 small: %.2fms %.2fGB/s, "

  1161.                "speedup: "

  1162.                "%0.2f (fp32 small/normal) %0.2f (small fp16/fp32)\n",

  1163.                s, src.to_string().c_str(), filter.to_string().c_str(),

  1164.                time_in_ms_fp32_normal, bandwith * 4 / time_in_ms_fp32_normal,

  1165.                time_in_ms_fp32_small, bandwith * 4 / time_in_ms_fp32_small,

  1166.                time_in_ms_fp16_small, bandwith * 2 / time_in_ms_fp16_small,

  1167.                time_in_ms_fp32_normal / time_in_ms_fp32_small,

  1168.                time_in_ms_fp32_small / time_in_ms_fp16_small);

  1169.     };

  1170. 

  1171.     // clang-format off

  1172.     for (size_t s : {1})

  1173.     for (size_t f : {3, 5})

  1174.     for (size_t batch : {64})

  1175.     for (size_t c : {16, 32, 64, 128})

  1176.     for (size_t ih: {8, 16, 32})

  1177.     for (size_t iw : {8, 16, 32})

  1178.         run(batch, c, ih, iw, f, s);

  1179.     // clang-format on

  1180. 

  1181.     run(128, 192, 28, 28, 3, 1);

  1182.     run(128, 576, 14, 14, 3, 1);

  1183.     run(128, 384, 14, 14, 3, 1);

  1184.     run(128, 960, 7, 7, 3, 1);

  1185.     run(128, 384, 14, 14, 3, 1);

  1186.     run(128, 384, 14, 14, 3, 1);

  1187.     run(128, 384, 14, 14, 3, 1);

  1188.     run(128, 192, 28, 28, 3, 1);

  1189.     run(128, 576, 14, 14, 3, 1);

  1190. }

  1191. 

  1192. TEST_F(CUDA, BENCHMARK_CHANWISE_CONV_BWD_DATA) {

  1193.     CUBenchmarker<ConvolutionBackwardData> bencher(handle_cuda());

  1194.     size_t RUNS = 1;

  1195.     bencher.set_display(false).set_times(RUNS);

  1196.     bencher.set_before_exec_callback(

  1197.             AlgoChecker<ConvolutionBackwardData>("CHANNEL_WISE"));

  1198. 

  1199.     Convolution::Param param;

  1200.     param.format = ConvBias::Param::Format::NCHW;

  1201.     param.sparse = Convolution::Param::Sparse::GROUP;

  1202.     NormalRNG rng;

  1203. 

  1204.     auto run = [&](size_t batch, size_t ocpg, size_t group, size_t ih, size_t iw,

  1205.                    size_t f, size_t p, size_t s) {

  1206.         param.pad_h = p;

  1207.         param.pad_w = p;

  1208.         param.stride_h = s;

  1209.         param.stride_w = s;

  1210.         size_t oh, ow;

  1211.         infer_conv_shape2d(ih, iw, f, f, s, s, p, p, oh, ow, true);

  1212.         param.compute_mode = param::Convolution::ComputeMode::DEFAULT;

  1213. 

  1214.         TensorShape src_grad = {batch, group, ih, iw},

  1215.                     dst_grad = {batch, group * ocpg, oh, ow},

  1216.                     flt = {group, ocpg, 1, f, f};

  1217. 

  1218.         auto opr = handle_cuda()->create_operator<Convolution>();

  1219.         opr->param() = param;

  1220.         float bandwith = static_cast<float>(

  1221.                                  flt.total_nr_elems() + dst_grad.total_nr_elems() +

  1222.                                  src_grad.total_nr_elems()) /

  1223.                          (1024 * 1024 * 1024) * 1e3;

  1224. 

  1225.         bencher.set_param(param)

  1226.                 .set_dtype(0, dtype::Float32())

  1227.                 .set_dtype(1, dtype::Float32())

  1228.                 .set_dtype(2, dtype::Float32())

  1229.                 .set_rng(0, &rng)

  1230.                 .set_rng(1, &rng);

  1231.         auto time_in_ms_fp32 = bencher.execs({flt, dst_grad, src_grad}) / RUNS;

  1232. 

  1233.         bencher.set_param(param)

  1234.                 .set_dtype(0, dtype::Float16())

  1235.                 .set_dtype(1, dtype::Float16())

  1236.                 .set_dtype(2, dtype::Float16())

  1237.                 .set_rng(0, &rng)

  1238.                 .set_rng(1, &rng);

  1239.         auto time_in_ms_fp16 = bencher.execs({flt, dst_grad, src_grad}) / RUNS;

  1240. 

  1241.         printf("stride=%zu, src_grad=%s, flt=%s, "

  1242.                "float32: %.2fms %.2fGB/s "

  1243.                "float16: %.2fms %.2fGB/s "

  1244.                "speedup: "

  1245.                "%0.2f (fp16/fp32)\n",

  1246.                s, src_grad.to_string().c_str(), flt.to_string().c_str(),

  1247.                time_in_ms_fp32, bandwith * 4 / time_in_ms_fp32, time_in_ms_fp16,

  1248.                bandwith * 2 / time_in_ms_fp16, time_in_ms_fp32 / time_in_ms_fp16);

  1249.     };

  1250. 

  1251.     // clang-format off

  1252.     for (size_t s : {1, 2})

  1253.     for (size_t f : {3, 5, 7})

  1254.     for (size_t p : {f / 2})

  1255.     for (size_t batch : {64})

  1256.     for (size_t ocpg : {1})

  1257.     for (size_t group : {16, 32, 64, 128})

  1258.     for (size_t ih : {8, 16, 32, 128, 256})

  1259.     for (size_t iw : {8, 16, 32, 128, 256})

  1260.         run(batch, ocpg, group, ih, iw, f, p, s);

  1261.     // clang-format on

  1262. }

  1263. 

  1264. TEST_F(CUDA, BENCHMARK_CHANWISE_CONV_BWD_FILTER) {

  1265.     CUBenchmarker<ConvolutionBackwardFilter> bencher(handle_cuda());

  1266.     size_t RUNS = 1;

  1267.     bencher.set_display(false).set_times(RUNS);

  1268.     bencher.set_before_exec_callback(

  1269.             AlgoChecker<ConvolutionBackwardFilter>("CHANNEL_WISE"));

  1270. 

  1271.     Convolution::Param param;

  1272.     param.format = ConvBias::Param::Format::NCHW;

  1273.     param.sparse = Convolution::Param::Sparse::GROUP;

  1274.     NormalRNG rng;

  1275. 

  1276.     auto run = [&](size_t batch, size_t ocpg, size_t group, size_t i, size_t f,

  1277.                    size_t p, size_t s) {

  1278.         param.pad_h = p;

  1279.         param.pad_w = p;

  1280.         param.stride_h = s;

  1281.         param.stride_w = s;

  1282.         size_t d = infer_conv_shape(i, f, s, p, true);

  1283.         param.compute_mode = param::Convolution::ComputeMode::DEFAULT;

  1284. 

  1285.         TensorShape src = {batch, group, i, i}, dst_grad = {batch, group * ocpg, d, d},

  1286.                     flt_grad = {group, ocpg, 1, f, f};

  1287. 

  1288.         auto opr = handle_cuda()->create_operator<Convolution>();

  1289.         opr->param() = param;

  1290.         float bandwith = static_cast<float>(

  1291.                                  flt_grad.total_nr_elems() + dst_grad.total_nr_elems() +

  1292.                                  src.total_nr_elems()) /

  1293.                          (1024 * 1024 * 1024) * 1e3;

  1294.         bencher.set_param(param)

  1295.                 .set_dtype(0, dtype::Float32())

  1296.                 .set_dtype(1, dtype::Float32())

  1297.                 .set_dtype(2, dtype::Float32())

  1298.                 .set_rng(0, &rng)

  1299.                 .set_rng(1, &rng);

  1300.         auto time_in_ms_fp32 = bencher.execs({src, dst_grad, flt_grad}) / RUNS;

  1301. 

  1302.         bencher.set_param(param)

  1303.                 .set_dtype(0, dtype::Float16())

  1304.                 .set_dtype(1, dtype::Float16())

  1305.                 .set_dtype(2, dtype::Float16())

  1306.                 .set_rng(0, &rng)

  1307.                 .set_rng(1, &rng);

  1308.         auto time_in_ms_fp16 = bencher.execs({src, dst_grad, flt_grad}) / RUNS;

  1309. 

  1310.         printf("stride=%zu, src=%s, flt_grad=%s, "

  1311.                "float32: %.2fms %.2fGB/s "

  1312.                "float16: %.2fms %.2fGB/s "

  1313.                "speedup: "

  1314.                "%.2f (fp16/fp32)\n",

  1315.                s, src.to_string().c_str(), flt_grad.to_string().c_str(),

  1316.                time_in_ms_fp32, bandwith * 4 / time_in_ms_fp32, time_in_ms_fp16,

  1317.                bandwith * 2 / time_in_ms_fp16, time_in_ms_fp32 / time_in_ms_fp16);

  1318.     };

  1319. 

  1320.     // clang-format off

  1321.     for (size_t s : {1, 2})

  1322.     for (size_t f : {3, 5, 7})

  1323.     for (size_t p : {f / 2})

  1324.     for (size_t batch : {64})

  1325.     for (size_t ocpg : {1})

  1326.     for (size_t group : {16, 32, 64, 128})

  1327.     for (size_t i : {8, 16, 32, 64, 128})

  1328.         run(batch, ocpg, group, i, f, p, s);

  1329.     // clang-format on

  1330. }

  1331. 

  1332. TEST_F(CUDA, BENCHMARK_CHANWISE_CONV_FORWARD_LARGE_KERNEL) {

  1333.     CUBenchmarker<ConvolutionForward> bencher(handle_cuda());

  1334.     size_t RUNS = 100;

  1335.     bencher.set_display(false).set_times(RUNS);

  1336.     std::unique_ptr<OprProxy<ConvolutionForward>> proxy{

  1337.             new OprProxy<ConvolutionForward>{true}};

  1338.     bencher.set_proxy(proxy);

  1339. 

  1340.     Convolution::Param param;

  1341.     param.format = ConvBias::Param::Format::NCHW;

  1342.     param.sparse = Convolution::Param::Sparse::GROUP;

  1343.     NormalRNG rng;

  1344. 

  1345.     auto run = [&](size_t batch, size_t c, size_t ih, size_t iw, size_t f, size_t s) {

  1346.         param.pad_h = f / 2;

  1347.         param.pad_w = f / 2;

  1348.         param.stride_h = s;

  1349.         param.stride_w = s;

  1350.         param.compute_mode = param::Convolution::ComputeMode::DEFAULT;

  1351. 

  1352.         TensorShape src = {batch, c, ih, iw}, filter = {c, 1, 1, f, f};

  1353. 

  1354.         TensorLayout dst_layout;

  1355.         auto opr = handle_cuda()->create_operator<Convolution>();

  1356.         opr->param() = param;

  1357.         opr->deduce_layout(

  1358.                 {src, dtype::Float32()}, {filter, dtype::Float32()}, dst_layout);

  1359.         float bandwith = static_cast<float>(

  1360.                                  src.total_nr_elems() + filter.total_nr_elems() +

  1361.                                  dst_layout.total_nr_elems()) /

  1362.                          (1024 * 1024 * 1024) * 1e3;

  1363. 

  1364.         bencher.set_param(param)

  1365.                 .set_dtype(0, dtype::Float32())

  1366.                 .set_dtype(1, dtype::Float32())

  1367.                 .set_dtype(2, dtype::Float32())

  1368.                 .set_rng(0, &rng)

  1369.                 .set_rng(1, &rng);

  1370.         bencher.proxy()->target_execution_policy = {};

  1371.         auto time_in_ms_fp32 = bencher.execs({src, filter, {}}) / RUNS;

  1372. 

  1373.         bencher.set_param(param)

  1374.                 .set_dtype(0, dtype::Float16())

  1375.                 .set_dtype(1, dtype::Float16())

  1376.                 .set_dtype(2, dtype::Float16())

  1377.                 .set_rng(0, &rng)

  1378.                 .set_rng(1, &rng);

  1379.         bencher.proxy()->target_execution_policy = {};

  1380.         auto time_in_ms_fp16 = bencher.execs({src, filter, {}}) / RUNS;

  1381. 

  1382.         bencher.proxy()->target_execution_policy.algo.reset();

  1383.         param.compute_mode = param::Convolution::ComputeMode::FLOAT32;

  1384.         bencher.set_param(param);

  1385.         auto time_in_ms_pseudo_fp16 = bencher.execs({src, filter, {}}) / RUNS;

  1386. 

  1387.         printf("stride=%zu src=%s, filter=%s, float32: %.2fms %.2fGB/s "

  1388.                "float16: %.2fms %.2fGB/s "

  1389.                "pseudo float16: %.2fms %.2fGB/s "

  1390.                "speedup: "

  1391.                "%0.2f (fp16/fp32) %.2f (fp16/pseudo fp16)\n",

  1392.                s, src.to_string().c_str(), filter.to_string().c_str(), time_in_ms_fp32,

  1393.                bandwith * 4 / time_in_ms_fp32, time_in_ms_fp16,

  1394.                bandwith * 2 / time_in_ms_fp16, time_in_ms_pseudo_fp16,

  1395.                bandwith * 2 / time_in_ms_pseudo_fp16, time_in_ms_fp32 / time_in_ms_fp16,

  1396.                time_in_ms_pseudo_fp16 / time_in_ms_fp16);

  1397.     };

  1398. 

  1399.     // clang-format off

  1400.     for (size_t b : {32, 64})

  1401.     for (size_t f : {3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31}) {

  1402.         run(b, 384, 32, 32, f, 1);

  1403.         run(b, 384, 64, 64, f, 1);

  1404.     }

  1405.     // clang-format on

  1406. }

  1407. 

  1408. TEST_F(CUDA, BENCHMARK_CHANWISE_CONV_BACKWARD_DATA_LARGE_KERNEL) {

  1409.     CUBenchmarker<ConvolutionBackwardData> bencher(handle_cuda());

  1410.     size_t RUNS = 100;

  1411.     bencher.set_display(false).set_times(RUNS);

  1412.     std::unique_ptr<OprProxy<ConvolutionBackwardData>> proxy{

  1413.             new OprProxy<ConvolutionBackwardData>{true}};

  1414.     bencher.set_proxy(proxy);

  1415. 

  1416.     Convolution::Param param;

  1417.     param.format = ConvBias::Param::Format::NCHW;

  1418.     param.sparse = Convolution::Param::Sparse::GROUP;

  1419.     NormalRNG rng;

  1420. 

  1421.     auto run = [&](size_t batch, size_t c, size_t ih, size_t iw, size_t f, size_t s) {

  1422.         param.pad_h = f / 2;

  1423.         param.pad_w = f / 2;

  1424.         param.stride_h = s;

  1425.         param.stride_w = s;

  1426.         param.compute_mode = param::Convolution::ComputeMode::DEFAULT;

  1427. 

  1428.         TensorShape src = {batch, c, ih, iw}, filter = {c, 1, 1, f, f};

  1429. 

  1430.         TensorLayout dst_layout;

  1431.         auto opr = handle_cuda()->create_operator<Convolution>();

  1432.         opr->param() = param;

  1433.         opr->deduce_layout(

  1434.                 {src, dtype::Float32()}, {filter, dtype::Float32()}, dst_layout);

  1435.         float bandwith = static_cast<float>(

  1436.                                  src.total_nr_elems() + filter.total_nr_elems() +

  1437.                                  dst_layout.total_nr_elems()) /

  1438.                          (1024 * 1024 * 1024) * 1e3;

  1439. 

  1440.         bencher.set_param(param)

  1441.                 .set_dtype(0, dtype::Float32())

  1442.                 .set_dtype(1, dtype::Float32())

  1443.                 .set_dtype(2, dtype::Float32())

  1444.                 .set_rng(0, &rng)

  1445.                 .set_rng(1, &rng);

  1446.         bencher.proxy()->target_execution_policy = {};

  1447.         auto time_in_ms_fp32 = bencher.execs({filter, src, src}) / RUNS;

  1448. 

  1449.         bencher.set_param(param)

  1450.                 .set_dtype(0, dtype::Float16())

  1451.                 .set_dtype(1, dtype::Float16())

  1452.                 .set_dtype(2, dtype::Float16())

  1453.                 .set_rng(0, &rng)

  1454.                 .set_rng(1, &rng);

  1455.         bencher.proxy()->target_execution_policy = {};

  1456.         auto time_in_ms_fp16 = bencher.execs({filter, src, src}) / RUNS;

  1457. 

  1458.         bencher.proxy()->target_execution_policy.algo.reset();

  1459.         param.compute_mode = param::Convolution::ComputeMode::FLOAT32;

  1460.         bencher.set_param(param);

  1461.         auto time_in_ms_pseudo_fp16 = bencher.execs({filter, src, src}) / RUNS;

  1462. 

  1463.         printf("stride=%zu src=%s, filter=%s, float32: %.2fms %.2fGB/s "

  1464.                "float16: %.2fms %.2fGB/s "

  1465.                "pseudo float16: %.2fms %.2fGB/s "

  1466.                "speedup: "

  1467.                "%0.2f (fp16/fp32) %.2f (fp16/pseudo fp16)\n",

  1468.                s, src.to_string().c_str(), filter.to_string().c_str(), time_in_ms_fp32,

  1469.                bandwith * 4 / time_in_ms_fp32, time_in_ms_fp16,

  1470.                bandwith * 2 / time_in_ms_fp16, time_in_ms_pseudo_fp16,

  1471.                bandwith * 2 / time_in_ms_pseudo_fp16, time_in_ms_fp32 / time_in_ms_fp16,

  1472.                time_in_ms_pseudo_fp16 / time_in_ms_fp16);

  1473.     };

  1474. 

  1475.     // clang-format off

  1476.     for (size_t b : {32, 64})

  1477.     for (size_t f : {3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31}) {

  1478.         run(b, 384, 32, 32, f, 1);

  1479.         run(b, 384, 64, 64, f, 1);

  1480.     }

  1481.     // clang-format on

  1482. }

  1483. 

  1484. TEST_F(CUDA, BENCHMARK_CHANWISE_CONV_BACKWARD_FILTER_LARGE_KERNEL) {

  1485.     CUBenchmarker<ConvolutionBackwardFilter> bencher(handle_cuda());

  1486.     size_t RUNS = 100;

  1487.     bencher.set_display(false).set_times(RUNS);

  1488.     std::unique_ptr<OprProxy<ConvolutionBackwardFilter>> proxy{

  1489.             new OprProxy<ConvolutionBackwardFilter>{true}};

  1490.     bencher.set_proxy(proxy);

  1491. 

  1492.     Convolution::Param param;

  1493.     param.format = ConvBias::Param::Format::NCHW;

  1494.     param.sparse = Convolution::Param::Sparse::GROUP;

  1495.     NormalRNG rng;

  1496. 

  1497.     auto run = [&](size_t batch, size_t c, size_t ih, size_t iw, size_t f, size_t s) {

  1498.         param.pad_h = f / 2;

  1499.         param.pad_w = f / 2;

  1500.         param.stride_h = s;

  1501.         param.stride_w = s;

  1502.         param.compute_mode = param::Convolution::ComputeMode::DEFAULT;

  1503. 

  1504.         TensorShape src = {batch, c, ih, iw}, filter = {c, 1, 1, f, f};

  1505. 

  1506.         TensorLayout dst_layout;

  1507.         auto opr = handle_cuda()->create_operator<Convolution>();

  1508.         opr->param() = param;

  1509.         opr->deduce_layout(

  1510.                 {src, dtype::Float32()}, {filter, dtype::Float32()}, dst_layout);

  1511.         float bandwith = static_cast<float>(

  1512.                                  src.total_nr_elems() + filter.total_nr_elems() +

  1513.                                  dst_layout.total_nr_elems()) /

  1514.                          (1024 * 1024 * 1024) * 1e3;

  1515. 

  1516.         bencher.set_param(param)

  1517.                 .set_dtype(0, dtype::Float32())

  1518.                 .set_dtype(1, dtype::Float32())

  1519.                 .set_dtype(2, dtype::Float32())

  1520.                 .set_rng(0, &rng)

  1521.                 .set_rng(1, &rng);

  1522.         bencher.proxy()->target_execution_policy = {};

  1523.         auto time_in_ms_fp32 = bencher.execs({src, src, filter}) / RUNS;

  1524. 

  1525.         bencher.set_param(param)

  1526.                 .set_dtype(0, dtype::Float16())

  1527.                 .set_dtype(1, dtype::Float16())

  1528.                 .set_dtype(2, dtype::Float16())

  1529.                 .set_rng(0, &rng)

  1530.                 .set_rng(1, &rng);

  1531.         bencher.proxy()->target_execution_policy = {};

  1532.         param.compute_mode = param::Convolution::ComputeMode::FLOAT32;

  1533.         bencher.set_param(param);

  1534.         auto time_in_ms_pseudo_fp16 = bencher.execs({src, src, filter}) / RUNS;

  1535. 

  1536.         printf("stride=%zu src=%s, filter=%s, float32: %.2fms %.2fGB/s "

  1537.                "pseudo float16: %.2fms %.2fGB/s "

  1538.                "speedup: "

  1539.                "%0.2f (fp16/fp32) \n",

  1540.                s, src.to_string().c_str(), filter.to_string().c_str(), time_in_ms_fp32,

  1541.                bandwith * 4 / time_in_ms_fp32, time_in_ms_pseudo_fp16,

  1542.                bandwith * 2 / time_in_ms_pseudo_fp16,

  1543.                time_in_ms_fp32 / time_in_ms_pseudo_fp16);

  1544.     };

  1545. 

  1546.     // clang-format off

  1547.     for (size_t b : {32, 64})

  1548.     for (size_t f : {3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31}) {

  1549.         run(b, 384, 32, 32, f, 1);

  1550.         run(b, 384, 64, 64, f, 1);

  1551.     }

  1552.     // clang-format on

  1553. }

  1554. #endif

  1555. 

  1556. // vim: syntax=cpp.doxygen