#include "test/fallback/fixture.h" #include "test/common/benchmarker.h" #include "test/common/checker.h" #include "test/common/pooling.h" #include "test/common/rng.h" #include "test/common/task_record_check.h" namespace megdnn { namespace test { namespace { std::vector> get_nchw44_pool_args( size_t filter, size_t stride) { constexpr size_t ic_step = 4; std::vector> args; for (size_t n : {1, 2}) for (size_t c : {4, 8}) for (size_t ih : {3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13}) for (size_t iw : {3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13}) for (size_t ph : {0, 1, 2}) for (size_t pw : {0, 1, 2}) for (auto mode : {param::Pooling::Mode::MAX, param::Pooling::Mode::AVERAGE}) if (ih + 2 * ph >= filter && iw + 2 * pw >= filter && filter > ph && filter > pw) { param::Pooling param; param.mode = mode; param.format = param::Pooling::Format::NCHW44; param.pad_h = ph; param.pad_w = pw; param.stride_h = param.stride_w = stride; param.window_h = param.window_w = filter; args.emplace_back(std::make_pair( param, TensorShapeArray{ {n, c / ic_step, ih, iw, ic_step}, {}})); } return args; } void run_pooling_check( Handle* handle, std::vector> args, bool is_int8) { Checker checker(handle); UniformIntRNG rng_int8{INT8_MIN >> 1, INT8_MAX >> 1}; UniformIntRNG rng_fp32{-10, 10}; if (is_int8) { checker.set_dtype(0, dtype::QuantizedS8(1.1f)); checker.set_rng(0, &rng_int8); } else { checker.set_rng(0, &rng_fp32); } for (auto arg : args) { checker.set_param(arg.first).exec(arg.second); } } } // namespace TEST_F(FALLBACK_MULTI_THREADS, POOLING_GI_NCHW44_FP32) { for (auto filter : {2, 3, 4, 5}) for (auto stride : {1, 2}) { run_pooling_check(handle(), get_nchw44_pool_args(filter, stride), false); } } TEST_F(FALLBACK, POOLING_GI) { using Param = param::Pooling; // clang-format off for (size_t ih: {2, 3, 5, 7, 11, 13, 17, 19, 23, 24, 25, 26, 27, 28, 29, 30}) for (size_t iw: {2, 3, 5, 7, 11, 13, 17, 19, 23, 24, 25, 26, 27, 28, 29, 30}) for (size_t p: {1, 2}) { Param param; param.mode = Param::Mode::MAX; param.window_h = param.window_w = 3; param.stride_h = param.stride_w = 2; param.pad_h = param.pad_w = p; Checker checker(handle()); checker.set_param(param).exec({{2, 3, ih, iw}, {}}); param.mode = Param::Mode::AVERAGE; param.window_h = param.window_w = 3; param.stride_h = param.stride_w = 2; param.pad_h = param.pad_w = p; checker.set_param(param).exec({{2, 3, ih, iw}, {}}); param.mode = Param::Mode::MAX; param.window_h = param.window_w = 4; param.stride_h = param.stride_w = 2; param.pad_h = param.pad_w = p; checker.set_param(param).exec({{2, 3, ih, iw}, {}}); param.mode = Param::Mode::MAX; param.window_h = param.window_w = 5; param.stride_h = param.stride_w = 2; param.pad_h = param.pad_w = p; if (ih + p * 2 >= 5 && iw + p * 2 >= 5) checker.set_param(param).exec({{2, 3, ih, iw}, {}}); } for (size_t ih: {2, 3, 5, 7, 11, 13, 17, 19, 23, 24, 25, 26, 27, 28, 29, 30}) for (size_t iw: {2, 3, 5, 7, 11, 13, 17, 19, 23, 24, 25, 26, 27, 28, 29, 30}) for (size_t p: {1, 2}) { Param param; param.mode = Param::Mode::AVERAGE_COUNT_EXCLUDE_PADDING; param.window_h = param.window_w = 3; param.stride_h = param.stride_w = 1; param.pad_h = param.pad_w = p; Checker checker(handle()); checker.set_param(param).exec({{2, 3, ih, iw}, {}}); } // clang-format on } TEST_F(FALLBACK, POOLING_GI_RECORD) { using Param = param::Pooling; TaskRecordChecker checker(0); // clang-format off for (size_t ih: {2, 3, 5, 7, 11, 13, 17}) for (size_t iw: {2, 3, 5, 7, 11, 13, 17}) for (size_t p: {1, 2}) { Param param; param.mode = Param::Mode::MAX; param.window_h = param.window_w = 3; param.stride_h = param.stride_w = 2; param.pad_h = param.pad_w = p; checker.set_param(param).exec({{2, 3, ih, iw}, {}}); param.mode = Param::Mode::AVERAGE; param.window_h = param.window_w = 3; param.stride_h = param.stride_w = 2; param.pad_h = param.pad_w = p; checker.set_param(param).exec({{2, 3, ih, iw}, {}}); param.mode = Param::Mode::MAX; param.window_h = param.window_w = 4; param.stride_h = param.stride_w = 2; param.pad_h = param.pad_w = p; checker.set_param(param).exec({{2, 3, ih, iw}, {}}); param.mode = Param::Mode::MAX; param.window_h = param.window_w = 5; param.stride_h = param.stride_w = 2; param.pad_h = param.pad_w = p; if (ih + p * 2 >= 5 && iw + p * 2 >= 5) checker.set_param(param).exec({{2, 3, ih, iw}, {}}); } for (size_t ih: {2, 3, 5, 7, 11, 13, 17, 19, 23, 24, 25, 26, 27, 28, 29, 30}) for (size_t iw: {2, 3, 5, 7, 11, 13, 17, 19, 23, 24, 25, 26, 27, 28, 29, 30}) for (size_t p: {1, 2}) { Param param; param.mode = Param::Mode::AVERAGE_COUNT_EXCLUDE_PADDING; param.window_h = param.window_w = 3; param.stride_h = param.stride_w = 1; param.pad_h = param.pad_w = p; Checker checker(handle()); checker.set_param(param).exec({{2, 3, ih, iw}, {}}); } // clang-format on } TEST_F(FALLBACK_MULTI_THREADS, POOLING_GI_RECORD) { using Param = param::Pooling; TaskRecordChecker checker(0); for (size_t ih : {2, 3, 5, 7, 11, 13, 17}) for (size_t iw : {2, 3, 5, 7, 11, 13, 17}) for (size_t p : {1, 2}) { Param param; param.mode = Param::Mode::MAX; param.window_h = param.window_w = 3; param.stride_h = param.stride_w = 2; param.pad_h = param.pad_w = p; checker.set_param(param).exec({{2, 3, ih, iw}, {}}); param.mode = Param::Mode::AVERAGE; param.window_h = param.window_w = 3; param.stride_h = param.stride_w = 2; param.pad_h = param.pad_w = p; checker.set_param(param).exec({{2, 3, ih, iw}, {}}); param.mode = Param::Mode::MAX; param.window_h = param.window_w = 4; param.stride_h = param.stride_w = 2; param.pad_h = param.pad_w = p; checker.set_param(param).exec({{2, 3, ih, iw}, {}}); param.mode = Param::Mode::MAX; param.window_h = param.window_w = 5; param.stride_h = param.stride_w = 2; param.pad_h = param.pad_w = p; if (ih + p * 2 >= 5 && iw + p * 2 >= 5) checker.set_param(param).exec({{2, 3, ih, iw}, {}}); } } TEST_F(FALLBACK_MULTI_THREADS, POOLING_GI_W9_w13_NCHW44) { UniformIntRNG rng{-10, 10}; Checker checker(handle()); checker.set_rng(0, &rng); // clang-format off for (size_t ih: {20, 15}) for (size_t iw: {15, 20}) for (size_t kernel: {9, 13}) for (size_t pad: {4, 6}) for(auto mode: {param::Pooling::Mode::MAX, param::Pooling::Mode::AVERAGE}) if (kernel > pad) { param::Pooling param; param.mode = mode; param.format = param::Pooling::Format::NCHW44; param.pad_h = pad; param.pad_w = pad; param.stride_h = param.stride_w = 1; param.window_h = param.window_w = kernel ; checker.set_param(param).exec(TensorShapeArray{{2, 8, ih, iw, 4}, {}}); } // clang-format on } TEST_F(FALLBACK_MULTI_THREADS, POOLING_GI_FALLBACK) { using Param = param::Pooling; for (size_t ih : {2, 3, 5, 7, 11, 13, 17, 19, 23, 24, 25, 26, 27, 28, 29, 30}) for (size_t iw : {2, 3, 5, 7, 11, 13, 17, 19, 23, 24, 25, 26, 27, 28, 29, 30}) for (size_t p : {1, 2}) { Param param; param.mode = Param::Mode::MAX; param.window_h = param.window_w = 3; param.stride_h = param.stride_w = 2; param.pad_h = param.pad_w = p; Checker checker(handle()); checker.set_param(param).exec({{2, 3, ih, iw}, {}}); } } TEST_F(FALLBACK_MULTI_THREADS, POOLING_GI) { using Param = param::Pooling; for (size_t ih : {2, 3, 5, 7, 11, 13, 17, 19, 23, 24, 25, 26, 27, 28, 29, 30}) for (size_t iw : {2, 3, 5, 7, 11, 13, 17, 19, 23, 24, 25, 26, 27, 28, 29, 30}) for (size_t p : {1, 2}) { Param param; param.mode = Param::Mode::MAX; param.window_h = param.window_w = 3; param.stride_h = param.stride_w = 2; param.pad_h = param.pad_w = p; Checker checker(handle()); checker.set_param(param).exec({{2, 3, ih, iw}, {}}); param.mode = Param::Mode::AVERAGE; param.window_h = param.window_w = 3; param.stride_h = param.stride_w = 2; param.pad_h = param.pad_w = p; checker.set_param(param).exec({{2, 3, ih, iw}, {}}); param.mode = Param::Mode::MAX; param.window_h = param.window_w = 4; param.stride_h = param.stride_w = 2; param.pad_h = param.pad_w = p; checker.set_param(param).exec({{2, 3, ih, iw}, {}}); param.mode = Param::Mode::MAX; param.window_h = param.window_w = 5; param.stride_h = param.stride_w = 2; param.pad_h = param.pad_w = p; if (ih + p * 2 >= 5 && iw + p * 2 >= 5) checker.set_param(param).exec({{2, 3, ih, iw}, {}}); } } #if MEGDNN_WITH_BENCHMARK namespace { void benchmark_nchw44_fp32(Handle* handle) { using Param = param::Pooling; auto run = [&](size_t n, size_t c, size_t h, size_t w, size_t filter, size_t stride, size_t pad, Param::Mode mode) { Param param; param.window_h = param.window_w = filter; param.stride_h = param.stride_w = stride; param.pad_h = param.pad_w = pad; param.format = Param::Format::NCHW; param.mode = mode; TensorShape nchw_shape = {n, c, h, w}; TensorShape nchw44_shape = {n, c / 4, h, w, 4}; TensorLayout dst_layout; auto opr = handle->create_operator(); opr->param() = param; opr->deduce_layout({nchw_shape, dtype::Float32()}, dst_layout); float calc_amount = dst_layout.total_nr_elems() * param.window_h * param.window_w; Benchmarker benchmarker_float_nchw(handle); Benchmarker benchmarker_float_nchw44(handle); Benchmarker benchmarker_int_nchw44(handle); size_t RUN = 500; auto t1 = benchmarker_float_nchw.set_display(false) .set_times(RUN) .set_param(param) .exec({nchw_shape, {}}); param.format = Param::Format::NCHW44; auto t2 = benchmarker_int_nchw44.set_display(false) .set_times(RUN) .set_param(param) .execl({{nchw44_shape, dtype::QuantizedS8(1.0)}, {{}, dtype::QuantizedS8(1.0)}}); auto t3 = benchmarker_float_nchw44.set_display(false) .set_times(RUN) .set_param(param) .exec({nchw44_shape, {}}); printf("{%zu %zu %zu %zu} filter = %zu, stride = %zu pad = %zu\n" "nchw_fp32={%.3f ms, %.3f Mflops}, " "nchw44_int={%.3f ms, %.3f Mflops}, " "nchw44_fp32={%.3f ms, %.3f Mflops, speed_up %f}\n\n", n, c, h, w, filter, stride, pad, t1 / RUN, calc_amount / (t1 / RUN * 1000), t2 / RUN, calc_amount / (t2 / RUN * 1000), t3 / RUN, calc_amount / (t3 / RUN * 1000), t1 / t3); }; // Resnet50 run(1, 64, 112, 112, 3, 2, 1, param::Pooling::Mode::MAX); run(1, 2048, 7, 7, 7, 1, 0, param::Pooling::Mode::AVERAGE); // VGG16 run(1, 64, 224, 224, 2, 2, 0, param::Pooling::Mode::MAX); run(1, 128, 112, 112, 2, 2, 0, param::Pooling::Mode::MAX); run(1, 256, 56, 56, 2, 2, 0, param::Pooling::Mode::MAX); run(1, 512, 28, 28, 2, 2, 0, param::Pooling::Mode::MAX); run(1, 512, 14, 14, 2, 2, 0, param::Pooling::Mode::MAX); } } // namespace TEST_F(FALLBACK, BENCHMARK_POOLING_GI_NCHW44_FP32) { benchmark_nchw44_fp32(handle()); } TEST_F(FALLBACK_MULTI_THREADS, BENCHMARK_POOLING_GI_NCHW44_FP32) { benchmark_nchw44_fp32(handle()); } TEST_F(FALLBACK, BENCHMARK_POOLING_GI_W4x4_S2x2) { using Param = param::Pooling; auto run = [&](const TensorShapeArray& shapes, Param param) { std::cout << "N:" << shapes[0][0] << " " << "IC:" << shapes[0][1] << " " << "IH:" << shapes[0][2] << " " << "IW:" << shapes[0][3] << std::endl; auto handle_naive = create_cpu_handle(2); Benchmarker benchmarker_naive(handle_naive.get()); Benchmarker benchmarker_float(handle()); size_t RUN = 10; auto t1 = benchmarker_naive.set_display(false) .set_times(RUN) .set_param(param) .exec(shapes); auto t2 = benchmarker_float.set_display(false) .set_times(RUN) .set_param(param) .exec(shapes); TensorLayout dst_layout; auto opr = handle()->create_operator(); opr->param() = param; opr->deduce_layout({shapes[0], dtype::Float32()}, dst_layout); float calc_amount = dst_layout.total_nr_elems() * param.window_h * param.window_w; printf("naive={%.3fms, %.3fMflops}, neon={%.3fms, %.3fMflops}\n", t1 / RUN, calc_amount / (t1 / RUN * 1000), t2 / RUN, calc_amount / (t2 / RUN * 1000)); }; Param param; param.window_h = param.window_w = 4; param.stride_h = param.stride_w = 2; param.pad_h = param.pad_w = 1; std::cout << "4x4 with 2x2 stride max pooling:" << std::endl; run({{1, 24, 160, 128}, {}}, param); run({{1, 4, 240, 135}, {}}, param); run({{1, 32, 120, 67}, {}}, param); run({{1, 64, 60, 33}, {}}, param); } TEST_F(FALLBACK, BENCHMARK_POOLING_GI_W5x5_S2x2) { using Param = param::Pooling; auto run = [&](const TensorShapeArray& shapes, Param param) { std::cout << "N:" << shapes[0][0] << " " << "IC:" << shapes[0][1] << " " << "IH:" << shapes[0][2] << " " << "IW:" << shapes[0][3] << std::endl; auto handle_naive = create_cpu_handle(2); Benchmarker benchmarker_naive(handle_naive.get()); Benchmarker benchmarker_float(handle()); size_t RUN = 10; auto t1 = benchmarker_naive.set_display(false) .set_times(RUN) .set_param(param) .exec(shapes); auto t2 = benchmarker_float.set_display(false) .set_times(RUN) .set_param(param) .exec(shapes); TensorLayout dst_layout; auto opr = handle()->create_operator(); opr->param() = param; opr->deduce_layout({shapes[0], dtype::Float32()}, dst_layout); float calc_amount = dst_layout.total_nr_elems() * param.window_h * param.window_w; printf("naive={%.3fms, %.3fMflops}, neon={%.3fms, %.3fMflops}\n", t1 / RUN, calc_amount / (t1 / RUN * 1000), t2 / RUN, calc_amount / (t2 / RUN * 1000)); }; Param param; param.window_h = param.window_w = 5; param.stride_h = param.stride_w = 2; param.pad_h = param.pad_w = 1; std::cout << "5x5 with 2x2 stride max pooling:" << std::endl; run({{1, 24, 160, 128}, {}}, param); run({{1, 4, 240, 135}, {}}, param); run({{1, 32, 120, 67}, {}}, param); run({{1, 64, 60, 33}, {}}, param); } namespace { template void benchmark_impl( const typename Opr::Param& param, std::vector> shapes, size_t RUNS, TaskExecutorConfig&& multi_thread_config, TaskExecutorConfig&& single_thread_config, DType data_type) { std::vector multi_thread_times, single_thread_times; { auto multi_thread_hanle = create_cpu_handle(0, true, &multi_thread_config); auto benchmarker = Benchmarker(multi_thread_hanle.get()); benchmarker.set_times(RUNS).set_display(false).set_param(param); benchmarker.set_dtype(0, data_type); for (auto shape : shapes) { multi_thread_times.push_back(benchmarker.exec(shape) / RUNS); } } { auto single_thread_handle = create_cpu_handle(0, true, &single_thread_config); auto benchmarker = Benchmarker(single_thread_handle.get()); benchmarker.set_times(RUNS).set_display(false).set_param(param); benchmarker.set_dtype(0, data_type); for (auto shape : shapes) { single_thread_times.push_back(benchmarker.exec(shape) / RUNS); } } printf("Benchmark : Multi threads %zu, ", multi_thread_config.nr_thread); printf("core_ids:"); for (size_t i = 0; i < multi_thread_config.affinity_core_set.size(); i++) { printf("%zu ", multi_thread_config.affinity_core_set[i]); } printf(", Single thread core_id %zu\n", single_thread_config.affinity_core_set[0]); for (size_t i = 0; i < shapes.size(); i++) { auto shape = shapes[i]; printf("Case: "); for (auto sh : shape) printf("%s ", sh.to_string().c_str()); printf("%zu threads time: %f,\n single thread time: " "%f. spead up = %f, speedup/cores=%f\n", multi_thread_config.nr_thread, multi_thread_times[i], single_thread_times[i], single_thread_times[i] / multi_thread_times[i], single_thread_times[i] / multi_thread_times[i] / multi_thread_config.nr_thread); } } } // namespace TEST_F(FALLBACK_MULTI_THREADS, BENCHMARK_POOLING_GI) { constexpr size_t RUNS = 50; using Param = param::Pooling; Param param; param.window_h = param.window_w = 3; param.stride_h = param.stride_w = 2; param.pad_h = param.pad_w = 1; std::vector> shapes; shapes.push_back({{32, 32, 215, 215}, {}}); shapes.push_back({{32, 32, 128, 128}, {}}); shapes.push_back({{8, 256, 100, 100}, {}}); shapes.push_back({{1, 256, 100, 100}, {}}); shapes.push_back({{1, 32, 100, 100}, {}}); shapes.push_back({{1, 256, 80, 80}, {}}); shapes.push_back({{1, 256, 60, 60}, {}}); shapes.push_back({{1, 256, 30, 30}, {}}); param.window_h = param.window_w = 3; param.stride_h = param.stride_w = 2; param.pad_h = param.pad_w = 1; printf("Benchmark POOLING kernel:%d*%d stride:%d,mode %d\n", param.window_h, param.window_w, param.stride_h, static_cast(param.mode)); benchmark_impl( param, shapes, RUNS, {4, {0, 1, 2, 3}}, {1, {0}}, dtype::Float32()); benchmark_impl( param, shapes, RUNS, {4, {4, 5, 6, 7}}, {1, {4}}, dtype::Float32()); benchmark_impl( param, shapes, RUNS, {2, {0, 1}}, {1, {0}}, dtype::Float32()); } TEST_F(FALLBACK_MULTI_THREADS, BENCHMARK_POOLING_GI_NCHW44) { constexpr size_t RUNS = 50; using Param = param::Pooling; Param param; param.pad_h = param.pad_w = 0; param.mode = Param::Mode::MAX; std::vector> shapes; std::vector> filter_and_stride = { {2, 1}, {2, 2}, {3, 1}, {3, 2}, {4, 1}, {4, 2}, {5, 1}, {5, 2}}; for (auto mode : {param::Pooling::Mode::MAX, param::Pooling::Mode::AVERAGE}) { for (auto filter : filter_and_stride) { shapes.push_back({{1, 32 * 4, 215, 215}, {}}); shapes.push_back({{1, 32 * 4, 128, 128}, {}}); shapes.push_back({{1, 16 * 4, 56, 56}, {}}); param.mode = mode; param.window_h = param.window_w = filter[0]; param.stride_h = param.stride_w = filter[1]; param.format = Param::Format::NCHW; printf("NCHW Benchmark POOLING kernel:%d*%d stride:%d,mode %d\n", param.window_h, param.window_h, param.stride_h, static_cast(param.mode)); benchmark_impl( param, shapes, RUNS, {4, {4, 5, 6, 7}}, {1, {4}}, dtype::QuantizedS8(1.1f)); shapes.clear(); shapes.push_back({{1, 32, 215, 215, 4}, {}}); shapes.push_back({{1, 32, 128, 128, 4}, {}}); shapes.push_back({{1, 16, 56, 56, 4}, {}}); param.format = Param::Format::NCHW44; printf("NCHW44 Benchmark POOLING kernel:%d*%d stride:%d,mode %d\n", param.window_h, param.window_w, param.stride_h, static_cast(param.mode)); benchmark_impl( param, shapes, RUNS, {4, {4, 5, 6, 7}}, {1, {4}}, dtype::QuantizedS8(1.1f)); shapes.clear(); } } } #endif } // namespace test } // namespace megdnn // vim: syntax=cpp.doxygen