Megvii Engine Team
Xu Xinran
5 years ago
12 changed files with 1645 additions and 2200 deletions
Split View
Diff Options
-
+6 -23dnn/src/arm_common/conv_bias/int8/algos.h
-
+0 -20dnn/src/arm_common/conv_bias/int8/direct.h
-
+149 -136dnn/src/arm_common/conv_bias/int8/direct_nchw44_algo.cpp
-
+1428 -0dnn/src/arm_common/conv_bias/int8/direct_nchw44_kern.h
-
+0 -393dnn/src/arm_common/conv_bias/int8/direct_stride1_nchw44_algo.cpp
-
+0 -791dnn/src/arm_common/conv_bias/int8/direct_stride1_nchw44_kern.cpp
-
+0 -793dnn/src/arm_common/conv_bias/int8/direct_stride2_nchw44_kern.cpp
-
+2 -4dnn/src/arm_common/conv_bias/opr_impl.cpp
-
+1 -2dnn/src/arm_common/conv_bias/opr_impl.h
-
+2 -0dnn/src/arm_common/elemwise_helper/kimpl/none.h
-
+9 -1dnn/test/arm_common/conv_bias.cpp
-
+48 -37dnn/test/arm_common/conv_bias_multi_thread.cpp
+ 6
- 23
dnn/src/arm_common/conv_bias/int8/algos.h
View File
| @@ -38,23 +38,6 @@ public: | |||
| const NCBKernSizeParam& param) const override; | |||
| }; | |||
| class ConvBiasImpl::AlgoS8DirectStride1NCHW44 final : public AlgoBase { | |||
| public: | |||
| AlgoS8DirectStride1NCHW44() {} | |||
| bool is_reproducible() const override { return true; } | |||
| const char* name() const override { return "S8_NCHW44_DIRECT_STRD1"; } | |||
| bool usable(fallback::ConvBiasImpl* opr, const NCBKernSizeParam& param, | |||
| AlgoSelectionStrategy algo_selection_strategy) const override; | |||
| size_t get_workspace(fallback::ConvBiasImpl*, | |||
| const NCBKernSizeParam& param) const override; | |||
| virtual SmallVector<NCBKern> dispatch_kerns( | |||
| fallback::ConvBiasImpl* opr, | |||
| const NCBKernSizeParam& param) const override; | |||
| bool is_preferred(megdnn::fallback::ConvBiasImpl*, | |||
| const NCBKernSizeParam& param) const override; | |||
| }; | |||
| class ConvBiasImpl::AlgoS8DirectStride2 final : public AlgoBase { | |||
| bool m_large_group; | |||
| @@ -74,11 +57,11 @@ public: | |||
| const NCBKernSizeParam& param) const override; | |||
| }; | |||
| class ConvBiasImpl::AlgoS8DirectStride2NCHW44 final : public AlgoBase { | |||
| class ConvBiasImpl::AlgoS8DirectNCHW44 final : public AlgoBase { | |||
| public: | |||
| AlgoS8DirectStride2NCHW44() {} | |||
| AlgoS8DirectNCHW44() {} | |||
| bool is_reproducible() const override { return true; } | |||
| const char* name() const override { return "S8_NCHW44_DIRECT_STRD2"; } | |||
| const char* name() const override { return "S8_NCHW44_DIRECT"; } | |||
| bool usable(fallback::ConvBiasImpl* opr, const NCBKernSizeParam& param, | |||
| AlgoSelectionStrategy algo_selection_strategy) const override; | |||
| size_t get_workspace(fallback::ConvBiasImpl*, | |||
| @@ -245,8 +228,8 @@ private: | |||
| //=======================input int8 compute fp32 output int8============ | |||
| class ConvBiasImpl::AlgoS8CF32WinogradF23_4x4_NCHW44 final : public AlgoBase { | |||
| public: | |||
| AlgoS8CF32WinogradF23_4x4_NCHW44(fallback::MatrixMulImpl::AlgoBase* matmul_algo, | |||
| uint32_t tile_size) | |||
| AlgoS8CF32WinogradF23_4x4_NCHW44( | |||
| fallback::MatrixMulImpl::AlgoBase* matmul_algo, uint32_t tile_size) | |||
| : m_matmul_algo{matmul_algo}, m_tile_size{tile_size} {} | |||
| bool is_reproducible() const override { return true; } | |||
| const char* name() const override { | |||
| @@ -277,7 +260,7 @@ private: | |||
| class ConvBiasImpl::AlgoS8WinogradF23_8x8_NCHW44 final : public AlgoBase { | |||
| public: | |||
| AlgoS8WinogradF23_8x8_NCHW44(fallback::MatrixMulImpl::AlgoBase* matmul_algo, | |||
| uint32_t tile_size) | |||
| uint32_t tile_size) | |||
| : m_matmul_algo{matmul_algo}, m_tile_size{tile_size} {} | |||
| bool is_reproducible() const override { return true; } | |||
| const char* name() const override { | |||
+ 0
- 20
dnn/src/arm_common/conv_bias/int8/direct.h
View File
| @@ -36,26 +36,6 @@ KERN(stride2, 7, nchw) | |||
| #undef KERN | |||
| #define KERN(stride, i, layout) \ | |||
| template <BiasMode bias_mode, typename Op, int remain_w> \ | |||
| void conv_direct_##stride##_##i##x##i##_int8_##layout( \ | |||
| const int8_t* src, const int8_t* filter, const int32_t* bias, \ | |||
| int32_t* temp, int8_t* dst, const size_t OC, const size_t IC, \ | |||
| const size_t IH, const size_t IW, const size_t OH, \ | |||
| const size_t OW, const Op& op); | |||
| KERN(stride1, 2, nchw44) | |||
| KERN(stride1, 3, nchw44) | |||
| KERN(stride1, 5, nchw44) | |||
| KERN(stride1, 7, nchw44) | |||
| KERN(stride2, 2, nchw44) | |||
| KERN(stride2, 3, nchw44) | |||
| KERN(stride2, 5, nchw44) | |||
| KERN(stride2, 7, nchw44) | |||
| #undef KERN | |||
| void nchw44_pack_filter(const int8_t* src, int8_t* dst, int filter); | |||
| void nchw44_pack_src(const int8_t* src, int8_t* dst, int length); | |||
| } // namespace conv_bias | |||
| } // namespace arm_common | |||
| } // namespace megdnn | |||
dnn/src/arm_common/conv_bias/int8/direct_stride2_nchw44_algo.cpp → dnn/src/arm_common/conv_bias/int8/direct_nchw44_algo.cpp
View File
| @@ -1,5 +1,5 @@ | |||
| /** | |||
| * \file dnn/src/arm_common/conv_bias/int8/direct_stride2_nchw44_algo.cpp | |||
| * \file dnn/src/arm_common/conv_bias/int8/direct_nchw44_algo.cpp | |||
| * MegEngine is Licensed under the Apache License, Version 2.0 (the "License") | |||
| * | |||
| * Copyright (c) 2014-2020 Megvii Inc. All rights reserved. | |||
| @@ -13,6 +13,7 @@ | |||
| #include "megdnn/oprs.h" | |||
| #include "src/arm_common/conv_bias/int8/algos.h" | |||
| #include "src/arm_common/conv_bias/int8/direct.h" | |||
| #include "src/arm_common/conv_bias/int8/direct_nchw44_kern.h" | |||
| #include "src/arm_common/conv_bias/int8/strategy.h" | |||
| #include "src/arm_common/elemwise_op.h" | |||
| #include "src/common/opr_delegate.h" | |||
| @@ -25,28 +26,19 @@ using conv_fun = std::function<void( | |||
| WorkspaceBundle bundle, const ConvBiasImpl::NCBKernParam& kern_param, | |||
| const ConvBiasImpl::NCBKernIndex& ncb_index, | |||
| const CpuNDRange& workspace_ids, const CpuNDRange& ncb_range)>; | |||
| MIDOUT_DECL(megdnn_arm_common_conv_bias_int8_nchw44_stride2) | |||
| MIDOUT_DECL(megdnn_arm_common_conv_bias_int8_nchw44) | |||
| static void get_rectified_size( | |||
| const megdnn::fallback::ConvBiasImpl::NCBKernSizeParam& param, | |||
| size_t& IH2, size_t& IW2, size_t& OH2, size_t& OW2) { | |||
| const megdnn::fallback::ConvBiasImpl::NCBKernSizeParam& param, int& ih2, | |||
| int& iw2) { | |||
| auto&& fm = param.filter_meta; | |||
| size_t SW = fm.stride[1]; | |||
| size_t IH = param.isz[0]; | |||
| size_t IW = param.isz[1]; | |||
| size_t OH = param.osz[0]; | |||
| size_t OW = param.osz[1]; | |||
| size_t FH = fm.spatial[0]; | |||
| size_t FW = fm.spatial[1]; | |||
| int ih = param.isz[0]; | |||
| int iw = param.isz[1]; | |||
| int ph = fm.padding[0]; | |||
| int pw = fm.padding[1]; | |||
| OH2 = OH; | |||
| OW2 = (OW + 7) & ~7; | |||
| IH2 = SW * OH + FH - SW; | |||
| IW2 = SW * OW2 + FW - SW; | |||
| // Because stride is 2, sometimes IW == IW2+1. Do a max update to | |||
| // handle this case. | |||
| IH2 = std::max(IH2, IH); | |||
| IW2 = std::max(IW2, IW); | |||
| ih2 = ih + ph * 2; | |||
| iw2 = iw + pw * 2; | |||
| } | |||
| static WorkspaceBundle get_bundle(const ConvBiasImpl::NCBKernSizeParam& param) { | |||
| constexpr size_t src_expand = 4; | |||
| @@ -57,8 +49,8 @@ static WorkspaceBundle get_bundle(const ConvBiasImpl::NCBKernSizeParam& param) { | |||
| size_t OC = fm.ocpg; | |||
| size_t FH = fm.spatial[0]; | |||
| size_t FW = fm.spatial[1]; | |||
| size_t IH2, IW2, OH2, OW2; | |||
| get_rectified_size(param, IH2, IW2, OH2, OW2); | |||
| int IH2, IW2; | |||
| get_rectified_size(param, IH2, IW2); | |||
| if (group == 1) { | |||
| size_t src_size = | |||
| batch * group * IC * IH2 * IW2 * sizeof(int8_t) * src_expand; | |||
| @@ -76,16 +68,16 @@ static void copy_padding_kern(WorkspaceBundle bundle, | |||
| const ConvBiasImpl::NCBKernParam& kern_param, | |||
| const ConvBiasImpl::NCBKernIndex& ncb_index, | |||
| const CpuNDRange& workspace_ids) { | |||
| size_t IH = kern_param.isz[0]; | |||
| size_t IW = kern_param.isz[1]; | |||
| size_t IC = kern_param.filter_meta.icpg; | |||
| size_t PH = kern_param.filter_meta.padding[0]; | |||
| size_t PW = kern_param.filter_meta.padding[1]; | |||
| size_t GROUP = kern_param.filter_meta.group; | |||
| int IH = kern_param.isz[0]; | |||
| int IW = kern_param.isz[1]; | |||
| int IC = kern_param.filter_meta.icpg; | |||
| int PH = kern_param.filter_meta.padding[0]; | |||
| int PW = kern_param.filter_meta.padding[1]; | |||
| int GROUP = kern_param.filter_meta.group; | |||
| size_t IH2, IW2, OH2, OW2; | |||
| get_rectified_size(kern_param, IH2, IW2, OH2, OW2); | |||
| size_t padding_group_size = IH2 * IW2 * IC; | |||
| int IH2, IW2; | |||
| get_rectified_size(kern_param, IH2, IW2); | |||
| int padding_group_size = IH2 * IW2 * IC; | |||
| bundle.set(kern_param.workspace_ptr); | |||
| //! Used for get the workspace offset | |||
| constexpr int pack_ic = 4; | |||
| @@ -100,16 +92,10 @@ static void copy_padding_kern(WorkspaceBundle bundle, | |||
| size_t group_id = ncb_index.ndrange_id[1]; | |||
| size_t group_pack_size = 1; | |||
| int nr_pad_h = PH * IW2 * pack_ic * expend_element; | |||
| int nr_pad_w = PW * pack_ic * expend_element; | |||
| int over_pad = std::max(0_z, IW2 - IW - 2 * PW) * pack_ic * expend_element; | |||
| int row_last_pad = ((int)IW2 - (int)IW - 2 * (int)PW) >= 0 | |||
| ? nr_pad_w + over_pad | |||
| : (IW2 - IW - PW) * pack_ic * expend_element; | |||
| int col_last_pad = | |||
| ((int)IH2 - (int)IH - 2 * (int)PH) >= 0 | |||
| ? nr_pad_h | |||
| : (IH2 - IH - PH) * IW2 * pack_ic * expend_element; | |||
| int nr_pad_h = PH * IW2 * pack_ic * expend_element; | |||
| int row_last_pad = (IW2 - IW - PW) * pack_ic * expend_element; | |||
| int col_last_pad = (IH2 - IH - PH) * IW2 * pack_ic * expend_element; | |||
| const int8_t* sptr = static_cast<const int8_t*>(kern_param.src<int8_t>( | |||
| batch_id, group_id, workspace_ic_id, group_pack_size, pack_ic)); | |||
| @@ -129,7 +115,7 @@ static void copy_padding_kern(WorkspaceBundle bundle, | |||
| rep(ih_idx, IH) { | |||
| std::memset(sptr_base, 0, nr_pad_w * sizeof(int8_t)); | |||
| sptr_base += nr_pad_w; | |||
| conv_bias::nchw44_pack_src(sptr, sptr_base, IW); | |||
| nchw44_pack_src(sptr, sptr_base, IW); | |||
| sptr_base += IW * pack_ic * expend_element; | |||
| sptr += IW * pack_ic; | |||
| std::memset(sptr_base, 0, row_last_pad * sizeof(int8_t)); | |||
| @@ -140,7 +126,8 @@ static void copy_padding_kern(WorkspaceBundle bundle, | |||
| } | |||
| } | |||
| template <size_t filter, BiasMode bias_mode, typename Op, int ow_remain> | |||
| template <size_t filter, BiasMode bias_mode, typename Op, int ow_remain, | |||
| typename DstType, int stride> | |||
| static void do_conv_kern(WorkspaceBundle bundle, | |||
| const ConvBiasImpl::NCBKernParam& kern_param, | |||
| const ConvBiasImpl::NCBKernIndex& ncb_index, | |||
| @@ -153,12 +140,12 @@ static void do_conv_kern(WorkspaceBundle bundle, | |||
| size_t IC = kern_param.filter_meta.icpg; | |||
| size_t OC = kern_param.filter_meta.ocpg; | |||
| size_t GROUP = kern_param.filter_meta.group; | |||
| size_t IH2, IW2, OH2, OW2; | |||
| get_rectified_size(kern_param, IH2, IW2, OH2, OW2); | |||
| int IH2, IW2; | |||
| get_rectified_size(kern_param, IH2, IW2); | |||
| bool need_post_process = | |||
| kern_param.dst_type.enumv() == DTypeEnum::QuantizedS8; | |||
| //! if dst_type is qint32, the op is not used, just fill with (1.0f,4.0f) | |||
| Op op = Op(1.0f, 4.0f); | |||
| Op op(1.f, 4.f); | |||
| if (need_post_process) { | |||
| float scale_bias = | |||
| kern_param.bias_type.param<dtype::QuantizedS32>().scale; | |||
| @@ -191,49 +178,43 @@ static void do_conv_kern(WorkspaceBundle bundle, | |||
| const int8_t* fptr = | |||
| kern_param.filter<dt_int8>(group_id) + oc_idx * FH * FW * IC; | |||
| void* dst = reinterpret_cast<void*>( | |||
| reinterpret_cast<ptrdiff_t>( | |||
| kern_param.dst<void>(batch_id, group_id)) + | |||
| oc_idx * OH * OW); | |||
| DstType* dst = reinterpret_cast<DstType*>( | |||
| kern_param.dst<void>(batch_id, group_id, oc_idx)); | |||
| const int32_t* bptr = | |||
| kern_param.bias<dt_int32>(batch_id, group_id) + oc_idx; | |||
| auto packed_weight = reinterpret_cast<int8_t*>(bundle.get(1)) + | |||
| group_id * OC * IC * FH * FW + oc_idx * IC * FH * FW; | |||
| conv_bias::nchw44_pack_filter(fptr, packed_weight, | |||
| oc_block / 4 * IC / 4 * FH * FW); | |||
| #define KERN1_NCHW44_CONV(filter) \ | |||
| conv_bias::conv_direct_stride2_##filter##x##filter##_int8_nchw44< \ | |||
| bias_mode, Op, ow_remain>(sptr, packed_weight, bptr, nullptr, \ | |||
| static_cast<int8_t*>(dst), oc_block, IC, \ | |||
| IH2, IW2, OH, OW, op) | |||
| DISPATCH_FILTER(filter, KERN1_NCHW44_CONV) | |||
| #undef KERN1_NCHW44_CONV | |||
| nchw44_pack_filter(fptr, packed_weight, oc_block / 4 * IC / 4 * FH * FW); | |||
| conv_direct_int8_nchw44<bias_mode, Op, ow_remain, filter, DstType, stride>( | |||
| sptr, packed_weight, bptr, nullptr, static_cast<DstType*>(dst), | |||
| oc_block, IC, IH2, IW2, OH, OW, op); | |||
| } | |||
| /* ===================== stride2 algo ===================== */ | |||
| bool ConvBiasImpl::AlgoS8DirectStride2NCHW44::usable( | |||
| bool ConvBiasImpl::AlgoS8DirectNCHW44::usable( | |||
| fallback::ConvBiasImpl*, const NCBKernSizeParam& param, | |||
| AlgoSelectionStrategy algo_selection_strategy) const { | |||
| MEGDNN_MARK_USED_VAR(algo_selection_strategy); | |||
| auto&& fm = param.filter_meta; | |||
| auto FH = fm.spatial[0]; | |||
| auto OC = fm.ocpg; | |||
| auto IC = fm.icpg; | |||
| bool avaible = //! src and filter are qint8, dst is qint8 or qint32 | |||
| const int fh = fm.spatial[0]; | |||
| const int fw = fm.spatial[1]; | |||
| const int oc = fm.ocpg; | |||
| const int ic = fm.icpg; | |||
| const bool avaible = //! src and filter are qint8, dst is qint8 or qint32 | |||
| ((param.src_type.enumv() == DTypeEnum::QuantizedS8 && | |||
| param.filter_type.enumv() == DTypeEnum::QuantizedS8 && | |||
| (param.dst_type.enumv() == DTypeEnum::QuantizedS8 || | |||
| param.dst_type.enumv() == DTypeEnum::QuantizedS32))) && | |||
| (fm.format == param::Convolution::Format::NCHW44) && | |||
| (OC % 4 == 0 && IC % 4 == 0 && OC >= 4) && !fm.should_flip && | |||
| (oc % 4 == 0 && ic % 4 == 0 && oc >= 4) && !fm.should_flip && | |||
| fm.spatial_ndim == 2 && fm.dilation[0] == 1 && | |||
| fm.dilation[1] == 1 && fm.stride[0] == 2 && fm.stride[1] == 2 && | |||
| FH == fm.spatial[1] && (FH == 2 || FH == 3 || FH == 5 || FH == 7) && | |||
| fm.dilation[1] == 1 && fm.stride[0] == fm.stride[1] && | |||
| (fm.stride[0] == 2 || fm.stride[0] == 1) && fh == fw && | |||
| (fh == 2 || fh == 3 || fh == 5 || fh == 7) && | |||
| param.bias_mode != BiasMode::BIAS; | |||
| return avaible; | |||
| } | |||
| bool ConvBiasImpl::AlgoS8DirectStride2NCHW44::is_preferred( | |||
| bool ConvBiasImpl::AlgoS8DirectNCHW44::is_preferred( | |||
| megdnn::fallback::ConvBiasImpl* conv_bias_impl_ptr, | |||
| const NCBKernSizeParam& param) const { | |||
| // TODO: benchmark and fix | |||
| @@ -242,13 +223,13 @@ bool ConvBiasImpl::AlgoS8DirectStride2NCHW44::is_preferred( | |||
| return false; | |||
| } | |||
| size_t ConvBiasImpl::AlgoS8DirectStride2NCHW44::get_workspace( | |||
| size_t ConvBiasImpl::AlgoS8DirectNCHW44::get_workspace( | |||
| fallback::ConvBiasImpl*, const NCBKernSizeParam& param) const { | |||
| return get_bundle(param).total_size_in_bytes(); | |||
| } | |||
| SmallVector<ConvBiasImpl::NCBKern> | |||
| ConvBiasImpl::AlgoS8DirectStride2NCHW44::dispatch_kerns( | |||
| ConvBiasImpl::AlgoS8DirectNCHW44::dispatch_kerns( | |||
| fallback::ConvBiasImpl*, const NCBKernSizeParam& param) const { | |||
| auto fm = param.filter_meta; | |||
| size_t N = param.n; | |||
| @@ -261,97 +242,129 @@ ConvBiasImpl::AlgoS8DirectStride2NCHW44::dispatch_kerns( | |||
| WorkspaceBundle wbundle = get_bundle(param); | |||
| conv_fun do_conv_fun = nullptr; | |||
| int ow_remain = OW % 8; | |||
| bool need_post_process = param.dst_type.enumv() == DTypeEnum::QuantizedS8; | |||
| // NOTE: remain_w is not used to gen hash of midout for compatible with changing | |||
| // shape runtime | |||
| #define DO_CONV_KERN_FUN(filter, bias_mode, remain_w, op) \ | |||
| MIDOUT_BEGIN(megdnn_arm_common_conv_bias_int8_nchw44_stride2, \ | |||
| midout_iv(#filter #bias_mode #op##_hash)) { \ | |||
| do_conv_fun = do_conv_kern<filter, bias_mode, op, remain_w>; \ | |||
| } \ | |||
| #define DO_CONV_KERN_FUN(stride, dst_type, filter, bias_mode, remain_w, op) \ | |||
| MIDOUT_BEGIN(megdnn_arm_common_conv_bias_int8_nchw44, \ | |||
| midout_iv(#stride #dst_type #filter #bias_mode #op##_hash)) { \ | |||
| do_conv_fun = do_conv_kern<filter, bias_mode, op, remain_w, dst_type, \ | |||
| stride>; \ | |||
| } \ | |||
| MIDOUT_END(); | |||
| #define GET_OP_PARAM(filter, bias_mode, remain_w) \ | |||
| switch (param.nonlineMode) { \ | |||
| case param::ConvBias::NonlineMode::IDENTITY: \ | |||
| DO_CONV_KERN_FUN(filter, bias_mode, remain_w, \ | |||
| TypeCvtOp<dt_qint32 MEGDNN_COMMA dt_qint8>) \ | |||
| break; \ | |||
| case param::ConvBias::NonlineMode::RELU: \ | |||
| DO_CONV_KERN_FUN(filter, bias_mode, remain_w, \ | |||
| ReluOp<dt_qint32 MEGDNN_COMMA dt_qint8>) \ | |||
| break; \ | |||
| case param::ConvBias::NonlineMode::H_SWISH: \ | |||
| DO_CONV_KERN_FUN(filter, bias_mode, remain_w, \ | |||
| HSwishOp<dt_qint32 MEGDNN_COMMA dt_qint8>) \ | |||
| break; \ | |||
| default: \ | |||
| megdnn_assert(0); \ | |||
| break; \ | |||
| #define GET_OP_PARAM(stride, filter, bias_mode, remain_w) \ | |||
| if (need_post_process) { \ | |||
| switch (param.nonlineMode) { \ | |||
| case param::ConvBias::NonlineMode::IDENTITY: \ | |||
| DO_CONV_KERN_FUN(stride, dt_qint8, filter, bias_mode, \ | |||
| remain_w, \ | |||
| TypeCvtOp<dt_qint32 MEGDNN_COMMA dt_qint8>) \ | |||
| break; \ | |||
| case param::ConvBias::NonlineMode::RELU: \ | |||
| DO_CONV_KERN_FUN(stride, dt_qint8, filter, bias_mode, \ | |||
| remain_w, \ | |||
| ReluOp<dt_qint32 MEGDNN_COMMA dt_qint8>) \ | |||
| break; \ | |||
| case param::ConvBias::NonlineMode::H_SWISH: \ | |||
| DO_CONV_KERN_FUN(stride, dt_qint8, filter, bias_mode, \ | |||
| remain_w, \ | |||
| HSwishOp<dt_qint32 MEGDNN_COMMA dt_qint8>) \ | |||
| break; \ | |||
| default: \ | |||
| megdnn_assert(0, "no supported noline mode"); \ | |||
| break; \ | |||
| } \ | |||
| } else { \ | |||
| switch (param.nonlineMode) { \ | |||
| case param::ConvBias::NonlineMode::IDENTITY: \ | |||
| DO_CONV_KERN_FUN(stride, dt_int32, filter, bias_mode, \ | |||
| remain_w, NoneOp<dt_int32>) \ | |||
| break; \ | |||
| default: \ | |||
| megdnn_assert( \ | |||
| 0, \ | |||
| "only support IDENTITY mode when dst is not qint8"); \ | |||
| break; \ | |||
| } \ | |||
| } | |||
| #define GET_REMAIN_W_PARAM(filter, bias_mode) \ | |||
| switch (ow_remain) { \ | |||
| case 0: \ | |||
| GET_OP_PARAM(filter, bias_mode, 0); \ | |||
| break; \ | |||
| case 1: \ | |||
| GET_OP_PARAM(filter, bias_mode, 1); \ | |||
| break; \ | |||
| case 2: \ | |||
| GET_OP_PARAM(filter, bias_mode, 2); \ | |||
| break; \ | |||
| case 3: \ | |||
| GET_OP_PARAM(filter, bias_mode, 3); \ | |||
| break; \ | |||
| case 4: \ | |||
| GET_OP_PARAM(filter, bias_mode, 4); \ | |||
| break; \ | |||
| case 5: \ | |||
| GET_OP_PARAM(filter, bias_mode, 5); \ | |||
| break; \ | |||
| case 6: \ | |||
| GET_OP_PARAM(filter, bias_mode, 6); \ | |||
| break; \ | |||
| case 7: \ | |||
| GET_OP_PARAM(filter, bias_mode, 7); \ | |||
| break; \ | |||
| default: \ | |||
| megdnn_assert(0); \ | |||
| #define GET_REMAIN_W_PARAM(stride, filter, bias_mode) \ | |||
| switch (ow_remain) { \ | |||
| case 0: \ | |||
| GET_OP_PARAM(stride, filter, bias_mode, 0); \ | |||
| break; \ | |||
| case 1: \ | |||
| GET_OP_PARAM(stride, filter, bias_mode, 1); \ | |||
| break; \ | |||
| case 2: \ | |||
| GET_OP_PARAM(stride, filter, bias_mode, 2); \ | |||
| break; \ | |||
| case 3: \ | |||
| GET_OP_PARAM(stride, filter, bias_mode, 3); \ | |||
| break; \ | |||
| case 4: \ | |||
| GET_OP_PARAM(stride, filter, bias_mode, 4); \ | |||
| break; \ | |||
| case 5: \ | |||
| GET_OP_PARAM(stride, filter, bias_mode, 5); \ | |||
| break; \ | |||
| case 6: \ | |||
| GET_OP_PARAM(stride, filter, bias_mode, 6); \ | |||
| break; \ | |||
| case 7: \ | |||
| GET_OP_PARAM(stride, filter, bias_mode, 7); \ | |||
| break; \ | |||
| default: \ | |||
| megdnn_assert(0); \ | |||
| } | |||
| #define GET_BIAS_MODE_PARAM(filter) \ | |||
| switch (param.bias_mode) { \ | |||
| case BiasMode::NO_BIAS: \ | |||
| GET_REMAIN_W_PARAM(filter, BiasMode::NO_BIAS) \ | |||
| break; \ | |||
| case BiasMode::BROADCAST_CHANNEL_BIAS: \ | |||
| GET_REMAIN_W_PARAM(filter, BiasMode::BROADCAST_CHANNEL_BIAS) \ | |||
| break; \ | |||
| default: \ | |||
| megdnn_assert(0); \ | |||
| break; \ | |||
| #define GET_BIAS_MODE_PARAM(stride, filter) \ | |||
| switch (param.bias_mode) { \ | |||
| case BiasMode::NO_BIAS: \ | |||
| GET_REMAIN_W_PARAM(stride, filter, BiasMode::NO_BIAS) \ | |||
| break; \ | |||
| case BiasMode::BROADCAST_CHANNEL_BIAS: \ | |||
| GET_REMAIN_W_PARAM(stride, filter, \ | |||
| BiasMode::BROADCAST_CHANNEL_BIAS) \ | |||
| break; \ | |||
| default: \ | |||
| megdnn_assert(0); \ | |||
| break; \ | |||
| } | |||
| #define DISPATCH_CONV_KERN() \ | |||
| #define DISPATCH_CONV_KERN(stride) \ | |||
| switch (param.filter_meta.spatial[0]) { \ | |||
| case 2: \ | |||
| GET_BIAS_MODE_PARAM(2) \ | |||
| GET_BIAS_MODE_PARAM(stride, 2) \ | |||
| break; \ | |||
| case 3: \ | |||
| GET_BIAS_MODE_PARAM(3) \ | |||
| GET_BIAS_MODE_PARAM(stride, 3) \ | |||
| break; \ | |||
| case 5: \ | |||
| GET_BIAS_MODE_PARAM(5) \ | |||
| GET_BIAS_MODE_PARAM(stride, 5) \ | |||
| break; \ | |||
| case 7: \ | |||
| GET_BIAS_MODE_PARAM(7) \ | |||
| GET_BIAS_MODE_PARAM(stride, 7) \ | |||
| break; \ | |||
| default: \ | |||
| megdnn_assert(0); \ | |||
| break; \ | |||
| } | |||
| DISPATCH_CONV_KERN(); | |||
| switch (param.filter_meta.stride[0]) { | |||
| case 1: | |||
| DISPATCH_CONV_KERN(1); | |||
| break; | |||
| case 2: | |||
| DISPATCH_CONV_KERN(2); | |||
| break; | |||
| default: | |||
| megdnn_throw(ssprintf("Unsupport stride size %u for the first conv", | |||
| param.filter_meta.stride[0]) | |||
| .c_str()); | |||
| break; | |||
| } | |||
| #undef DO_CONV_KERN_FUN | |||
| #undef GET_REMAIN_W_PARAM | |||
+ 1428
- 0
dnn/src/arm_common/conv_bias/int8/direct_nchw44_kern.h
File diff suppressed because it is too large
View File
+ 0
- 393
dnn/src/arm_common/conv_bias/int8/direct_stride1_nchw44_algo.cpp
View File
| @@ -1,393 +0,0 @@ | |||
| /** | |||
| * \file dnn/src/arm_common/conv_bias/int8/direct_stride1_nchw44_algo.cpp | |||
| * MegEngine is Licensed under the Apache License, Version 2.0 (the "License") | |||
| * | |||
| * Copyright (c) 2014-2020 Megvii Inc. All rights reserved. | |||
| * | |||
| * Unless required by applicable law or agreed to in writing, | |||
| * software distributed under the License is distributed on an | |||
| * "AS IS" BASIS, WITHOUT ARRANTIES OR CONDITIONS OF ANY KIND, either express or | |||
| * implied. | |||
| */ | |||
| #include "megdnn/oprs.h" | |||
| #include "src/arm_common/conv_bias/int8/algos.h" | |||
| #include "src/arm_common/conv_bias/int8/direct.h" | |||
| #include "src/arm_common/conv_bias/int8/strategy.h" | |||
| #include "src/arm_common/elemwise_op.h" | |||
| #include "src/common/opr_delegate.h" | |||
| #include "midout.h" | |||
| using namespace megdnn; | |||
| using namespace arm_common; | |||
| using conv_fun = std::function<void( | |||
| WorkspaceBundle bundle, const ConvBiasImpl::NCBKernParam& kern_param, | |||
| const ConvBiasImpl::NCBKernIndex& ncb_index, | |||
| const CpuNDRange& workspace_ids, const CpuNDRange& ncb_range)>; | |||
| MIDOUT_DECL(megdnn_arm_common_conv_bias_int8_nchw44_stride1) | |||
| static void get_rectified_size( | |||
| const megdnn::fallback::ConvBiasImpl::NCBKernSizeParam& param, | |||
| size_t& IH2, size_t& IW2, size_t& OH2, size_t& OW2) { | |||
| auto&& fm = param.filter_meta; | |||
| auto SW = fm.stride[1]; | |||
| auto OH = param.osz[0]; | |||
| auto OW = param.osz[1]; | |||
| auto FH = fm.spatial[0]; | |||
| auto FW = fm.spatial[1]; | |||
| OH2 = OH; | |||
| OW2 = (OW + 7) & ~7; | |||
| IH2 = SW * OH + FH - SW; | |||
| IW2 = SW * OW2 + FW - SW; | |||
| } | |||
| static WorkspaceBundle get_bundle(const ConvBiasImpl::NCBKernSizeParam& param) { | |||
| constexpr size_t src_expand = 4; | |||
| auto&& fm = param.filter_meta; | |||
| size_t group = fm.group; | |||
| size_t batch = param.n; | |||
| size_t IC = fm.icpg; | |||
| size_t OC = fm.ocpg; | |||
| size_t FH = fm.spatial[0]; | |||
| size_t FW = fm.spatial[1]; | |||
| size_t IH2, IW2, OH2, OW2; | |||
| get_rectified_size(param, IH2, IW2, OH2, OW2); | |||
| if (group == 1) { | |||
| size_t src_size = | |||
| batch * group * IC * IH2 * IW2 * sizeof(int8_t) * src_expand; | |||
| size_t weight_size = group * OC * IC * FH * FW * sizeof(int8_t); | |||
| return {nullptr, {src_size, weight_size}}; | |||
| } else { | |||
| size_t src_size = | |||
| param.nr_threads * IC * IH2 * IW2 * sizeof(int8_t) * src_expand; | |||
| size_t weight_size = group * OC * IC * FH * FW * sizeof(int8_t); | |||
| return {nullptr, {src_size, weight_size}}; | |||
| } | |||
| }; | |||
| static void copy_padding_kern(WorkspaceBundle bundle, | |||
| const ConvBiasImpl::NCBKernParam& kern_param, | |||
| const ConvBiasImpl::NCBKernIndex& ncb_index, | |||
| const CpuNDRange& workspace_ids) { | |||
| size_t IH = kern_param.isz[0]; | |||
| size_t IW = kern_param.isz[1]; | |||
| size_t IC = kern_param.filter_meta.icpg; | |||
| size_t PH = kern_param.filter_meta.padding[0]; | |||
| size_t PW = kern_param.filter_meta.padding[1]; | |||
| size_t GROUP = kern_param.filter_meta.group; | |||
| size_t IH2, IW2, OH2, OW2; | |||
| get_rectified_size(kern_param, IH2, IW2, OH2, OW2); | |||
| size_t padding_group_size = IH2 * IW2 * IC; | |||
| bundle.set(kern_param.workspace_ptr); | |||
| //! Used for get the workspace offset | |||
| constexpr int pack_ic = 4; | |||
| constexpr int expend_element = 4; | |||
| // TODO: block dim is better to get from arg | |||
| size_t workspace_ic_block = 4; | |||
| size_t workspace_batch_id = workspace_ids[0]; | |||
| size_t workspace_group_id = workspace_ids[1]; | |||
| size_t workspace_ic_id = workspace_ids[2]; | |||
| size_t workspace_ic = workspace_ic_id * workspace_ic_block; | |||
| size_t batch_id = ncb_index.ndrange_id[0]; | |||
| size_t group_id = ncb_index.ndrange_id[1]; | |||
| size_t group_pack_size = 1; | |||
| int nr_pad_h = PH * IW2 * pack_ic * expend_element; | |||
| int nr_pad_w = PW * pack_ic * expend_element; | |||
| int over_pad = std::max(0_z, IW2 - IW - 2 * PW) * pack_ic * expend_element; | |||
| //! copy to sptr_base to eliminate padding effect | |||
| const int8_t* sptr = static_cast<const int8_t*>(kern_param.src<int8_t>( | |||
| batch_id, group_id, workspace_ic_id, group_pack_size, pack_ic)); | |||
| int8_t* sptr_base = static_cast<int8_t*>(bundle.get(0)) + | |||
| (workspace_batch_id * GROUP * padding_group_size + | |||
| workspace_group_id * padding_group_size + | |||
| workspace_ic * IH2 * IW2) * | |||
| expend_element; | |||
| size_t nr_ic = workspace_ic_block; | |||
| if (GROUP > 1) { | |||
| nr_ic = IC; | |||
| } | |||
| rep_step(ic_idx, nr_ic, pack_ic) { | |||
| std::memset(sptr_base, 0, nr_pad_h * sizeof(int8_t)); | |||
| sptr_base += nr_pad_h; | |||
| rep(ih_idx, IH) { | |||
| std::memset(sptr_base, 0, nr_pad_w * sizeof(int8_t)); | |||
| sptr_base += nr_pad_w; | |||
| conv_bias::nchw44_pack_src(sptr, sptr_base, IW); | |||
| sptr_base += IW * pack_ic * expend_element; | |||
| sptr += IW * pack_ic; | |||
| std::memset(sptr_base, 0, (nr_pad_w + over_pad) * sizeof(int8_t)); | |||
| sptr_base += nr_pad_w + over_pad; | |||
| } | |||
| std::memset(sptr_base, 0, nr_pad_h * sizeof(int8_t)); | |||
| sptr_base += nr_pad_h; | |||
| } | |||
| } | |||
| template <size_t filter, BiasMode bias_mode, typename Op, int ow_remain> | |||
| static void do_conv_kern(WorkspaceBundle bundle, | |||
| const ConvBiasImpl::NCBKernParam& kern_param, | |||
| const ConvBiasImpl::NCBKernIndex& ncb_index, | |||
| const CpuNDRange& workspace_ids, | |||
| const CpuNDRange& ncb_range) { | |||
| size_t OH = kern_param.osz[0]; | |||
| size_t OW = kern_param.osz[1]; | |||
| size_t FH = kern_param.filter_meta.spatial[0]; | |||
| size_t FW = kern_param.filter_meta.spatial[1]; | |||
| size_t IC = kern_param.filter_meta.icpg; | |||
| size_t OC = kern_param.filter_meta.ocpg; | |||
| size_t GROUP = kern_param.filter_meta.group; | |||
| size_t IH2, IW2, OH2, OW2; | |||
| get_rectified_size(kern_param, IH2, IW2, OH2, OW2); | |||
| bool need_post_process = | |||
| kern_param.dst_type.enumv() == DTypeEnum::QuantizedS8; | |||
| //! if dst_type is qint32, the op is not used, just fill with (1.0f,4.0f) | |||
| Op op = Op(1.0f, 4.0f); | |||
| if (need_post_process) { | |||
| float scale_bias = | |||
| kern_param.bias_type.param<dtype::QuantizedS32>().scale; | |||
| float scale_dst = kern_param.dst_type.param<dtype::QuantizedS8>().scale; | |||
| op = Op(scale_bias, scale_dst); | |||
| } | |||
| size_t padding_group_size = IH2 * IW2 * IC; | |||
| bundle.set(kern_param.workspace_ptr); | |||
| constexpr size_t pack_c = 4; | |||
| constexpr size_t src_expand_size = 4; | |||
| const size_t workspace_batch_id = workspace_ids[0]; | |||
| const size_t workspace_group_id = workspace_ids[1]; | |||
| const size_t batch_id = ncb_index.ndrange_id[0]; | |||
| const size_t group_id = ncb_index.ndrange_id[1]; | |||
| const size_t oc_id = ncb_index.ndrange_id[2]; | |||
| const size_t oc_block_num = ncb_range[2]; | |||
| size_t nr_pack_per_step = div_ceil(div_ceil(OC, pack_c), oc_block_num); | |||
| size_t oc_block = nr_pack_per_step * pack_c; | |||
| const size_t oc_idx = oc_id * oc_block; | |||
| if (oc_id == (oc_block_num - 1)) { | |||
| oc_block = OC - oc_id * nr_pack_per_step * pack_c; | |||
| } | |||
| megdnn_assert(oc_block % pack_c == 0, | |||
| "oc must be devisible by 4, but oc = %zu", oc_block); | |||
| const int8_t* sptr = | |||
| static_cast<int8_t*>(bundle.get(0)) + | |||
| workspace_batch_id * GROUP * padding_group_size * src_expand_size + | |||
| workspace_group_id * padding_group_size * src_expand_size; | |||
| const int8_t* fptr = | |||
| kern_param.filter<dt_int8>(group_id) + oc_idx * FH * FW * IC; | |||
| void* dst = reinterpret_cast<void*>( | |||
| reinterpret_cast<ptrdiff_t>( | |||
| kern_param.dst<void>(batch_id, group_id)) + | |||
| oc_idx * OH * OW); | |||
| const int32_t* bptr = | |||
| kern_param.bias<dt_int32>(batch_id, group_id) + oc_idx; | |||
| auto packed_weight = reinterpret_cast<int8_t*>(bundle.get(1)) + | |||
| group_id * OC * IC * FH * FW + oc_idx * IC * FH * FW; | |||
| conv_bias::nchw44_pack_filter(fptr, packed_weight, | |||
| oc_block / 4 * IC / 4 * FH * FW); | |||
| #define KERN1_NCHW44_CONV(filter) \ | |||
| conv_bias::conv_direct_stride1_##filter##x##filter##_int8_nchw44< \ | |||
| bias_mode, Op, ow_remain>(sptr, packed_weight, bptr, nullptr, \ | |||
| static_cast<int8_t*>(dst), oc_block, IC, \ | |||
| IH2, IW2, OH, OW, op) | |||
| DISPATCH_FILTER(filter, KERN1_NCHW44_CONV) | |||
| #undef KERN1_NCHW44_CONV | |||
| } | |||
| /* ===================== stride1 algo ===================== */ | |||
| bool ConvBiasImpl::AlgoS8DirectStride1NCHW44::usable( | |||
| fallback::ConvBiasImpl*, const NCBKernSizeParam& param, | |||
| AlgoSelectionStrategy algo_selection_strategy) const { | |||
| MEGDNN_MARK_USED_VAR(algo_selection_strategy); | |||
| auto&& fm = param.filter_meta; | |||
| auto FH = fm.spatial[0]; | |||
| auto OC = fm.ocpg; | |||
| auto IC = fm.icpg; | |||
| bool avaible = //! src and filter are qint8, dst is qint8 or qint32 | |||
| ((param.src_type.enumv() == DTypeEnum::QuantizedS8 && | |||
| param.filter_type.enumv() == DTypeEnum::QuantizedS8 && | |||
| (param.dst_type.enumv() == DTypeEnum::QuantizedS8 || | |||
| param.dst_type.enumv() == DTypeEnum::QuantizedS32))) && | |||
| (fm.format == param::Convolution::Format::NCHW44) && | |||
| (OC % 4 == 0 && IC % 4 == 0 && OC >= 4) && !fm.should_flip && | |||
| fm.spatial_ndim == 2 && fm.dilation[0] == 1 && | |||
| fm.dilation[1] == 1 && fm.stride[0] == 1 && fm.stride[1] == 1 && | |||
| FH == fm.spatial[1] && (FH == 2 || FH == 3 || FH == 5 || FH == 7) && | |||
| param.bias_mode != BiasMode::BIAS; | |||
| return avaible; | |||
| } | |||
| bool ConvBiasImpl::AlgoS8DirectStride1NCHW44::is_preferred( | |||
| megdnn::fallback::ConvBiasImpl* conv_bias_impl_ptr, | |||
| const NCBKernSizeParam& param) const { | |||
| // TODO: benchmark and fix | |||
| MEGDNN_MARK_USED_VAR(conv_bias_impl_ptr); | |||
| MEGDNN_MARK_USED_VAR(param); | |||
| return false; | |||
| } | |||
| size_t ConvBiasImpl::AlgoS8DirectStride1NCHW44::get_workspace( | |||
| fallback::ConvBiasImpl*, const NCBKernSizeParam& param) const { | |||
| return get_bundle(param).total_size_in_bytes(); | |||
| } | |||
| SmallVector<ConvBiasImpl::NCBKern> | |||
| ConvBiasImpl::AlgoS8DirectStride1NCHW44::dispatch_kerns( | |||
| fallback::ConvBiasImpl*, const NCBKernSizeParam& param) const { | |||
| auto fm = param.filter_meta; | |||
| size_t N = param.n; | |||
| size_t IC = fm.icpg; | |||
| size_t OC = fm.ocpg; | |||
| size_t OW = param.osz[1]; | |||
| size_t group = fm.group; | |||
| size_t fh = fm.spatial[0]; | |||
| size_t fw = fm.spatial[1]; | |||
| WorkspaceBundle wbundle = get_bundle(param); | |||
| conv_fun do_conv_fun = nullptr; | |||
| int ow_remain = OW % 8; | |||
| // NOTE: remain_w is not used to gen hash of midout for compatible with changing | |||
| // shape runtime | |||
| #define DO_CONV_KERN_FUN(filter, bias_mode, remain_w, op) \ | |||
| MIDOUT_BEGIN(megdnn_arm_common_conv_bias_int8_nchw44_stride1, \ | |||
| midout_iv(#filter #bias_mode #op##_hash)) { \ | |||
| do_conv_fun = do_conv_kern<filter, bias_mode, op, remain_w>; \ | |||
| } \ | |||
| MIDOUT_END(); | |||
| #define GET_OP_PARAM(filter, bias_mode, remain_w) \ | |||
| switch (param.nonlineMode) { \ | |||
| case param::ConvBias::NonlineMode::IDENTITY: \ | |||
| DO_CONV_KERN_FUN(filter, bias_mode, remain_w, \ | |||
| TypeCvtOp<dt_qint32 MEGDNN_COMMA dt_qint8>) \ | |||
| break; \ | |||
| case param::ConvBias::NonlineMode::RELU: \ | |||
| DO_CONV_KERN_FUN(filter, bias_mode, remain_w, \ | |||
| ReluOp<dt_qint32 MEGDNN_COMMA dt_qint8>) \ | |||
| break; \ | |||
| case param::ConvBias::NonlineMode::H_SWISH: \ | |||
| DO_CONV_KERN_FUN(filter, bias_mode, remain_w, \ | |||
| HSwishOp<dt_qint32 MEGDNN_COMMA dt_qint8>) \ | |||
| break; \ | |||
| default: \ | |||
| megdnn_assert(0); \ | |||
| break; \ | |||
| } | |||
| #define GET_REMAIN_W_PARAM(filter, bias_mode) \ | |||
| switch (ow_remain) { \ | |||
| case 0: \ | |||
| GET_OP_PARAM(filter, bias_mode, 0); \ | |||
| break; \ | |||
| case 1: \ | |||
| GET_OP_PARAM(filter, bias_mode, 1); \ | |||
| break; \ | |||
| case 2: \ | |||
| GET_OP_PARAM(filter, bias_mode, 2); \ | |||
| break; \ | |||
| case 3: \ | |||
| GET_OP_PARAM(filter, bias_mode, 3); \ | |||
| break; \ | |||
| case 4: \ | |||
| GET_OP_PARAM(filter, bias_mode, 4); \ | |||
| break; \ | |||
| case 5: \ | |||
| GET_OP_PARAM(filter, bias_mode, 5); \ | |||
| break; \ | |||
| case 6: \ | |||
| GET_OP_PARAM(filter, bias_mode, 6); \ | |||
| break; \ | |||
| case 7: \ | |||
| GET_OP_PARAM(filter, bias_mode, 7); \ | |||
| break; \ | |||
| default: \ | |||
| megdnn_assert(0); \ | |||
| } | |||
| #define GET_BIAS_MODE_PARAM(filter) \ | |||
| switch (param.bias_mode) { \ | |||
| case BiasMode::NO_BIAS: \ | |||
| GET_REMAIN_W_PARAM(filter, BiasMode::NO_BIAS) \ | |||
| break; \ | |||
| case BiasMode::BROADCAST_CHANNEL_BIAS: \ | |||
| GET_REMAIN_W_PARAM(filter, BiasMode::BROADCAST_CHANNEL_BIAS) \ | |||
| break; \ | |||
| default: \ | |||
| megdnn_assert(0); \ | |||
| break; \ | |||
| } | |||
| #define DISPATCH_CONV_KERN() \ | |||
| switch (param.filter_meta.spatial[0]) { \ | |||
| case 2: \ | |||
| GET_BIAS_MODE_PARAM(2) \ | |||
| break; \ | |||
| case 3: \ | |||
| GET_BIAS_MODE_PARAM(3) \ | |||
| break; \ | |||
| case 5: \ | |||
| GET_BIAS_MODE_PARAM(5) \ | |||
| break; \ | |||
| case 7: \ | |||
| GET_BIAS_MODE_PARAM(7) \ | |||
| break; \ | |||
| default: \ | |||
| megdnn_assert(0); \ | |||
| break; \ | |||
| } | |||
| DISPATCH_CONV_KERN(); | |||
| #undef DO_CONV_KERN_FUN | |||
| #undef GET_REMAIN_W_PARAM | |||
| #undef GET_OP_PARAM | |||
| #undef GET_BIAS_MODE_PARAM | |||
| #undef DISPATCH_CONV_KERN | |||
| megdnn_assert(do_conv_fun); | |||
| SmallVector<ConvBiasImpl::NCBKern> ret_kerns; | |||
| WorkspaceBundle bundle = wbundle; | |||
| constexpr size_t pack_oc = 4; | |||
| size_t oc_step = pack_oc; | |||
| if (fh == 2 && fw == 2 && OC >= 8) { | |||
| oc_step = 8; | |||
| } | |||
| if (group == 1) { | |||
| CpuNDRange ncb_range = {N, group, div_ceil(OC, oc_step)}; | |||
| auto copy_padding = [bundle](const NCBKernParam& kern_param, | |||
| const NCBKernIndex& ncb_index) { | |||
| copy_padding_kern(bundle, kern_param, ncb_index, | |||
| ncb_index.ndrange_id); | |||
| }; | |||
| constexpr size_t pack_ic = 4; | |||
| ret_kerns.push_back({copy_padding, {N, group, div_ceil(IC, pack_ic)}}); | |||
| auto do_conv = [bundle, do_conv_fun, ncb_range]( | |||
| const NCBKernParam& kern_param, | |||
| const NCBKernIndex& ncb_index) { | |||
| do_conv_fun(bundle, kern_param, ncb_index, ncb_index.ndrange_id, | |||
| ncb_range); | |||
| }; | |||
| ret_kerns.push_back({do_conv, ncb_range}); | |||
| } else { | |||
| CpuNDRange ncb_range = {N, group, 1}; | |||
| auto do_conv = [bundle, do_conv_fun, ncb_range]( | |||
| const NCBKernParam& kern_param, | |||
| const NCBKernIndex& ncb_index) { | |||
| copy_padding_kern(bundle, kern_param, ncb_index, | |||
| {0, ncb_index.thread_id, 0}); | |||
| do_conv_fun(bundle, kern_param, ncb_index, | |||
| {0, ncb_index.thread_id, 0}, ncb_range); | |||
| }; | |||
| ret_kerns.push_back({do_conv, ncb_range}); | |||
| } | |||
| return ret_kerns; | |||
| } | |||
| // vim: syntax=cpp.doxygen | |||
+ 0
- 791
dnn/src/arm_common/conv_bias/int8/direct_stride1_nchw44_kern.cpp
View File
| @@ -1,791 +0,0 @@ | |||
| /** | |||
| * \file dnn/src/arm_common/conv_bias/int8/direct_stride1_nchw44_kern.cpp | |||
| * MegEngine is Licensed under the Apache License, Version 2.0 (the "License") | |||
| * | |||
| * Copyright (c) 2014-2020 Megvii Inc. All rights reserved. | |||
| * | |||
| * Unless required by applicable law or agreed to in writing, | |||
| * software distributed under the License is distributed on an | |||
| * "AS IS" BASIS, WITHOUT ARRANTIES OR CONDITIONS OF ANY KIND, either express or | |||
| * implied. | |||
| */ | |||
| #include "src/arm_common/conv_bias/int8/direct.h" | |||
| #include "src/arm_common/conv_bias/intrinsic_helper.h" | |||
| #include "src/arm_common/elemwise_op.h" | |||
| #include "src/arm_common/simd_macro/marm_neon.h" | |||
| #include "src/common/utils.h" | |||
| #include "src/fallback/conv_bias/common.h" | |||
| using namespace megdnn; | |||
| using namespace arm_common; | |||
| namespace { | |||
| /** | |||
| dot like impl. dot 4 ic to 1 oc, accumale to c <ow, oc> | |||
| example: (format like weight<oc, ic>) | |||
| packed weight | |||
| low 64 bit <0, 0> <0, 1> <1, 2> <1, 3> | <2, 0> <2, 1> <3, 2> <3, 3> | |||
| --------------------------------------------------------------------- | |||
| high 64 bit <0, 3> <0, 2> <1, 1> <1, 0> | <2, 3> <2, 2> <3, 1> <3, 0> | |||
| dot: (<0, 0> + <0, 3>) + (<0, 1> + <0, 2>) -> <0> | |||
| **/ | |||
| // TODO: can try oh = 2 impl, oc = 8 impl | |||
| template <BiasMode bias_mode, typename Op, int remain_w, int filter_size> | |||
| static void ker_neon_dirctconv_3x3s1_oc4_ow8(const int8_t* src_ptr, | |||
| const int8_t* weight_ptr, | |||
| const int32_t* bias_ptr, | |||
| int8_t* dst_ptr, int ic, int ih, | |||
| int iw, const Op& op) { | |||
| constexpr int fh = filter_size; | |||
| constexpr int fw = filter_size; | |||
| constexpr int ic_step = 4; | |||
| constexpr int loop_ic_step = 4; | |||
| constexpr int ld_weight_ic4 = 16; | |||
| constexpr int pack_iw_len = 4; | |||
| const int ic_stride = ih * iw * pack_iw_len; | |||
| int32x4_t c[2 * 4]; | |||
| int8x16_t weight[3]; | |||
| int8x16_t src[8 + 2]; | |||
| int16x8_t temp_c[2]; | |||
| init_oc4_ow8<bias_mode>(c, bias_ptr); | |||
| for (int ic_idx = 0; ic_idx < ic; ic_idx += loop_ic_step) { | |||
| for (int fh_idx = 0; fh_idx < fh; ++fh_idx) { | |||
| const int8_t* src_ic_0_3 = src_ptr + ic_idx * ic_stride + | |||
| fh_idx * iw * ic_step * pack_iw_len; | |||
| src[0] = vld1q_s8(src_ic_0_3); | |||
| src[1] = vld1q_s8((src_ic_0_3 + 16)); | |||
| src[2] = vld1q_s8((src_ic_0_3 + 2 * 16)); | |||
| src[3] = vld1q_s8((src_ic_0_3 + 3 * 16)); | |||
| src[4] = vld1q_s8((src_ic_0_3 + 4 * 16)); | |||
| src[5] = vld1q_s8((src_ic_0_3 + 5 * 16)); | |||
| src[6] = vld1q_s8((src_ic_0_3 + 6 * 16)); | |||
| src[7] = vld1q_s8((src_ic_0_3 + 7 * 16)); | |||
| src[8] = vld1q_s8((src_ic_0_3 + 8 * 16)); | |||
| src[9] = vld1q_s8((src_ic_0_3 + 9 * 16)); | |||
| // oc == 0 | |||
| const int8_t* read_weight_ptr = | |||
| weight_ptr + fh_idx * fw * ld_weight_ic4; | |||
| weight[0] = vld1q_s8(read_weight_ptr); | |||
| weight[1] = vld1q_s8(read_weight_ptr + 16); | |||
| weight[2] = vld1q_s8(read_weight_ptr + 2 * 16); | |||
| c[0] = vdotq_s32_h(weight[0], src[0], c[0], temp_c[0]); | |||
| c[1] = vdotq_s32_h(weight[0], src[1], c[1], temp_c[1]); | |||
| c[0] = vdotq_s32_h(weight[1], src[1], c[0], temp_c[0]); | |||
| c[1] = vdotq_s32_h(weight[1], src[2], c[1], temp_c[1]); | |||
| c[0] = vdotq_s32_h(weight[2], src[2], c[0], temp_c[0]); | |||
| c[1] = vdotq_s32_h(weight[2], src[3], c[1], temp_c[1]); | |||
| c[2] = vdotq_s32_h(weight[0], src[2], c[2], temp_c[0]); | |||
| c[3] = vdotq_s32_h(weight[0], src[3], c[3], temp_c[1]); | |||
| c[2] = vdotq_s32_h(weight[1], src[3], c[2], temp_c[0]); | |||
| c[3] = vdotq_s32_h(weight[1], src[4], c[3], temp_c[1]); | |||
| c[2] = vdotq_s32_h(weight[2], src[4], c[2], temp_c[0]); | |||
| c[3] = vdotq_s32_h(weight[2], src[5], c[3], temp_c[1]); | |||
| c[4] = vdotq_s32_h(weight[0], src[4], c[4], temp_c[0]); | |||
| c[5] = vdotq_s32_h(weight[0], src[5], c[5], temp_c[1]); | |||
| c[4] = vdotq_s32_h(weight[1], src[5], c[4], temp_c[0]); | |||
| c[5] = vdotq_s32_h(weight[1], src[6], c[5], temp_c[1]); | |||
| c[4] = vdotq_s32_h(weight[2], src[6], c[4], temp_c[0]); | |||
| c[5] = vdotq_s32_h(weight[2], src[7], c[5], temp_c[1]); | |||
| c[6] = vdotq_s32_h(weight[0], src[6], c[6], temp_c[0]); | |||
| c[7] = vdotq_s32_h(weight[0], src[7], c[7], temp_c[1]); | |||
| c[6] = vdotq_s32_h(weight[1], src[7], c[6], temp_c[0]); | |||
| c[7] = vdotq_s32_h(weight[1], src[8], c[7], temp_c[1]); | |||
| c[6] = vdotq_s32_h(weight[2], src[8], c[6], temp_c[0]); | |||
| c[7] = vdotq_s32_h(weight[2], src[9], c[7], temp_c[1]); | |||
| } | |||
| weight_ptr += fh * fw * ld_weight_ic4; | |||
| } | |||
| store_oc4_ow8_remain_static<remain_w, Op>(c, op, dst_ptr); | |||
| } | |||
| template <BiasMode bias_mode, typename Op, int remain_w, int filter_size> | |||
| static void ker_neon_dirctconv_2x2s1_oc8_ow8(const int8_t* src_ptr, | |||
| const int8_t* weight_ptr, | |||
| const int32_t* bias_ptr, | |||
| int8_t* dst_ptr, int ic, int ih, | |||
| int iw, int ld_dst_oc, | |||
| const Op& op) { | |||
| constexpr int fh = filter_size; | |||
| constexpr int fw = filter_size; | |||
| constexpr int ic_step = 4; | |||
| constexpr int oc_step = 4; | |||
| constexpr int loop_ic_step = 4; | |||
| constexpr int ld_weight_ic4 = 16; | |||
| constexpr int pack_iw_len = 4; | |||
| const int ic_stride = ih * iw * pack_iw_len; | |||
| const int ld_weight_oc4 = oc_step * fh * fw * ic; | |||
| int32x4_t c[2][8]; | |||
| int8x16_t weight[2][2]; | |||
| int8x16_t src[8 + 1]; | |||
| int16x8_t temp_c[4]; | |||
| init_oc8_ow8<bias_mode>(c, bias_ptr, oc_step); | |||
| for (int ic_idx = 0; ic_idx < ic; ic_idx += loop_ic_step) { | |||
| for (int fh_idx = 0; fh_idx < fh; ++fh_idx) { | |||
| const int8_t* src_ic_0_3 = src_ptr + ic_idx * ic_stride + | |||
| fh_idx * iw * ic_step * pack_iw_len; | |||
| src[0] = vld1q_s8(src_ic_0_3); | |||
| src[1] = vld1q_s8((src_ic_0_3 + 16)); | |||
| src[2] = vld1q_s8((src_ic_0_3 + 2 * 16)); | |||
| src[3] = vld1q_s8((src_ic_0_3 + 3 * 16)); | |||
| src[4] = vld1q_s8((src_ic_0_3 + 4 * 16)); | |||
| src[5] = vld1q_s8((src_ic_0_3 + 5 * 16)); | |||
| src[6] = vld1q_s8((src_ic_0_3 + 6 * 16)); | |||
| src[7] = vld1q_s8((src_ic_0_3 + 7 * 16)); | |||
| src[8] = vld1q_s8((src_ic_0_3 + 8 * 16)); | |||
| // oc == 0 | |||
| const int8_t* read_weight_ptr = | |||
| weight_ptr + fh_idx * fw * ld_weight_ic4; | |||
| weight[0][0] = vld1q_s8(read_weight_ptr); | |||
| weight[0][1] = vld1q_s8(read_weight_ptr + 16); | |||
| weight[1][0] = vld1q_s8(read_weight_ptr + ld_weight_oc4); | |||
| weight[1][1] = vld1q_s8(read_weight_ptr + ld_weight_oc4 + 16); | |||
| c[0][0] = vdotq_s32_h(weight[0][0], src[0], c[0][0], temp_c[0]); | |||
| c[1][0] = vdotq_s32_h(weight[1][0], src[0], c[1][0], temp_c[1]); | |||
| c[0][1] = vdotq_s32_h(weight[0][0], src[1], c[0][1], temp_c[2]); | |||
| c[1][1] = vdotq_s32_h(weight[1][0], src[1], c[1][1], temp_c[3]); | |||
| c[0][0] = vdotq_s32_h(weight[0][1], src[1], c[0][0], temp_c[0]); | |||
| c[1][0] = vdotq_s32_h(weight[1][1], src[1], c[1][0], temp_c[1]); | |||
| c[0][1] = vdotq_s32_h(weight[0][1], src[2], c[0][1], temp_c[2]); | |||
| c[1][1] = vdotq_s32_h(weight[1][1], src[2], c[1][1], temp_c[3]); | |||
| c[0][2] = vdotq_s32_h(weight[0][0], src[2], c[0][2], temp_c[0]); | |||
| c[1][2] = vdotq_s32_h(weight[1][0], src[2], c[1][2], temp_c[1]); | |||
| c[0][3] = vdotq_s32_h(weight[0][0], src[3], c[0][3], temp_c[2]); | |||
| c[1][3] = vdotq_s32_h(weight[1][0], src[3], c[1][3], temp_c[3]); | |||
| c[0][2] = vdotq_s32_h(weight[0][1], src[3], c[0][2], temp_c[0]); | |||
| c[1][2] = vdotq_s32_h(weight[1][1], src[3], c[1][2], temp_c[1]); | |||
| c[0][3] = vdotq_s32_h(weight[0][1], src[4], c[0][3], temp_c[2]); | |||
| c[1][3] = vdotq_s32_h(weight[1][1], src[4], c[1][3], temp_c[3]); | |||
| c[0][4] = vdotq_s32_h(weight[0][0], src[4], c[0][4], temp_c[0]); | |||
| c[1][4] = vdotq_s32_h(weight[1][0], src[4], c[1][4], temp_c[1]); | |||
| c[0][5] = vdotq_s32_h(weight[0][0], src[5], c[0][5], temp_c[2]); | |||
| c[1][5] = vdotq_s32_h(weight[1][0], src[5], c[1][5], temp_c[3]); | |||
| c[0][4] = vdotq_s32_h(weight[0][1], src[5], c[0][4], temp_c[0]); | |||
| c[1][4] = vdotq_s32_h(weight[1][1], src[5], c[1][4], temp_c[1]); | |||
| c[0][5] = vdotq_s32_h(weight[0][1], src[6], c[0][5], temp_c[2]); | |||
| c[1][5] = vdotq_s32_h(weight[1][1], src[6], c[1][5], temp_c[3]); | |||
| c[0][6] = vdotq_s32_h(weight[0][0], src[6], c[0][6], temp_c[0]); | |||
| c[1][6] = vdotq_s32_h(weight[1][0], src[6], c[1][6], temp_c[1]); | |||
| c[0][7] = vdotq_s32_h(weight[0][0], src[7], c[0][7], temp_c[2]); | |||
| c[1][7] = vdotq_s32_h(weight[1][0], src[7], c[1][7], temp_c[3]); | |||
| c[0][6] = vdotq_s32_h(weight[0][1], src[7], c[0][6], temp_c[0]); | |||
| c[1][6] = vdotq_s32_h(weight[1][1], src[7], c[1][6], temp_c[1]); | |||
| c[0][7] = vdotq_s32_h(weight[0][1], src[8], c[0][7], temp_c[2]); | |||
| c[1][7] = vdotq_s32_h(weight[1][1], src[8], c[1][7], temp_c[3]); | |||
| } | |||
| weight_ptr += fh * fw * ld_weight_ic4; | |||
| } | |||
| store_oc8_ow8_remain_static<remain_w>(c, op, dst_ptr, ld_dst_oc); | |||
| } | |||
| template <BiasMode bias_mode, typename Op, int remain_w, int filter_size> | |||
| static void ker_neon_dirctconv_2x2s1_oc4_ow8(const int8_t* src_ptr, | |||
| const int8_t* weight_ptr, | |||
| const int32_t* bias_ptr, | |||
| int8_t* dst_ptr, int ic, int ih, | |||
| int iw, const Op& op) { | |||
| constexpr int fh = filter_size; | |||
| constexpr int fw = filter_size; | |||
| constexpr int ic_step = 4; | |||
| constexpr int loop_ic_step = 4; | |||
| constexpr int ld_weight_ic4 = 16; | |||
| constexpr int pack_iw_len = 4; | |||
| const int ic_stride = ih * iw * pack_iw_len; | |||
| int32x4_t c[2 * 4]; | |||
| int8x16_t weight[2]; | |||
| int8x16_t src[8 + 1]; | |||
| int16x8_t temp_c[2]; | |||
| init_oc4_ow8<bias_mode>(c, bias_ptr); | |||
| for (int ic_idx = 0; ic_idx < ic; ic_idx += loop_ic_step) { | |||
| for (int fh_idx = 0; fh_idx < fh; ++fh_idx) { | |||
| const int8_t* src_ic_0_3 = src_ptr + ic_idx * ic_stride + | |||
| fh_idx * iw * ic_step * pack_iw_len; | |||
| src[0] = vld1q_s8(src_ic_0_3); | |||
| src[1] = vld1q_s8((src_ic_0_3 + 16)); | |||
| src[2] = vld1q_s8((src_ic_0_3 + 2 * 16)); | |||
| src[3] = vld1q_s8((src_ic_0_3 + 3 * 16)); | |||
| src[4] = vld1q_s8((src_ic_0_3 + 4 * 16)); | |||
| src[5] = vld1q_s8((src_ic_0_3 + 5 * 16)); | |||
| src[6] = vld1q_s8((src_ic_0_3 + 6 * 16)); | |||
| src[7] = vld1q_s8((src_ic_0_3 + 7 * 16)); | |||
| src[8] = vld1q_s8((src_ic_0_3 + 8 * 16)); | |||
| // oc == 0 | |||
| const int8_t* read_weight_ptr = | |||
| weight_ptr + fh_idx * fw * ld_weight_ic4; | |||
| weight[0] = vld1q_s8(read_weight_ptr); | |||
| weight[1] = vld1q_s8(read_weight_ptr + 16); | |||
| c[0] = vdotq_s32_h(weight[0], src[0], c[0], temp_c[0]); | |||
| c[1] = vdotq_s32_h(weight[0], src[1], c[1], temp_c[1]); | |||
| c[0] = vdotq_s32_h(weight[1], src[1], c[0], temp_c[0]); | |||
| c[1] = vdotq_s32_h(weight[1], src[2], c[1], temp_c[1]); | |||
| c[2] = vdotq_s32_h(weight[0], src[2], c[2], temp_c[0]); | |||
| c[3] = vdotq_s32_h(weight[0], src[3], c[3], temp_c[1]); | |||
| c[2] = vdotq_s32_h(weight[1], src[3], c[2], temp_c[0]); | |||
| c[3] = vdotq_s32_h(weight[1], src[4], c[3], temp_c[1]); | |||
| c[4] = vdotq_s32_h(weight[0], src[4], c[4], temp_c[0]); | |||
| c[5] = vdotq_s32_h(weight[0], src[5], c[5], temp_c[1]); | |||
| c[4] = vdotq_s32_h(weight[1], src[5], c[4], temp_c[0]); | |||
| c[5] = vdotq_s32_h(weight[1], src[6], c[5], temp_c[1]); | |||
| c[6] = vdotq_s32_h(weight[0], src[6], c[6], temp_c[0]); | |||
| c[7] = vdotq_s32_h(weight[0], src[7], c[7], temp_c[1]); | |||
| c[6] = vdotq_s32_h(weight[1], src[7], c[6], temp_c[0]); | |||
| c[7] = vdotq_s32_h(weight[1], src[8], c[7], temp_c[1]); | |||
| } | |||
| weight_ptr += fh * fw * ld_weight_ic4; | |||
| } | |||
| store_oc4_ow8_remain_static<remain_w, Op>(c, op, dst_ptr); | |||
| } | |||
| template <BiasMode bias_mode, typename Op, int remain_w, int filter_size> | |||
| static void ker_neon_dirctconv_5x5s1_oc4_ow8(const int8_t* src_ptr, | |||
| const int8_t* weight_ptr, | |||
| const int32_t* bias_ptr, | |||
| int8_t* dst_ptr, int ic, int ih, | |||
| int iw, const Op& op) { | |||
| constexpr int fh = filter_size; | |||
| constexpr int fw = filter_size; | |||
| constexpr int ic_step = 4; | |||
| constexpr int loop_ic_step = 4; | |||
| constexpr int ld_weight_ic4 = 16; | |||
| constexpr int pack_iw_len = 4; | |||
| const int ic_stride = ih * iw * pack_iw_len; | |||
| int32x4_t c[2 * 4]; | |||
| int8x16_t weight[5]; | |||
| int8x16_t src[8 + 2]; | |||
| int16x8_t temp_c[2]; | |||
| init_oc4_ow8<bias_mode>(c, bias_ptr); | |||
| for (int ic_idx = 0; ic_idx < ic; ic_idx += loop_ic_step) { | |||
| for (int fh_idx = 0; fh_idx < fh; ++fh_idx) { | |||
| const int8_t* src_ic_0_3 = src_ptr + ic_idx * ic_stride + | |||
| fh_idx * iw * ic_step * pack_iw_len; | |||
| src[0] = vld1q_s8(src_ic_0_3); | |||
| src[1] = vld1q_s8((src_ic_0_3 + 16)); | |||
| src[2] = vld1q_s8((src_ic_0_3 + 2 * 16)); | |||
| src[3] = vld1q_s8((src_ic_0_3 + 3 * 16)); | |||
| src[4] = vld1q_s8((src_ic_0_3 + 4 * 16)); | |||
| src[5] = vld1q_s8((src_ic_0_3 + 5 * 16)); | |||
| src[6] = vld1q_s8((src_ic_0_3 + 6 * 16)); | |||
| src[7] = vld1q_s8((src_ic_0_3 + 7 * 16)); | |||
| src[8] = vld1q_s8((src_ic_0_3 + 8 * 16)); | |||
| src[9] = vld1q_s8((src_ic_0_3 + 9 * 16)); | |||
| // oc == 0 | |||
| const int8_t* read_weight_ptr = | |||
| weight_ptr + fh_idx * fw * ld_weight_ic4; | |||
| weight[0] = vld1q_s8(read_weight_ptr); | |||
| weight[1] = vld1q_s8(read_weight_ptr + 16); | |||
| weight[2] = vld1q_s8(read_weight_ptr + 2 * 16); | |||
| weight[3] = vld1q_s8(read_weight_ptr + 3 * 16); | |||
| weight[4] = vld1q_s8(read_weight_ptr + 4 * 16); | |||
| c[0] = vdotq_s32_h(weight[0], src[0], c[0], temp_c[0]); | |||
| c[1] = vdotq_s32_h(weight[0], src[1], c[1], temp_c[1]); | |||
| c[0] = vdotq_s32_h(weight[1], src[1], c[0], temp_c[0]); | |||
| c[1] = vdotq_s32_h(weight[1], src[2], c[1], temp_c[1]); | |||
| c[0] = vdotq_s32_h(weight[2], src[2], c[0], temp_c[0]); | |||
| c[1] = vdotq_s32_h(weight[2], src[3], c[1], temp_c[1]); | |||
| c[0] = vdotq_s32_h(weight[3], src[3], c[0], temp_c[0]); | |||
| c[1] = vdotq_s32_h(weight[3], src[4], c[1], temp_c[1]); | |||
| c[0] = vdotq_s32_h(weight[4], src[4], c[0], temp_c[0]); | |||
| c[1] = vdotq_s32_h(weight[4], src[5], c[1], temp_c[1]); | |||
| c[2] = vdotq_s32_h(weight[0], src[2], c[2], temp_c[0]); | |||
| c[3] = vdotq_s32_h(weight[0], src[3], c[3], temp_c[1]); | |||
| c[2] = vdotq_s32_h(weight[1], src[3], c[2], temp_c[0]); | |||
| c[3] = vdotq_s32_h(weight[1], src[4], c[3], temp_c[1]); | |||
| c[2] = vdotq_s32_h(weight[2], src[4], c[2], temp_c[0]); | |||
| c[3] = vdotq_s32_h(weight[2], src[5], c[3], temp_c[1]); | |||
| c[2] = vdotq_s32_h(weight[3], src[5], c[2], temp_c[0]); | |||
| c[3] = vdotq_s32_h(weight[3], src[6], c[3], temp_c[1]); | |||
| c[2] = vdotq_s32_h(weight[4], src[6], c[2], temp_c[0]); | |||
| c[3] = vdotq_s32_h(weight[4], src[7], c[3], temp_c[1]); | |||
| c[4] = vdotq_s32_h(weight[0], src[4], c[4], temp_c[0]); | |||
| c[5] = vdotq_s32_h(weight[0], src[5], c[5], temp_c[1]); | |||
| c[4] = vdotq_s32_h(weight[1], src[5], c[4], temp_c[0]); | |||
| c[5] = vdotq_s32_h(weight[1], src[6], c[5], temp_c[1]); | |||
| c[4] = vdotq_s32_h(weight[2], src[6], c[4], temp_c[0]); | |||
| c[5] = vdotq_s32_h(weight[2], src[7], c[5], temp_c[1]); | |||
| c[4] = vdotq_s32_h(weight[3], src[7], c[4], temp_c[0]); | |||
| c[5] = vdotq_s32_h(weight[3], src[8], c[5], temp_c[1]); | |||
| c[4] = vdotq_s32_h(weight[4], src[8], c[4], temp_c[0]); | |||
| c[5] = vdotq_s32_h(weight[4], src[9], c[5], temp_c[1]); | |||
| src[0] = vld1q_s8(src_ic_0_3 + 10 * 16); | |||
| src[1] = vld1q_s8((src_ic_0_3 + 11 * 16)); | |||
| c[6] = vdotq_s32_h(weight[0], src[6], c[6], temp_c[0]); | |||
| c[7] = vdotq_s32_h(weight[0], src[7], c[7], temp_c[1]); | |||
| c[6] = vdotq_s32_h(weight[1], src[7], c[6], temp_c[0]); | |||
| c[7] = vdotq_s32_h(weight[1], src[8], c[7], temp_c[1]); | |||
| c[6] = vdotq_s32_h(weight[2], src[8], c[6], temp_c[0]); | |||
| c[7] = vdotq_s32_h(weight[2], src[9], c[7], temp_c[1]); | |||
| c[6] = vdotq_s32_h(weight[3], src[9], c[6], temp_c[0]); | |||
| c[7] = vdotq_s32_h(weight[3], src[0], c[7], temp_c[1]); | |||
| c[6] = vdotq_s32_h(weight[4], src[0], c[6], temp_c[0]); | |||
| c[7] = vdotq_s32_h(weight[4], src[1], c[7], temp_c[1]); | |||
| } | |||
| weight_ptr += fh * fw * ld_weight_ic4; | |||
| } | |||
| store_oc4_ow8_remain_static<remain_w, Op>(c, op, dst_ptr); | |||
| } | |||
| template <BiasMode bias_mode, typename Op, int remain_w, int filter_size> | |||
| static void ker_neon_dirctconv_7x7s1_oc4_ow8(const int8_t* src_ptr, | |||
| const int8_t* weight_ptr, | |||
| const int32_t* bias_ptr, | |||
| int8_t* dst_ptr, int ic, int ih, | |||
| int iw, const Op& op) { | |||
| constexpr int fh = filter_size; | |||
| constexpr int fw = filter_size; | |||
| constexpr int ic_step = 4; | |||
| constexpr int loop_ic_step = 4; | |||
| constexpr int ld_weight_ic4 = 16; | |||
| constexpr int pack_iw_len = 4; | |||
| const int ic_stride = ih * iw * pack_iw_len; | |||
| int32x4_t c[2 * 4]; | |||
| int8x16_t weight[7]; | |||
| int8x16_t src[8 + 2]; | |||
| int16x8_t temp_c[2]; | |||
| init_oc4_ow8<bias_mode>(c, bias_ptr); | |||
| for (int ic_idx = 0; ic_idx < ic; ic_idx += loop_ic_step) { | |||
| for (int fh_idx = 0; fh_idx < fh; ++fh_idx) { | |||
| const int8_t* src_ic_0_3 = src_ptr + ic_idx * ic_stride + | |||
| fh_idx * iw * ic_step * pack_iw_len; | |||
| src[0] = vld1q_s8(src_ic_0_3); | |||
| src[1] = vld1q_s8((src_ic_0_3 + 16)); | |||
| src[2] = vld1q_s8((src_ic_0_3 + 2 * 16)); | |||
| src[3] = vld1q_s8((src_ic_0_3 + 3 * 16)); | |||
| src[4] = vld1q_s8((src_ic_0_3 + 4 * 16)); | |||
| src[5] = vld1q_s8((src_ic_0_3 + 5 * 16)); | |||
| src[6] = vld1q_s8((src_ic_0_3 + 6 * 16)); | |||
| src[7] = vld1q_s8((src_ic_0_3 + 7 * 16)); | |||
| src[8] = vld1q_s8((src_ic_0_3 + 8 * 16)); | |||
| src[9] = vld1q_s8((src_ic_0_3 + 9 * 16)); | |||
| // oc == 0 | |||
| const int8_t* read_weight_ptr = | |||
| weight_ptr + fh_idx * fw * ld_weight_ic4; | |||
| weight[0] = vld1q_s8(read_weight_ptr); | |||
| weight[1] = vld1q_s8(read_weight_ptr + 16); | |||
| weight[2] = vld1q_s8(read_weight_ptr + 2 * 16); | |||
| weight[3] = vld1q_s8(read_weight_ptr + 3 * 16); | |||
| weight[4] = vld1q_s8(read_weight_ptr + 4 * 16); | |||
| weight[5] = vld1q_s8(read_weight_ptr + 5 * 16); | |||
| weight[6] = vld1q_s8(read_weight_ptr + 6 * 16); | |||
| c[0] = vdotq_s32_h(weight[0], src[0], c[0], temp_c[0]); | |||
| c[1] = vdotq_s32_h(weight[0], src[1], c[1], temp_c[1]); | |||
| c[0] = vdotq_s32_h(weight[1], src[1], c[0], temp_c[0]); | |||
| c[1] = vdotq_s32_h(weight[1], src[2], c[1], temp_c[1]); | |||
| c[0] = vdotq_s32_h(weight[2], src[2], c[0], temp_c[0]); | |||
| c[1] = vdotq_s32_h(weight[2], src[3], c[1], temp_c[1]); | |||
| c[0] = vdotq_s32_h(weight[3], src[3], c[0], temp_c[0]); | |||
| c[1] = vdotq_s32_h(weight[3], src[4], c[1], temp_c[1]); | |||
| c[0] = vdotq_s32_h(weight[4], src[4], c[0], temp_c[0]); | |||
| c[1] = vdotq_s32_h(weight[4], src[5], c[1], temp_c[1]); | |||
| c[0] = vdotq_s32_h(weight[5], src[5], c[0], temp_c[0]); | |||
| c[1] = vdotq_s32_h(weight[5], src[6], c[1], temp_c[1]); | |||
| c[0] = vdotq_s32_h(weight[6], src[6], c[0], temp_c[0]); | |||
| c[1] = vdotq_s32_h(weight[6], src[7], c[1], temp_c[1]); | |||
| c[2] = vdotq_s32_h(weight[0], src[2], c[2], temp_c[0]); | |||
| c[3] = vdotq_s32_h(weight[0], src[3], c[3], temp_c[1]); | |||
| c[2] = vdotq_s32_h(weight[1], src[3], c[2], temp_c[0]); | |||
| c[3] = vdotq_s32_h(weight[1], src[4], c[3], temp_c[1]); | |||
| c[2] = vdotq_s32_h(weight[2], src[4], c[2], temp_c[0]); | |||
| c[3] = vdotq_s32_h(weight[2], src[5], c[3], temp_c[1]); | |||
| c[2] = vdotq_s32_h(weight[3], src[5], c[2], temp_c[0]); | |||
| c[3] = vdotq_s32_h(weight[3], src[6], c[3], temp_c[1]); | |||
| c[2] = vdotq_s32_h(weight[4], src[6], c[2], temp_c[0]); | |||
| c[3] = vdotq_s32_h(weight[4], src[7], c[3], temp_c[1]); | |||
| c[2] = vdotq_s32_h(weight[5], src[7], c[2], temp_c[0]); | |||
| c[3] = vdotq_s32_h(weight[5], src[8], c[3], temp_c[1]); | |||
| c[2] = vdotq_s32_h(weight[6], src[8], c[2], temp_c[0]); | |||
| c[3] = vdotq_s32_h(weight[6], src[9], c[3], temp_c[1]); | |||
| src[0] = vld1q_s8(src_ic_0_3 + 10 * 16); | |||
| src[1] = vld1q_s8((src_ic_0_3 + 11 * 16)); | |||
| c[4] = vdotq_s32_h(weight[0], src[4], c[4], temp_c[0]); | |||
| c[5] = vdotq_s32_h(weight[0], src[5], c[5], temp_c[1]); | |||
| c[4] = vdotq_s32_h(weight[1], src[5], c[4], temp_c[0]); | |||
| c[5] = vdotq_s32_h(weight[1], src[6], c[5], temp_c[1]); | |||
| c[4] = vdotq_s32_h(weight[2], src[6], c[4], temp_c[0]); | |||
| c[5] = vdotq_s32_h(weight[2], src[7], c[5], temp_c[1]); | |||
| c[4] = vdotq_s32_h(weight[3], src[7], c[4], temp_c[0]); | |||
| c[5] = vdotq_s32_h(weight[3], src[8], c[5], temp_c[1]); | |||
| c[4] = vdotq_s32_h(weight[4], src[8], c[4], temp_c[0]); | |||
| c[5] = vdotq_s32_h(weight[4], src[9], c[5], temp_c[1]); | |||
| c[4] = vdotq_s32_h(weight[5], src[9], c[4], temp_c[0]); | |||
| c[5] = vdotq_s32_h(weight[5], src[0], c[5], temp_c[1]); | |||
| c[4] = vdotq_s32_h(weight[6], src[0], c[4], temp_c[0]); | |||
| c[5] = vdotq_s32_h(weight[6], src[1], c[5], temp_c[1]); | |||
| src[2] = vld1q_s8(src_ic_0_3 + 12 * 16); | |||
| src[3] = vld1q_s8((src_ic_0_3 + 13 * 16)); | |||
| c[6] = vdotq_s32_h(weight[0], src[6], c[6], temp_c[0]); | |||
| c[7] = vdotq_s32_h(weight[0], src[7], c[7], temp_c[1]); | |||
| c[6] = vdotq_s32_h(weight[1], src[7], c[6], temp_c[0]); | |||
| c[7] = vdotq_s32_h(weight[1], src[8], c[7], temp_c[1]); | |||
| c[6] = vdotq_s32_h(weight[2], src[8], c[6], temp_c[0]); | |||
| c[7] = vdotq_s32_h(weight[2], src[9], c[7], temp_c[1]); | |||
| c[6] = vdotq_s32_h(weight[3], src[9], c[6], temp_c[0]); | |||
| c[7] = vdotq_s32_h(weight[3], src[0], c[7], temp_c[1]); | |||
| c[6] = vdotq_s32_h(weight[4], src[0], c[6], temp_c[0]); | |||
| c[7] = vdotq_s32_h(weight[4], src[1], c[7], temp_c[1]); | |||
| c[6] = vdotq_s32_h(weight[5], src[1], c[6], temp_c[0]); | |||
| c[7] = vdotq_s32_h(weight[5], src[2], c[7], temp_c[1]); | |||
| c[6] = vdotq_s32_h(weight[6], src[2], c[6], temp_c[0]); | |||
| c[7] = vdotq_s32_h(weight[6], src[3], c[7], temp_c[1]); | |||
| } | |||
| weight_ptr += fh * fw * ld_weight_ic4; | |||
| } | |||
| store_oc4_ow8_remain_static<remain_w, Op>(c, op, dst_ptr); | |||
| } | |||
| } // namespace | |||
| /** | |||
| origin weight shape <oc/4, ic/4, fh, fw, 4, 4> | |||
| packed weight shape <oc/4, ic/4, fh, fw, 16> | |||
| example: (format like weight<oc, ic>) | |||
| origin | |||
| <0, 0> <1, 0> <2, 0> <3, 0> | |||
| <0, 1> <1, 1> <2, 1> <3, 1> | |||
| <0, 2> <1, 2> <2, 2> <3, 2> | |||
| <0, 3> <1, 3> <2, 3> <3, 3> | |||
| packed | |||
| low 64 bit <0, 0> <0, 1> <1, 2> <1, 3> | <2, 0> <2, 1> <3, 2> <3, 3> | |||
| --------------------------------------------------------------------- | |||
| high 64 bit <0, 3> <0, 2> <1, 1> <1, 0> | <2, 3> <2, 2> <3, 1> <3, 0> | |||
| **/ | |||
| void conv_bias::nchw44_pack_filter(const int8_t* src, int8_t* dst, int length) { | |||
| static const uint8_t weight_idx_buffer[16] = {0, 4, 9, 13, 2, 6, 11, 15, | |||
| 12, 8, 5, 1, 14, 10, 7, 3}; | |||
| constexpr int simd_len = 16; | |||
| uint8x16_t weight_idx = vld1q_u8(weight_idx_buffer); | |||
| for (int i = 0; i < length; i++) { | |||
| int8x16_t result = vldq_tbl_s8(src + i * simd_len, weight_idx); | |||
| vst1q_s8(dst + i * simd_len, result); | |||
| } | |||
| } | |||
| /** | |||
| origin src shape <n, ic/4, h, w, 4> | |||
| packed src shape <n, ic/4, h, w, 16> | |||
| example: (format like <ic>) | |||
| origin | |||
| <0> <0> <0> <0> | |||
| packed | |||
| low 64 bit <0> <1> <2> <3> | <0> <1> <2> <3> | |||
| --------------------------------------------------------------------- | |||
| high 64 bit <3> <2> <1> <0> | <3> <2> <1> <0> | |||
| **/ | |||
| void conv_bias::nchw44_pack_src(const int8_t* src, int8_t* dst, int length) { | |||
| static const uint8_t src_idx_buffer[16] = {0, 1, 2, 3, 0, 1, 2, 3, | |||
| 3, 2, 1, 0, 3, 2, 1, 0}; | |||
| constexpr int pack_ic = 4; | |||
| constexpr int simd_len = 16; | |||
| uint8x16_t src_idx = vld1q_u8(src_idx_buffer); | |||
| for (int i = 0; i < length; i++) { | |||
| int8x16_t result = vld_dup_tbl_s32(src + i * pack_ic, src_idx); | |||
| vst1q_s8(dst + i * simd_len, result); | |||
| } | |||
| } | |||
| template <BiasMode bias_mode, typename Op, int remain_w> | |||
| void conv_bias::conv_direct_stride1_2x2_int8_nchw44( | |||
| const int8_t* src, const int8_t* filter, const int32_t* bias, | |||
| int32_t* temp, int8_t* dst, const size_t oc, const size_t ic, | |||
| const size_t ih, const size_t iw, const size_t oh, const size_t ow, | |||
| const Op& op) { | |||
| MEGDNN_MARK_USED_VAR(temp); | |||
| constexpr size_t filter_size = 2; | |||
| constexpr size_t fh = filter_size; | |||
| constexpr size_t fw = filter_size; | |||
| constexpr size_t ic_step = 4; | |||
| constexpr size_t oc_step = 4; | |||
| constexpr size_t big_oc_step = 8; | |||
| constexpr size_t oh_step = 1; | |||
| constexpr size_t ow_step = 8; | |||
| constexpr int pack_iw_len = 4; | |||
| const size_t img_stride = oh * ow; | |||
| const size_t ow_end = ow / ow_step * ow_step; | |||
| const size_t ow_remain = ow - ow_end; | |||
| const size_t oc_end = oc / big_oc_step * big_oc_step; | |||
| const size_t oc_remain = oc - oc_end; | |||
| const int ld_oc = oh * ow * ic_step; | |||
| for (size_t oc_idx = 0; oc_idx < oc_end; oc_idx += big_oc_step) { | |||
| const size_t weight_offset = oc_idx * ic * fh * fw; | |||
| for (size_t oh_idx = 0; oh_idx < oh; oh_idx += oh_step) { | |||
| for (size_t ow_idx = 0; ow_idx < ow_end; ow_idx += ow_step) { | |||
| const size_t src_offset = | |||
| (oh_idx * iw + ow_idx) * ic_step * pack_iw_len; | |||
| const size_t dst_offset = | |||
| oc_idx * img_stride + (oh_idx * ow + ow_idx) * oc_step; | |||
| ker_neon_dirctconv_2x2s1_oc8_ow8<bias_mode, Op, 0, filter_size>( | |||
| src + src_offset, filter + weight_offset, bias + oc_idx, | |||
| dst + dst_offset, ic, ih, iw, ld_oc, op); | |||
| } | |||
| if (ow_remain > 0) { | |||
| const size_t src_offset = | |||
| (oh_idx * iw + ow_end) * ic_step * pack_iw_len; | |||
| const size_t dst_offset = | |||
| oc_idx * img_stride + (oh_idx * ow + ow_end) * oc_step; | |||
| ker_neon_dirctconv_2x2s1_oc8_ow8<bias_mode, Op, remain_w, | |||
| filter_size>( | |||
| src + src_offset, filter + weight_offset, bias + oc_idx, | |||
| dst + dst_offset, ic, ih, iw, ld_oc, op); | |||
| } | |||
| } | |||
| } | |||
| if (oc_remain > 0) { | |||
| const size_t oc_idx = oc_end; | |||
| const size_t weight_offset = oc_idx * ic * fh * fw; | |||
| for (size_t oh_idx = 0; oh_idx < oh; oh_idx += oh_step) { | |||
| for (size_t ow_idx = 0; ow_idx < ow_end; ow_idx += ow_step) { | |||
| const size_t src_offset = | |||
| (oh_idx * iw + ow_idx) * ic_step * pack_iw_len; | |||
| const size_t dst_offset = | |||
| oc_idx * img_stride + (oh_idx * ow + ow_idx) * oc_step; | |||
| ker_neon_dirctconv_2x2s1_oc4_ow8<bias_mode, Op, 0, filter_size>( | |||
| src + src_offset, filter + weight_offset, bias + oc_idx, | |||
| dst + dst_offset, ic, ih, iw, op); | |||
| } | |||
| if (ow_remain > 0) { | |||
| const size_t src_offset = | |||
| (oh_idx * iw + ow_end) * ic_step * pack_iw_len; | |||
| const size_t dst_offset = | |||
| oc_idx * img_stride + (oh_idx * ow + ow_end) * oc_step; | |||
| ker_neon_dirctconv_2x2s1_oc4_ow8<bias_mode, Op, remain_w, | |||
| filter_size>( | |||
| src + src_offset, filter + weight_offset, bias + oc_idx, | |||
| dst + dst_offset, ic, ih, iw, op); | |||
| } | |||
| } | |||
| } | |||
| } | |||
| template <BiasMode bias_mode, typename Op, int remain_w> | |||
| void conv_bias::conv_direct_stride1_3x3_int8_nchw44( | |||
| const int8_t* src, const int8_t* filter, const int32_t* bias, | |||
| int32_t* temp, int8_t* dst, const size_t oc, const size_t ic, | |||
| const size_t ih, const size_t iw, const size_t oh, const size_t ow, | |||
| const Op& op) { | |||
| MEGDNN_MARK_USED_VAR(temp); | |||
| constexpr size_t filter_size = 3; | |||
| constexpr size_t fh = filter_size; | |||
| constexpr size_t fw = filter_size; | |||
| constexpr size_t ic_step = 4; | |||
| constexpr size_t oc_step = 4; | |||
| constexpr size_t oh_step = 1; | |||
| constexpr size_t ow_step = 8; | |||
| constexpr int pack_iw_len = 4; | |||
| const size_t img_stride = oh * ow; | |||
| const size_t ow_end = ow / ow_step * ow_step; | |||
| const size_t ow_remain = ow - ow_end; | |||
| for (size_t oc_idx = 0; oc_idx < oc; oc_idx += oc_step) { | |||
| const size_t weight_offset = oc_idx * ic * fh * fw; | |||
| for (size_t oh_idx = 0; oh_idx < oh; oh_idx += oh_step) { | |||
| for (size_t ow_idx = 0; ow_idx < ow_end; ow_idx += ow_step) { | |||
| const size_t src_offset = | |||
| (oh_idx * iw + ow_idx) * ic_step * pack_iw_len; | |||
| const size_t dst_offset = | |||
| oc_idx * img_stride + (oh_idx * ow + ow_idx) * oc_step; | |||
| ker_neon_dirctconv_3x3s1_oc4_ow8<bias_mode, Op, 0, filter_size>( | |||
| src + src_offset, filter + weight_offset, bias + oc_idx, | |||
| dst + dst_offset, ic, ih, iw, op); | |||
| } | |||
| if (ow_remain > 0) { | |||
| const size_t src_offset = | |||
| (oh_idx * iw + ow_end) * ic_step * pack_iw_len; | |||
| const size_t dst_offset = | |||
| oc_idx * img_stride + (oh_idx * ow + ow_end) * oc_step; | |||
| ker_neon_dirctconv_3x3s1_oc4_ow8<bias_mode, Op, remain_w, | |||
| filter_size>( | |||
| src + src_offset, filter + weight_offset, bias + oc_idx, | |||
| dst + dst_offset, ic, ih, iw, op); | |||
| } | |||
| } | |||
| } | |||
| } | |||
| template <BiasMode bias_mode, typename Op, int remain_w> | |||
| void conv_bias::conv_direct_stride1_5x5_int8_nchw44( | |||
| const int8_t* src, const int8_t* filter, const int32_t* bias, | |||
| int32_t* temp, int8_t* dst, const size_t oc, const size_t ic, | |||
| const size_t ih, const size_t iw, const size_t oh, const size_t ow, | |||
| const Op& op) { | |||
| MEGDNN_MARK_USED_VAR(temp); | |||
| constexpr size_t filter_size = 5; | |||
| constexpr size_t fh = filter_size; | |||
| constexpr size_t fw = filter_size; | |||
| constexpr size_t ic_step = 4; | |||
| constexpr size_t oc_step = 4; | |||
| constexpr size_t oh_step = 1; | |||
| constexpr size_t ow_step = 8; | |||
| constexpr int pack_iw_len = 4; | |||
| const size_t img_stride = oh * ow; | |||
| const size_t ow_end = ow / ow_step * ow_step; | |||
| const size_t ow_remain = ow - ow_end; | |||
| for (size_t oc_idx = 0; oc_idx < oc; oc_idx += oc_step) { | |||
| const size_t weight_offset = oc_idx * ic * fh * fw; | |||
| for (size_t oh_idx = 0; oh_idx < oh; oh_idx += oh_step) { | |||
| for (size_t ow_idx = 0; ow_idx < ow_end; ow_idx += ow_step) { | |||
| const size_t src_offset = | |||
| (oh_idx * iw + ow_idx) * ic_step * pack_iw_len; | |||
| const size_t dst_offset = | |||
| oc_idx * img_stride + (oh_idx * ow + ow_idx) * oc_step; | |||
| ker_neon_dirctconv_5x5s1_oc4_ow8<bias_mode, Op, 0, filter_size>( | |||
| src + src_offset, filter + weight_offset, bias + oc_idx, | |||
| dst + dst_offset, ic, ih, iw, op); | |||
| } | |||
| if (ow_remain > 0) { | |||
| const size_t src_offset = | |||
| (oh_idx * iw + ow_end) * ic_step * pack_iw_len; | |||
| const size_t dst_offset = | |||
| oc_idx * img_stride + (oh_idx * ow + ow_end) * oc_step; | |||
| ker_neon_dirctconv_5x5s1_oc4_ow8<bias_mode, Op, remain_w, | |||
| filter_size>( | |||
| src + src_offset, filter + weight_offset, bias + oc_idx, | |||
| dst + dst_offset, ic, ih, iw, op); | |||
| } | |||
| } | |||
| } | |||
| } | |||
| template <BiasMode bias_mode, typename Op, int remain_w> | |||
| void conv_bias::conv_direct_stride1_7x7_int8_nchw44( | |||
| const int8_t* src, const int8_t* filter, const int32_t* bias, | |||
| int32_t* temp, int8_t* dst, const size_t oc, const size_t ic, | |||
| const size_t ih, const size_t iw, const size_t oh, const size_t ow, | |||
| const Op& op) { | |||
| MEGDNN_MARK_USED_VAR(temp); | |||
| constexpr size_t filter_size = 7; | |||
| constexpr size_t fh = filter_size; | |||
| constexpr size_t fw = filter_size; | |||
| constexpr size_t ic_step = 4; | |||
| constexpr size_t oc_step = 4; | |||
| constexpr size_t oh_step = 1; | |||
| constexpr size_t ow_step = 8; | |||
| constexpr int pack_iw_len = 4; | |||
| const size_t img_stride = oh * ow; | |||
| const size_t ow_end = ow / ow_step * ow_step; | |||
| const size_t ow_remain = ow - ow_end; | |||
| for (size_t oc_idx = 0; oc_idx < oc; oc_idx += oc_step) { | |||
| const size_t weight_offset = oc_idx * ic * fh * fw; | |||
| for (size_t oh_idx = 0; oh_idx < oh; oh_idx += oh_step) { | |||
| for (size_t ow_idx = 0; ow_idx < ow_end; ow_idx += ow_step) { | |||
| const size_t src_offset = | |||
| (oh_idx * iw + ow_idx) * ic_step * pack_iw_len; | |||
| const size_t dst_offset = | |||
| oc_idx * img_stride + (oh_idx * ow + ow_idx) * oc_step; | |||
| ker_neon_dirctconv_7x7s1_oc4_ow8<bias_mode, Op, 0, filter_size>( | |||
| src + src_offset, filter + weight_offset, bias + oc_idx, | |||
| dst + dst_offset, ic, ih, iw, op); | |||
| } | |||
| if (ow_remain > 0) { | |||
| const size_t src_offset = | |||
| (oh_idx * iw + ow_end) * ic_step * pack_iw_len; | |||
| const size_t dst_offset = | |||
| oc_idx * img_stride + (oh_idx * ow + ow_end) * oc_step; | |||
| ker_neon_dirctconv_7x7s1_oc4_ow8<bias_mode, Op, remain_w, | |||
| filter_size>( | |||
| src + src_offset, filter + weight_offset, bias + oc_idx, | |||
| dst + dst_offset, ic, ih, iw, op); | |||
| } | |||
| } | |||
| } | |||
| } | |||
| #define INSTANTIATION(stride, i, bias, remain_w, Op) \ | |||
| template void conv_bias::conv_direct_##stride##_##i##x##i##_int8_nchw44< \ | |||
| bias, Op, remain_w>(const int8_t*, const int8_t*, const int32_t*, \ | |||
| int32_t*, int8_t*, const size_t, const size_t, \ | |||
| const size_t, const size_t, const size_t, \ | |||
| const size_t, const Op&); | |||
| #define FOR_OP(stride, i, bias, remain_w) \ | |||
| INSTANTIATION(stride, i, bias, remain_w, \ | |||
| TypeCvtOp<dt_qint32 MEGDNN_COMMA dt_qint8>) \ | |||
| INSTANTIATION(stride, i, bias, remain_w, \ | |||
| ReluOp<dt_qint32 MEGDNN_COMMA dt_qint8>) \ | |||
| INSTANTIATION(stride, i, bias, remain_w, \ | |||
| HSwishOp<dt_qint32 MEGDNN_COMMA dt_qint8>) | |||
| #define FOR_REMAIN(stride, i, bias) \ | |||
| FOR_OP(stride, i, bias, 0) \ | |||
| FOR_OP(stride, i, bias, 1) \ | |||
| FOR_OP(stride, i, bias, 2) \ | |||
| FOR_OP(stride, i, bias, 3) \ | |||
| FOR_OP(stride, i, bias, 4) \ | |||
| FOR_OP(stride, i, bias, 5) \ | |||
| FOR_OP(stride, i, bias, 6) \ | |||
| FOR_OP(stride, i, bias, 7) | |||
| #define FOR_BIAS(stride, i) \ | |||
| FOR_REMAIN(stride, i, BiasMode::NO_BIAS) \ | |||
| FOR_REMAIN(stride, i, BiasMode::BROADCAST_CHANNEL_BIAS) | |||
| #define FOR_FILTER(stride) \ | |||
| FOR_BIAS(stride, 2) \ | |||
| FOR_BIAS(stride, 3) \ | |||
| FOR_BIAS(stride, 5) \ | |||
| FOR_BIAS(stride, 7) | |||
| FOR_FILTER(stride1) | |||
| #undef FOR_STRIDE | |||
| #undef FOR_FILTER | |||
| #undef FOR_IC | |||
| #undef FOR_BIAS | |||
| #undef FOR_NONLINEAR | |||
| #undef FOR_REMAIN | |||
| #undef INSTANTIATION | |||
+ 0
- 793
dnn/src/arm_common/conv_bias/int8/direct_stride2_nchw44_kern.cpp
View File
| @@ -1,793 +0,0 @@ | |||
| /** | |||
| * \file dnn/src/arm_common/conv_bias/int8/direct_stride2_nchw44_kern.cpp | |||
| * MegEngine is Licensed under the Apache License, Version 2.0 (the "License") | |||
| * | |||
| * Copyright (c) 2014-2020 Megvii Inc. All rights reserved. | |||
| * | |||
| * Unless required by applicable law or agreed to in writing, | |||
| * software distributed under the License is distributed on an | |||
| * "AS IS" BASIS, WITHOUT ARRANTIES OR CONDITIONS OF ANY KIND, either express or | |||
| * implied. | |||
| */ | |||
| #include "src/arm_common/conv_bias/int8/direct.h" | |||
| #include "src/arm_common/conv_bias/intrinsic_helper.h" | |||
| #include "src/arm_common/elemwise_op.h" | |||
| #include "src/arm_common/simd_macro/marm_neon.h" | |||
| #include "src/common/utils.h" | |||
| #include "src/fallback/conv_bias/common.h" | |||
| using namespace megdnn; | |||
| using namespace arm_common; | |||
| namespace { | |||
| /** | |||
| dot like impl. dot 4 ic to 1 oc, accumale to c <ow, oc> | |||
| example: (format like weight<oc, ic>) | |||
| packed weight | |||
| low 64 bit <0, 0> <0, 1> <1, 2> <1, 3> | <2, 0> <2, 1> <3, 2> <3, 3> | |||
| --------------------------------------------------------------------- | |||
| high 64 bit <0, 3> <0, 2> <1, 1> <1, 0> | <2, 3> <2, 2> <3, 1> <3, 0> | |||
| dot: (<0, 0> + <0, 3>) + (<0, 1> + <0, 2>) -> <0> | |||
| **/ | |||
| // TODO: can try oh = 2 impl, oc = 8 impl | |||
| template <BiasMode bias_mode, typename Op, int remain_w, int filter_size> | |||
| static void ker_neon_dirctconv_3x3s2_oc4_ow8(const int8_t* src_ptr, | |||
| const int8_t* weight_ptr, | |||
| const int32_t* bias_ptr, | |||
| int8_t* dst_ptr, int ic, int ih, | |||
| int iw, const Op& op) { | |||
| constexpr int fh = filter_size; | |||
| constexpr int fw = filter_size; | |||
| constexpr int ic_step = 4; | |||
| constexpr int loop_ic_step = 4; | |||
| constexpr int ld_weight_ic4 = 16; | |||
| constexpr int pack_iw_len = 4; | |||
| const int ic_stride = ih * iw * pack_iw_len; | |||
| int32x4_t c[2 * 4]; | |||
| int8x16_t weight[3]; | |||
| int8x16_t src[8 + 2]; | |||
| int16x8_t temp_c[2]; | |||
| init_oc4_ow8<bias_mode>(c, bias_ptr); | |||
| for (int ic_idx = 0; ic_idx < ic; ic_idx += loop_ic_step) { | |||
| for (int fh_idx = 0; fh_idx < fh; ++fh_idx) { | |||
| const int8_t* src_ic_0_3 = src_ptr + ic_idx * ic_stride + | |||
| fh_idx * iw * ic_step * pack_iw_len; | |||
| src[0] = vld1q_s8(src_ic_0_3); | |||
| src[1] = vld1q_s8((src_ic_0_3 + 16)); | |||
| src[2] = vld1q_s8((src_ic_0_3 + 2 * 16)); | |||
| src[3] = vld1q_s8((src_ic_0_3 + 3 * 16)); | |||
| src[4] = vld1q_s8((src_ic_0_3 + 4 * 16)); | |||
| src[5] = vld1q_s8((src_ic_0_3 + 5 * 16)); | |||
| src[6] = vld1q_s8((src_ic_0_3 + 6 * 16)); | |||
| src[7] = vld1q_s8((src_ic_0_3 + 7 * 16)); | |||
| src[8] = vld1q_s8((src_ic_0_3 + 8 * 16)); | |||
| src[9] = vld1q_s8((src_ic_0_3 + 9 * 16)); | |||
| // oc == 0 | |||
| const int8_t* read_weight_ptr = | |||
| weight_ptr + fh_idx * fw * ld_weight_ic4; | |||
| weight[0] = vld1q_s8(read_weight_ptr); | |||
| weight[1] = vld1q_s8(read_weight_ptr + 16); | |||
| weight[2] = vld1q_s8(read_weight_ptr + 2 * 16); | |||
| c[0] = vdotq_s32_h(weight[0], src[0], c[0], temp_c[0]); | |||
| c[1] = vdotq_s32_h(weight[0], src[2], c[1], temp_c[1]); | |||
| c[0] = vdotq_s32_h(weight[1], src[1], c[0], temp_c[0]); | |||
| c[1] = vdotq_s32_h(weight[1], src[3], c[1], temp_c[1]); | |||
| c[0] = vdotq_s32_h(weight[2], src[2], c[0], temp_c[0]); | |||
| c[1] = vdotq_s32_h(weight[2], src[4], c[1], temp_c[1]); | |||
| c[2] = vdotq_s32_h(weight[0], src[4], c[2], temp_c[0]); | |||
| c[3] = vdotq_s32_h(weight[0], src[6], c[3], temp_c[1]); | |||
| c[2] = vdotq_s32_h(weight[1], src[5], c[2], temp_c[0]); | |||
| c[3] = vdotq_s32_h(weight[1], src[7], c[3], temp_c[1]); | |||
| c[2] = vdotq_s32_h(weight[2], src[6], c[2], temp_c[0]); | |||
| c[3] = vdotq_s32_h(weight[2], src[8], c[3], temp_c[1]); | |||
| src[0] = vld1q_s8(src_ic_0_3 + 10 * 16); | |||
| src[1] = vld1q_s8((src_ic_0_3 + 11 * 16)); | |||
| src[2] = vld1q_s8((src_ic_0_3 + 12 * 16)); | |||
| c[4] = vdotq_s32_h(weight[0], src[8], c[4], temp_c[0]); | |||
| c[5] = vdotq_s32_h(weight[0], src[0], c[5], temp_c[1]); | |||
| c[4] = vdotq_s32_h(weight[1], src[9], c[4], temp_c[0]); | |||
| c[5] = vdotq_s32_h(weight[1], src[1], c[5], temp_c[1]); | |||
| c[4] = vdotq_s32_h(weight[2], src[0], c[4], temp_c[0]); | |||
| c[5] = vdotq_s32_h(weight[2], src[2], c[5], temp_c[1]); | |||
| src[3] = vld1q_s8((src_ic_0_3 + 13 * 16)); | |||
| src[4] = vld1q_s8((src_ic_0_3 + 14 * 16)); | |||
| src[5] = vld1q_s8((src_ic_0_3 + 15 * 16)); | |||
| src[6] = vld1q_s8((src_ic_0_3 + 16 * 16)); | |||
| c[6] = vdotq_s32_h(weight[0], src[2], c[6], temp_c[0]); | |||
| c[7] = vdotq_s32_h(weight[0], src[4], c[7], temp_c[1]); | |||
| c[6] = vdotq_s32_h(weight[1], src[3], c[6], temp_c[0]); | |||
| c[7] = vdotq_s32_h(weight[1], src[5], c[7], temp_c[1]); | |||
| c[6] = vdotq_s32_h(weight[2], src[4], c[6], temp_c[0]); | |||
| c[7] = vdotq_s32_h(weight[2], src[6], c[7], temp_c[1]); | |||
| } | |||
| weight_ptr += fh * fw * ld_weight_ic4; | |||
| } | |||
| store_oc4_ow8_remain_static<remain_w, Op>(c, op, dst_ptr); | |||
| } | |||
| template <BiasMode bias_mode, typename Op, int remain_w, int filter_size> | |||
| static void ker_neon_dirctconv_2x2s2_oc8_ow8(const int8_t* src_ptr, | |||
| const int8_t* weight_ptr, | |||
| const int32_t* bias_ptr, | |||
| int8_t* dst_ptr, int ic, int ih, | |||
| int iw, int ld_dst_oc, | |||
| const Op& op) { | |||
| constexpr int fh = filter_size; | |||
| constexpr int fw = filter_size; | |||
| constexpr int ic_step = 4; | |||
| constexpr int oc_step = 4; | |||
| constexpr int loop_ic_step = 4; | |||
| constexpr int ld_weight_ic4 = 16; | |||
| constexpr int pack_iw_len = 4; | |||
| const int ic_stride = ih * iw * pack_iw_len; | |||
| const int ld_weight_oc4 = oc_step * fh * fw * ic; | |||
| int32x4_t c[2][8]; | |||
| int8x16_t weight[2][2]; | |||
| int8x16_t src[8 + 1]; | |||
| int16x8_t temp_c[4]; | |||
| init_oc8_ow8<bias_mode>(c, bias_ptr, oc_step); | |||
| for (int ic_idx = 0; ic_idx < ic; ic_idx += loop_ic_step) { | |||
| for (int fh_idx = 0; fh_idx < fh; ++fh_idx) { | |||
| const int8_t* src_ic_0_3 = src_ptr + ic_idx * ic_stride + | |||
| fh_idx * iw * ic_step * pack_iw_len; | |||
| src[0] = vld1q_s8(src_ic_0_3); | |||
| src[1] = vld1q_s8(src_ic_0_3 + 16); | |||
| src[2] = vld1q_s8(src_ic_0_3 + 2 * 16); | |||
| src[3] = vld1q_s8(src_ic_0_3 + 3 * 16); | |||
| src[4] = vld1q_s8(src_ic_0_3 + 4 * 16); | |||
| src[5] = vld1q_s8(src_ic_0_3 + 5 * 16); | |||
| src[6] = vld1q_s8(src_ic_0_3 + 6 * 16); | |||
| src[7] = vld1q_s8(src_ic_0_3 + 7 * 16); | |||
| src[8] = vld1q_s8(src_ic_0_3 + 8 * 16); | |||
| // oc == 0 | |||
| const int8_t* read_weight_ptr = | |||
| weight_ptr + fh_idx * fw * ld_weight_ic4; | |||
| weight[0][0] = vld1q_s8(read_weight_ptr); | |||
| weight[0][1] = vld1q_s8(read_weight_ptr + 16); | |||
| weight[1][0] = vld1q_s8(read_weight_ptr + ld_weight_oc4); | |||
| weight[1][1] = vld1q_s8(read_weight_ptr + ld_weight_oc4 + 16); | |||
| c[0][0] = vdotq_s32_h(weight[0][0], src[0], c[0][0], temp_c[0]); | |||
| c[1][0] = vdotq_s32_h(weight[1][0], src[0], c[1][0], temp_c[1]); | |||
| c[0][1] = vdotq_s32_h(weight[0][0], src[2], c[0][1], temp_c[2]); | |||
| c[1][1] = vdotq_s32_h(weight[1][0], src[2], c[1][1], temp_c[3]); | |||
| c[0][0] = vdotq_s32_h(weight[0][1], src[1], c[0][0], temp_c[0]); | |||
| c[1][0] = vdotq_s32_h(weight[1][1], src[1], c[1][0], temp_c[1]); | |||
| c[0][1] = vdotq_s32_h(weight[0][1], src[3], c[0][1], temp_c[2]); | |||
| c[1][1] = vdotq_s32_h(weight[1][1], src[3], c[1][1], temp_c[3]); | |||
| c[0][2] = vdotq_s32_h(weight[0][0], src[4], c[0][2], temp_c[0]); | |||
| c[1][2] = vdotq_s32_h(weight[1][0], src[4], c[1][2], temp_c[1]); | |||
| c[0][3] = vdotq_s32_h(weight[0][0], src[6], c[0][3], temp_c[2]); | |||
| c[1][3] = vdotq_s32_h(weight[1][0], src[6], c[1][3], temp_c[3]); | |||
| c[0][2] = vdotq_s32_h(weight[0][1], src[5], c[0][2], temp_c[0]); | |||
| c[1][2] = vdotq_s32_h(weight[1][1], src[5], c[1][2], temp_c[1]); | |||
| c[0][3] = vdotq_s32_h(weight[0][1], src[7], c[0][3], temp_c[2]); | |||
| c[1][3] = vdotq_s32_h(weight[1][1], src[7], c[1][3], temp_c[3]); | |||
| src[0] = vld1q_s8(src_ic_0_3 + 9 * 16); | |||
| src[1] = vld1q_s8(src_ic_0_3 + 10 * 16); | |||
| src[2] = vld1q_s8(src_ic_0_3 + 11 * 16); | |||
| c[0][4] = vdotq_s32_h(weight[0][0], src[8], c[0][4], temp_c[0]); | |||
| c[1][4] = vdotq_s32_h(weight[1][0], src[8], c[1][4], temp_c[1]); | |||
| c[0][5] = vdotq_s32_h(weight[0][0], src[1], c[0][5], temp_c[2]); | |||
| c[1][5] = vdotq_s32_h(weight[1][0], src[1], c[1][5], temp_c[3]); | |||
| c[0][4] = vdotq_s32_h(weight[0][1], src[0], c[0][4], temp_c[0]); | |||
| c[1][4] = vdotq_s32_h(weight[1][1], src[0], c[1][4], temp_c[1]); | |||
| c[0][5] = vdotq_s32_h(weight[0][1], src[2], c[0][5], temp_c[2]); | |||
| c[1][5] = vdotq_s32_h(weight[1][1], src[2], c[1][5], temp_c[3]); | |||
| src[3] = vld1q_s8(src_ic_0_3 + 12 * 16); | |||
| src[4] = vld1q_s8(src_ic_0_3 + 13 * 16); | |||
| src[5] = vld1q_s8(src_ic_0_3 + 14 * 16); | |||
| src[6] = vld1q_s8(src_ic_0_3 + 15 * 16); | |||
| c[0][6] = vdotq_s32_h(weight[0][0], src[3], c[0][6], temp_c[0]); | |||
| c[1][6] = vdotq_s32_h(weight[1][0], src[3], c[1][6], temp_c[1]); | |||
| c[0][7] = vdotq_s32_h(weight[0][0], src[5], c[0][7], temp_c[2]); | |||
| c[1][7] = vdotq_s32_h(weight[1][0], src[5], c[1][7], temp_c[3]); | |||
| c[0][6] = vdotq_s32_h(weight[0][1], src[4], c[0][6], temp_c[0]); | |||
| c[1][6] = vdotq_s32_h(weight[1][1], src[4], c[1][6], temp_c[1]); | |||
| c[0][7] = vdotq_s32_h(weight[0][1], src[6], c[0][7], temp_c[2]); | |||
| c[1][7] = vdotq_s32_h(weight[1][1], src[6], c[1][7], temp_c[3]); | |||
| } | |||
| weight_ptr += fh * fw * ld_weight_ic4; | |||
| } | |||
| store_oc8_ow8_remain_static<remain_w>(c, op, dst_ptr, ld_dst_oc); | |||
| } | |||
| template <BiasMode bias_mode, typename Op, int remain_w, int filter_size> | |||
| static void ker_neon_dirctconv_2x2s2_oc4_ow8(const int8_t* src_ptr, | |||
| const int8_t* weight_ptr, | |||
| const int32_t* bias_ptr, | |||
| int8_t* dst_ptr, int ic, int ih, | |||
| int iw, const Op& op) { | |||
| constexpr int fh = filter_size; | |||
| constexpr int fw = filter_size; | |||
| constexpr int ic_step = 4; | |||
| constexpr int loop_ic_step = 4; | |||
| constexpr int ld_weight_ic4 = 16; | |||
| constexpr int pack_iw_len = 4; | |||
| const int ic_stride = ih * iw * pack_iw_len; | |||
| int32x4_t c[2 * 4]; | |||
| int8x16_t weight[2]; | |||
| int8x16_t src[8 + 1]; | |||
| int16x8_t temp_c[2]; | |||
| init_oc4_ow8<bias_mode>(c, bias_ptr); | |||
| for (int ic_idx = 0; ic_idx < ic; ic_idx += loop_ic_step) { | |||
| for (int fh_idx = 0; fh_idx < fh; ++fh_idx) { | |||
| const int8_t* src_ic_0_3 = src_ptr + ic_idx * ic_stride + | |||
| fh_idx * iw * ic_step * pack_iw_len; | |||
| src[0] = vld1q_s8(src_ic_0_3); | |||
| src[1] = vld1q_s8((src_ic_0_3 + 16)); | |||
| src[2] = vld1q_s8((src_ic_0_3 + 2 * 16)); | |||
| src[3] = vld1q_s8((src_ic_0_3 + 3 * 16)); | |||
| src[4] = vld1q_s8((src_ic_0_3 + 4 * 16)); | |||
| src[5] = vld1q_s8((src_ic_0_3 + 5 * 16)); | |||
| src[6] = vld1q_s8((src_ic_0_3 + 6 * 16)); | |||
| src[7] = vld1q_s8((src_ic_0_3 + 7 * 16)); | |||
| src[8] = vld1q_s8((src_ic_0_3 + 8 * 16)); | |||
| // oc == 0 | |||
| const int8_t* read_weight_ptr = | |||
| weight_ptr + fh_idx * fw * ld_weight_ic4; | |||
| weight[0] = vld1q_s8(read_weight_ptr); | |||
| weight[1] = vld1q_s8(read_weight_ptr + 16); | |||
| c[0] = vdotq_s32_h(weight[0], src[0], c[0], temp_c[0]); | |||
| c[1] = vdotq_s32_h(weight[0], src[2], c[1], temp_c[1]); | |||
| c[0] = vdotq_s32_h(weight[1], src[1], c[0], temp_c[0]); | |||
| c[1] = vdotq_s32_h(weight[1], src[3], c[1], temp_c[1]); | |||
| c[2] = vdotq_s32_h(weight[0], src[4], c[2], temp_c[0]); | |||
| c[3] = vdotq_s32_h(weight[0], src[6], c[3], temp_c[1]); | |||
| c[2] = vdotq_s32_h(weight[1], src[5], c[2], temp_c[0]); | |||
| c[3] = vdotq_s32_h(weight[1], src[7], c[3], temp_c[1]); | |||
| src[0] = vld1q_s8(src_ic_0_3 + 9 * 16); | |||
| src[1] = vld1q_s8(src_ic_0_3 + 10 * 16); | |||
| src[2] = vld1q_s8(src_ic_0_3 + 11 * 16); | |||
| c[4] = vdotq_s32_h(weight[0], src[8], c[4], temp_c[0]); | |||
| c[5] = vdotq_s32_h(weight[0], src[1], c[5], temp_c[1]); | |||
| c[4] = vdotq_s32_h(weight[1], src[0], c[4], temp_c[0]); | |||
| c[5] = vdotq_s32_h(weight[1], src[2], c[5], temp_c[1]); | |||
| src[3] = vld1q_s8(src_ic_0_3 + 12 * 16); | |||
| src[4] = vld1q_s8(src_ic_0_3 + 13 * 16); | |||
| src[5] = vld1q_s8(src_ic_0_3 + 14 * 16); | |||
| src[6] = vld1q_s8(src_ic_0_3 + 15 * 16); | |||
| c[6] = vdotq_s32_h(weight[0], src[3], c[6], temp_c[0]); | |||
| c[7] = vdotq_s32_h(weight[0], src[5], c[7], temp_c[1]); | |||
| c[6] = vdotq_s32_h(weight[1], src[4], c[6], temp_c[0]); | |||
| c[7] = vdotq_s32_h(weight[1], src[6], c[7], temp_c[1]); | |||
| } | |||
| weight_ptr += fh * fw * ld_weight_ic4; | |||
| } | |||
| store_oc4_ow8_remain_static<remain_w, Op>(c, op, dst_ptr); | |||
| } | |||
| template <BiasMode bias_mode, typename Op, int remain_w, int filter_size> | |||
| static void ker_neon_dirctconv_5x5s2_oc4_ow8(const int8_t* src_ptr, | |||
| const int8_t* weight_ptr, | |||
| const int32_t* bias_ptr, | |||
| int8_t* dst_ptr, int ic, int ih, | |||
| int iw, const Op& op) { | |||
| constexpr int fh = filter_size; | |||
| constexpr int fw = filter_size; | |||
| constexpr int ic_step = 4; | |||
| constexpr int loop_ic_step = 4; | |||
| constexpr int ld_weight_ic4 = 16; | |||
| constexpr int pack_iw_len = 4; | |||
| const int ic_stride = ih * iw * pack_iw_len; | |||
| int32x4_t c[2 * 4]; | |||
| int8x16_t weight[5]; | |||
| int8x16_t src[8 + 2]; | |||
| int16x8_t temp_c[2]; | |||
| init_oc4_ow8<bias_mode>(c, bias_ptr); | |||
| for (int ic_idx = 0; ic_idx < ic; ic_idx += loop_ic_step) { | |||
| for (int fh_idx = 0; fh_idx < fh; ++fh_idx) { | |||
| const int8_t* src_ic_0_3 = src_ptr + ic_idx * ic_stride + | |||
| fh_idx * iw * ic_step * pack_iw_len; | |||
| src[0] = vld1q_s8(src_ic_0_3); | |||
| src[1] = vld1q_s8((src_ic_0_3 + 16)); | |||
| src[2] = vld1q_s8((src_ic_0_3 + 2 * 16)); | |||
| src[3] = vld1q_s8((src_ic_0_3 + 3 * 16)); | |||
| src[4] = vld1q_s8((src_ic_0_3 + 4 * 16)); | |||
| src[5] = vld1q_s8((src_ic_0_3 + 5 * 16)); | |||
| src[6] = vld1q_s8((src_ic_0_3 + 6 * 16)); | |||
| src[7] = vld1q_s8((src_ic_0_3 + 7 * 16)); | |||
| src[8] = vld1q_s8((src_ic_0_3 + 8 * 16)); | |||
| src[9] = vld1q_s8((src_ic_0_3 + 9 * 16)); | |||
| // oc == 0 | |||
| const int8_t* read_weight_ptr = | |||
| weight_ptr + fh_idx * fw * ld_weight_ic4; | |||
| weight[0] = vld1q_s8(read_weight_ptr); | |||
| weight[1] = vld1q_s8(read_weight_ptr + 16); | |||
| weight[2] = vld1q_s8(read_weight_ptr + 2 * 16); | |||
| weight[3] = vld1q_s8(read_weight_ptr + 3 * 16); | |||
| weight[4] = vld1q_s8(read_weight_ptr + 4 * 16); | |||
| c[0] = vdotq_s32_h(weight[0], src[0], c[0], temp_c[0]); | |||
| c[1] = vdotq_s32_h(weight[0], src[2], c[1], temp_c[1]); | |||
| c[0] = vdotq_s32_h(weight[1], src[1], c[0], temp_c[0]); | |||
| c[1] = vdotq_s32_h(weight[1], src[3], c[1], temp_c[1]); | |||
| c[0] = vdotq_s32_h(weight[2], src[2], c[0], temp_c[0]); | |||
| c[1] = vdotq_s32_h(weight[2], src[4], c[1], temp_c[1]); | |||
| c[0] = vdotq_s32_h(weight[3], src[3], c[0], temp_c[0]); | |||
| c[1] = vdotq_s32_h(weight[3], src[5], c[1], temp_c[1]); | |||
| c[0] = vdotq_s32_h(weight[4], src[4], c[0], temp_c[0]); | |||
| c[1] = vdotq_s32_h(weight[4], src[6], c[1], temp_c[1]); | |||
| src[0] = vld1q_s8(src_ic_0_3 + 10 * 16); | |||
| c[2] = vdotq_s32_h(weight[0], src[4], c[2], temp_c[0]); | |||
| c[3] = vdotq_s32_h(weight[0], src[6], c[3], temp_c[1]); | |||
| c[2] = vdotq_s32_h(weight[1], src[5], c[2], temp_c[0]); | |||
| c[3] = vdotq_s32_h(weight[1], src[7], c[3], temp_c[1]); | |||
| c[2] = vdotq_s32_h(weight[2], src[6], c[2], temp_c[0]); | |||
| c[3] = vdotq_s32_h(weight[2], src[8], c[3], temp_c[1]); | |||
| c[2] = vdotq_s32_h(weight[3], src[7], c[2], temp_c[0]); | |||
| c[3] = vdotq_s32_h(weight[3], src[9], c[3], temp_c[1]); | |||
| c[2] = vdotq_s32_h(weight[4], src[8], c[2], temp_c[0]); | |||
| c[3] = vdotq_s32_h(weight[4], src[0], c[3], temp_c[1]); | |||
| src[1] = vld1q_s8((src_ic_0_3 + 11 * 16)); | |||
| src[2] = vld1q_s8((src_ic_0_3 + 12 * 16)); | |||
| src[3] = vld1q_s8((src_ic_0_3 + 13 * 16)); | |||
| src[4] = vld1q_s8((src_ic_0_3 + 14 * 16)); | |||
| c[4] = vdotq_s32_h(weight[0], src[8], c[4], temp_c[0]); | |||
| c[5] = vdotq_s32_h(weight[0], src[0], c[5], temp_c[1]); | |||
| c[4] = vdotq_s32_h(weight[1], src[9], c[4], temp_c[0]); | |||
| c[5] = vdotq_s32_h(weight[1], src[1], c[5], temp_c[1]); | |||
| c[4] = vdotq_s32_h(weight[2], src[0], c[4], temp_c[0]); | |||
| c[5] = vdotq_s32_h(weight[2], src[2], c[5], temp_c[1]); | |||
| c[4] = vdotq_s32_h(weight[3], src[1], c[4], temp_c[0]); | |||
| c[5] = vdotq_s32_h(weight[3], src[3], c[5], temp_c[1]); | |||
| c[4] = vdotq_s32_h(weight[4], src[2], c[4], temp_c[0]); | |||
| c[5] = vdotq_s32_h(weight[4], src[4], c[5], temp_c[1]); | |||
| src[5] = vld1q_s8((src_ic_0_3 + 15 * 16)); | |||
| src[6] = vld1q_s8((src_ic_0_3 + 16 * 16)); | |||
| src[7] = vld1q_s8((src_ic_0_3 + 17 * 16)); | |||
| src[8] = vld1q_s8((src_ic_0_3 + 18 * 16)); | |||
| c[6] = vdotq_s32_h(weight[0], src[2], c[6], temp_c[0]); | |||
| c[7] = vdotq_s32_h(weight[0], src[4], c[7], temp_c[1]); | |||
| c[6] = vdotq_s32_h(weight[1], src[3], c[6], temp_c[0]); | |||
| c[7] = vdotq_s32_h(weight[1], src[5], c[7], temp_c[1]); | |||
| c[6] = vdotq_s32_h(weight[2], src[4], c[6], temp_c[0]); | |||
| c[7] = vdotq_s32_h(weight[2], src[6], c[7], temp_c[1]); | |||
| c[6] = vdotq_s32_h(weight[3], src[5], c[6], temp_c[0]); | |||
| c[7] = vdotq_s32_h(weight[3], src[7], c[7], temp_c[1]); | |||
| c[6] = vdotq_s32_h(weight[4], src[6], c[6], temp_c[0]); | |||
| c[7] = vdotq_s32_h(weight[4], src[8], c[7], temp_c[1]); | |||
| } | |||
| weight_ptr += fh * fw * ld_weight_ic4; | |||
| } | |||
| store_oc4_ow8_remain_static<remain_w, Op>(c, op, dst_ptr); | |||
| } | |||
| template <BiasMode bias_mode, typename Op, int remain_w, int filter_size> | |||
| static void ker_neon_dirctconv_7x7s2_oc4_ow8(const int8_t* src_ptr, | |||
| const int8_t* weight_ptr, | |||
| const int32_t* bias_ptr, | |||
| int8_t* dst_ptr, int ic, int ih, | |||
| int iw, const Op& op) { | |||
| constexpr int fh = filter_size; | |||
| constexpr int fw = filter_size; | |||
| constexpr int ic_step = 4; | |||
| constexpr int loop_ic_step = 4; | |||
| constexpr int ld_weight_ic4 = 16; | |||
| constexpr int pack_iw_len = 4; | |||
| const int ic_stride = ih * iw * pack_iw_len; | |||
| int32x4_t c[2 * 4]; | |||
| int8x16_t weight[7]; | |||
| int8x16_t src[8 + 2]; | |||
| int16x8_t temp_c[2]; | |||
| init_oc4_ow8<bias_mode>(c, bias_ptr); | |||
| for (int ic_idx = 0; ic_idx < ic; ic_idx += loop_ic_step) { | |||
| for (int fh_idx = 0; fh_idx < fh; ++fh_idx) { | |||
| const int8_t* src_ic_0_3 = src_ptr + ic_idx * ic_stride + | |||
| fh_idx * iw * ic_step * pack_iw_len; | |||
| src[0] = vld1q_s8(src_ic_0_3); | |||
| src[1] = vld1q_s8(src_ic_0_3 + 1 * 16); | |||
| src[2] = vld1q_s8(src_ic_0_3 + 2 * 16); | |||
| src[3] = vld1q_s8(src_ic_0_3 + 3 * 16); | |||
| src[4] = vld1q_s8(src_ic_0_3 + 4 * 16); | |||
| src[5] = vld1q_s8(src_ic_0_3 + 5 * 16); | |||
| src[6] = vld1q_s8(src_ic_0_3 + 6 * 16); | |||
| src[7] = vld1q_s8(src_ic_0_3 + 7 * 16); | |||
| src[8] = vld1q_s8(src_ic_0_3 + 8 * 16); | |||
| src[9] = vld1q_s8(src_ic_0_3 + 9 * 16); | |||
| // oc == 0 | |||
| const int8_t* read_weight_ptr = | |||
| weight_ptr + fh_idx * fw * ld_weight_ic4; | |||
| weight[0] = vld1q_s8(read_weight_ptr); | |||
| weight[1] = vld1q_s8(read_weight_ptr + 16); | |||
| weight[2] = vld1q_s8(read_weight_ptr + 2 * 16); | |||
| weight[3] = vld1q_s8(read_weight_ptr + 3 * 16); | |||
| weight[4] = vld1q_s8(read_weight_ptr + 4 * 16); | |||
| weight[5] = vld1q_s8(read_weight_ptr + 5 * 16); | |||
| weight[6] = vld1q_s8(read_weight_ptr + 6 * 16); | |||
| c[0] = vdotq_s32_h(weight[0], src[0], c[0], temp_c[0]); | |||
| c[1] = vdotq_s32_h(weight[0], src[2], c[1], temp_c[1]); | |||
| c[0] = vdotq_s32_h(weight[1], src[1], c[0], temp_c[0]); | |||
| c[1] = vdotq_s32_h(weight[1], src[3], c[1], temp_c[1]); | |||
| c[0] = vdotq_s32_h(weight[2], src[2], c[0], temp_c[0]); | |||
| c[1] = vdotq_s32_h(weight[2], src[4], c[1], temp_c[1]); | |||
| c[0] = vdotq_s32_h(weight[3], src[3], c[0], temp_c[0]); | |||
| c[1] = vdotq_s32_h(weight[3], src[5], c[1], temp_c[1]); | |||
| c[0] = vdotq_s32_h(weight[4], src[4], c[0], temp_c[0]); | |||
| c[1] = vdotq_s32_h(weight[4], src[6], c[1], temp_c[1]); | |||
| c[0] = vdotq_s32_h(weight[5], src[5], c[0], temp_c[0]); | |||
| c[1] = vdotq_s32_h(weight[5], src[7], c[1], temp_c[1]); | |||
| c[0] = vdotq_s32_h(weight[6], src[6], c[0], temp_c[0]); | |||
| c[1] = vdotq_s32_h(weight[6], src[8], c[1], temp_c[1]); | |||
| src[0] = vld1q_s8(src_ic_0_3 + 10 * 16); | |||
| src[1] = vld1q_s8(src_ic_0_3 + 11 * 16); | |||
| src[2] = vld1q_s8(src_ic_0_3 + 12 * 16); | |||
| c[2] = vdotq_s32_h(weight[0], src[4], c[2], temp_c[0]); | |||
| c[3] = vdotq_s32_h(weight[0], src[6], c[3], temp_c[1]); | |||
| c[2] = vdotq_s32_h(weight[1], src[5], c[2], temp_c[0]); | |||
| c[3] = vdotq_s32_h(weight[1], src[7], c[3], temp_c[1]); | |||
| c[2] = vdotq_s32_h(weight[2], src[6], c[2], temp_c[0]); | |||
| c[3] = vdotq_s32_h(weight[2], src[8], c[3], temp_c[1]); | |||
| c[2] = vdotq_s32_h(weight[3], src[7], c[2], temp_c[0]); | |||
| c[3] = vdotq_s32_h(weight[3], src[9], c[3], temp_c[1]); | |||
| c[2] = vdotq_s32_h(weight[4], src[8], c[2], temp_c[0]); | |||
| c[3] = vdotq_s32_h(weight[4], src[0], c[3], temp_c[1]); | |||
| c[2] = vdotq_s32_h(weight[5], src[9], c[2], temp_c[0]); | |||
| c[3] = vdotq_s32_h(weight[5], src[1], c[3], temp_c[1]); | |||
| c[2] = vdotq_s32_h(weight[6], src[0], c[2], temp_c[0]); | |||
| c[3] = vdotq_s32_h(weight[6], src[2], c[3], temp_c[1]); | |||
| src[3] = vld1q_s8(src_ic_0_3 + 13 * 16); | |||
| src[4] = vld1q_s8(src_ic_0_3 + 14 * 16); | |||
| src[5] = vld1q_s8(src_ic_0_3 + 15 * 16); | |||
| src[6] = vld1q_s8(src_ic_0_3 + 16 * 16); | |||
| c[4] = vdotq_s32_h(weight[0], src[8], c[4], temp_c[0]); | |||
| c[5] = vdotq_s32_h(weight[0], src[0], c[5], temp_c[1]); | |||
| c[4] = vdotq_s32_h(weight[1], src[9], c[4], temp_c[0]); | |||
| c[5] = vdotq_s32_h(weight[1], src[1], c[5], temp_c[1]); | |||
| c[4] = vdotq_s32_h(weight[2], src[0], c[4], temp_c[0]); | |||
| c[5] = vdotq_s32_h(weight[2], src[2], c[5], temp_c[1]); | |||
| c[4] = vdotq_s32_h(weight[3], src[1], c[4], temp_c[0]); | |||
| c[5] = vdotq_s32_h(weight[3], src[3], c[5], temp_c[1]); | |||
| c[4] = vdotq_s32_h(weight[4], src[2], c[4], temp_c[0]); | |||
| c[5] = vdotq_s32_h(weight[4], src[4], c[5], temp_c[1]); | |||
| c[4] = vdotq_s32_h(weight[5], src[3], c[4], temp_c[0]); | |||
| c[5] = vdotq_s32_h(weight[5], src[5], c[5], temp_c[1]); | |||
| c[4] = vdotq_s32_h(weight[6], src[4], c[4], temp_c[0]); | |||
| c[5] = vdotq_s32_h(weight[6], src[6], c[5], temp_c[1]); | |||
| src[7] = vld1q_s8(src_ic_0_3 + 17 * 16); | |||
| src[8] = vld1q_s8(src_ic_0_3 + 18 * 16); | |||
| src[9] = vld1q_s8(src_ic_0_3 + 19 * 16); | |||
| src[0] = vld1q_s8(src_ic_0_3 + 20 * 16); | |||
| c[6] = vdotq_s32_h(weight[0], src[2], c[6], temp_c[0]); | |||
| c[7] = vdotq_s32_h(weight[0], src[4], c[7], temp_c[1]); | |||
| c[6] = vdotq_s32_h(weight[1], src[3], c[6], temp_c[0]); | |||
| c[7] = vdotq_s32_h(weight[1], src[5], c[7], temp_c[1]); | |||
| c[6] = vdotq_s32_h(weight[2], src[4], c[6], temp_c[0]); | |||
| c[7] = vdotq_s32_h(weight[2], src[6], c[7], temp_c[1]); | |||
| c[6] = vdotq_s32_h(weight[3], src[5], c[6], temp_c[0]); | |||
| c[7] = vdotq_s32_h(weight[3], src[7], c[7], temp_c[1]); | |||
| c[6] = vdotq_s32_h(weight[4], src[6], c[6], temp_c[0]); | |||
| c[7] = vdotq_s32_h(weight[4], src[8], c[7], temp_c[1]); | |||
| c[6] = vdotq_s32_h(weight[5], src[7], c[6], temp_c[0]); | |||
| c[7] = vdotq_s32_h(weight[5], src[9], c[7], temp_c[1]); | |||
| c[6] = vdotq_s32_h(weight[6], src[8], c[6], temp_c[0]); | |||
| c[7] = vdotq_s32_h(weight[6], src[0], c[7], temp_c[1]); | |||
| } | |||
| weight_ptr += fh * fw * ld_weight_ic4; | |||
| } | |||
| store_oc4_ow8_remain_static<remain_w, Op>(c, op, dst_ptr); | |||
| } | |||
| } // namespace | |||
| template <BiasMode bias_mode, typename Op, int remain_w> | |||
| void conv_bias::conv_direct_stride2_2x2_int8_nchw44( | |||
| const int8_t* src, const int8_t* filter, const int32_t* bias, | |||
| int32_t* temp, int8_t* dst, const size_t oc, const size_t ic, | |||
| const size_t ih, const size_t iw, const size_t oh, const size_t ow, | |||
| const Op& op) { | |||
| MEGDNN_MARK_USED_VAR(temp); | |||
| constexpr size_t filter_size = 2; | |||
| constexpr size_t fh = filter_size; | |||
| constexpr size_t fw = filter_size; | |||
| constexpr size_t ic_step = 4; | |||
| constexpr size_t oc_step = 4; | |||
| constexpr size_t big_oc_step = 8; | |||
| constexpr size_t oh_step = 1; | |||
| constexpr size_t ow_step = 8; | |||
| constexpr size_t stride_h = 2; | |||
| constexpr size_t stride_w = 2; | |||
| constexpr int pack_iw_len = 4; | |||
| const size_t out_img_stride = oh * ow; | |||
| const size_t ow_end = ow / ow_step * ow_step; | |||
| const size_t ow_remain = ow - ow_end; | |||
| const size_t oc_end = oc / big_oc_step * big_oc_step; | |||
| const size_t oc_remain = oc - oc_end; | |||
| const int ld_oc = oh * ow * ic_step; | |||
| for (size_t oc_idx = 0; oc_idx < oc_end; oc_idx += big_oc_step) { | |||
| const size_t weight_offset = oc_idx * ic * fh * fw; | |||
| for (size_t oh_idx = 0; oh_idx < oh; oh_idx += oh_step) { | |||
| for (size_t ow_idx = 0; ow_idx < ow_end; ow_idx += ow_step) { | |||
| const size_t src_offset = | |||
| (oh_idx * stride_h * iw + ow_idx * stride_w) * ic_step * | |||
| pack_iw_len; | |||
| const size_t dst_offset = oc_idx * out_img_stride + | |||
| (oh_idx * ow + ow_idx) * oc_step; | |||
| ker_neon_dirctconv_2x2s2_oc8_ow8<bias_mode, Op, 0, filter_size>( | |||
| src + src_offset, filter + weight_offset, bias + oc_idx, | |||
| dst + dst_offset, ic, ih, iw, ld_oc, op); | |||
| } | |||
| if (ow_remain > 0) { | |||
| const size_t src_offset = | |||
| (oh_idx * stride_h * iw + ow_end * stride_w) * ic_step * | |||
| pack_iw_len; | |||
| const size_t dst_offset = oc_idx * out_img_stride + | |||
| (oh_idx * ow + ow_end) * oc_step; | |||
| ker_neon_dirctconv_2x2s2_oc8_ow8<bias_mode, Op, remain_w, | |||
| filter_size>( | |||
| src + src_offset, filter + weight_offset, bias + oc_idx, | |||
| dst + dst_offset, ic, ih, iw, ld_oc, op); | |||
| } | |||
| } | |||
| } | |||
| if (oc_remain > 0) { | |||
| const size_t oc_idx = oc_end; | |||
| const size_t weight_offset = oc_idx * ic * fh * fw; | |||
| for (size_t oh_idx = 0; oh_idx < oh; oh_idx += oh_step) { | |||
| for (size_t ow_idx = 0; ow_idx < ow_end; ow_idx += ow_step) { | |||
| const size_t src_offset = | |||
| (oh_idx * stride_h * iw + ow_idx * stride_w) * ic_step * | |||
| pack_iw_len; | |||
| const size_t dst_offset = oc_idx * out_img_stride + | |||
| (oh_idx * ow + ow_idx) * oc_step; | |||
| ker_neon_dirctconv_2x2s2_oc4_ow8<bias_mode, Op, 0, filter_size>( | |||
| src + src_offset, filter + weight_offset, bias + oc_idx, | |||
| dst + dst_offset, ic, ih, iw, op); | |||
| } | |||
| if (ow_remain > 0) { | |||
| const size_t src_offset = | |||
| (oh_idx * stride_h * iw + ow_end * stride_w) * ic_step * | |||
| pack_iw_len; | |||
| const size_t dst_offset = oc_idx * out_img_stride + | |||
| (oh_idx * ow + ow_end) * oc_step; | |||
| ker_neon_dirctconv_2x2s2_oc4_ow8<bias_mode, Op, remain_w, | |||
| filter_size>( | |||
| src + src_offset, filter + weight_offset, bias + oc_idx, | |||
| dst + dst_offset, ic, ih, iw, op); | |||
| } | |||
| } | |||
| } | |||
| } | |||
| template <BiasMode bias_mode, typename Op, int remain_w> | |||
| void conv_bias::conv_direct_stride2_3x3_int8_nchw44( | |||
| const int8_t* src, const int8_t* filter, const int32_t* bias, | |||
| int32_t* temp, int8_t* dst, const size_t oc, const size_t ic, | |||
| const size_t ih, const size_t iw, const size_t oh, const size_t ow, | |||
| const Op& op) { | |||
| MEGDNN_MARK_USED_VAR(temp); | |||
| constexpr size_t filter_size = 3; | |||
| constexpr size_t fh = filter_size; | |||
| constexpr size_t fw = filter_size; | |||
| constexpr size_t ic_step = 4; | |||
| constexpr size_t oc_step = 4; | |||
| constexpr size_t oh_step = 1; | |||
| constexpr size_t ow_step = 8; | |||
| constexpr size_t stride_h = 2; | |||
| constexpr size_t stride_w = 2; | |||
| constexpr int pack_iw_len = 4; | |||
| const size_t img_stride = oh * ow; | |||
| const size_t ow_end = ow / ow_step * ow_step; | |||
| const size_t ow_remain = ow - ow_end; | |||
| for (size_t oc_idx = 0; oc_idx < oc; oc_idx += oc_step) { | |||
| const size_t weight_offset = oc_idx * ic * fh * fw; | |||
| for (size_t oh_idx = 0; oh_idx < oh; oh_idx += oh_step) { | |||
| for (size_t ow_idx = 0; ow_idx < ow_end; ow_idx += ow_step) { | |||
| const size_t src_offset = | |||
| (oh_idx * stride_h * iw + ow_idx * stride_w) * ic_step * | |||
| pack_iw_len; | |||
| const size_t dst_offset = | |||
| oc_idx * img_stride + (oh_idx * ow + ow_idx) * oc_step; | |||
| ker_neon_dirctconv_3x3s2_oc4_ow8<bias_mode, Op, 0, filter_size>( | |||
| src + src_offset, filter + weight_offset, bias + oc_idx, | |||
| dst + dst_offset, ic, ih, iw, op); | |||
| } | |||
| if (ow_remain > 0) { | |||
| const size_t src_offset = | |||
| (oh_idx * stride_h * iw + ow_end * stride_w) * ic_step * | |||
| pack_iw_len; | |||
| const size_t dst_offset = | |||
| oc_idx * img_stride + (oh_idx * ow + ow_end) * oc_step; | |||
| ker_neon_dirctconv_3x3s2_oc4_ow8<bias_mode, Op, remain_w, | |||
| filter_size>( | |||
| src + src_offset, filter + weight_offset, bias + oc_idx, | |||
| dst + dst_offset, ic, ih, iw, op); | |||
| } | |||
| } | |||
| } | |||
| } | |||
| template <BiasMode bias_mode, typename Op, int remain_w> | |||
| void conv_bias::conv_direct_stride2_5x5_int8_nchw44( | |||
| const int8_t* src, const int8_t* filter, const int32_t* bias, | |||
| int32_t* temp, int8_t* dst, const size_t oc, const size_t ic, | |||
| const size_t ih, const size_t iw, const size_t oh, const size_t ow, | |||
| const Op& op) { | |||
| MEGDNN_MARK_USED_VAR(temp); | |||
| constexpr size_t filter_size = 5; | |||
| constexpr size_t fh = filter_size; | |||
| constexpr size_t fw = filter_size; | |||
| constexpr size_t ic_step = 4; | |||
| constexpr size_t oc_step = 4; | |||
| constexpr size_t oh_step = 1; | |||
| constexpr size_t ow_step = 8; | |||
| constexpr size_t stride_h = 2; | |||
| constexpr size_t stride_w = 2; | |||
| constexpr int pack_iw_len = 4; | |||
| const size_t img_stride = oh * ow; | |||
| const size_t ow_end = ow / ow_step * ow_step; | |||
| const size_t ow_remain = ow - ow_end; | |||
| for (size_t oc_idx = 0; oc_idx < oc; oc_idx += oc_step) { | |||
| const size_t weight_offset = oc_idx * ic * fh * fw; | |||
| for (size_t oh_idx = 0; oh_idx < oh; oh_idx += oh_step) { | |||
| for (size_t ow_idx = 0; ow_idx < ow_end; ow_idx += ow_step) { | |||
| const size_t src_offset = | |||
| (oh_idx * stride_h * iw + ow_idx * stride_w) * ic_step * | |||
| pack_iw_len; | |||
| const size_t dst_offset = | |||
| oc_idx * img_stride + (oh_idx * ow + ow_idx) * oc_step; | |||
| ker_neon_dirctconv_5x5s2_oc4_ow8<bias_mode, Op, 0, filter_size>( | |||
| src + src_offset, filter + weight_offset, bias + oc_idx, | |||
| dst + dst_offset, ic, ih, iw, op); | |||
| } | |||
| if (ow_remain > 0) { | |||
| const size_t src_offset = | |||
| (oh_idx * stride_h * iw + ow_end * stride_w) * ic_step * | |||
| pack_iw_len; | |||
| const size_t dst_offset = | |||
| oc_idx * img_stride + (oh_idx * ow + ow_end) * oc_step; | |||
| ker_neon_dirctconv_5x5s2_oc4_ow8<bias_mode, Op, remain_w, | |||
| filter_size>( | |||
| src + src_offset, filter + weight_offset, bias + oc_idx, | |||
| dst + dst_offset, ic, ih, iw, op); | |||
| } | |||
| } | |||
| } | |||
| } | |||
| template <BiasMode bias_mode, typename Op, int remain_w> | |||
| void conv_bias::conv_direct_stride2_7x7_int8_nchw44( | |||
| const int8_t* src, const int8_t* filter, const int32_t* bias, | |||
| int32_t* temp, int8_t* dst, const size_t oc, const size_t ic, | |||
| const size_t ih, const size_t iw, const size_t oh, const size_t ow, | |||
| const Op& op) { | |||
| MEGDNN_MARK_USED_VAR(temp); | |||
| constexpr size_t filter_size = 7; | |||
| constexpr size_t fh = filter_size; | |||
| constexpr size_t fw = filter_size; | |||
| constexpr size_t ic_step = 4; | |||
| constexpr size_t oc_step = 4; | |||
| constexpr size_t oh_step = 1; | |||
| constexpr size_t ow_step = 8; | |||
| constexpr size_t stride_h = 2; | |||
| constexpr size_t stride_w = 2; | |||
| constexpr int pack_iw_len = 4; | |||
| const size_t img_stride = oh * ow; | |||
| const size_t ow_end = ow / ow_step * ow_step; | |||
| const size_t ow_remain = ow - ow_end; | |||
| for (size_t oc_idx = 0; oc_idx < oc; oc_idx += oc_step) { | |||
| const size_t weight_offset = oc_idx * ic * fh * fw; | |||
| for (size_t oh_idx = 0; oh_idx < oh; oh_idx += oh_step) { | |||
| for (size_t ow_idx = 0; ow_idx < ow_end; ow_idx += ow_step) { | |||
| const size_t src_offset = | |||
| (oh_idx * stride_h * iw + ow_idx * stride_w) * ic_step * | |||
| pack_iw_len; | |||
| const size_t dst_offset = | |||
| oc_idx * img_stride + (oh_idx * ow + ow_idx) * oc_step; | |||
| ker_neon_dirctconv_7x7s2_oc4_ow8<bias_mode, Op, 0, filter_size>( | |||
| src + src_offset, filter + weight_offset, bias + oc_idx, | |||
| dst + dst_offset, ic, ih, iw, op); | |||
| } | |||
| if (ow_remain > 0) { | |||
| const size_t src_offset = | |||
| (oh_idx * stride_h * iw + ow_end * stride_w) * ic_step * | |||
| pack_iw_len; | |||
| const size_t dst_offset = | |||
| oc_idx * img_stride + (oh_idx * ow + ow_end) * oc_step; | |||
| ker_neon_dirctconv_7x7s2_oc4_ow8<bias_mode, Op, remain_w, | |||
| filter_size>( | |||
| src + src_offset, filter + weight_offset, bias + oc_idx, | |||
| dst + dst_offset, ic, ih, iw, op); | |||
| } | |||
| } | |||
| } | |||
| } | |||
| #define INSTANTIATION(stride, i, bias, remain_w, Op) \ | |||
| template void conv_bias::conv_direct_##stride##_##i##x##i##_int8_nchw44< \ | |||
| bias, Op, remain_w>(const int8_t*, const int8_t*, const int32_t*, \ | |||
| int32_t*, int8_t*, const size_t, const size_t, \ | |||
| const size_t, const size_t, const size_t, \ | |||
| const size_t, const Op&); | |||
| #define FOR_OP(stride, i, bias, remain_w) \ | |||
| INSTANTIATION(stride, i, bias, remain_w, \ | |||
| TypeCvtOp<dt_qint32 MEGDNN_COMMA dt_qint8>) \ | |||
| INSTANTIATION(stride, i, bias, remain_w, \ | |||
| ReluOp<dt_qint32 MEGDNN_COMMA dt_qint8>) \ | |||
| INSTANTIATION(stride, i, bias, remain_w, \ | |||
| HSwishOp<dt_qint32 MEGDNN_COMMA dt_qint8>) | |||
| #define FOR_REMAIN(stride, i, bias) \ | |||
| FOR_OP(stride, i, bias, 0) \ | |||
| FOR_OP(stride, i, bias, 1) \ | |||
| FOR_OP(stride, i, bias, 2) \ | |||
| FOR_OP(stride, i, bias, 3) \ | |||
| FOR_OP(stride, i, bias, 4) \ | |||
| FOR_OP(stride, i, bias, 5) \ | |||
| FOR_OP(stride, i, bias, 6) \ | |||
| FOR_OP(stride, i, bias, 7) | |||
| #define FOR_BIAS(stride, i) \ | |||
| FOR_REMAIN(stride, i, BiasMode::NO_BIAS) \ | |||
| FOR_REMAIN(stride, i, BiasMode::BROADCAST_CHANNEL_BIAS) | |||
| #define FOR_FILTER(stride) \ | |||
| FOR_BIAS(stride, 2) \ | |||
| FOR_BIAS(stride, 3) \ | |||
| FOR_BIAS(stride, 5) \ | |||
| FOR_BIAS(stride, 7) | |||
| FOR_FILTER(stride2) | |||
| #undef FOR_STRIDE | |||
| #undef FOR_FILTER | |||
| #undef FOR_IC | |||
| #undef FOR_BIAS | |||
| #undef FOR_NONLINEAR | |||
| #undef FOR_REMAIN | |||
| #undef INSTANTIATION | |||
+ 2
- 4
dnn/src/arm_common/conv_bias/opr_impl.cpp
View File
| @@ -46,11 +46,10 @@ class ConvBiasImpl::AlgoPack : NonCopyableObj { | |||
| AlgoQU8DirectStride1 qu8_direct_stride1_small_group{false}; | |||
| AlgoS8DirectStride2 s8_direct_stride2_large_group{true}; | |||
| AlgoS8DirectStride2 s8_direct_stride2_small_group{false}; | |||
| AlgoS8DirectStride2NCHW44 s8_direct_stride2_nchw44; | |||
| AlgoS8DirectNCHW44 s8_direct_nchw44; | |||
| AlgoS8DirectNCHWNCHW44 s8_direct_nchw_nchw44; | |||
| AlgoS8DirectStride1 s8_direct_stride1_large_group{true}; | |||
| AlgoS8DirectStride1 s8_direct_stride1_small_group{false}; | |||
| AlgoS8DirectStride1NCHW44 s8_direct_stride1_nchw44; | |||
| AlgoS8ChanWiseStride1NCHW44 s8_channel_wise_stride1_nchw44; | |||
| AlgoS8ChanWiseStride2NCHW44 s8_channel_wise_stride2_nchw44; | |||
| @@ -114,11 +113,10 @@ public: | |||
| direct_algos.emplace_back(&qu8_direct_stride1_small_group); | |||
| direct_algos.emplace_back(&s8_direct_stride2_large_group); | |||
| direct_algos.emplace_back(&s8_direct_stride2_small_group); | |||
| direct_algos.emplace_back(&s8_direct_stride2_nchw44); | |||
| direct_algos.emplace_back(&s8_direct_nchw44); | |||
| direct_algos.emplace_back(&s8_direct_nchw_nchw44); | |||
| direct_algos.emplace_back(&s8_direct_stride1_large_group); | |||
| direct_algos.emplace_back(&s8_direct_stride1_small_group); | |||
| direct_algos.emplace_back(&s8_direct_stride1_nchw44); | |||
| direct_algos.emplace_back(&s8_channel_wise_stride1_nchw44); | |||
| direct_algos.emplace_back(&s8_channel_wise_stride2_nchw44); | |||
+ 1
- 2
dnn/src/arm_common/conv_bias/opr_impl.h
View File
| @@ -37,9 +37,8 @@ protected: | |||
| private: | |||
| class AlgoS8DirectStride1; | |||
| class AlgoS8DirectStride1NCHW44; | |||
| class AlgoS8DirectStride2; | |||
| class AlgoS8DirectStride2NCHW44; | |||
| class AlgoS8DirectNCHW44; | |||
| class AlgoS8DirectNCHWNCHW44; | |||
| class AlgoQU8DirectStride1; | |||
| class AlgoQU8DirectStride2; | |||
+ 2
- 0
dnn/src/arm_common/elemwise_helper/kimpl/none.h
View File
| @@ -27,6 +27,8 @@ struct NoneOp; | |||
| #define OP(_ctype, _neon_type, _neon_type2, _func_suffix, _simd_width) \ | |||
| template <> \ | |||
| struct NoneOp<_ctype> : NoneOpBase<_ctype> { \ | |||
| NoneOp(){}; \ | |||
| NoneOp(float, float){}; \ | |||
| using NoneOpBase::NoneOpBase; \ | |||
| using NoneOpBase::operator(); \ | |||
| constexpr static size_t SIMD_WIDTH = _simd_width; \ | |||
+ 9
- 1
dnn/test/arm_common/conv_bias.cpp
View File
| @@ -226,7 +226,15 @@ static void benchmark_convbias(Handle* handle, std::string int_name, | |||
| run(1, 3, 32, 224, 224, 5, 1, true); | |||
| run(1, 3, 64, 224, 224, 7, 1, true); | |||
| for (size_t stride : {1, 2}) { | |||
| run(1, 64, 128, 56, 56, 3, 2, false); | |||
| run(1, 128, 256, 28, 28, 3, 2, false); | |||
| run(1, 256, 512, 14, 14, 3, 2, false); | |||
| run(1, 128, 128, 28, 28, 3, 1, false); | |||
| run(1, 256, 256, 14, 14, 3, 1, false); | |||
| run(1, 512, 512, 7, 7, 3, 1, false); | |||
| for (size_t stride : {1}) { | |||
| printf("stride %zu\n", stride); | |||
| for (size_t filter_size : {2, 3, 5, 7}) { | |||
| for (size_t img_size : {32}) { | |||
+ 48
- 37
dnn/test/arm_common/conv_bias_multi_thread.cpp
View File
| @@ -527,12 +527,22 @@ TEST_F(ARM_COMMON_MULTI_THREADS, CONV_BIAS_INT8_STRIDE2_SMALL_GROUP) { | |||
| TEST_F(ARM_COMMON_MULTI_THREADS, CONV_BIAS_INT8_STRIDE1_NCHW44) { | |||
| checker_conv_bias_qint8x8x8( | |||
| get_nchw44_conv_bias_args({2, 3, 5, 7}, 1, false, false, false), | |||
| handle(), "S8_NCHW44_DIRECT_STRD1"); | |||
| handle(), "S8_NCHW44_DIRECT"); | |||
| } | |||
| TEST_F(ARM_COMMON_MULTI_THREADS, CONV_BIAS_INT8_STRIDE1_NCHW44_8832) { | |||
| checker_conv_bias_qint8x8x32( | |||
| get_nchw44_conv_bias_args({2, 3, 5, 7}, 1, false, false, true), | |||
| handle(), "S8_NCHW44_DIRECT"); | |||
| } | |||
| TEST_F(ARM_COMMON_MULTI_THREADS, CONV_BIAS_INT8_STRIDE2_NCHW44_8832) { | |||
| checker_conv_bias_qint8x8x32( | |||
| get_nchw44_conv_bias_args({2, 3, 5, 7}, 2, false, false, true), | |||
| handle(), "S8_NCHW44_DIRECT"); | |||
| } | |||
| TEST_F(ARM_COMMON_MULTI_THREADS, CONV_BIAS_INT8_STRIDE2_NCHW44) { | |||
| checker_conv_bias_qint8x8x8( | |||
| get_nchw44_conv_bias_args({2, 3, 5, 7}, 2, false, false, false), | |||
| handle(), "S8_NCHW44_DIRECT_STRD2"); | |||
| handle(), "S8_NCHW44_DIRECT"); | |||
| } | |||
| TEST_F(ARM_COMMON_MULTI_THREADS, CONV_BIAS_QS8_CHANNEL_WISE_DIRECT1_NCHW44) { | |||
| checker_conv_bias_qint8x8x8( | |||
| @@ -1085,7 +1095,6 @@ TEST_F(ARM_COMMON_MULTI_THREADS, CONV_BIAS_WINOGRAD_MK_PACKED_INT8) { | |||
| dtype::QuantizedS8(60.25f), param::MatrixMul::Format::MK8, 1e-3); | |||
| } | |||
| TEST_F(ARM_COMMON_MULTI_THREADS, CONV_BIAS_WINOGRAD_NCHW44_MK_PACKED_INT8) { | |||
| using namespace conv_bias; | |||
| @@ -1096,17 +1105,17 @@ TEST_F(ARM_COMMON_MULTI_THREADS, CONV_BIAS_WINOGRAD_NCHW44_MK_PACKED_INT8) { | |||
| param::MatrixMul::Format format, float eps) { | |||
| for (auto&& arg : args) { | |||
| for (uint32_t m : out_size) { | |||
| checker.set_extra_opr_impl(std::bind( | |||
| winograd_algo_extra_impl, std::placeholders::_1, m, | |||
| arg.param, handle, format)); | |||
| checker.set_dtype(0, A_dtype) | |||
| .set_dtype(1, B_dtype) | |||
| .set_dtype(2, C_dtype) | |||
| .set_dtype(4, D_dtype) | |||
| .set_epsilon(eps) | |||
| .set_param(arg.param) | |||
| .execs({arg.src, arg.filter, arg.bias, {}, {}}); | |||
| } | |||
| checker.set_extra_opr_impl(std::bind( | |||
| winograd_algo_extra_impl, std::placeholders::_1, m, | |||
| arg.param, handle, format)); | |||
| checker.set_dtype(0, A_dtype) | |||
| .set_dtype(1, B_dtype) | |||
| .set_dtype(2, C_dtype) | |||
| .set_dtype(4, D_dtype) | |||
| .set_epsilon(eps) | |||
| .set_param(arg.param) | |||
| .execs({arg.src, arg.filter, arg.bias, {}, {}}); | |||
| } | |||
| } | |||
| }; | |||
| @@ -1118,7 +1127,7 @@ TEST_F(ARM_COMMON_MULTI_THREADS, CONV_BIAS_WINOGRAD_NCHW44_MK_PACKED_INT8) { | |||
| checker.set_before_exec_callback(conv_bias::ConvBiasAlgoChecker<ConvBias>( | |||
| ssprintf("WINOGRAD_NCHW44:%s:8:2:32", matmul_name).c_str())); | |||
| std::vector<TestArg> quantized_args = get_int8_nchw44_args (3,4); | |||
| std::vector<TestArg> quantized_args = get_int8_nchw44_args(3, 4); | |||
| UniformIntRNG int_rng{-50, 50}; | |||
| checker.set_rng(0, &int_rng).set_rng(1, &int_rng).set_rng(2, &int_rng); | |||
| run(handle(), quantized_args, {2}, dtype::QuantizedS8(2.5f), | |||
| @@ -1126,8 +1135,8 @@ TEST_F(ARM_COMMON_MULTI_THREADS, CONV_BIAS_WINOGRAD_NCHW44_MK_PACKED_INT8) { | |||
| dtype::QuantizedS8(60.25f), param::MatrixMul::Format::MK8, 1e-3); | |||
| } | |||
| TEST_F(ARM_COMMON_MULTI_THREADS, CONV_BIAS_WINOGRAD_NCHW44_MK_PACKED_INT8_GROUPMODE) { | |||
| TEST_F(ARM_COMMON_MULTI_THREADS, | |||
| CONV_BIAS_WINOGRAD_NCHW44_MK_PACKED_INT8_GROUPMODE) { | |||
| using namespace conv_bias; | |||
| Checker<ConvBiasForward> checker(handle()); | |||
| @@ -1137,17 +1146,17 @@ TEST_F(ARM_COMMON_MULTI_THREADS, CONV_BIAS_WINOGRAD_NCHW44_MK_PACKED_INT8_GROUPM | |||
| param::MatrixMul::Format format, float eps) { | |||
| for (auto&& arg : args) { | |||
| for (uint32_t m : out_size) { | |||
| checker.set_extra_opr_impl(std::bind( | |||
| winograd_algo_extra_impl, std::placeholders::_1, m, | |||
| arg.param, handle, format)); | |||
| checker.set_dtype(0, A_dtype) | |||
| .set_dtype(1, B_dtype) | |||
| .set_dtype(2, C_dtype) | |||
| .set_dtype(4, D_dtype) | |||
| .set_epsilon(eps) | |||
| .set_param(arg.param) | |||
| .execs({arg.src, arg.filter, arg.bias, {}, {}}); | |||
| } | |||
| checker.set_extra_opr_impl(std::bind( | |||
| winograd_algo_extra_impl, std::placeholders::_1, m, | |||
| arg.param, handle, format)); | |||
| checker.set_dtype(0, A_dtype) | |||
| .set_dtype(1, B_dtype) | |||
| .set_dtype(2, C_dtype) | |||
| .set_dtype(4, D_dtype) | |||
| .set_epsilon(eps) | |||
| .set_param(arg.param) | |||
| .execs({arg.src, arg.filter, arg.bias, {}, {}}); | |||
| } | |||
| } | |||
| }; | |||
| @@ -1168,7 +1177,8 @@ TEST_F(ARM_COMMON_MULTI_THREADS, CONV_BIAS_WINOGRAD_NCHW44_MK_PACKED_INT8_GROUPM | |||
| dtype::QuantizedS8(60.25f), param::MatrixMul::Format::MK8, 1e-3); | |||
| } | |||
| TEST_F(ARM_COMMON_MULTI_THREADS, CONV_BIAS_WINOGRAD_NCHW44_MK_PACKED_INT8_COMP_F32) { | |||
| TEST_F(ARM_COMMON_MULTI_THREADS, | |||
| CONV_BIAS_WINOGRAD_NCHW44_MK_PACKED_INT8_COMP_F32) { | |||
| using namespace conv_bias; | |||
| Checker<ConvBiasForward> checker(handle()); | |||
| @@ -1196,21 +1206,22 @@ TEST_F(ARM_COMMON_MULTI_THREADS, CONV_BIAS_WINOGRAD_NCHW44_MK_PACKED_INT8_COMP_F | |||
| #if MEGDNN_AARCH64 | |||
| const char* matmul_name = "AARCH64_F32_MK4_4x16"; | |||
| #else | |||
| const char* matmul_name = "ARMV7_F32_MK4_4x8"; | |||
| const char* matmul_name = "ARMV7_F32_MK4_4x8"; | |||
| #endif | |||
| checker.set_before_exec_callback(conv_bias::ConvBiasAlgoChecker<ConvBias>( | |||
| ssprintf("WINOGRAD_NCHW44:%s:4:2:32", matmul_name).c_str())); | |||
| std::vector<TestArg> quantized_args = | |||
| get_int8_nchw44_args(3, 4, true); | |||
| std::vector<TestArg> quantized_args = get_int8_nchw44_args(3, 4, true); | |||
| UniformIntRNG int_rng{-50, 50}; | |||
| checker.set_rng(0, &int_rng).set_rng(1, &int_rng).set_rng(2, &int_rng); | |||
| run(handle(), quantized_args, {2}, dtype::QuantizedS8(0.41113496f), | |||
| dtype::QuantizedS8(0.01887994f), | |||
| dtype::QuantizedS32(0.41113496f * 0.01887994f), | |||
| dtype::QuantizedS8(0.49550694f), param::MatrixMul::Format::MK4, epsilon); | |||
| dtype::QuantizedS8(0.49550694f), param::MatrixMul::Format::MK4, | |||
| epsilon); | |||
| } | |||
| TEST_F(ARM_COMMON_MULTI_THREADS, CONV_BIAS_WINOGRAD_NCHW44_MK_PACKED_INT8_COMP_F32_GROUPMODE) { | |||
| TEST_F(ARM_COMMON_MULTI_THREADS, | |||
| CONV_BIAS_WINOGRAD_NCHW44_MK_PACKED_INT8_COMP_F32_GROUPMODE) { | |||
| using namespace conv_bias; | |||
| Checker<ConvBiasForward> checker(handle()); | |||
| @@ -1238,7 +1249,7 @@ TEST_F(ARM_COMMON_MULTI_THREADS, CONV_BIAS_WINOGRAD_NCHW44_MK_PACKED_INT8_COMP_F | |||
| #if MEGDNN_AARCH64 | |||
| const char* matmul_name = "AARCH64_F32_MK4_4x16"; | |||
| #else | |||
| const char* matmul_name = "ARMV7_F32_MK4_4x8"; | |||
| const char* matmul_name = "ARMV7_F32_MK4_4x8"; | |||
| #endif | |||
| checker.set_before_exec_callback(conv_bias::ConvBiasAlgoChecker<ConvBias>( | |||
| ssprintf("WINOGRAD_NCHW44:%s:4:2:32", matmul_name).c_str())); | |||
| @@ -1249,10 +1260,10 @@ TEST_F(ARM_COMMON_MULTI_THREADS, CONV_BIAS_WINOGRAD_NCHW44_MK_PACKED_INT8_COMP_F | |||
| run(handle(), quantized_args, {2}, dtype::QuantizedS8(0.41113496f), | |||
| dtype::QuantizedS8(0.01887994f), | |||
| dtype::QuantizedS32(0.41113496f * 0.01887994f), | |||
| dtype::QuantizedS8(0.49550694f), param::MatrixMul::Format::MK4, epsilon); | |||
| dtype::QuantizedS8(0.49550694f), param::MatrixMul::Format::MK4, | |||
| epsilon); | |||
| } | |||
| #if __ARM_FEATURE_FP16_VECTOR_ARITHMETIC | |||
| TEST_F(ARM_COMMON_MULTI_THREADS, CONV_BIAS_WINOGRAD_F16_F23) { | |||
| using namespace conv_bias; | |||