123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348 |
- /*
- * Copyright (C) 2016-2022 T-Head Semiconductor Co., Ltd. All rights reserved.
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Licensed under the Apache License, Version 2.0 (the License); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an AS IS BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
- /* CSI-NN2 version 2.0.x */
- #ifndef INCLUDE_SHL_C908_H_
- #define INCLUDE_SHL_C908_H_
- #include "csi_nn.h"
- #include "shl_gref.h"
- #include "shl_ref.h"
- #include "shl_thead_rvv.h"
- /*********************************** initialization ***********************************/
- int shl_c908_conv2d_init_fp32(struct csinn_tensor *input, struct csinn_tensor *output,
- struct csinn_tensor *kernel, struct csinn_tensor *bias,
- struct csinn_conv2d_params *params);
- int shl_c908_conv2d_init_fp16(struct csinn_tensor *input, struct csinn_tensor *output,
- struct csinn_tensor *kernel, struct csinn_tensor *bias,
- struct csinn_conv2d_params *params);
- int shl_c908_conv2d_init_int8(struct csinn_tensor *input, struct csinn_tensor *output,
- struct csinn_tensor *kernel, struct csinn_tensor *bias,
- struct csinn_conv2d_params *params);
- int shl_c908_depthwise_conv2d_init_fp32(struct csinn_tensor *input, struct csinn_tensor *output,
- struct csinn_tensor *kernel, struct csinn_tensor *bias,
- struct csinn_conv2d_params *params);
- int shl_c908_depthwise_conv2d_init_fp16(struct csinn_tensor *input, struct csinn_tensor *output,
- struct csinn_tensor *kernel, struct csinn_tensor *bias,
- struct csinn_conv2d_params *params);
- int shl_c908_depthwise_conv2d_init_int8(struct csinn_tensor *input, struct csinn_tensor *output,
- struct csinn_tensor *kernel, struct csinn_tensor *bias,
- struct csinn_conv2d_params *params);
- int shl_c908_avgpool2d_init_fp32(struct csinn_tensor *input, struct csinn_tensor *output,
- struct csinn_pool_params *params);
- int shl_c908_avgpool2d_init_fp16(struct csinn_tensor *input, struct csinn_tensor *output,
- struct csinn_pool_params *params);
- int shl_c908_avgpool2d_init_int8(struct csinn_tensor *input, struct csinn_tensor *output,
- struct csinn_pool_params *params);
- int shl_c908_maxpool2d_init_fp32(struct csinn_tensor *input, struct csinn_tensor *output,
- struct csinn_pool_params *params);
- int shl_c908_maxpool2d_init_fp16(struct csinn_tensor *input, struct csinn_tensor *output,
- struct csinn_pool_params *params);
- int shl_c908_maxpool2d_init_int8(struct csinn_tensor *input, struct csinn_tensor *output,
- struct csinn_pool_params *params);
- int shl_c908_fullyconnected_init(struct csinn_tensor *input, struct csinn_tensor *output,
- struct csinn_tensor *weights, struct csinn_tensor *bias,
- struct csinn_fc_params *params);
- /************************************ convolution *********************************/
- /*********************************** im2col + gemm ********************************/
- void shl_c908_conv_im2col_gemm_reorder_kernel_fp32(struct csinn_tensor *kernel,
- struct csinn_conv2d_params *params);
- void shl_c908_conv_im2col_gemm_reorder_kernel_fp16(struct csinn_tensor *kernel,
- struct csinn_conv2d_params *params);
- void shl_c908_conv_im2col_gemm_reorder_kernel_int8(struct csinn_tensor *kernel,
- struct csinn_conv2d_params *params);
- int shl_c908_conv_im2col_gemm_fp32(struct csinn_tensor *input, struct csinn_tensor *output,
- struct csinn_tensor *kernel, struct csinn_tensor *bias,
- struct csinn_conv2d_params *params);
- int shl_c908_conv_im2col_gemm_fp16(struct csinn_tensor *input, struct csinn_tensor *output,
- struct csinn_tensor *kernel, struct csinn_tensor *bias,
- struct csinn_conv2d_params *params);
- int shl_c908_conv_im2col_gemm_int8(struct csinn_tensor *input, struct csinn_tensor *output,
- struct csinn_tensor *kernel, struct csinn_tensor *bias,
- struct csinn_conv2d_params *params);
- void shl_c908_conv_im2col_gemm_reorder_kernel_packn_fp32(struct csinn_tensor *kernel,
- struct csinn_conv2d_params *params);
- void shl_c908_conv_im2col_gemm_reorder_kernel_packn_fp16(struct csinn_tensor *kernel,
- struct csinn_conv2d_params *params);
- void shl_c908_conv_im2col_gemm_reorder_kernel_packn_int8(struct csinn_tensor *kernel,
- struct csinn_conv2d_params *params);
- int shl_c908_conv_im2col_gemm_packn_fp32(struct csinn_tensor *input, struct csinn_tensor *output,
- struct csinn_tensor *kernel, struct csinn_tensor *bias,
- struct csinn_conv2d_params *params);
- int shl_c908_conv_im2col_gemm_packn_fp16(struct csinn_tensor *input, struct csinn_tensor *output,
- struct csinn_tensor *kernel, struct csinn_tensor *bias,
- struct csinn_conv2d_params *params);
- int shl_c908_conv_im2col_gemm_packn_int8(struct csinn_tensor *input, struct csinn_tensor *output,
- struct csinn_tensor *kernel, struct csinn_tensor *bias,
- struct csinn_conv2d_params *params);
- void shl_c908_conv_im2col_gemm_reorder_kernel_pack1ton_fp32(struct csinn_tensor *kernel,
- struct csinn_conv2d_params *params);
- void shl_c908_conv_im2col_gemm_reorder_kernel_pack1ton_fp16(struct csinn_tensor *kernel,
- struct csinn_conv2d_params *params);
- void shl_c908_conv_im2col_gemm_reorder_kernel_pack1ton_int8(struct csinn_tensor *kernel,
- struct csinn_conv2d_params *params);
- int shl_c908_conv_im2col_gemm_pack1ton_fp32(struct csinn_tensor *input, struct csinn_tensor *output,
- struct csinn_tensor *kernel, struct csinn_tensor *bias,
- struct csinn_conv2d_params *params);
- int shl_c908_conv_im2col_gemm_pack1ton_fp16(struct csinn_tensor *input, struct csinn_tensor *output,
- struct csinn_tensor *kernel, struct csinn_tensor *bias,
- struct csinn_conv2d_params *params);
- int shl_c908_conv_im2col_gemm_pack1ton_int8(struct csinn_tensor *input, struct csinn_tensor *output,
- struct csinn_tensor *kernel, struct csinn_tensor *bias,
- struct csinn_conv2d_params *params);
- void shl_c908_conv_im2col_gemm_reorder_kernel_packnto1_fp32(struct csinn_tensor *kernel,
- struct csinn_conv2d_params *params);
- void shl_c908_conv_im2col_gemm_reorder_kernel_packnto1_fp16(struct csinn_tensor *kernel,
- struct csinn_conv2d_params *params);
- void shl_c908_conv_im2col_gemm_reorder_kernel_packnto1_int8(struct csinn_tensor *kernel,
- struct csinn_conv2d_params *params);
- int shl_c908_conv_im2col_gemm_packnto1_fp32(struct csinn_tensor *input, struct csinn_tensor *output,
- struct csinn_tensor *kernel, struct csinn_tensor *bias,
- struct csinn_conv2d_params *params);
- int shl_c908_conv_im2col_gemm_packnto1_fp16(struct csinn_tensor *input, struct csinn_tensor *output,
- struct csinn_tensor *kernel, struct csinn_tensor *bias,
- struct csinn_conv2d_params *params);
- int shl_c908_conv_im2col_gemm_packnto1_int8(struct csinn_tensor *input, struct csinn_tensor *output,
- struct csinn_tensor *kernel, struct csinn_tensor *bias,
- struct csinn_conv2d_params *params);
- /******************************** conv2d1x1s1 + gemm ******************************/
- void shl_c908_conv1x1s1_gemm_reorder_kernel_fp32(struct csinn_tensor *kernel,
- struct csinn_conv2d_params *params);
- void shl_c908_conv1x1s1_gemm_reorder_kernel_fp16(struct csinn_tensor *kernel,
- struct csinn_conv2d_params *params);
- void shl_c908_conv1x1s1_gemm_reorder_kernel_int8(struct csinn_tensor *kernel,
- struct csinn_conv2d_params *params);
- int shl_c908_conv1x1s1_gemm_fp32(struct csinn_tensor *input, struct csinn_tensor *output,
- struct csinn_tensor *kernel, struct csinn_tensor *bias,
- struct csinn_conv2d_params *params);
- int shl_c908_conv1x1s1_gemm_fp16(struct csinn_tensor *input, struct csinn_tensor *output,
- struct csinn_tensor *kernel, struct csinn_tensor *bias,
- struct csinn_conv2d_params *params);
- int shl_c908_conv1x1s1_gemm_int8(struct csinn_tensor *input, struct csinn_tensor *output,
- struct csinn_tensor *kernel, struct csinn_tensor *bias,
- struct csinn_conv2d_params *params);
- void shl_c908_conv1x1s1_gemm_reorder_kernel_packn_fp32(struct csinn_tensor *kernel,
- struct csinn_conv2d_params *params);
- void shl_c908_conv1x1s1_gemm_reorder_kernel_packn_fp16(struct csinn_tensor *kernel,
- struct csinn_conv2d_params *params);
- void shl_c908_conv1x1s1_gemm_reorder_kernel_packn_int8(struct csinn_tensor *kernel,
- struct csinn_conv2d_params *params);
- int shl_c908_conv1x1s1_gemm_packn_fp32(struct csinn_tensor *input, struct csinn_tensor *output,
- struct csinn_tensor *kernel, struct csinn_tensor *bias,
- struct csinn_conv2d_params *params);
- int shl_c908_conv1x1s1_gemm_packn_fp16(struct csinn_tensor *input, struct csinn_tensor *output,
- struct csinn_tensor *kernel, struct csinn_tensor *bias,
- struct csinn_conv2d_params *params);
- int shl_c908_conv1x1s1_gemm_packn_int8(struct csinn_tensor *input, struct csinn_tensor *output,
- struct csinn_tensor *kernel, struct csinn_tensor *bias,
- struct csinn_conv2d_params *params);
- void shl_c908_conv1x1s1_gemm_reorder_kernel_pack1ton_fp32(struct csinn_tensor *kernel,
- struct csinn_conv2d_params *params);
- void shl_c908_conv1x1s1_gemm_reorder_kernel_pack1ton_fp16(struct csinn_tensor *kernel,
- struct csinn_conv2d_params *params);
- void shl_c908_conv1x1s1_gemm_reorder_kernel_pack1ton_int8(struct csinn_tensor *kernel,
- struct csinn_conv2d_params *params);
- int shl_c908_conv1x1s1_gemm_pack1ton_fp32(struct csinn_tensor *input, struct csinn_tensor *output,
- struct csinn_tensor *kernel, struct csinn_tensor *bias,
- struct csinn_conv2d_params *params);
- int shl_c908_conv1x1s1_gemm_pack1ton_fp16(struct csinn_tensor *input, struct csinn_tensor *output,
- struct csinn_tensor *kernel, struct csinn_tensor *bias,
- struct csinn_conv2d_params *params);
- int shl_c908_conv1x1s1_gemm_pack1ton_int8(struct csinn_tensor *input, struct csinn_tensor *output,
- struct csinn_tensor *kernel, struct csinn_tensor *bias,
- struct csinn_conv2d_params *params);
- void shl_c908_conv1x1s1_gemm_reorder_kernel_packnto1_fp32(struct csinn_tensor *kernel,
- struct csinn_conv2d_params *params);
- void shl_c908_conv1x1s1_gemm_reorder_kernel_packnto1_fp16(struct csinn_tensor *kernel,
- struct csinn_conv2d_params *params);
- void shl_c908_conv1x1s1_gemm_reorder_kernel_packnto1_int8(struct csinn_tensor *kernel,
- struct csinn_conv2d_params *params);
- int shl_c908_conv1x1s1_gemm_packnto1_fp32(struct csinn_tensor *input, struct csinn_tensor *output,
- struct csinn_tensor *kernel, struct csinn_tensor *bias,
- struct csinn_conv2d_params *params);
- int shl_c908_conv1x1s1_gemm_packnto1_fp16(struct csinn_tensor *input, struct csinn_tensor *output,
- struct csinn_tensor *kernel, struct csinn_tensor *bias,
- struct csinn_conv2d_params *params);
- int shl_c908_conv1x1s1_gemm_packnto1_int8(struct csinn_tensor *input, struct csinn_tensor *output,
- struct csinn_tensor *kernel, struct csinn_tensor *bias,
- struct csinn_conv2d_params *params);
- /*********************************** winograd ***********************************/
- void shl_c908_wg_b6f3s1_trans_kernel_pack8_fp32(struct csinn_tensor *src_kernel,
- struct csinn_tensor *dst_kernel);
- void shl_c908_wg_b6f3s1_trans_kernel_pack8_fp16(struct csinn_tensor *src_kernel,
- struct csinn_tensor *dst_kernel);
- void shl_c908_wg_b6f3s1_trans_kernel_pack16_fp16(struct csinn_tensor *src_kernel,
- struct csinn_tensor *dst_kernel);
- void shl_c908_wg_b4f3s1_trans_kernel_pack8_fp32(struct csinn_tensor *src_kernel,
- struct csinn_tensor *dst_kernel);
- void shl_c908_wg_b4f3s1_trans_kernel_pack8_fp16(struct csinn_tensor *src_kernel,
- struct csinn_tensor *dst_kernel);
- void shl_c908_wg_b4f3s1_trans_kernel_pack16_fp16(struct csinn_tensor *src_kernel,
- struct csinn_tensor *dst_kernel);
- void shl_c908_wg_b4f3s1_trans_kernel_pack8_int8(struct csinn_tensor *src_kernel,
- struct csinn_tensor *dst_kernel);
- int shl_c908_wg_b6f3s1_pack8_fp32(struct csinn_tensor *input, struct csinn_tensor *output,
- struct csinn_tensor *kernel, struct csinn_tensor *bias,
- struct csinn_conv2d_params *params);
- int shl_c908_wg_b6f3s1_pack8_fp16(struct csinn_tensor *input, struct csinn_tensor *output,
- struct csinn_tensor *kernel, struct csinn_tensor *bias,
- struct csinn_conv2d_params *params);
- int shl_c908_wg_b6f3s1_pack16_fp16(struct csinn_tensor *input, struct csinn_tensor *output,
- struct csinn_tensor *kernel, struct csinn_tensor *bias,
- struct csinn_conv2d_params *params);
- int shl_c908_wg_b4f3s1_pack8_fp32(struct csinn_tensor *input, struct csinn_tensor *output,
- struct csinn_tensor *kernel, struct csinn_tensor *bias,
- struct csinn_conv2d_params *params);
- int shl_c908_wg_b4f3s1_pack8_fp16(struct csinn_tensor *input, struct csinn_tensor *output,
- struct csinn_tensor *kernel, struct csinn_tensor *bias,
- struct csinn_conv2d_params *params);
- int shl_c908_wg_b4f3s1_pack16_fp16(struct csinn_tensor *input, struct csinn_tensor *output,
- struct csinn_tensor *kernel, struct csinn_tensor *bias,
- struct csinn_conv2d_params *params);
- int shl_c908_wg_b4f3s1_pack8_int8(struct csinn_tensor *input, struct csinn_tensor *output,
- struct csinn_tensor *kernel, struct csinn_tensor *bias,
- struct csinn_conv2d_params *params);
- void shl_c908_ncxhwx_wg_b6f3s1_trans_kernel_packn_fp32(struct csinn_tensor *src_kernel,
- struct csinn_tensor *dst_kernel);
- void shl_c908_ncxhwx_wg_b6f3s1_trans_kernel_packn_fp16(struct csinn_tensor *src_kernel,
- struct csinn_tensor *dst_kernel);
- int shl_c908_ncxhwx_wg_b6f3s1_packn_fp32(struct csinn_tensor *input, struct csinn_tensor *output,
- struct csinn_tensor *kernel, struct csinn_tensor *bias,
- struct csinn_conv2d_params *params);
- int shl_c908_ncxhwx_wg_b6f3s1_packn_fp16(struct csinn_tensor *input, struct csinn_tensor *output,
- struct csinn_tensor *kernel, struct csinn_tensor *bias,
- struct csinn_conv2d_params *params);
- void shl_c908_ncxhwx_wg_b4f3s1_trans_kernel_packn_fp32(struct csinn_tensor *src_kernel,
- struct csinn_tensor *dst_kernel);
- void shl_c908_ncxhwx_wg_b4f3s1_trans_kernel_packn_fp16(struct csinn_tensor *src_kernel,
- struct csinn_tensor *dst_kernel);
- void shl_c908_ncxhwx_wg_b4f3s1_trans_kernel_packn_int8(struct csinn_tensor *src_kernel,
- struct csinn_tensor *dst_kernel);
- int shl_c908_ncxhwx_wg_b4f3s1_packn_fp32(struct csinn_tensor *input, struct csinn_tensor *output,
- struct csinn_tensor *kernel, struct csinn_tensor *bias,
- struct csinn_conv2d_params *params);
- int shl_c908_ncxhwx_wg_b4f3s1_packn_fp16(struct csinn_tensor *input, struct csinn_tensor *output,
- struct csinn_tensor *kernel, struct csinn_tensor *bias,
- struct csinn_conv2d_params *params);
- int shl_c908_ncxhwx_wg_b4f3s1_packn_int8(struct csinn_tensor *input, struct csinn_tensor *output,
- struct csinn_tensor *kernel, struct csinn_tensor *bias,
- struct csinn_conv2d_params *params);
- /*********************************** gemm ncxhwx kernel ***********************************/
- void shl_c908_ncxhwx_gemm_12xpack2n_fp32(float *dst, const float *sa, const float *sb,
- const float *bias, int m, int k, int n, bool fuse_relu);
- void shl_c908_ncxhwx_gemm_12xpack2n_fp16(__fp16 *dst, const __fp16 *sa, const __fp16 *sb,
- const __fp16 *bias, int m, int k, int n, bool fuse_relu);
- void shl_c908_ncxhwx_gemm_12xpackn_int8(int8_t *dst, const int8_t *sa, const int8_t *sb,
- const int32_t *bias, int m, int k, int n, int32_t out_zp,
- int32_t *mult, int32_t *shift);
- void shl_c908_ncxhwx_gemm_12xpackn_int16(int32_t *dst, const int16_t *sa, const int16_t *sb, int m,
- int k, int n);
- /*********************************** gemm kernel ***********************************/
- void shl_c908_reorder_kernel_n8_fp32(float *src, float *dst, int m, int k, int ldc);
- void shl_c908_reorder_input_z12_fp32(float *src, float *dst, int k, int n, int ldc);
- void shl_c908_gemm_8x12_fp32(float *dst, const float *sa, const float *sb, float *bias, int m,
- int k, int n, int ldc);
- void shl_c908_reorder_kernel_n8_fp16(__fp16 *src, __fp16 *dst, int m, int k, int ldc);
- void shl_c908_reorder_input_z24_fp16(__fp16 *src, __fp16 *dst, int k, int n, int ldc);
- void shl_c908_gemm_8x24_fp16(__fp16 *dst, const __fp16 *sa, const __fp16 *sb, __fp16 *bias, int m,
- int k, int n, int ldc);
- void shl_c908_reorder_kernel_n8_int8(int8_t *src, int8_t *dst, int m, int k, int ldc);
- void shl_c908_reorder_input_z8_int8(int8_t *src, int8_t *dst, int k, int n, int ldc);
- void shl_c908_gemm_8x8_int8(int8_t *dst, const int8_t *sa, const int8_t *sb, int32_t *bias, int m,
- int k, int n, int ldc, int32_t out_zp, int32_t *mult, int32_t *shift);
- void shl_c908_reorder_input_z12_int8(int8_t *src, int8_t *dst, int k, int n, int ldc);
- /*********************************** VLEN = 256 ***********************************/
- /*********************************** VLEN = 256 ***********************************/
- /*********************************** VLEN = 256 ***********************************/
- void shl_c908_reorder_input_z16_fp32_v256(float *src, float *dst, int k, int n, int ldc);
- void shl_c908_gemm_8x16_fp32_v256(float *dst, const float *sa, const float *sb, float *bias, int m,
- int k, int n, int ldc);
- void shl_c908_reorder_input_z32_fp16_v256(__fp16 *src, __fp16 *dst, int k, int n, int ldc);
- void shl_c908_gemm_8x32_fp16_v256(__fp16 *dst, const __fp16 *sa, const __fp16 *sb, __fp16 *bias,
- int m, int k, int n, int ldc);
- void shl_c908_reorder_input_z16_int8_v256(int8_t *src, int8_t *dst, int k, int n, int ldc);
- void shl_c908_gemm_8x16_int8_v256(int8_t *dst, const int8_t *sa, const int8_t *sb, int32_t *bias,
- int m, int k, int n, int ldc, int32_t out_zp, int32_t *mult,
- int32_t *shift);
- #ifdef SHL_UNUSED_REGISTER_BLK
- void shl_c908_reorder_input_z8_fp32(float *src, float *dst, int k, int n, int ldc);
- void shl_c908_gemm_8x8_fp32(float *dst, const float *sa, const float *sb, float *bias, int m, int k,
- int n, int ldc);
- void shl_c908_reorder_input_z16_fp16(__fp16 *src, __fp16 *dst, int k, int n, int ldc);
- void shl_c908_gemm_8x16_fp16(__fp16 *dst, const __fp16 *sa, const __fp16 *sb, __fp16 *bias, int m,
- int k, int n, int ldc);
- void shl_c908_reorder_input_z24_fp32_v256(float *src, float *dst, int k, int n, int ldc);
- void shl_c908_gemm_8x24_fp32_v256(float *dst, const float *sa, const float *sb, float *bias, int m,
- int k, int n, int ldc);
- void shl_c908_reorder_input_z48_fp16_v256(__fp16 *src, __fp16 *dst, int k, int n, int ldc);
- void shl_c908_gemm_8x48_fp16_v256(__fp16 *dst, const __fp16 *sa, const __fp16 *sb, __fp16 *bias,
- int m, int k, int n, int ldc);
- #endif
- #ifdef SHL_USE_DOT_INT4
- int shl_c908_conv2d_init_int4(struct csinn_tensor *input, struct csinn_tensor *output,
- struct csinn_tensor *kernel, struct csinn_tensor *bias,
- struct csinn_conv2d_params *params);
- int shl_c908_depthwise_conv2d_init_int4(struct csinn_tensor *input, struct csinn_tensor *output,
- struct csinn_tensor *kernel, struct csinn_tensor *bias,
- struct csinn_conv2d_params *params);
- #endif
- #endif // INCLUDE_SHL_C908_H_
|