diff --git a/ggml/src/ggml-cpu/arch-fallback.h b/ggml/src/ggml-cpu/arch-fallback.h index 0775c87f98..8af7512805 100644 --- a/ggml/src/ggml-cpu/arch-fallback.h +++ b/ggml/src/ggml-cpu/arch-fallback.h @@ -35,6 +35,8 @@ #define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8 #define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4 #define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8 +#define ggml_repack_mat_f16_7x1_generic ggml_repack_mat_f16_7x1 +#define ggml_repack_mat_f32_7x1_generic ggml_repack_mat_f32_7x1 #define ggml_gemv_q4_0_4x4_q8_0_generic ggml_gemv_q4_0_4x4_q8_0 #define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0 #define ggml_gemv_q4_0_8x8_q8_0_generic ggml_gemv_q4_0_8x8_q8_0 @@ -43,6 +45,14 @@ #define ggml_gemv_q2_K_8x8_q8_K_generic ggml_gemv_q2_K_8x8_q8_K #define ggml_gemv_iq4_nl_4x4_q8_0_generic ggml_gemv_iq4_nl_4x4_q8_0 #define ggml_gemv_iq4_nl_8x8_q8_0_generic ggml_gemv_iq4_nl_8x8_q8_0 +#define ggml_gemv_f16_1x16_f16_generic ggml_gemv_f16_1x16_f16 +#define ggml_gemv_f16_1x32_f16_generic ggml_gemv_f16_1x32_f16 +#define ggml_gemv_f16_1x64_f16_generic ggml_gemv_f16_1x64_f16 +#define ggml_gemv_f16_1x128_f16_generic ggml_gemv_f16_1x128_f16 +#define ggml_gemv_f32_1x16_f32_generic ggml_gemv_f32_1x16_f32 +#define ggml_gemv_f32_1x32_f32_generic ggml_gemv_f32_1x32_f32 +#define ggml_gemv_f32_1x64_f32_generic ggml_gemv_f32_1x64_f32 +#define ggml_gemv_f32_1x128_f32_generic ggml_gemv_f32_1x128_f32 #define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0 #define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0 #define ggml_gemm_q4_0_8x8_q8_0_generic ggml_gemm_q4_0_8x8_q8_0 @@ -51,26 +61,68 @@ #define ggml_gemm_q2_K_8x8_q8_K_generic ggml_gemm_q2_K_8x8_q8_K #define ggml_gemm_iq4_nl_4x4_q8_0_generic ggml_gemm_iq4_nl_4x4_q8_0 #define ggml_gemm_iq4_nl_8x8_q8_0_generic ggml_gemm_iq4_nl_8x8_q8_0 +#define ggml_gemm_f16_7x1x16_f16_generic ggml_gemm_f16_7x1x16_f16 +#define ggml_gemm_f16_7x1x32_f16_generic ggml_gemm_f16_7x1x32_f16 +#define ggml_gemm_f16_7x1x64_f16_generic ggml_gemm_f16_7x1x64_f16 +#define ggml_gemm_f16_7x1x128_f16_generic ggml_gemm_f16_7x1x128_f16 +#define ggml_gemm_f32_7x1x16_f32_generic ggml_gemm_f32_7x1x16_f32 +#define ggml_gemm_f32_7x1x32_f32_generic ggml_gemm_f32_7x1x32_f32 +#define ggml_gemm_f32_7x1x64_f32_generic ggml_gemm_f32_7x1x64_f32 +#define ggml_gemm_f32_7x1x128_f32_generic ggml_gemm_f32_7x1x128_f32 #elif defined(__aarch64__) || defined(__arm__) || defined(_M_ARM) || defined(_M_ARM64) // repack.cpp #define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4 #define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8 #define ggml_gemv_iq4_nl_8x8_q8_0_generic ggml_gemv_iq4_nl_8x8_q8_0 #define ggml_gemv_q2_K_8x8_q8_K_generic ggml_gemv_q2_K_8x8_q8_K +#define ggml_gemv_f16_1x16_f16_generic ggml_gemv_f16_1x16_f16 +#define ggml_gemv_f16_1x32_f16_generic ggml_gemv_f16_1x32_f16 +#define ggml_gemv_f16_1x64_f16_generic ggml_gemv_f16_1x64_f16 +#define ggml_gemv_f16_1x128_f16_generic ggml_gemv_f16_1x128_f16 +#define ggml_gemv_f32_1x16_f32_generic ggml_gemv_f32_1x16_f32 +#define ggml_gemv_f32_1x32_f32_generic ggml_gemv_f32_1x32_f32 +#define ggml_gemv_f32_1x64_f32_generic ggml_gemv_f32_1x64_f32 +#define ggml_gemv_f32_1x128_f32_generic ggml_gemv_f32_1x128_f32 #define ggml_gemm_iq4_nl_8x8_q8_0_generic ggml_gemm_iq4_nl_8x8_q8_0 #define ggml_gemm_q2_K_8x8_q8_K_generic ggml_gemm_q2_K_8x8_q8_K +#define ggml_gemm_f16_7x1x16_f16_generic ggml_gemm_f16_7x1x16_f16 +#define ggml_gemm_f16_7x1x32_f16_generic ggml_gemm_f16_7x1x32_f16 +#define ggml_gemm_f16_7x1x64_f16_generic ggml_gemm_f16_7x1x64_f16 +#define ggml_gemm_f16_7x1x128_f16_generic ggml_gemm_f16_7x1x128_f16 +#define ggml_gemm_f32_7x1x16_f32_generic ggml_gemm_f32_7x1x16_f32 +#define ggml_gemm_f32_7x1x32_f32_generic ggml_gemm_f32_7x1x32_f32 +#define ggml_gemm_f32_7x1x64_f32_generic ggml_gemm_f32_7x1x64_f32 +#define ggml_gemm_f32_7x1x128_f32_generic ggml_gemm_f32_7x1x128_f32 #elif defined(__x86_64__) || defined(__i386__) || defined(_M_IX86) || defined(_M_X64) // repack.cpp #define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4 #define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4 +#define ggml_repack_mat_f16_7x1_generic ggml_repack_mat_f16_7x1 +#define ggml_repack_mat_f32_7x1_generic ggml_repack_mat_f32_7x1 #define ggml_gemv_q4_0_4x4_q8_0_generic ggml_gemv_q4_0_4x4_q8_0 #define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0 #define ggml_gemv_q4_K_8x4_q8_K_generic ggml_gemv_q4_K_8x4_q8_K #define ggml_gemv_iq4_nl_4x4_q8_0_generic ggml_gemv_iq4_nl_4x4_q8_0 +#define ggml_gemv_f16_1x16_f16_generic ggml_gemv_f16_1x16_f16 +#define ggml_gemv_f16_1x32_f16_generic ggml_gemv_f16_1x32_f16 +#define ggml_gemv_f16_1x64_f16_generic ggml_gemv_f16_1x64_f16 +#define ggml_gemv_f16_1x128_f16_generic ggml_gemv_f16_1x128_f16 +#define ggml_gemv_f32_1x16_f32_generic ggml_gemv_f32_1x16_f32 +#define ggml_gemv_f32_1x32_f32_generic ggml_gemv_f32_1x32_f32 +#define ggml_gemv_f32_1x64_f32_generic ggml_gemv_f32_1x64_f32 +#define ggml_gemv_f32_1x128_f32_generic ggml_gemv_f32_1x128_f32 #define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0 #define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0 #define ggml_gemm_q4_K_8x4_q8_K_generic ggml_gemm_q4_K_8x4_q8_K #define ggml_gemm_iq4_nl_4x4_q8_0_generic ggml_gemm_iq4_nl_4x4_q8_0 +#define ggml_gemm_f16_7x1x16_f16_generic ggml_gemm_f16_7x1x16_f16 +#define ggml_gemm_f16_7x1x32_f16_generic ggml_gemm_f16_7x1x32_f16 +#define ggml_gemm_f16_7x1x64_f16_generic ggml_gemm_f16_7x1x64_f16 +#define ggml_gemm_f16_7x1x128_f16_generic ggml_gemm_f16_7x1x128_f16 +#define ggml_gemm_f32_7x1x16_f32_generic ggml_gemm_f32_7x1x16_f32 +#define ggml_gemm_f32_7x1x32_f32_generic ggml_gemm_f32_7x1x32_f32 +#define ggml_gemm_f32_7x1x64_f32_generic ggml_gemm_f32_7x1x64_f32 +#define ggml_gemm_f32_7x1x128_f32_generic ggml_gemm_f32_7x1x128_f32 #elif defined(__POWERPC__) || defined(__powerpc__) // ref: https://github.com/ggml-org/llama.cpp/pull/14146#issuecomment-2972561679 // quants.c @@ -83,6 +135,8 @@ #define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8 #define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4 #define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8 +#define ggml_repack_mat_f16_7x1_generic ggml_repack_mat_f16_7x1 +#define ggml_repack_mat_f32_7x1_generic ggml_repack_mat_f32_7x1 #define ggml_gemv_q4_0_4x4_q8_0_generic ggml_gemv_q4_0_4x4_q8_0 #define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0 #define ggml_gemv_q4_0_8x8_q8_0_generic ggml_gemv_q4_0_8x8_q8_0 @@ -91,6 +145,14 @@ #define ggml_gemv_q2_K_8x8_q8_K_generic ggml_gemv_q2_K_8x8_q8_K #define ggml_gemv_iq4_nl_4x4_q8_0_generic ggml_gemv_iq4_nl_4x4_q8_0 #define ggml_gemv_iq4_nl_8x8_q8_0_generic ggml_gemv_iq4_nl_8x8_q8_0 +#define ggml_gemv_f16_1x16_f16_generic ggml_gemv_f16_1x16_f16 +#define ggml_gemv_f16_1x32_f16_generic ggml_gemv_f16_1x32_f16 +#define ggml_gemv_f16_1x64_f16_generic ggml_gemv_f16_1x64_f16 +#define ggml_gemv_f16_1x128_f16_generic ggml_gemv_f16_1x128_f16 +#define ggml_gemv_f32_1x16_f32_generic ggml_gemv_f32_1x16_f32 +#define ggml_gemv_f32_1x32_f32_generic ggml_gemv_f32_1x32_f32 +#define ggml_gemv_f32_1x64_f32_generic ggml_gemv_f32_1x64_f32 +#define ggml_gemv_f32_1x128_f32_generic ggml_gemv_f32_1x128_f32 #define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0 #define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0 #define ggml_gemm_q4_0_8x8_q8_0_generic ggml_gemm_q4_0_8x8_q8_0 @@ -99,6 +161,14 @@ #define ggml_gemm_q2_K_8x8_q8_K_generic ggml_gemm_q2_K_8x8_q8_K #define ggml_gemm_iq4_nl_4x4_q8_0_generic ggml_gemm_iq4_nl_4x4_q8_0 #define ggml_gemm_iq4_nl_8x8_q8_0_generic ggml_gemm_iq4_nl_8x8_q8_0 +#define ggml_gemm_f16_7x1x16_f16_generic ggml_gemm_f16_7x1x16_f16 +#define ggml_gemm_f16_7x1x32_f16_generic ggml_gemm_f16_7x1x32_f16 +#define ggml_gemm_f16_7x1x64_f16_generic ggml_gemm_f16_7x1x64_f16 +#define ggml_gemm_f16_7x1x128_f16_generic ggml_gemm_f16_7x1x128_f16 +#define ggml_gemm_f32_7x1x16_f32_generic ggml_gemm_f32_7x1x16_f32 +#define ggml_gemm_f32_7x1x32_f32_generic ggml_gemm_f32_7x1x32_f32 +#define ggml_gemm_f32_7x1x64_f32_generic ggml_gemm_f32_7x1x64_f32 +#define ggml_gemm_f32_7x1x128_f32_generic ggml_gemm_f32_7x1x128_f32 #elif defined(__loongarch64) // quants.c #define quantize_row_q8_K_generic quantize_row_q8_K @@ -111,6 +181,8 @@ #define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8 #define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4 #define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8 +#define ggml_repack_mat_f16_7x1_generic ggml_repack_mat_f16_7x1 +#define ggml_repack_mat_f32_7x1_generic ggml_repack_mat_f32_7x1 #define ggml_gemv_q4_0_4x4_q8_0_generic ggml_gemv_q4_0_4x4_q8_0 #define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0 #define ggml_gemv_q4_0_8x8_q8_0_generic ggml_gemv_q4_0_8x8_q8_0 @@ -119,6 +191,14 @@ #define ggml_gemv_q2_K_8x8_q8_K_generic ggml_gemv_q2_K_8x8_q8_K #define ggml_gemv_iq4_nl_4x4_q8_0_generic ggml_gemv_iq4_nl_4x4_q8_0 #define ggml_gemv_iq4_nl_8x8_q8_0_generic ggml_gemv_iq4_nl_8x8_q8_0 +#define ggml_gemv_f16_1x16_f16_generic ggml_gemv_f16_1x16_f16 +#define ggml_gemv_f16_1x32_f16_generic ggml_gemv_f16_1x32_f16 +#define ggml_gemv_f16_1x64_f16_generic ggml_gemv_f16_1x64_f16 +#define ggml_gemv_f16_1x128_f16_generic ggml_gemv_f16_1x128_f16 +#define ggml_gemv_f32_1x16_f32_generic ggml_gemv_f32_1x16_f32 +#define ggml_gemv_f32_1x32_f32_generic ggml_gemv_f32_1x32_f32 +#define ggml_gemv_f32_1x64_f32_generic ggml_gemv_f32_1x64_f32 +#define ggml_gemv_f32_1x128_f32_generic ggml_gemv_f32_1x128_f32 #define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0 #define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0 #define ggml_gemm_q4_0_8x8_q8_0_generic ggml_gemm_q4_0_8x8_q8_0 @@ -127,6 +207,14 @@ #define ggml_gemm_q2_K_8x8_q8_K_generic ggml_gemm_q2_K_8x8_q8_K #define ggml_gemm_iq4_nl_4x4_q8_0_generic ggml_gemm_iq4_nl_4x4_q8_0 #define ggml_gemm_iq4_nl_8x8_q8_0_generic ggml_gemm_iq4_nl_8x8_q8_0 +#define ggml_gemm_f16_7x1x16_f16_generic ggml_gemm_f16_7x1x16_f16 +#define ggml_gemm_f16_7x1x32_f16_generic ggml_gemm_f16_7x1x32_f16 +#define ggml_gemm_f16_7x1x64_f16_generic ggml_gemm_f16_7x1x64_f16 +#define ggml_gemm_f16_7x1x128_f16_generic ggml_gemm_f16_7x1x128_f16 +#define ggml_gemm_f32_7x1x16_f32_generic ggml_gemm_f32_7x1x16_f32 +#define ggml_gemm_f32_7x1x32_f32_generic ggml_gemm_f32_7x1x32_f32 +#define ggml_gemm_f32_7x1x64_f32_generic ggml_gemm_f32_7x1x64_f32 +#define ggml_gemm_f32_7x1x128_f32_generic ggml_gemm_f32_7x1x128_f32 #elif defined(__riscv) // quants.c #define quantize_row_q8_K_generic quantize_row_q8_K @@ -147,6 +235,8 @@ #define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8 #define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4 #define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8 +#define ggml_repack_mat_f16_7x1_generic ggml_repack_mat_f16_7x1 +#define ggml_repack_mat_f32_7x1_generic ggml_repack_mat_f32_7x1 #define ggml_gemv_q4_0_4x4_q8_0_generic ggml_gemv_q4_0_4x4_q8_0 #define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0 #define ggml_gemv_q4_K_8x4_q8_K_generic ggml_gemv_q4_K_8x4_q8_K @@ -179,6 +269,8 @@ #define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8 #define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4 #define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8 +#define ggml_repack_mat_f16_7x1_generic ggml_repack_mat_f16_7x1 +#define ggml_repack_mat_f32_7x1_generic ggml_repack_mat_f32_7x1 #define ggml_gemv_q4_0_4x4_q8_0_generic ggml_gemv_q4_0_4x4_q8_0 #define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0 #define ggml_gemv_q4_0_8x8_q8_0_generic ggml_gemv_q4_0_8x8_q8_0 @@ -187,6 +279,14 @@ #define ggml_gemv_q2_K_8x8_q8_K_generic ggml_gemv_q2_K_8x8_q8_K #define ggml_gemv_iq4_nl_4x4_q8_0_generic ggml_gemv_iq4_nl_4x4_q8_0 #define ggml_gemv_iq4_nl_8x8_q8_0_generic ggml_gemv_iq4_nl_8x8_q8_0 +#define ggml_gemv_f16_1x16_f16_generic ggml_gemv_f16_1x16_f16 +#define ggml_gemv_f16_1x32_f16_generic ggml_gemv_f16_1x32_f16 +#define ggml_gemv_f16_1x64_f16_generic ggml_gemv_f16_1x64_f16 +#define ggml_gemv_f16_1x128_f16_generic ggml_gemv_f16_1x128_f16 +#define ggml_gemv_f32_1x16_f32_generic ggml_gemv_f32_1x16_f32 +#define ggml_gemv_f32_1x32_f32_generic ggml_gemv_f32_1x32_f32 +#define ggml_gemv_f32_1x64_f32_generic ggml_gemv_f32_1x64_f32 +#define ggml_gemv_f32_1x128_f32_generic ggml_gemv_f32_1x128_f32 #define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0 #define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0 #define ggml_gemm_q4_0_8x8_q8_0_generic ggml_gemm_q4_0_8x8_q8_0 @@ -195,6 +295,14 @@ #define ggml_gemm_q2_K_8x8_q8_K_generic ggml_gemm_q2_K_8x8_q8_K #define ggml_gemm_iq4_nl_4x4_q8_0_generic ggml_gemm_iq4_nl_4x4_q8_0 #define ggml_gemm_iq4_nl_8x8_q8_0_generic ggml_gemm_iq4_nl_8x8_q8_0 +#define ggml_gemm_f16_7x1x16_f16_generic ggml_gemm_f16_7x1x16_f16 +#define ggml_gemm_f16_7x1x32_f16_generic ggml_gemm_f16_7x1x32_f16 +#define ggml_gemm_f16_7x1x64_f16_generic ggml_gemm_f16_7x1x64_f16 +#define ggml_gemm_f16_7x1x128_f16_generic ggml_gemm_f16_7x1x128_f16 +#define ggml_gemm_f32_7x1x16_f32_generic ggml_gemm_f32_7x1x16_f32 +#define ggml_gemm_f32_7x1x32_f32_generic ggml_gemm_f32_7x1x32_f32 +#define ggml_gemm_f32_7x1x64_f32_generic ggml_gemm_f32_7x1x64_f32 +#define ggml_gemm_f32_7x1x128_f32_generic ggml_gemm_f32_7x1x128_f32 #elif defined(__wasm__) // quants.c #define ggml_vec_dot_q4_1_q8_1_generic ggml_vec_dot_q4_1_q8_1 @@ -215,6 +323,8 @@ #define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8 #define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4 #define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8 +#define ggml_repack_mat_f16_7x1_generic ggml_repack_mat_f16_7x1 +#define ggml_repack_mat_f32_7x1_generic ggml_repack_mat_f32_7x1 #define ggml_gemv_q4_0_4x4_q8_0_generic ggml_gemv_q4_0_4x4_q8_0 #define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0 #define ggml_gemv_q4_0_8x8_q8_0_generic ggml_gemv_q4_0_8x8_q8_0 @@ -223,6 +333,14 @@ #define ggml_gemv_q2_K_8x8_q8_K_generic ggml_gemv_q2_K_8x8_q8_K #define ggml_gemv_iq4_nl_4x4_q8_0_generic ggml_gemv_iq4_nl_4x4_q8_0 #define ggml_gemv_iq4_nl_8x8_q8_0_generic ggml_gemv_iq4_nl_8x8_q8_0 +#define ggml_gemv_f16_1x16_f16_generic ggml_gemv_f16_1x16_f16 +#define ggml_gemv_f16_1x32_f16_generic ggml_gemv_f16_1x32_f16 +#define ggml_gemv_f16_1x64_f16_generic ggml_gemv_f16_1x64_f16 +#define ggml_gemv_f16_1x128_f16_generic ggml_gemv_f16_1x128_f16 +#define ggml_gemv_f32_1x16_f32_generic ggml_gemv_f32_1x16_f32 +#define ggml_gemv_f32_1x32_f32_generic ggml_gemv_f32_1x32_f32 +#define ggml_gemv_f32_1x64_f32_generic ggml_gemv_f32_1x64_f32 +#define ggml_gemv_f32_1x128_f32_generic ggml_gemv_f32_1x128_f32 #define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0 #define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0 #define ggml_gemm_q4_0_8x8_q8_0_generic ggml_gemm_q4_0_8x8_q8_0 @@ -231,4 +349,12 @@ #define ggml_gemm_q2_K_8x8_q8_K_generic ggml_gemm_q2_K_8x8_q8_K #define ggml_gemm_iq4_nl_4x4_q8_0_generic ggml_gemm_iq4_nl_4x4_q8_0 #define ggml_gemm_iq4_nl_8x8_q8_0_generic ggml_gemm_iq4_nl_8x8_q8_0 +#define ggml_gemm_f16_7x1x16_f16_generic ggml_gemm_f16_7x1x16_f16 +#define ggml_gemm_f16_7x1x32_f16_generic ggml_gemm_f16_7x1x32_f16 +#define ggml_gemm_f16_7x1x64_f16_generic ggml_gemm_f16_7x1x64_f16 +#define ggml_gemm_f16_7x1x128_f16_generic ggml_gemm_f16_7x1x128_f16 +#define ggml_gemm_f32_7x1x16_f32_generic ggml_gemm_f32_7x1x16_f32 +#define ggml_gemm_f32_7x1x32_f32_generic ggml_gemm_f32_7x1x32_f32 +#define ggml_gemm_f32_7x1x64_f32_generic ggml_gemm_f32_7x1x64_f32 +#define ggml_gemm_f32_7x1x128_f32_generic ggml_gemm_f32_7x1x128_f32 #endif diff --git a/ggml/src/ggml-cpu/arch/riscv/repack.cpp b/ggml/src/ggml-cpu/arch/riscv/repack.cpp index 2a35ff9ad8..358e43f8d2 100644 --- a/ggml/src/ggml-cpu/arch/riscv/repack.cpp +++ b/ggml/src/ggml-cpu/arch/riscv/repack.cpp @@ -340,3 +340,277 @@ void ggml_gemm_q4_0_8x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const vo #endif ggml_gemm_q4_0_8x8_q8_0_generic(n, s, bs, vx, vy, nr, nc); } + +template +static inline void ggml_gemv_f16_1xM_f16(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { + const int nb = n / 1; + + assert (nr == 1); + assert(n % 1 == 0); + assert(nc % ncols_interleaved == 0); + + const _Float16 * a_ptr = (const _Float16 *) vy; + for (int x = 0; x < nc / ncols_interleaved; x++) { + const block_f16 * b_ptr = (const block_f16 *) vx + (x * nb); + + // Accumulators + vfloat32m4_t sumf_0 = __riscv_vfmv_v_f_f32m4(0.0f, ncols_interleaved); + + for (int l = 0; l < nb; l++) { + vfloat16m2_t b_0 = __riscv_vle16_v_f16m2((const _Float16 *)&b_ptr[l].d[0], ncols_interleaved); + + sumf_0 = __riscv_vfwmacc_vf_f32m4(sumf_0, *(const _Float16*)(&a_ptr[l]), b_0, ncols_interleaved); + } + + __riscv_vse32_v_f32m4(&s[x * ncols_interleaved], sumf_0, ncols_interleaved); + } + + return; +} + +void ggml_gemv_f16_1x16_f16(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { +#if defined __riscv_v_intrinsic + ggml_gemv_f16_1xM_f16<16>(n, s, bs, vx, vy, nr, nc); + return; +#endif + ggml_gemv_f16_1x16_f16_generic(n, s, bs, vx, vy, nr, nc); +} + +void ggml_gemv_f16_1x32_f16(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { +#if defined __riscv_v_intrinsic + ggml_gemv_f16_1xM_f16<32>(n, s, bs, vx, vy, nr, nc); + return; +#endif + ggml_gemv_f16_1x32_f16_generic(n, s, bs, vx, vy, nr, nc); +} + +void ggml_gemv_f16_1x64_f16(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { +#if defined __riscv_v_intrinsic + ggml_gemv_f16_1xM_f16<64>(n, s, bs, vx, vy, nr, nc); + return; +#endif + ggml_gemv_f16_1x64_f16_generic(n, s, bs, vx, vy, nr, nc); +} + +void ggml_gemv_f16_1x128_f16(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { +#if defined __riscv_v_intrinsic + ggml_gemv_f16_1xM_f16<128>(n, s, bs, vx, vy, nr, nc); + return; +#endif + ggml_gemv_f16_1x128_f16_generic(n, s, bs, vx, vy, nr, nc); +} + +template +static inline void ggml_gemv_f32_1xM_f32(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { + const int nb = n / 1; + + assert (nr == 1); + assert(n % 1 == 0); + assert(nc % ncols_interleaved == 0); + + const float * a_ptr = (const float *) vy; + for (int x = 0; x < nc / ncols_interleaved; x++) { + const block_f32 * b_ptr = (const block_f32 *) vx + (x * nb); + + // Accumulators + vfloat32m4_t sumf_0 = __riscv_vfmv_v_f_f32m4(0.0f, ncols_interleaved); + + for (int l = 0; l < nb; l++) { + vfloat32m4_t b_0 = __riscv_vle32_v_f32m4((const float *)&b_ptr[l].d[0], ncols_interleaved); + + sumf_0 = __riscv_vfmacc_vf_f32m4(sumf_0, *(const float*)(&a_ptr[l]), b_0, ncols_interleaved); + } + + __riscv_vse32_v_f32m4(&s[x * ncols_interleaved], sumf_0, ncols_interleaved); + } + + return; +} + +void ggml_gemv_f32_1x16_f32(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { +#if defined __riscv_v_intrinsic + ggml_gemv_f32_1xM_f32<16>(n, s, bs, vx, vy, nr, nc); + return; +#endif + ggml_gemv_f32_1x16_f32_generic(n, s, bs, vx, vy, nr, nc); +} + +void ggml_gemv_f32_1x32_f32(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { +#if defined __riscv_v_intrinsic + ggml_gemv_f32_1xM_f32<32>(n, s, bs, vx, vy, nr, nc); + return; +#endif + ggml_gemv_f32_1x32_f32_generic(n, s, bs, vx, vy, nr, nc); +} + +void ggml_gemv_f32_1x64_f32(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { +#if defined __riscv_v_intrinsic + ggml_gemv_f32_1xM_f32<64>(n, s, bs, vx, vy, nr, nc); + return; +#endif + ggml_gemv_f32_1x64_f32_generic(n, s, bs, vx, vy, nr, nc); +} + +void ggml_gemv_f32_1x128_f32(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { +#if defined __riscv_v_intrinsic + ggml_gemv_f32_1xM_f32<128>(n, s, bs, vx, vy, nr, nc); + return; +#endif + ggml_gemv_f32_1x128_f32_generic(n, s, bs, vx, vy, nr, nc); +} + +template +static inline void ggml_gemm_f16_7x1xM_f16(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { + const int nb = n / 1; + + assert (nr % 7 == 0); + assert(n % 1 == 0); + assert(nc % ncols_interleaved == 0); + + for (int y = 0; y < nr / 7; y++) { + const block_f16_7x1 * a_ptr = (const block_f16_7x1*) vy + (y * nb); + for (int x = 0; x < nc / ncols_interleaved; x++) { + const block_f16 * b_ptr = (const block_f16 *) vx + (x * nb); + + // Accumulators + vfloat32m4_t sumf_0 = __riscv_vfmv_v_f_f32m4(0.0f, ncols_interleaved); + vfloat32m4_t sumf_1 = __riscv_vfmv_v_f_f32m4(0.0f, ncols_interleaved); + vfloat32m4_t sumf_2 = __riscv_vfmv_v_f_f32m4(0.0f, ncols_interleaved); + vfloat32m4_t sumf_3 = __riscv_vfmv_v_f_f32m4(0.0f, ncols_interleaved); + vfloat32m4_t sumf_4 = __riscv_vfmv_v_f_f32m4(0.0f, ncols_interleaved); + vfloat32m4_t sumf_5 = __riscv_vfmv_v_f_f32m4(0.0f, ncols_interleaved); + vfloat32m4_t sumf_6 = __riscv_vfmv_v_f_f32m4(0.0f, ncols_interleaved); + + for (int l = 0; l < nb; l++) { + vfloat16m2_t b_0 = __riscv_vle16_v_f16m2((const _Float16 *)&b_ptr[l].d[0], ncols_interleaved); + + sumf_0 = __riscv_vfwmacc_vf_f32m4(sumf_0, *(const _Float16*)&a_ptr[l].d[0], b_0, ncols_interleaved); + sumf_1 = __riscv_vfwmacc_vf_f32m4(sumf_1, *(const _Float16*)&a_ptr[l].d[1], b_0, ncols_interleaved); + sumf_2 = __riscv_vfwmacc_vf_f32m4(sumf_2, *(const _Float16*)&a_ptr[l].d[2], b_0, ncols_interleaved); + sumf_3 = __riscv_vfwmacc_vf_f32m4(sumf_3, *(const _Float16*)&a_ptr[l].d[3], b_0, ncols_interleaved); + sumf_4 = __riscv_vfwmacc_vf_f32m4(sumf_4, *(const _Float16*)&a_ptr[l].d[4], b_0, ncols_interleaved); + sumf_5 = __riscv_vfwmacc_vf_f32m4(sumf_5, *(const _Float16*)&a_ptr[l].d[5], b_0, ncols_interleaved); + sumf_6 = __riscv_vfwmacc_vf_f32m4(sumf_6, *(const _Float16*)&a_ptr[l].d[6], b_0, ncols_interleaved); + } + + __riscv_vse32_v_f32m4(&s[(y * 7 + 0) * bs + x * ncols_interleaved], sumf_0, ncols_interleaved); + __riscv_vse32_v_f32m4(&s[(y * 7 + 1) * bs + x * ncols_interleaved], sumf_1, ncols_interleaved); + __riscv_vse32_v_f32m4(&s[(y * 7 + 2) * bs + x * ncols_interleaved], sumf_2, ncols_interleaved); + __riscv_vse32_v_f32m4(&s[(y * 7 + 3) * bs + x * ncols_interleaved], sumf_3, ncols_interleaved); + __riscv_vse32_v_f32m4(&s[(y * 7 + 4) * bs + x * ncols_interleaved], sumf_4, ncols_interleaved); + __riscv_vse32_v_f32m4(&s[(y * 7 + 5) * bs + x * ncols_interleaved], sumf_5, ncols_interleaved); + __riscv_vse32_v_f32m4(&s[(y * 7 + 6) * bs + x * ncols_interleaved], sumf_6, ncols_interleaved); + } + } + return; +} + +void ggml_gemm_f16_7x1x16_f16(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { +#if defined __riscv_v_intrinsic + ggml_gemm_f16_7x1xM_f16<16>(n, s, bs, vx, vy, nr, nc); + return; +#endif + ggml_gemm_f16_7x1x16_f16_generic(n, s, bs, vx, vy, nr, nc); +} + +void ggml_gemm_f16_7x1x32_f16(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { +#if defined __riscv_v_intrinsic + ggml_gemm_f16_7x1xM_f16<32>(n, s, bs, vx, vy, nr, nc); + return; +#endif + ggml_gemm_f16_7x1x32_f16_generic(n, s, bs, vx, vy, nr, nc); +} + +void ggml_gemm_f16_7x1x64_f16(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { +#if defined __riscv_v_intrinsic + ggml_gemm_f16_7x1xM_f16<64>(n, s, bs, vx, vy, nr, nc); + return; +#endif + ggml_gemm_f16_7x1x64_f16_generic(n, s, bs, vx, vy, nr, nc); +} + +void ggml_gemm_f16_7x1x128_f16(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { +#if defined __riscv_v_intrinsic + ggml_gemm_f16_7x1xM_f16<128>(n, s, bs, vx, vy, nr, nc); + return; +#endif + ggml_gemm_f16_7x1x128_f16_generic(n, s, bs, vx, vy, nr, nc); +} + +template +static inline void ggml_gemm_f32_7x1xM_f32(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { + const int nb = n / 1; + + assert (nr % 7 == 0); + assert(n % 1 == 0); + assert(nc % ncols_interleaved == 0); + + for (int y = 0; y < nr / 7; y++) { + const block_f32_7x1 * a_ptr = (const block_f32_7x1*) vy + (y * nb); + for (int x = 0; x < nc / ncols_interleaved; x++) { + const block_f32 * b_ptr = (const block_f32 *) vx + (x * nb); + + // Accumulators + vfloat32m4_t sumf_0 = __riscv_vfmv_v_f_f32m4(0.0f, ncols_interleaved); + vfloat32m4_t sumf_1 = __riscv_vfmv_v_f_f32m4(0.0f, ncols_interleaved); + vfloat32m4_t sumf_2 = __riscv_vfmv_v_f_f32m4(0.0f, ncols_interleaved); + vfloat32m4_t sumf_3 = __riscv_vfmv_v_f_f32m4(0.0f, ncols_interleaved); + vfloat32m4_t sumf_4 = __riscv_vfmv_v_f_f32m4(0.0f, ncols_interleaved); + vfloat32m4_t sumf_5 = __riscv_vfmv_v_f_f32m4(0.0f, ncols_interleaved); + vfloat32m4_t sumf_6 = __riscv_vfmv_v_f_f32m4(0.0f, ncols_interleaved); + + for (int l = 0; l < nb; l++) { + vfloat32m4_t b_0 = __riscv_vle32_v_f32m4((const float*)&b_ptr[l].d[0], ncols_interleaved); + + sumf_0 = __riscv_vfmacc_vf_f32m4(sumf_0, *(const float*)&a_ptr[l].d[0], b_0, ncols_interleaved); + sumf_1 = __riscv_vfmacc_vf_f32m4(sumf_1, *(const float*)&a_ptr[l].d[1], b_0, ncols_interleaved); + sumf_2 = __riscv_vfmacc_vf_f32m4(sumf_2, *(const float*)&a_ptr[l].d[2], b_0, ncols_interleaved); + sumf_3 = __riscv_vfmacc_vf_f32m4(sumf_3, *(const float*)&a_ptr[l].d[3], b_0, ncols_interleaved); + sumf_4 = __riscv_vfmacc_vf_f32m4(sumf_4, *(const float*)&a_ptr[l].d[4], b_0, ncols_interleaved); + sumf_5 = __riscv_vfmacc_vf_f32m4(sumf_5, *(const float*)&a_ptr[l].d[5], b_0, ncols_interleaved); + sumf_6 = __riscv_vfmacc_vf_f32m4(sumf_6, *(const float*)&a_ptr[l].d[6], b_0, ncols_interleaved); + } + + __riscv_vse32_v_f32m4(&s[(y * 7 + 0) * bs + x * ncols_interleaved], sumf_0, ncols_interleaved); + __riscv_vse32_v_f32m4(&s[(y * 7 + 1) * bs + x * ncols_interleaved], sumf_1, ncols_interleaved); + __riscv_vse32_v_f32m4(&s[(y * 7 + 2) * bs + x * ncols_interleaved], sumf_2, ncols_interleaved); + __riscv_vse32_v_f32m4(&s[(y * 7 + 3) * bs + x * ncols_interleaved], sumf_3, ncols_interleaved); + __riscv_vse32_v_f32m4(&s[(y * 7 + 4) * bs + x * ncols_interleaved], sumf_4, ncols_interleaved); + __riscv_vse32_v_f32m4(&s[(y * 7 + 5) * bs + x * ncols_interleaved], sumf_5, ncols_interleaved); + __riscv_vse32_v_f32m4(&s[(y * 7 + 6) * bs + x * ncols_interleaved], sumf_6, ncols_interleaved); + } + } + return; +} + +void ggml_gemm_f32_7x1x16_f32(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { +#if defined __riscv_v_intrinsic + ggml_gemm_f32_7x1xM_f32<16>(n, s, bs, vx, vy, nr, nc); + return; +#endif + ggml_gemm_f32_7x1x16_f32_generic(n, s, bs, vx, vy, nr, nc); +} + +void ggml_gemm_f32_7x1x32_f32(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { +#if defined __riscv_v_intrinsic + ggml_gemm_f32_7x1xM_f32<32>(n, s, bs, vx, vy, nr, nc); + return; +#endif + ggml_gemm_f32_7x1x32_f32_generic(n, s, bs, vx, vy, nr, nc); +} + +void ggml_gemm_f32_7x1x64_f32(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { +#if defined __riscv_v_intrinsic + ggml_gemm_f32_7x1xM_f32<64>(n, s, bs, vx, vy, nr, nc); + return; +#endif + ggml_gemm_f32_7x1x64_f32_generic(n, s, bs, vx, vy, nr, nc); +} + +void ggml_gemm_f32_7x1x128_f32(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { +#if defined __riscv_v_intrinsic + ggml_gemm_f32_7x1xM_f32<128>(n, s, bs, vx, vy, nr, nc); + return; +#endif + ggml_gemm_f32_7x1x128_f32_generic(n, s, bs, vx, vy, nr, nc); +} diff --git a/ggml/src/ggml-cpu/repack.cpp b/ggml/src/ggml-cpu/repack.cpp index 9f0d449bd6..0cc5b3221c 100644 --- a/ggml/src/ggml-cpu/repack.cpp +++ b/ggml/src/ggml-cpu/repack.cpp @@ -31,6 +31,40 @@ static inline int nearest_int(float fval) { return (i & 0x007fffff) - 0x00400000; } +// Helper template functions for `fp16` and `fp32`. + +template +static inline void ggml_repack_mat_f16_NxK_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) { + assert(k % interleave_size == 0); + const int nb = k / interleave_size; + + block_f16 * GGML_RESTRICT y = (block_f16 *) vy; + + for (int i = 0; i < nb; i++) { + for (int j = 0; j < nrows_interleaved; j++) { + for (int l = 0; l < interleave_size; l++) { + y[i].d[j * interleave_size + l] = GGML_CPU_FP32_TO_FP16(x[j * k + i * interleave_size + l]); + } + } + } +} + +template +static inline void ggml_repack_mat_f32_NxK_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) { + assert(k % interleave_size == 0); + const int nb = k / interleave_size; + + block_f32 * GGML_RESTRICT y = (block_f32 *) vy; + + for (int i = 0; i < nb; i++) { + for (int j = 0; j < nrows_interleaved; j++) { + for (int l = 0; l < interleave_size; l++) { + y[i].d[j * interleave_size + l] = x[j * k + i * interleave_size + l]; + } + } + } +} + // Functions to create the interleaved data layout formats // interleave 4 block_q4_0s in blocks of blck_size_interleave @@ -46,6 +80,7 @@ static inline int nearest_int(float fval) { // operations durin unpacking) // + extern "C" { void ggml_quantize_mat_q8_0_4x4_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) { @@ -227,35 +262,189 @@ void ggml_quantize_mat_q8_K_4x8_generic(const float * GGML_RESTRICT x, void * GG } } +void ggml_repack_mat_f16_7x1_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) { + ggml_repack_mat_f16_NxK_generic<7, 1>(x, vy, k); +} + +void ggml_repack_mat_f32_7x1_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) { + ggml_repack_mat_f32_NxK_generic<7, 1>(x, vy, k); +} + } // extern "C" -template -void ggml_quantize_mat_t(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t nrow, int64_t n_per_row); +template +void ggml_repack_mat_t(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t nrow, int64_t n_per_row); -template <> void ggml_quantize_mat_t<4, GGML_TYPE_Q8_0>(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t nrow, int64_t n_per_row) { +template <> void ggml_repack_mat_t<4, 4, GGML_TYPE_Q8_0>(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t nrow, int64_t n_per_row) { assert(nrow == 4); UNUSED(nrow); ggml_quantize_mat_q8_0_4x4(x, vy, n_per_row); } -template <> void ggml_quantize_mat_t<8, GGML_TYPE_Q8_0>(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t nrow, int64_t n_per_row) { +template <> void ggml_repack_mat_t<4, 8, GGML_TYPE_Q8_0>(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t nrow, int64_t n_per_row) { assert(nrow == 4); UNUSED(nrow); ggml_quantize_mat_q8_0_4x8(x, vy, n_per_row); } -template <> void ggml_quantize_mat_t<4, GGML_TYPE_Q8_K>(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t nrow, int64_t n_per_row) { +template <> void ggml_repack_mat_t<4, 4, GGML_TYPE_Q8_K>(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t nrow, int64_t n_per_row) { assert(nrow == 4); UNUSED(nrow); ggml_quantize_mat_q8_K_4x4(x, vy, n_per_row); } -template <> void ggml_quantize_mat_t<8, GGML_TYPE_Q8_K>(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t nrow, int64_t n_per_row) { +template <> void ggml_repack_mat_t<4, 8, GGML_TYPE_Q8_K>(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t nrow, int64_t n_per_row) { assert(nrow == 4); UNUSED(nrow); ggml_quantize_mat_q8_K_4x8(x, vy, n_per_row); } +template <> void ggml_repack_mat_t<7, 1, GGML_TYPE_F16>(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t nrow, int64_t n_per_row) { + assert(nrow == 7); + UNUSED(nrow); + ggml_repack_mat_f16_7x1(x, vy, n_per_row); +} + +template <> void ggml_repack_mat_t<7, 1, GGML_TYPE_F32>(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t nrow, int64_t n_per_row) { + assert(nrow == 7); + UNUSED(nrow); + ggml_repack_mat_f32_7x1(x, vy, n_per_row); +} + +template +static inline void ggml_gemv_f16_KxM_f16_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { + const int nb = n / interleave_size; + + assert(nr == 1); + assert(n % interleave_size == 0); + assert(nc % ncols_interleaved == 0); + + UNUSED(bs); + UNUSED(nr); + + float sumf[ncols_interleaved]; + + const ggml_half * a_ptr = (const ggml_half *) vy; + for (int x = 0; x < nc / ncols_interleaved; x++) { + const block_f16 * b_ptr = + (const block_f16 *) vx + (x * nb); + + for (int j = 0; j < ncols_interleaved; j++) { sumf[j] = 0.0f; } + for (int l = 0; l < nb; l++) { + for (int j = 0; j < ncols_interleaved; j++) { + for (int k = 0; k < interleave_size; k++) { + sumf[j] += GGML_FP16_TO_FP32(b_ptr[l].d[j * interleave_size + k]) * GGML_FP16_TO_FP32(a_ptr[l + k]); + } + } + } + for (int j = 0; j < ncols_interleaved; j++) { s[x * ncols_interleaved + j] = sumf[j]; } + } +} + +template +static inline void ggml_gemv_f32_KxM_f32_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { + const int nb = n / interleave_size; + + assert(nr == 1); + assert(n % interleave_size == 0); + assert(nc % ncols_interleaved == 0); + + UNUSED(bs); + UNUSED(nr); + + float sumf[ncols_interleaved]; + + const float * a_ptr = (const float *) vy; + for (int x = 0; x < nc / ncols_interleaved; x++) { + const block_f32 * b_ptr = + (const block_f32 *) vx + (x * nb); + + for (int j = 0; j < ncols_interleaved; j++) { sumf[j] = 0.0f; } + for (int l = 0; l < nb; l++) { + for (int j = 0; j < ncols_interleaved; j++) { + for (int k = 0; k < interleave_size; k++) { + sumf[j] += b_ptr[l].d[j * interleave_size + k] * a_ptr[l + k]; + } + } + } + for (int j = 0; j < ncols_interleaved; j++) { s[x * ncols_interleaved + j] = sumf[j]; } + } +} + +template +static inline void ggml_gemm_f16_NxKxM_f16_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { + const int nb = n / interleave_size; + + assert (nr % nrows == 0); + assert(n % interleave_size == 0); + assert(nc % ncols_interleaved == 0); + + float sumf[nrows][ncols_interleaved]; + + for (int y = 0; y < nr / nrows; y++) { + const block_f16 * a_ptr = + (const block_f16 *) vy + (y * nb); + for (int x = 0; x < nc / ncols_interleaved; x++) { + const block_f16 * b_ptr = + (const block_f16 *) vx + (x * nb); + + for (int m = 0; m < nrows; m++) { + for (int j = 0; j < ncols_interleaved; j++) { sumf[m][j] = 0.0f; } + } + for (int l = 0; l < nb; l++) { + for (int m = 0; m < nrows; m++) { + for (int j = 0; j < ncols_interleaved; j++) { + for (int k = 0; k < interleave_size; k++) { + sumf[m][j] += b_ptr[l].d[j * interleave_size + k] * a_ptr[l].d[m * interleave_size + k]; + } + } + } + } + for (int m = 0; m < nrows; m++) { + for (int j = 0; j < ncols_interleaved; j++) + { s[(y * nrows + m) * bs + x * ncols_interleaved + j] = sumf[m][j]; } + } + } + } +} + +template +static inline void ggml_gemm_f32_NxKxM_f32_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { + const int nb = n / interleave_size; + + assert (nr % nrows == 0); + assert(n % interleave_size == 0); + assert(nc % ncols_interleaved == 0); + + float sumf[nrows][ncols_interleaved]; + + for (int y = 0; y < nr / nrows; y++) { + const block_f32 * a_ptr = + (const block_f32 *) vy + (y * nb); + for (int x = 0; x < nc / ncols_interleaved; x++) { + const block_f32 * b_ptr = + (const block_f32 *) vx + (x * nb); + + for (int m = 0; m < nrows; m++) { + for (int j = 0; j < ncols_interleaved; j++) { sumf[m][j] = 0.0f; } + } + for (int l = 0; l < nb; l++) { + for (int m = 0; m < nrows; m++) { + for (int j = 0; j < ncols_interleaved; j++) { + for (int k = 0; k < interleave_size; k++) { + sumf[m][j] += b_ptr[l].d[j * interleave_size + k] * a_ptr[l].d[m * interleave_size + k]; + } + } + } + } + for (int m = 0; m < nrows; m++) { + for (int j = 0; j < ncols_interleaved; j++) + { s[(y * nrows + m) * bs + x * ncols_interleaved + j] = sumf[m][j]; } + } + } + } +} + extern "C" { void ggml_gemv_q4_0_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { @@ -692,6 +881,39 @@ void ggml_gemv_iq4_nl_8x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs } } + +void ggml_gemv_f16_1x16_f16_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { + ggml_gemv_f16_KxM_f16_generic<1, 16>(n, s, bs, vx, vy, nr, nc); +} + +void ggml_gemv_f16_1x32_f16_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { + ggml_gemv_f16_KxM_f16_generic<1, 32>(n, s, bs, vx, vy, nr, nc); +} + +void ggml_gemv_f16_1x64_f16_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { + ggml_gemv_f16_KxM_f16_generic<1, 64>(n, s, bs, vx, vy, nr, nc); +} + +void ggml_gemv_f16_1x128_f16_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { + ggml_gemv_f16_KxM_f16_generic<1, 128>(n, s, bs, vx, vy, nr, nc); +} + +void ggml_gemv_f32_1x16_f32_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { + ggml_gemv_f32_KxM_f32_generic<1, 16>(n, s, bs, vx, vy, nr, nc); +} + +void ggml_gemv_f32_1x32_f32_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { + ggml_gemv_f32_KxM_f32_generic<1, 32>(n, s, bs, vx, vy, nr, nc); +} + +void ggml_gemv_f32_1x64_f32_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { + ggml_gemv_f32_KxM_f32_generic<1, 64>(n, s, bs, vx, vy, nr, nc); +} + +void ggml_gemv_f32_1x128_f32_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { + ggml_gemv_f32_KxM_f32_generic<1, 128>(n, s, bs, vx, vy, nr, nc); +} + void ggml_gemm_q4_0_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { const int qk = QK8_0; const int nb = n / qk; @@ -1219,6 +1441,38 @@ void ggml_gemm_iq4_nl_8x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs } } +void ggml_gemm_f16_7x1x16_f16_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { + ggml_gemm_f16_NxKxM_f16_generic<7, 1, 16>(n, s, bs, vx, vy, nr, nc); +} + +void ggml_gemm_f16_7x1x32_f16_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { + ggml_gemm_f16_NxKxM_f16_generic<7, 1, 32>(n, s, bs, vx, vy, nr, nc); +} + +void ggml_gemm_f16_7x1x64_f16_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { + ggml_gemm_f16_NxKxM_f16_generic<7, 1, 64>(n, s, bs, vx, vy, nr, nc); +} + +void ggml_gemm_f16_7x1x128_f16_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { + ggml_gemm_f16_NxKxM_f16_generic<7, 1, 128>(n, s, bs, vx, vy, nr, nc); +} + +void ggml_gemm_f32_7x1x16_f32_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { + ggml_gemm_f32_NxKxM_f32_generic<7, 1, 16>(n, s, bs, vx, vy, nr, nc); +} + +void ggml_gemm_f32_7x1x32_f32_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { + ggml_gemm_f32_NxKxM_f32_generic<7, 1, 32>(n, s, bs, vx, vy, nr, nc); +} + +void ggml_gemm_f32_7x1x64_f32_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { + ggml_gemm_f32_NxKxM_f32_generic<7, 1, 64>(n, s, bs, vx, vy, nr, nc); +} + +void ggml_gemm_f32_7x1x128_f32_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { + ggml_gemm_f32_NxKxM_f32_generic<7, 1, 128>(n, s, bs, vx, vy, nr, nc); +} + } // extern "C" static block_q4_0x4 make_block_q4_0x4(block_q4_0 * in, unsigned int blck_size_interleave) { @@ -1659,6 +1913,78 @@ static int repack_iq4_nl_to_iq4_nl_8_bl(struct ggml_tensor * t, int interleave_b GGML_UNUSED(data_size); } +template +static int repack_f16_to_f16_N_bl(struct ggml_tensor * t, const void * GGML_RESTRICT data, size_t data_size) { + GGML_ASSERT(t->type == GGML_TYPE_F16); + + const ggml_half * src = (const ggml_half *)data; + block_f16 * dst = ( block_f16 *)t->data; + + ggml_half dst_tmp[nrows_interleaved * interleave_size]; + + int nrow = ggml_nrows(t); + int row_size = t->ne[0]; + int nblocks = row_size / interleave_size; + + GGML_ASSERT(data_size == nrow * nblocks * interleave_size * sizeof(ggml_half)); + + if (t->ne[1] % nrows_interleaved != 0 || t->ne[0] % interleave_size != 0) { + return -1; + } + + for (int b = 0; b < nrow; b += nrows_interleaved) { + for (int i = 0; i < nblocks; i++) { + for (int j = 0; j < nrows_interleaved; j++) { + for (int k = 0; k < interleave_size; k++) { + dst_tmp[j * interleave_size + k] = src[(j + b) * row_size + i * interleave_size + k]; + } + } + block_f16 out; + memcpy(&out.d, dst_tmp, sizeof(ggml_half) * nrows_interleaved * interleave_size); + *dst = out; + dst++; + } + } + + return 0; +} + +template +static int repack_f32_to_f32_N_bl(struct ggml_tensor * t, const void * GGML_RESTRICT data, size_t data_size) { + GGML_ASSERT(t->type == GGML_TYPE_F32); + + const float * src = (const float *)data; + block_f32 * dst = ( block_f32 *)t->data; + + float dst_tmp[nrows_interleaved * interleave_size]; + + int nrow = ggml_nrows(t); + int row_size = t->ne[0]; + int nblocks = row_size / interleave_size; + + GGML_ASSERT(data_size == nrow * nblocks * interleave_size * sizeof(float)); + + if (t->ne[1] % nrows_interleaved != 0 || t->ne[0] % interleave_size != 0) { + return -1; + } + + for (int b = 0; b < nrow; b += nrows_interleaved) { + for (int i = 0; i < nblocks; i++) { + for (int j = 0; j < nrows_interleaved; j++) { + for (int k = 0; k < interleave_size; k++) { + dst_tmp[j * interleave_size + k] = src[(j + b) * row_size + i * interleave_size + k]; + } + } + block_f32 out; + memcpy(&out.d, dst_tmp, sizeof(float) * nrows_interleaved * interleave_size); + *dst = out; + dst++; + } + } + + return 0; +} + namespace ggml::cpu::repack { // repack template @@ -1702,6 +2028,32 @@ template <> int repack(struct ggml_tensor * t, const void * return repack_iq4_nl_to_iq4_nl_8_bl(t, 8, data, data_size); } +template <> int repack(struct ggml_tensor * t, const void * data, size_t data_size) { + return repack_f16_to_f16_N_bl<16, 1>(t, data, data_size); +} +template <> int repack(struct ggml_tensor * t, const void * data, size_t data_size) { + return repack_f16_to_f16_N_bl<32, 1>(t, data, data_size); +} +template <> int repack(struct ggml_tensor * t, const void * data, size_t data_size) { + return repack_f16_to_f16_N_bl<64, 1>(t, data, data_size); +} +template <> int repack(struct ggml_tensor * t, const void * data, size_t data_size) { + return repack_f16_to_f16_N_bl<128, 1>(t, data, data_size); +} + +template <> int repack(struct ggml_tensor * t, const void * data, size_t data_size) { + return repack_f32_to_f32_N_bl<16, 1>(t, data, data_size); +} +template <> int repack(struct ggml_tensor * t, const void * data, size_t data_size) { + return repack_f32_to_f32_N_bl<32, 1>(t, data, data_size); +} +template <> int repack(struct ggml_tensor * t, const void * data, size_t data_size) { + return repack_f32_to_f32_N_bl<64, 1>(t, data, data_size); +} +template <> int repack(struct ggml_tensor * t, const void * data, size_t data_size) { + return repack_f32_to_f32_N_bl<128, 1>(t, data, data_size); +} + // gemv template void gemv(int, float *, size_t, const void *, const void *, int, int); @@ -1738,48 +2090,112 @@ template <> void gemv(int n, float * s, size ggml_gemv_iq4_nl_8x8_q8_0(n, s, bs, vx, vy, nr, nc); } +template <> void gemv(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) { + ggml_gemv_f16_1x16_f16(n, s, bs, vx, vy, nr, nc); +} + +template <> void gemv(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) { + ggml_gemv_f16_1x32_f16(n, s, bs, vx, vy, nr, nc); +} + +template <> void gemv(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) { + ggml_gemv_f16_1x64_f16(n, s, bs, vx, vy, nr, nc); +} + +template <> void gemv(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) { + ggml_gemv_f16_1x128_f16(n, s, bs, vx, vy, nr, nc); +} + +template <> void gemv(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) { + ggml_gemv_f32_1x16_f32(n, s, bs, vx, vy, nr, nc); +} + +template <> void gemv(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) { + ggml_gemv_f32_1x32_f32(n, s, bs, vx, vy, nr, nc); +} + +template <> void gemv(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) { + ggml_gemv_f32_1x64_f32(n, s, bs, vx, vy, nr, nc); +} + +template <> void gemv(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) { + ggml_gemv_f32_1x128_f32(n, s, bs, vx, vy, nr, nc); +} + // gemm -template +template void gemm(int, float *, size_t, const void *, const void *, int, int); -template <> void gemm(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) { +template <> void gemm(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) { ggml_gemm_q4_0_4x4_q8_0(n, s, bs, vx, vy, nr, nc); } -template <> void gemm(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) { +template <> void gemm(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) { ggml_gemm_q4_0_4x8_q8_0(n, s, bs, vx, vy, nr, nc); } -template <> void gemm(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) { +template <> void gemm(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) { ggml_gemm_q4_K_8x4_q8_K(n, s, bs, vx, vy, nr, nc); } -template <> void gemm(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) { +template <> void gemm(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) { ggml_gemm_q4_0_8x8_q8_0(n, s, bs, vx, vy, nr, nc); } -template <> void gemm(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) { +template <> void gemm(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) { ggml_gemm_q4_K_8x8_q8_K(n, s, bs, vx, vy, nr, nc); } -template <> void gemm(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) { +template <> void gemm(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) { ggml_gemm_q2_K_8x8_q8_K(n, s, bs, vx, vy, nr, nc); } -template <> void gemm(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) { +template <> void gemm(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) { ggml_gemm_iq4_nl_4x4_q8_0(n, s, bs, vx, vy, nr, nc); } -template <> void gemm(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) { +template <> void gemm(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) { ggml_gemm_iq4_nl_8x8_q8_0(n, s, bs, vx, vy, nr, nc); } +template <> void gemm(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) { + ggml_gemm_f16_7x1x16_f16(n, s, bs, vx, vy, nr, nc); +} + +template <> void gemm(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) { + ggml_gemm_f16_7x1x32_f16(n, s, bs, vx, vy, nr, nc); +} + +template <> void gemm(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) { + ggml_gemm_f16_7x1x64_f16(n, s, bs, vx, vy, nr, nc); +} + +template <> void gemm(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) { + ggml_gemm_f16_7x1x128_f16(n, s, bs, vx, vy, nr, nc); +} + +template <> void gemm(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) { + ggml_gemm_f32_7x1x16_f32(n, s, bs, vx, vy, nr, nc); +} + +template <> void gemm(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) { + ggml_gemm_f32_7x1x32_f32(n, s, bs, vx, vy, nr, nc); +} + +template <> void gemm(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) { + ggml_gemm_f32_7x1x64_f32(n, s, bs, vx, vy, nr, nc); +} + +template <> void gemm(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) { + ggml_gemm_f32_7x1x128_f32(n, s, bs, vx, vy, nr, nc); +} + class tensor_traits_base : public ggml::cpu::tensor_traits { public: virtual int repack(struct ggml_tensor * t, const void * data, size_t data_size) = 0; }; -template class tensor_traits : public tensor_traits_base { +template class tensor_traits : public tensor_traits_base { bool work_size(int /* n_threads */, const struct ggml_tensor * op, size_t & size) override { // not realy a GGML_TYPE_Q8_0 but same size. @@ -1862,12 +2278,12 @@ template wdata + params->wsize); // If there are more than three rows in src1, use gemm; otherwise, use gemv. - if (nrows > 3) { - gemm(ne00, (float *) (dst_ptr) + src0_start, nb1 / nb0, + if (nrows > (NB_ROWS - 1)) { + gemm(ne00, (float *) (dst_ptr) + src0_start, nb1 / nb0, src0_ptr + src0_start * nb01, src1_ptr, - nrows - (nrows % 4), ncols); + nrows - (nrows % NB_ROWS), ncols); } - for (int iter = nrows - (nrows % 4); iter < nrows; iter++) { + for (int iter = nrows - (nrows % NB_ROWS); iter < nrows; iter++) { gemv(ne00, (float *) (dst_ptr + (iter * nb1)) + src0_start, ne01, src0_ptr + src0_start * nb01, src1_ptr + (src1_col_stride * iter), 1 /* nrows */, ncols); @@ -1921,12 +2337,12 @@ template data + i12 * nb12; char * wdata_ptr = wdata + i12 * nbw2; - for (int64_t i11 = ith * 4; i11 < ne11 - ne11 % 4; i11 += nth * 4) { - ggml_quantize_mat_t((float *) (data_ptr + i11 * nb11), - (void *) (wdata_ptr + i11 * nbw1), 4, ne10); + for (int64_t i11 = ith * NB_ROWS; i11 < ne11 - ne11 % NB_ROWS; i11 += nth * NB_ROWS) { + ggml_repack_mat_t((float *) (data_ptr + i11 * nb11), + (void *) (wdata_ptr + i11 * nbw1), NB_ROWS, ne10); } - const int64_t i11_processed = ne11 - ne11 % 4; + const int64_t i11_processed = ne11 - ne11 % NB_ROWS; for (int64_t i11 = i11_processed + ith; i11 < ne11; i11 += nth) { from_float((float *) (data_ptr + i11 * nb11), (void *) (wdata_ptr + i11 * nbw1), ne10); } @@ -1938,7 +2354,7 @@ template src[0]); - int nth_scaled = nth * 4; + int nth_scaled = nth * NB_ROWS; int64_t chunk_size0 = (nr0 + nth_scaled - 1) / nth_scaled; int64_t nchunk0 = (nr0 + chunk_size0 - 1) / chunk_size0; @@ -2153,20 +2569,36 @@ template q4_0_4x4_q8_0; - static const ggml::cpu::repack::tensor_traits q4_0_4x8_q8_0; - static const ggml::cpu::repack::tensor_traits q4_0_8x8_q8_0; + static const ggml::cpu::repack::tensor_traits q4_0_4x4_q8_0; + static const ggml::cpu::repack::tensor_traits q4_0_4x8_q8_0; + static const ggml::cpu::repack::tensor_traits q4_0_8x8_q8_0; // instance for Q4_K - static const ggml::cpu::repack::tensor_traits q4_K_8x4_q8_K; - static const ggml::cpu::repack::tensor_traits q4_K_8x8_q8_K; + static const ggml::cpu::repack::tensor_traits q4_K_8x4_q8_K; + static const ggml::cpu::repack::tensor_traits q4_K_8x8_q8_K; // instance for Q2 - static const ggml::cpu::repack::tensor_traits q2_K_8x8_q8_K; + static const ggml::cpu::repack::tensor_traits q2_K_8x8_q8_K; // instance for IQ4 - static const ggml::cpu::repack::tensor_traits iq4_nl_4x4_q8_0; - static const ggml::cpu::repack::tensor_traits iq4_nl_8x8_q8_0; + static const ggml::cpu::repack::tensor_traits iq4_nl_4x4_q8_0; + static const ggml::cpu::repack::tensor_traits iq4_nl_8x8_q8_0; + + // instance for F16 +#if defined __riscv_zvfh + static const ggml::cpu::repack::tensor_traits f16_7x16x1_f16; + static const ggml::cpu::repack::tensor_traits f16_7x32x1_f16; + static const ggml::cpu::repack::tensor_traits f16_7x64x1_f16; + static const ggml::cpu::repack::tensor_traits f16_7x128x1_f16; +#endif + + // instance for F32 +#if defined __riscv_zvfh + static const ggml::cpu::repack::tensor_traits f32_7x16x1_f32; + static const ggml::cpu::repack::tensor_traits f32_7x32x1_f32; + static const ggml::cpu::repack::tensor_traits f32_7x64x1_f32; + static const ggml::cpu::repack::tensor_traits f32_7x128x1_f32; +#endif if (cur->type == GGML_TYPE_Q4_0) { if (ggml_cpu_has_avx2() || (ggml_cpu_has_sve() && ggml_cpu_has_matmul_int8() && ggml_cpu_get_sve_cnt() == QK8_0)) { @@ -2217,6 +2649,30 @@ static const ggml::cpu::tensor_traits * ggml_repack_get_optimal_repack_type(cons return &iq4_nl_4x4_q8_0; } } + } else if (cur->type == GGML_TYPE_F16) { + if (ggml_cpu_has_riscv_v()) { + #if defined __riscv_zvfh + switch (__riscv_vlenb() * 8) { + case 128: { if (cur->ne[1] % 16 == 0) { return &f16_7x16x1_f16; } break; } + case 256: { if (cur->ne[1] % 32 == 0) { return &f16_7x32x1_f16; } break; } + case 512: { if (cur->ne[1] % 64 == 0) { return &f16_7x64x1_f16; } break; } + case 1024: { if (cur->ne[1] % 128 == 0) { return &f16_7x128x1_f16; } break; } + default: return nullptr; + } + #endif + } + } else if (cur->type == GGML_TYPE_F32) { + if (ggml_cpu_has_riscv_v()) { + #if defined __riscv_zvfh + switch (__riscv_vlenb() * 8) { + case 128: { if (cur->ne[1] % 16 == 0) { return &f32_7x16x1_f32; } break; } + case 256: { if (cur->ne[1] % 32 == 0) { return &f32_7x32x1_f32; } break; } + case 512: { if (cur->ne[1] % 64 == 0) { return &f32_7x64x1_f32; } break; } + case 1024: { if (cur->ne[1] % 128 == 0) { return &f32_7x128x1_f32; } break; } + default: return nullptr; + } + #endif + } } return nullptr; diff --git a/ggml/src/ggml-cpu/repack.h b/ggml/src/ggml-cpu/repack.h index c4d928cd15..79dfd28115 100644 --- a/ggml/src/ggml-cpu/repack.h +++ b/ggml/src/ggml-cpu/repack.h @@ -74,6 +74,23 @@ struct block_iq4_nlx8 { static_assert(sizeof(block_iq4_nlx8) == 8 * sizeof(ggml_half) + QK4_NL * 4, "wrong iq4_nlx8 block size/padding"); +template +struct block_f16 { + ggml_half d[N * K]; +}; + +using block_f16_32x1 = block_f16<32, 1>; +using block_f16_7x1 = block_f16<7, 1>; +using block_f16_4x1 = block_f16<4, 1>; + +template +struct block_f32 { + float d[N * K]; +}; + +using block_f32_32x1 = block_f32<32, 1>; +using block_f32_7x1 = block_f32<7, 1>; + #if defined(__cplusplus) extern "C" { #endif @@ -121,6 +138,50 @@ void ggml_gemm_q2_K_8x8_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, void ggml_gemm_iq4_nl_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemm_iq4_nl_8x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); +// FP16 +void ggml_repack_mat_f16_4x1_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k); +void ggml_repack_mat_f16_7x1_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k); +void ggml_gemv_f16_1x16_f16_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); +void ggml_gemv_f16_1x32_f16_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); +void ggml_gemv_f16_1x64_f16_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); +void ggml_gemv_f16_1x128_f16_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); +void ggml_gemm_f16_4x1x32_f16_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); +void ggml_gemm_f16_7x1x16_f16_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); +void ggml_gemm_f16_7x1x32_f16_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); +void ggml_gemm_f16_7x1x64_f16_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); +void ggml_gemm_f16_7x1x128_f16_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); +void ggml_repack_mat_f16_4x1(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k); +void ggml_repack_mat_f16_7x1(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k); +void ggml_gemv_f16_1x16_f16(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); +void ggml_gemv_f16_1x32_f16(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); +void ggml_gemv_f16_1x64_f16(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); +void ggml_gemv_f16_1x128_f16(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); +void ggml_gemm_f16_4x1x32_f16(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); +void ggml_gemm_f16_7x1x16_f16(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); +void ggml_gemm_f16_7x1x32_f16(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); +void ggml_gemm_f16_7x1x64_f16(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); +void ggml_gemm_f16_7x1x128_f16(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); + +// FP32 +void ggml_repack_mat_f32_7x1_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k); +void ggml_gemv_f32_1x16_f32_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); +void ggml_gemv_f32_1x32_f32_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); +void ggml_gemv_f32_1x64_f32_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); +void ggml_gemv_f32_1x128_f32_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); +void ggml_gemm_f32_7x1x16_f32_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); +void ggml_gemm_f32_7x1x32_f32_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); +void ggml_gemm_f32_7x1x64_f32_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); +void ggml_gemm_f32_7x1x128_f32_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); +void ggml_repack_mat_f32_7x1(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k); +void ggml_gemv_f32_1x16_f32(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); +void ggml_gemv_f32_1x32_f32(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); +void ggml_gemv_f32_1x64_f32(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); +void ggml_gemv_f32_1x128_f32(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); +void ggml_gemm_f32_7x1x16_f32(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); +void ggml_gemm_f32_7x1x32_f32(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); +void ggml_gemm_f32_7x1x64_f32(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); +void ggml_gemm_f32_7x1x128_f32(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); + #if defined(__cplusplus) } // extern "C" #endif