From a75a790e8ffbc4c34d2e07c623a1bba9497961c7 Mon Sep 17 00:00:00 2001 From: Iwan Kawrakow Date: Tue, 17 Dec 2024 10:06:02 +0100 Subject: [PATCH] iq2_k_r4: better matrix x vector multiplication on NEON --- ggml/src/iqk/iqk_mul_mat.cpp | 75 +++++++++++++++++++++--------------- 1 file changed, 44 insertions(+), 31 deletions(-) diff --git a/ggml/src/iqk/iqk_mul_mat.cpp b/ggml/src/iqk/iqk_mul_mat.cpp index 60c94f62..d08491c3 100644 --- a/ggml/src/iqk/iqk_mul_mat.cpp +++ b/ggml/src/iqk/iqk_mul_mat.cpp @@ -9084,11 +9084,19 @@ void mul_mat_iq4_xs_r4_q8_k(int n, const void * vx, size_t bx, const DataInfo& i } } -template +template inline void iq3_4_add_shift(int ibl, const Q8& q8, const int8x16x4_t& i8scales, uint8x16_t extra, - uint8x16_t ms, int32x4_t * isum) { - auto s8_1 = vmulq_s8(i8scales.val[0], vandq_u8(ms, vshlq_n_u8(extra, 2))); - auto s8_2 = vmulq_s8(i8scales.val[1], vandq_u8(ms, extra)); + int32x4_t * isum) { + auto ms = is_iq2k ? vdupq_n_s8(5) : vdupq_n_s8(4); + int8x16_t s8_1, s8_2; + if constexpr (is_iq2k) { + auto m1 = vdupq_n_u8(1); + s8_1 = vmulq_s8(i8scales.val[0], vandq_s8(ms, vceqq_u8(vandq_u8(extra, m1), m1))); extra = vshrq_n_u8(extra, 2); + s8_2 = vmulq_s8(i8scales.val[1], vandq_s8(ms, vceqq_u8(vandq_u8(extra, m1), m1))); extra = vshrq_n_u8(extra, 2); + } else { + s8_1 = vmulq_s8(i8scales.val[0], vandq_u8(ms, vshlq_n_u8(extra, 2))); + s8_2 = vmulq_s8(i8scales.val[1], vandq_u8(ms, extra)); + } auto s16_1 = vmovl_s8(vget_low_s8 (s8_1)); auto s16_2 = vmovl_s8(vget_high_s8(s8_1)); auto s16_3 = vmovl_s8(vget_low_s8 (s8_2)); @@ -9105,8 +9113,14 @@ inline void iq3_4_add_shift(int ibl, const Q8& q8, const int8 isum[iy] = vmlal_lane_s16(isum[iy], vget_low_s16 (s16_4), b8, 2); isum[iy] = vmlal_lane_s16(isum[iy], vget_high_s16(s16_4), b8, 3); } - s8_1 = vmulq_s8(i8scales.val[2], vandq_u8(ms, vshrq_n_u8(extra, 2))); - s8_2 = vmulq_s8(i8scales.val[3], vandq_u8(ms, vshrq_n_u8(extra, 4))); + if constexpr (is_iq2k) { + auto m1 = vdupq_n_u8(1); + s8_1 = vmulq_s8(i8scales.val[2], vandq_s8(ms, vceqq_u8(vandq_u8(extra, m1), m1))); extra = vshrq_n_u8(extra, 2); + s8_2 = vmulq_s8(i8scales.val[3], vandq_s8(ms, vceqq_u8(vandq_u8(extra, m1), m1))); extra = vshrq_n_u8(extra, 2); + } else { + s8_1 = vmulq_s8(i8scales.val[2], vandq_u8(ms, vshrq_n_u8(extra, 2))); + s8_2 = vmulq_s8(i8scales.val[3], vandq_u8(ms, vshrq_n_u8(extra, 4))); + } s16_1 = vmovl_s8(vget_low_s8 (s8_1)); s16_2 = vmovl_s8(vget_high_s8(s8_1)); s16_3 = vmovl_s8(vget_low_s8 (s8_2)); @@ -9131,7 +9145,6 @@ void mul_mat_iq2_k_r4_q8_k(int n, const void * vx, size_t bx, const DataInfo& in Q8 q8(info); auto m4 = vdupq_n_u8(0xf); auto m03 = vdupq_n_u8(0x03); - //auto ms = nrc_y == 1 ? vdupq_n_u8(4) : vdupq_n_u8(8); auto ms = vdupq_n_u8(4); uint8x16x2_t shift_shuffle = { vreinterpretq_u8_u64(uint64x2_t{0x0101010100000000, 0x0303030302020202}), @@ -9150,20 +9163,20 @@ void mul_mat_iq2_k_r4_q8_k(int n, const void * vx, size_t bx, const DataInfo& in auto d4 = vcvt_f32_f16(vld1_f16((const float16_t *)iq2[ibl].d)); auto extra8 = vld1_u8(iq2[ibl].extra); uint8x16_t extra; - //if constexpr (nrc_y == 1) { + if constexpr (nrc_y == 1) { extra = vcombine_u8(extra8, vshr_n_u8(extra8,1)); - //} else { - // extra = vcombine_u8(extra8, extra8); - //} + } else { + extra = vcombine_u8(extra8, extra8); + } auto sl = vld1q_u8_x2(iq2[ibl].scales); i8scales.val[0] = vaddq_s8(vandq_u8(sl.val[0], m4), vdupq_n_s8(-8)); i8scales.val[1] = vaddq_s8(vandq_u8(sl.val[1], m4), vdupq_n_s8(-8)); i8scales.val[2] = vaddq_s8(vshrq_n_u8(sl.val[0], 4), vdupq_n_s8(-8)); i8scales.val[3] = vaddq_s8(vshrq_n_u8(sl.val[1], 4), vdupq_n_s8(-8)); int32x4_t isum[nrc_y] = {}; - //if constexpr (nrc_y == 1) { - // iq3_4_add_shift(ibl, q8, i8scales, extra, ms, isum); - //} + if constexpr (nrc_y == 1) { + iq3_4_add_shift(ibl, q8, i8scales, extra, isum); + } for (int is = 0; is < 2; ++is) { i16scales.val[0] = vmovl_s8(vget_low_s8 (i8scales.val[2*is+0])); i16scales.val[1] = vmovl_s8(vget_high_s8(i8scales.val[2*is+0])); @@ -9177,12 +9190,12 @@ void mul_mat_iq2_k_r4_q8_k(int n, const void * vx, size_t bx, const DataInfo& in qx[2] = vandq_u8(vshrq_n_u8(bits.val[0], 4), m03); qx[3] = vandq_u8(vshrq_n_u8(bits.val[0], 6), m03); uint8x16_t shifts; - //if constexpr (nrc_y == 1) { - // qx[0] = vqtbl1q_s8(values, qx[0]); // 0...3 from the 4 rows - // qx[1] = vqtbl1q_s8(values, qx[1]); // 4...7 - // qx[2] = vqtbl1q_s8(values, qx[2]); // 8..11 - // qx[3] = vqtbl1q_s8(values, qx[3]); // 12..15 - //} else { + if constexpr (nrc_y == 1) { + qx[0] = vqtbl1q_s8(values, qx[0]); // 0...3 from the 4 rows + qx[1] = vqtbl1q_s8(values, qx[1]); // 4...7 + qx[2] = vqtbl1q_s8(values, qx[2]); // 8..11 + qx[3] = vqtbl1q_s8(values, qx[3]); // 12..15 + } else { shifts = vandq_u8(ms, vshlq_n_u8(extra, 2)); auto shift = vqtbl1q_u8(shifts, shift_shuffle.val[0]); extra = vshrq_n_u8(extra, 1); @@ -9190,7 +9203,7 @@ void mul_mat_iq2_k_r4_q8_k(int n, const void * vx, size_t bx, const DataInfo& in qx[1] = vqtbl1q_s8(values, vaddq_u8(shift, qx[1])); // 4...7 qx[2] = vqtbl1q_s8(values, vaddq_u8(shift, qx[2])); // 8..11 qx[3] = vqtbl1q_s8(values, vaddq_u8(shift, qx[3])); // 12..15 - //} + } for (int iy = 0; iy < nrc_y; ++iy) { auto y = vld1q_s8(q8.y[iy][ibl].qs+128*is+32*ib); auto sumi = interleaved_dotq(qx, y); @@ -9200,18 +9213,18 @@ void mul_mat_iq2_k_r4_q8_k(int n, const void * vx, size_t bx, const DataInfo& in qx[1] = vandq_u8(vshrq_n_u8(bits.val[1], 2), m03); qx[2] = vandq_u8(vshrq_n_u8(bits.val[1], 4), m03); qx[3] = vandq_u8(vshrq_n_u8(bits.val[1], 6), m03); - //if constexpr (nrc_y == 1) { - // qx[0] = vqtbl1q_s8(values, qx[0]); // 0...3 from the 4 rows - // qx[1] = vqtbl1q_s8(values, qx[1]); // 4...7 - // qx[2] = vqtbl1q_s8(values, qx[2]); // 8..11 - // qx[3] = vqtbl1q_s8(values, qx[3]); // 12..15 - //} else { - shift = vqtbl1q_u8(shifts, shift_shuffle.val[1]); + if constexpr (nrc_y == 1) { + qx[0] = vqtbl1q_s8(values, qx[0]); // 0...3 from the 4 rows + qx[1] = vqtbl1q_s8(values, qx[1]); // 4...7 + qx[2] = vqtbl1q_s8(values, qx[2]); // 8..11 + qx[3] = vqtbl1q_s8(values, qx[3]); // 12..15 + } else { + auto shift = vqtbl1q_u8(shifts, shift_shuffle.val[1]); qx[0] = vqtbl1q_s8(values, vaddq_u8(shift, qx[0])); // 0...3 from the 4 rows qx[1] = vqtbl1q_s8(values, vaddq_u8(shift, qx[1])); // 4...7 qx[2] = vqtbl1q_s8(values, vaddq_u8(shift, qx[2])); // 8..11 qx[3] = vqtbl1q_s8(values, vaddq_u8(shift, qx[3])); // 12..15 - //} + } scales = vmovl_s16(vget_high_s16(i16scales.val[ib])); for (int iy = 0; iy < nrc_y; ++iy) { auto y = vld1q_s8(q8.y[iy][ibl].qs+128*is+32*ib+16); @@ -9275,7 +9288,7 @@ void mul_mat_iq3_k_r4_q8_k(int n, const void * vx, size_t bx, const DataInfo& in i8scales.val[3] = vmulq_s8(i8scales.val[3], vorrq_u8(vceqq_u8(vandq_u8(sh, smask.val[1]), smask.val[1]), vdupq_n_u8(1))); int32x4_t isum[nrc_y] = {}; if constexpr (nrc_y == 1) { - iq3_4_add_shift(ibl, q8, i8scales, extra, ms, isum); + iq3_4_add_shift(ibl, q8, i8scales, extra, isum); } for (int is = 0; is < 2; ++is) { i16scales.val[0] = vmovl_s8(vget_low_s8 (i8scales.val[2*is+0])); @@ -9382,7 +9395,7 @@ void mul_mat_iq4_k_r4_q8_k(int n, const void * vx, size_t bx, const DataInfo& in i8scales.val[3] = vaddq_s8(vorrq_u8(vshrq_n_u8(sl.val[1], 4), vandq_u8(vshrq_n_u8(sh, 2), m3)), m32); int32x4_t isum[nrc_y] = {}; if constexpr (nrc_y == 1) { - iq3_4_add_shift(ibl, q8, i8scales, extra, ms, isum); + iq3_4_add_shift(ibl, q8, i8scales, extra, isum); } for (int is = 0; is < 2; ++is) { i16scales.val[0] = vmovl_s8(vget_low_s8 (i8scales.val[2*is+0]));