From ea27251b560a71e4cad0e7120c5596da3ca1bf47 Mon Sep 17 00:00:00 2001 From: Iwan Kawrakow Date: Wed, 20 Aug 2025 11:54:53 +0300 Subject: [PATCH] q8_k_r16: iq3_s now uses q8_k_r16 on Zen4+ --- ggml/src/iqk/iqk_gemm_iquants.cpp | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/ggml/src/iqk/iqk_gemm_iquants.cpp b/ggml/src/iqk/iqk_gemm_iquants.cpp index 3657ac43..6148637d 100644 --- a/ggml/src/iqk/iqk_gemm_iquants.cpp +++ b/ggml/src/iqk/iqk_gemm_iquants.cpp @@ -2550,14 +2550,21 @@ void iqk_convert_iq3_xxs_q8_0_r8(int n, const void * vx, size_t bx, void * vy, i } void iqk_convert_iq3_s_q8_k_r8(int n, const void * vx, size_t bx, void * vy, int nrc_x) { +#ifdef HAVE_FANCY_SIMD + constexpr int k_nr = 16; + using block_q8_k_r = block_q8_k_r16; +#else + constexpr int k_nr = 8; + using block_q8_k_r = block_q8_k_r8; +#endif GGML_ASSERT(n%QK_K == 0); - GGML_ASSERT(nrc_x%8 == 0); + GGML_ASSERT(nrc_x%k_nr == 0); int nb = n/QK_K; - const block_iq3_s * x8[8]; + const block_iq3_s * x8[k_nr]; - block_q8_k_r8 * y = (block_q8_k_r8 *)vy; + block_q8_k_r * y = (block_q8_k_r *)vy; int16_t ls[16]; SignHelper sh; @@ -2566,10 +2573,10 @@ void iqk_convert_iq3_s_q8_k_r8(int n, const void * vx, size_t bx, void * vy, int uint32_t block[8]; __m256i values[8]; - for (int ix = 0; ix < nrc_x; ix += 8) { - for (int k = 0; k < 8; ++k) x8[k] = (const block_iq3_s *)((const char *)vx + (ix + k)*bx); + for (int ix = 0; ix < nrc_x; ix += k_nr) { + for (int k = 0; k < k_nr; ++k) x8[k] = (const block_iq3_s *)((const char *)vx + (ix + k)*bx); for (int i = 0; i < nb; ++i) { - for (int k = 0; k < 8; ++k) { + for (int k = 0; k < k_nr; ++k) { float d = GGML_FP16_TO_FP32(x8[k][i].d); auto qs = x8[k][i].qs; auto qh = x8[k][i].qh; @@ -2584,9 +2591,15 @@ void iqk_convert_iq3_s_q8_k_r8(int n, const void * vx, size_t bx, void * vy, int ls[2*ib32 + 0] = (2*((x8[k][i].scales[ib32/2] >> 4*(ib32%2)) & 0xf) + 1); ls[2*ib32 + 1] = ls[2*ib32 + 0]; } - float dnew = convert_to_q8_k_r8(k, 1.f/127, values, ls, block, y[i].qs); + float dnew = convert_to_q8_k_r8(k, 1.f/127, values, ls, block, y[i].qs); y[i].d[k] = GGML_FP32_TO_FP16(d*dnew); } +#ifdef HAVE_FANCY_SIMD + for (int l = 0; l < 64; ++l) { + auto v = _mm512_xor_si512(_mm512_loadu_si512((const __m512i *)y[i].qs + l), _mm512_set1_epi8(-128)); + _mm512_storeu_si512((__m512i *)y[i].qs + l, v); + } +#endif } y += nb; }