From bdb266baf165e30e1135a8601056fb025946dc4f Mon Sep 17 00:00:00 2001 From: Iwan Kawrakow Date: Wed, 20 Aug 2025 11:49:35 +0300 Subject: [PATCH] q8_k_r16: iq3_xxs now uses q8_k_r16 on Zen4+ --- ggml/src/iqk/iqk_gemm_iquants.cpp | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/ggml/src/iqk/iqk_gemm_iquants.cpp b/ggml/src/iqk/iqk_gemm_iquants.cpp index 476101dc..3657ac43 100644 --- a/ggml/src/iqk/iqk_gemm_iquants.cpp +++ b/ggml/src/iqk/iqk_gemm_iquants.cpp @@ -2444,14 +2444,21 @@ static void mul_mat_iq2_s_q8_2_X4(int n, const void * vx, size_t bx, const DataI } void iqk_convert_iq3_xxs_q8_k_r8(int n, const void * vx, size_t bx, void * vy, int nrc_x) { +#ifdef HAVE_FANCY_SIMD + constexpr int k_nr = 16; + using block_q8_k_r = block_q8_k_r16; +#else + constexpr int k_nr = 8; + using block_q8_k_r = block_q8_k_r8; +#endif GGML_ASSERT(n%QK_K == 0); - GGML_ASSERT(nrc_x%8 == 0); + GGML_ASSERT(nrc_x%k_nr == 0); int nb = n/QK_K; - const block_iq3_xxs * x8[8]; + const block_iq3_xxs * x8[k_nr]; - block_q8_k_r8 * y = (block_q8_k_r8 *)vy; + block_q8_k_r * y = (block_q8_k_r *)vy; int16_t ls[16]; EvenSignHelper esh; @@ -2460,10 +2467,10 @@ void iqk_convert_iq3_xxs_q8_k_r8(int n, const void * vx, size_t bx, void * vy, i uint32_t block[8]; uint32_t aux32; - for (int ix = 0; ix < nrc_x; ix += 8) { - for (int k = 0; k < 8; ++k) x8[k] = (const block_iq3_xxs *)((const char *)vx + (ix + k)*bx); + for (int ix = 0; ix < nrc_x; ix += k_nr) { + for (int k = 0; k < k_nr; ++k) x8[k] = (const block_iq3_xxs *)((const char *)vx + (ix + k)*bx); for (int i = 0; i < nb; ++i) { - for (int k = 0; k < 8; ++k) { + for (int k = 0; k < k_nr; ++k) { float d = 0.25f * GGML_FP16_TO_FP32(x8[k][i].d); auto qs = x8[k][i].qs; auto sas = qs + QK_K/4; @@ -2476,9 +2483,15 @@ void iqk_convert_iq3_xxs_q8_k_r8(int n, const void * vx, size_t bx, void * vy, i esh.sign_value(aux32, values[ib32]); qs += 8; } - float dnew = convert_to_q8_k_r8(k, 1.f/124, values, ls, block, y[i].qs); + float dnew = convert_to_q8_k_r8(k, 1.f/124, values, ls, block, y[i].qs); y[i].d[k] = GGML_FP32_TO_FP16(d*dnew); } +#ifdef HAVE_FANCY_SIMD + for (int l = 0; l < 64; ++l) { + auto v = _mm512_xor_si512(_mm512_loadu_si512((const __m512i *)y[i].qs + l), _mm512_set1_epi8(-128)); + _mm512_storeu_si512((__m512i *)y[i].qs + l, v); + } +#endif } y += nb; }