diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c index 773871ce..7267d9a7 100644 --- a/ggml/src/ggml.c +++ b/ggml/src/ggml.c @@ -1132,11 +1132,7 @@ static const ggml_type_traits_t type_traits[GGML_TYPE_COUNT] = { .from_float = quantize_row_iq3_xxs, .from_float_ref = (ggml_from_float_t)quantize_row_iq3_xxs_ref, .vec_dot = ggml_vec_dot_iq3_xxs_q8_K, -#ifdef __AVX2__ - .vec_dot_type = GGML_TYPE_Q8_2_X4, -#else .vec_dot_type = GGML_TYPE_Q8_K, -#endif .nrows = 1, .row_meta_size = 0, }, diff --git a/ggml/src/iqk/iqk_gemm_iquants.cpp b/ggml/src/iqk/iqk_gemm_iquants.cpp index 3862fa6a..d51dca66 100644 --- a/ggml/src/iqk/iqk_gemm_iquants.cpp +++ b/ggml/src/iqk/iqk_gemm_iquants.cpp @@ -2460,6 +2460,47 @@ static void mul_mat_iq2_s_q8_2_X4(int n, const void * vx, size_t bx, const DataI } } +void iqk_convert_iq3_xxs_q8_k_r8(int n, const void * vx, size_t bx, void * vy, int nrc_x) { + GGML_ASSERT(n%QK_K == 0); + GGML_ASSERT(nrc_x%8 == 0); + + int nb = n/QK_K; + + const block_iq3_xxs * x8[8]; + + block_q8_k_r8 * y = (block_q8_k_r8 *)vy; + + int16_t ls[16]; + EvenSignHelper esh; + + __m256i values[8]; + uint32_t block[8]; + uint32_t aux32; + + for (int ix = 0; ix < nrc_x; ix += 8) { + for (int k = 0; k < 8; ++k) x8[k] = (const block_iq3_xxs *)((const char *)vx + (ix + k)*bx); + for (int i = 0; i < nb; ++i) { + // TODO: simdify + for (int k = 0; k < 8; ++k) { + float d = 0.25f * GGML_FP16_TO_FP32(x8[k][i].d); + auto qs = x8[k][i].qs; + auto sas = qs + QK_K/4; + for (int ib32 = 0; ib32 < 8; ++ib32) { + std::memcpy(&aux32, sas + 4*ib32, sizeof(uint32_t)); + ls[2*ib32 + 0] = (2*(aux32 >> 28) + 1); + ls[2*ib32 + 1] = ls[2*ib32 + 0]; + values[ib32] = _mm256_set_epi32(iq3xxs_grid[qs[7]], iq3xxs_grid[qs[6]], iq3xxs_grid[qs[5]], iq3xxs_grid[qs[4]], + iq3xxs_grid[qs[3]], iq3xxs_grid[qs[2]], iq3xxs_grid[qs[1]], iq3xxs_grid[qs[0]]); + esh.sign_value(aux32, values[ib32]); + qs += 8; + } + float dnew = convert_to_q8_k_r8(k, 124, values, ls, block, y[i].qs); + y[i].d[k] = GGML_FP32_TO_FP16(d*dnew); + } + } + y += nb; + } +} void iqk_convert_iq3_xxs_q8_0_r8(int n, const void * vx, size_t bx, void * vy, int nrc_x) { GGML_ASSERT(n%QK_K == 0); @@ -2611,14 +2652,14 @@ bool iqk_set_kernels_iquants(int ne00, int typeA, int typeB, std::array= 32 ? GGML_TYPE_Q8_K_R8 : type; case GGML_TYPE_IQ2_XS : return nrc_y >= 32 ? GGML_TYPE_Q8_K_R8 : type; case GGML_TYPE_IQ2_S : return nrc_y >= 16 ? GGML_TYPE_Q8_K_R8 : type; - case GGML_TYPE_IQ3_XXS: return nrc_y >= 32 ? GGML_TYPE_Q8_0_R8 : type; + case GGML_TYPE_IQ3_XXS: return nrc_y >= 32 ? GGML_TYPE_Q8_K_R8 : type; case GGML_TYPE_IQ3_S : return nrc_y >= 32 ? GGML_TYPE_Q8_0_R8 : type; case GGML_TYPE_IQ1_S : return nrc_y >= 32 ? GGML_TYPE_Q8_0_R8 : type; case GGML_TYPE_Q3_K : return nrc_y >= 32 ? GGML_TYPE_Q8_K_R8 : type;