ikawrakow/ik_llama.cpp
1
0
Fork 0
mirror of https://github.com/ikawrakow/ik_llama.cpp.git synced 2026-04-30 19:31:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix q4_0_r8

Browse Source
This commit is contained in:
Iwan Kawrakow
2025-11-13 12:12:53 +02:00
parent 742328bfcd
commit 64126305da
1 changed files with 9 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
ggml/src/iqk/iqk_gemm_legacy_quants.cpp
View File

@@ -1003,11 +1003,10 @@ inline __m256i accum_q4_0_quants(const __m256i * v, const int8_t * qs) {
template <int nrc_y> template <int nrc_y>
static void mul_mat_q4_0_r8_q8_2_avx2(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { static void mul_mat_q4_0_r8_q8_2_avx2(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) {
GGML_ASSERT(nrc_x%8 == 0); GGML_ASSERT(nrc_x%8 == 0);
Q8<nrc_y, block_q8_1_x4> q8(info); Q8<nrc_y, block_q8_2_x4> q8(info);
auto m4 = _mm256_set1_epi8(0xf); auto m4 = _mm256_set1_epi8(0xf);
int nb = n / QK4_NL; int nb = n / QK4_NL;
__m256i v[8]; __m256i v[8];
GGML_ASSERT(nb%4 == 0);
if constexpr (nrc_y == 1) { if constexpr (nrc_y == 1) {
union { __m256 vec; float val[8]; } helper; union { __m256 vec; float val[8]; } helper;
for (int ix = 0; ix < nrc_x; ix += 8) { for (int ix = 0; ix < nrc_x; ix += 8) {
@@ -1026,14 +1025,14 @@ static void mul_mat_q4_0_r8_q8_2_avx2(int n, const void * vx, size_t bx, const D
} }
} }
for (int ib = 4*(nb/4); ib < nb; ++ib) { for (int ib = 4*(nb/4); ib < nb; ++ib) {
auto qy = (const block_q8_1 *)q8.y[0]; auto qy = (const block_q8_2 *)q8.y[0];
auto scales = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)iq4[ib].d)); auto scales = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)iq4[ib].d));
prepare_q4_0_quants_avx2(iq4[ib].qs, v, m4); prepare_q4_0_quants_avx2(iq4[ib].qs, v, m4);
auto sumi = accum_q4_0_quants(v, qy[ib].qs); auto sumi = accum_q4_0_quants(v, qy[ib].qs);
ggml_bf16_t d{qy[ib].d}, s{qy[ib].s}; auto [d8, m8] = ScaleHelperQ8_2::prepare1(qy + ib);
auto d4d8 = _mm256_mul_ps(scales, _mm256_set1_ps(GGML_BF16_TO_FP32(d))); auto d4d8 = _mm256_mul_ps(scales, _mm256_set1_ps(d8));
acc1 = _mm256_fmadd_ps(d4d8, _mm256_cvtepi32_ps(sumi), acc1); acc1 = _mm256_fmadd_ps(d4d8, _mm256_cvtepi32_ps(sumi), acc1);
acc2 = _mm256_fmadd_ps(scales, _mm256_set1_ps(GGML_BF16_TO_FP32(s)), acc2); acc2 = _mm256_fmadd_ps(scales, _mm256_set1_ps(m8), acc2);
} }
acc1 = _mm256_fmadd_ps(acc2, _mm256_set1_ps(-8.f), acc1); acc1 = _mm256_fmadd_ps(acc2, _mm256_set1_ps(-8.f), acc1);
info.store(ix, 0, acc1); info.store(ix, 0, acc1);
@@ -1077,12 +1076,12 @@ static void mul_mat_q4_0_r8_q8_2_avx2(int n, const void * vx, size_t bx, const D
auto scales_m = _mm256_mul_ps(scales, _mm256_set1_ps(-8.f)); auto scales_m = _mm256_mul_ps(scales, _mm256_set1_ps(-8.f));
prepare_q4_0_quants_avx2(iq4[ib].qs, v, m4); prepare_q4_0_quants_avx2(iq4[ib].qs, v, m4);
for (int iy = 0; iy < nrc_y; ++iy) { for (int iy = 0; iy < nrc_y; ++iy) {
auto qy = (const block_q8_1 *)q8.y[iy]; auto qy = (const block_q8_2 *)q8.y[iy];
auto sumi = accum_q4_0_quants(v, qy[ib].qs); auto sumi = accum_q4_0_quants(v, qy[ib].qs);
ggml_bf16_t d{qy[ib].d}, s{qy[ib].s}; auto [d8, m8] = ScaleHelperQ8_2::prepare1(qy + ib);
auto d4d8 = _mm256_mul_ps(scales, _mm256_set1_ps(GGML_BF16_TO_FP32(d))); auto d4d8 = _mm256_mul_ps(scales, _mm256_set1_ps(d8));
acc[iy] = _mm256_fmadd_ps(d4d8, _mm256_cvtepi32_ps(sumi), acc[iy]); acc[iy] = _mm256_fmadd_ps(d4d8, _mm256_cvtepi32_ps(sumi), acc[iy]);
acc[iy] = _mm256_fmadd_ps(scales_m, _mm256_set1_ps(GGML_BF16_TO_FP32(s)), acc[iy]); acc[iy] = _mm256_fmadd_ps(scales_m, _mm256_set1_ps(m8), acc[iy]);
} }
} }
for (int iy = 0; iy < nrc_y; ++iy) { for (int iy = 0; iy < nrc_y; ++iy) {