ikawrakow/ik_llama.cpp
1
0
Fork 0
mirror of https://github.com/ikawrakow/ik_llama.cpp.git synced 2026-02-28 00:54:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

bitnet: fix scalar dot product

Browse Source 
I had forgotten to adjust for the change to q8_K64.
On the M2 I'm getting 10.8 t/s with the scalar version!
This commit is contained in:
Iwan Kawrakow
2024-06-16 16:55:49 +02:00
parent 4fcfcd05d1
commit d1c40ff7e2
1 changed files with 7 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
iqk-quantize.cpp
View File

@@ -401,33 +401,31 @@ void ggml_vec_dot_iq1_bn_q8_K64(int n, float * s, size_t bs, const void * vx, si
#else
uint16_t u = x[0].extra & 0xff;
scale.i = ((((u >> 4) | 0xf0) - 132) << 23) | ((u & 0x0f) << 19);
for (int i = 0; i < nblock; ++i) {
uint16_t u = x[i].extra & 0xff;
scale.i = ((((u >> 4) | 0xf0) - 132) << 23) | ((u & 0x0f) << 19);
uint8_t extra = x[i].extra >> 8;
auto qh = x[i].qh;
auto ql = x[i].ql;
auto q8 = y[2*i+0].qs;
int16_t sumi1 = 0;
auto q8 = y[i].qs;
int sumi = 0;
for (int k = 0; k < 4; ++k) {
uint16_t idx = ql[k] | ((qh[k/2] << (8 - 4*(k%2))) & 0x0f00);
uint16_t val = iq1bn_grid_u16[idx];
int16_t sl = 0;
for (int j = 0; j < 8; ++j) sl += q8[j] * (((val >> 2*j) & 3) - 1);
sumi1 += extra & (1 << k) ? -sl : sl;
sumi += extra & (1 << k) ? -sl : sl;
q8 += 8;
}
q8 = y[2*i+1].qs;
int16_t sumi2 = 0;
for (int k = 4; k < 8; ++k) {
uint16_t idx = ql[k] | ((qh[k/2] << (8 - 4*(k%2))) & 0x0f00);
uint16_t val = iq1bn_grid_u16[idx];
int16_t sl = 0;
for (int j = 0; j < 8; ++j) sl += q8[j] * (((val >> 2*j) & 3) - 1);
sumi2 += extra & (1 << k) ? -sl : sl;
sumi += extra & (1 << k) ? -sl : sl;
q8 += 8;
}
sumf += scale.f * (GGML_FP16_TO_FP32(y[2*i+0].d) * sumi1 + GGML_FP16_TO_FP32(y[2*i+1].d) * sumi2);
sumf += scale.f * (y[i].d) * sumi;
}
#endif