ikawrakow/ik_llama.cpp
1
0
Fork 0
mirror of https://github.com/ikawrakow/ik_llama.cpp.git synced 2026-02-05 05:50:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

iq1_bn(Metal): 64 -> 66.2 t/s for TG

Browse Source 
This should be good enough. One cannot ask
Apple Silicon to do too much work.
This commit is contained in:
Kawrakow
2024-06-17 11:51:20 +02:00
parent 7cb77d7a67
commit bd266036b6
1 changed files with 13 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
ggml-metal.metal
View File

@@ -5046,8 +5046,10 @@ void kernel_mul_mv_iq1_bn_f32_impl(
d1bn[row] = scale.f;
}
uint16_t aux16;
thread const uint8_t * aux8 = (thread const uint8_t *)&aux16;
//uint32_t aux32;
//thread const uint8_t * aux8 = (thread const uint8_t *)&aux32;
uint32_t aux32[2];
thread const uint8_t * aux8 = (thread const uint8_t *)aux32;
for (int ib32 = ix; ib32 < nb32; ib32 += 8) {
@@ -5069,15 +5071,15 @@ void kernel_mul_mv_iq1_bn_f32_impl(
uint8_t signs = extra[0] >> (8 + 4*ib + ir);
float acc = 0.f;
uint16_t v = iq1bn_grid_u16[ql[0] | ((qh[0] << (8 - 4*(ir%2))) & 0x0f00)];
aux16 = v & 0x0303;
acc += yl[0] * aux8[0] + yl[4] * aux8[1];
aux16 = (v >> 2) & 0x0303;
acc += yl[1] * aux8[0] + yl[5] * aux8[1];
aux16 = (v >> 4) & 0x0303;
acc += yl[2] * aux8[0] + yl[6] * aux8[1];
aux16 = (v >> 6) & 0x0303;
acc += yl[3] * aux8[0] + yl[7] * aux8[1];
uint32_t v = iq1bn_grid_u16[ql[0] | ((qh[0] << (8 - 4*(ir%2))) & 0x0f00)];
uint32_t v32 = v | (v << 12);
//aux32 = v32 & 0x03030303;
//acc += yl[0] * aux8[0] + yl[4] * aux8[1] + yl[2]*aux8[2] + yl[6]*aux8[3];
//aux32 = v32 & 0x0c0c0c0c;
//acc += (yl[1] * aux8[0] + yl[5] * aux8[1] + yl[3]*aux8[2] + yl[7]*aux8[3]) * 0.25f;
aux32[0] = v32 & 0x03030303; aux32[1] = v32 & 0x0c0c0c0c;
acc = yl[0] * aux8[0] + yl[4] * aux8[1] + yl[2]*aux8[2] + yl[6]*aux8[3];
acc += (yl[1] * aux8[4] + yl[5] * aux8[5] + yl[3]*aux8[6] + yl[7]*aux8[7]) * 0.25f;
sumf[row] += (signs & 1 ? sumy-acc : acc-sumy);