ikawrakow/ik_llama.cpp
1
0
Fork 0
mirror of https://github.com/ikawrakow/ik_llama.cpp.git synced 2026-04-30 11:21:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

New iq4_kt: slightly faster NEON

Browse Source
This commit is contained in:
Iwan Kawrakow
2025-06-08 08:43:02 +03:00
parent 68ef8a7ae9
commit dcb464a4cb
1 changed files with 22 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
ggml/src/iqk/iqk_gemm_ktquants.cpp
View File

@@ -1237,17 +1237,32 @@ struct Trellis3 {
// return result; // return result;
//} //}
// This works: // This works:
//inline int8x16x2_t next32(const uint32_t * val) const {
// uint16x8_t aux[4];
// for (int i = 0; i < 4; ++i) {
// auto i8 = next8(val[2*i+0], val[2*i+1]);
// i8.val[0] = vandq_u32(i8.val[0], vdupq_n_u32(0x3f3f3f3f));
// i8.val[1] = vandq_u32(i8.val[1], vdupq_n_u32(0x3f3f3f3f));
// auto s1 = vdotq_s32(vdupq_n_s32(-126), vdupq_n_s8(1), vreinterpretq_s8_u32(i8.val[0]));
// auto s2 = vdotq_s32(vdupq_n_s32(-126), vdupq_n_s8(1), vreinterpretq_s8_u32(i8.val[1]));
// aux[i] = vcombine_s16(vmovn_s32(s1), vmovn_s32(s2));
// }
// int8x16x2_t result = {vcombine_s8(vmovn_s16(aux[0]), vmovn_s16(aux[1])), vcombine_s8(vmovn_s16(aux[2]), vmovn_s16(aux[3]))};
// return result;
//}
inline int8x16x2_t next32(const uint32_t * val) const { inline int8x16x2_t next32(const uint32_t * val) const {
uint16x8_t aux[4]; int8x16x2_t result = {vdupq_n_s8(-126), vdupq_n_s8(-126)};
for (int i = 0; i < 4; ++i) { for (int i = 0; i < 2; ++i) {
auto i8 = next8(val[2*i+0], val[2*i+1]); auto i8 = next8(val[4*i+0], val[4*i+1]);
i8.val[0] = vandq_u32(i8.val[0], vdupq_n_u32(0x3f3f3f3f)); i8.val[0] = vandq_u32(i8.val[0], vdupq_n_u32(0x3f3f3f3f));
i8.val[1] = vandq_u32(i8.val[1], vdupq_n_u32(0x3f3f3f3f)); i8.val[1] = vandq_u32(i8.val[1], vdupq_n_u32(0x3f3f3f3f));
auto s1 = vdotq_s32(vdupq_n_s32(-126), vdupq_n_s8(1), vreinterpretq_s8_u32(i8.val[0])); auto s1 = vpaddq_s8(vreinterpretq_s8_u32(i8.val[0]), vreinterpretq_s8_u32(i8.val[1]));
auto s2 = vdotq_s32(vdupq_n_s32(-126), vdupq_n_s8(1), vreinterpretq_s8_u32(i8.val[1])); i8 = next8(val[4*i+2], val[4*i+3]);
aux[i] = vcombine_s16(vmovn_s32(s1), vmovn_s32(s2)); i8.val[0] = vandq_u32(i8.val[0], vdupq_n_u32(0x3f3f3f3f));
i8.val[1] = vandq_u32(i8.val[1], vdupq_n_u32(0x3f3f3f3f));
auto s2 = vpaddq_s8(vreinterpretq_s8_u32(i8.val[0]), vreinterpretq_s8_u32(i8.val[1]));
result.val[i] = vaddq_s8(result.val[i], vpaddq_s8(s1, s2));
} }
int8x16x2_t result = {vcombine_s8(vmovn_s16(aux[0]), vmovn_s16(aux[1])), vcombine_s8(vmovn_s16(aux[2]), vmovn_s16(aux[3]))};
return result; return result;
} }
static uint8x16_t load_shuffle() { static uint8x16_t load_shuffle() {