ikawrakow/ik_llama.cpp
1
0
Fork 0
mirror of https://github.com/ikawrakow/ik_llama.cpp.git synced 2026-04-30 19:31:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

Adding q6_0: CUDA mmvq works

Browse Source
This commit is contained in:
Iwan Kawrakow
2024-10-02 10:34:24 +03:00
parent 6b5c7c378e
commit 4cdf9b333f
1 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
ggml/src/ggml-cuda/vecdotq.cuh
View File

@@ -58,8 +58,8 @@ template <int vdr> static __device__ __forceinline__ float vec_dot_q6_0_q8_1_imp
#pragma unroll #pragma unroll
for (int i = 0; i < vdr; ++i) { for (int i = 0; i < vdr; ++i) {
const int vi0 = ((vl[i] >> 0) & 0x0F0F0F0F) | ((vh[i/2] << 4) & 0x30303030); const int vi0 = ((vl[i] >> 0) & 0x0F0F0F0F) | ((vh[i] << 4) & 0x30303030);
const int vi1 = ((vl[i] >> 4) & 0x0F0F0F0F) | ((vh[i/2] << 2) & 0x30303030); const int vi1 = ((vl[i] >> 4) & 0x0F0F0F0F) | ((vh[i] << 2) & 0x30303030);
// SIMD dot product of quantized values // SIMD dot product of quantized values
sumi = ggml_cuda_dp4a(vi0, u[2*i+0], sumi); sumi = ggml_cuda_dp4a(vi0, u[2*i+0], sumi);