ikawrakow/ik_llama.cpp
1
0
Fork 0
mirror of https://github.com/ikawrakow/ik_llama.cpp.git synced 2026-03-05 19:40:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix bug in iqk_mul_mat

Browse Source 
I recently added the possibility to have a matrix multiplication
kernel that processes 16 columns in the right matrix per iteration.
This introduced a bug that shows up when batch size is greater
than 16, is not a multiple of 16, and the remainder is not a multiple
of the maximum columns being processed by the regular kernels
(and so, never showed up in my testing using TG-128 and PP-512).

This commit fixes the issue.
This commit is contained in:
Iwan Kawrakow
2025-01-12 11:00:04 +02:00
parent 7553989dd8
commit 4e7ce22614
2 changed files with 10 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
examples/batched-bench/batched-bench.cpp
View File

@@ -139,6 +139,8 @@ int main(int argc, char ** argv) {
const int n_ctx_req = is_pp_shared ? pp + pl*tg : pl*(pp + tg);
if (n_ctx_req > n_kv_max) {
printf("n_ctx_req = %d is greater than n_kv_max = %d for pp = %d, tg = %d, pl = %d\n",
n_ctx_req, n_kv_max, pp, tg, pl);
continue;
}

14
ggml/src/iqk/iqk_mul_mat.cpp
View File

@@ -142,13 +142,14 @@ struct MulMat {
}
int ny = funcs.size();
while (!funcs[ny-1] && ny > 0) --ny;
int n_step = (nrc_y - info.cur_y)/ny;
int n_left = nrc_y - info.cur_y;
int n_step = n_left/ny;
if (n_step > 0) {
if (n_step*ny != nrc_y) {
if (n_step*ny != n_left) {
++n_step;
int ny1 = nrc_y/n_step;
int ny1 = n_left/n_step;
int ny2 = ny1 + 1;
int my1 = n_step*ny2 - nrc_y;
int my1 = n_step*ny2 - n_left;
int my2 = n_step - my1;
for (int ix = 0; ix < nrc_x; ix += k_x_step) {
auto this_info = info;
@@ -163,7 +164,7 @@ struct MulMat {
this_info.cur_y += ny2;
}
}
info.cur_y += nrc_y;
info.cur_y += n_left;
}
else {
for (int ix = 0; ix < nrc_x; ix += k_x_step) {
@@ -178,7 +179,7 @@ struct MulMat {
info.cur_y += ny * n_step;
}
}
int n_left = nrc_y - info.cur_y;
n_left = nrc_y - info.cur_y;
if (n_left > 0) {
funcs[n_left-1](n, vx, bx, info, nrc_x);
}
@@ -13597,6 +13598,7 @@ void compute_helper(KHelper& kh, VHelper& vh, int nq1, int nk1, int stride_q, in
#ifdef __aarch64__
float16_t q_f16[D*q_step];
#endif
for (int i1 = 0; i1 < nq1/q_step; ++i1) {
fms.init_qstep();
kh.reset_block();