ikawrakow/ik_llama.cpp
1
0
Fork 0
mirror of https://github.com/ikawrakow/ik_llama.cpp.git synced 2026-02-24 07:04:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

Make sure the bias really is 1 row to use fusion

Browse Source
This commit is contained in:
Iwan Kawrakow
2025-10-27 07:10:03 +02:00
parent 1cf4f21463
commit 444782523d
1 changed files with 4 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
ggml/src/ggml-cuda.cu
View File

@@ -2103,7 +2103,8 @@ static int ggml_cuda_mul_mat_q(ggml_backend_cuda_context & ctx, const ggml_tenso
cgraph->nodes[node_n+1]->op == GGML_OP_ADD &&
dst == cgraph->nodes[node_n+1]->src[0] &&
dst->ne[0] == cgraph->nodes[node_n+1]->src[1]->ne[0] &&
cgraph->nodes[node_n+1]->src[1]->type == GGML_TYPE_F32) {
cgraph->nodes[node_n+1]->src[1]->type == GGML_TYPE_F32 &&
ggml_nrows(cgraph->nodes[node_n+1]->src[1]) == 1) {
// We have a bias applied after the matrix multiplication and we can fuse it
ggml_cuda_op_mul_mat_vec_q_biased(ctx, dst->src[0], src1, cgraph->nodes[node_n+1], cgraph->nodes[node_n+1]->src[1],
(const char *)dst->src[0]->data, nullptr, src1_quantized.get(), (float *)cgraph->nodes[node_n+1]->data,
@@ -2138,7 +2139,8 @@ static int ggml_cuda_mul_mat_q(ggml_backend_cuda_context & ctx, const ggml_tenso
cgraph->nodes[node_n+1]->op == GGML_OP_ADD &&
dst == cgraph->nodes[node_n+1]->src[0] &&
dst->ne[0] == cgraph->nodes[node_n+1]->src[1]->ne[0] &&
cgraph->nodes[node_n+1]->src[1]->type == GGML_TYPE_F32) {
cgraph->nodes[node_n+1]->src[1]->type == GGML_TYPE_F32 &&
ggml_nrows(cgraph->nodes[node_n+1]->src[1]) == 1) {
// We have a bias applied after the matrix multiplication and we can fuse it
ggml_cuda_op_mul_mat_vec_q_biased(ctx, dst->src[0], src1, cgraph->nodes[node_n+1], cgraph->nodes[node_n+1]->src[1],
(const char *)dst->src[0]->data, nullptr, src1_quantized.get(), (float *)cgraph->nodes[node_n+1]->data,