ikawrakow/ik_llama.cpp
1
0
Fork 0
mirror of https://github.com/ikawrakow/ik_llama.cpp.git synced 2026-04-30 19:31:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

Much faster long context TG for Minimax-M2 (#1194)

Browse Source
This commit is contained in:
Kawrakow
2026-01-28 10:43:11 +02:00
committed by GitHub
parent f9b5420e6a
commit 68cd52e583
2 changed files with 7 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
ggml/src/ggml-cuda/fattn-new-mma.cu
View File

@@ -2148,11 +2148,15 @@ void ggml_cuda_flash_attn_ext_mma_new(ggml_backend_cuda_context & ctx, ggml_tens
GGML_ASSERT(Q->ne[2] % K->ne[2] == 0); GGML_ASSERT(Q->ne[2] % K->ne[2] == 0);
const int gqa_ratio = Q->ne[2] / K->ne[2]; const int gqa_ratio = Q->ne[2] / K->ne[2];
if (K->ne[0] == 128 && gqa_ratio == 12) { if (K->ne[0] == 128 && (gqa_ratio == 12 || gqa_ratio == 6)) {
GGML_ASSERT(Q->ne[0] == 128 && V->ne[0] == 128); GGML_ASSERT(Q->ne[0] == 128 && V->ne[0] == 128);
//GGML_ASSERT(Q->ne[1] <= 4); //GGML_ASSERT(Q->ne[1] <= 4);
//ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1<128, 128, 16>(ctx, dst); //ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1<128, 128, 16>(ctx, dst);
ggml_cuda_flash_attn_ext_mma_f16_case<128, 128, 1, 16>(ctx, dst); if (gqa_ratio == 12) {
ggml_cuda_flash_attn_ext_mma_f16_case<128, 128, 1, 16>(ctx, dst);
} else {
ggml_cuda_flash_attn_ext_mma_f16_case<128, 128, 1, 8>(ctx, dst);
}
return; return;
} }
//if (K->ne[0] == 64 && V->ne[0] == 64) { //if (K->ne[0] == 64 && V->ne[0] == 64) {

2
ggml/src/ggml-cuda/fattn.cu
View File

@@ -90,7 +90,7 @@ void ggml_cuda_flash_attn_ext(ggml_backend_cuda_context & ctx, ggml_tensor * dst
} }
if (new_mma_available(cc) && K->ne[0] == 128 && V->ne[0] == 128 && Q->ne[0] == 128 && Q->ne[1] == 1 && if (new_mma_available(cc) && K->ne[0] == 128 && V->ne[0] == 128 && Q->ne[0] == 128 && Q->ne[1] == 1 &&
Q->ne[2] / K->ne[2] == 12) { (Q->ne[2] / K->ne[2] == 12 || Q->ne[2] / K->ne[2] == 6)) {
ggml_cuda_flash_attn_ext_mma_new(ctx, dst); ggml_cuda_flash_attn_ext_mma_new(ctx, dst);
return; return;
} }