From 3f6b6980b1a204136d6214df7460620eeeae605f Mon Sep 17 00:00:00 2001 From: Iwan Kawrakow Date: Tue, 28 Oct 2025 11:09:42 +0200 Subject: [PATCH] Don't use vector kernels if K or V are quantized --- ggml/src/ggml-cuda/fattn.cu | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/ggml/src/ggml-cuda/fattn.cu b/ggml/src/ggml-cuda/fattn.cu index 90a369d2..1cd3535a 100644 --- a/ggml/src/ggml-cuda/fattn.cu +++ b/ggml/src/ggml-cuda/fattn.cu @@ -91,12 +91,8 @@ void ggml_cuda_flash_attn_ext(ggml_backend_cuda_context & ctx, ggml_tensor * dst //const bool mma_faster_for_bs1 = new_mma_available(cc) && gqa_opt_applies && cc < CC_ADA_LOVELACE && !mma_needs_data_conversion; const bool mma_faster_for_bs1 = new_mma_available(cc) && gqa_opt_applies && !(Q->ne[1] == 1 && n_swa > 0); const bool can_use_vector_kernel = Q->ne[0] <= 256 && Q->ne[0] % (2*WARP_SIZE) == 0; - if (Q->ne[1] == 1 && can_use_vector_kernel && !mma_faster_for_bs1) { - //if (precision == GGML_PREC_DEFAULT) { - // ggml_cuda_flash_attn_ext_vec_f16(ctx, dst); - //} else { - ggml_cuda_flash_attn_ext_vec_f32(ctx, dst); - //} + if (Q->ne[1] == 1 && can_use_vector_kernel && !mma_faster_for_bs1 && !ggml_is_quantized(K->type) && !ggml_is_quantized(V->type)) { + ggml_cuda_flash_attn_ext_vec_f32(ctx, dst); return; }