From f4489897085bba5c187b43f578b7fec324ecbc1a Mon Sep 17 00:00:00 2001 From: Jim Date: Fri, 7 Feb 2025 16:03:53 +0800 Subject: [PATCH 01/10] template bwd v3 api --- .../ck_tile/01_fmha/codegen/ops/fmha_bwd.py | 545 +++--------------- example/ck_tile/01_fmha/generate.py | 2 +- 2 files changed, 82 insertions(+), 465 deletions(-) diff --git a/example/ck_tile/01_fmha/codegen/ops/fmha_bwd.py b/example/ck_tile/01_fmha/codegen/ops/fmha_bwd.py index ed70446cad..3d0a2dd52a 100644 --- a/example/ck_tile/01_fmha/codegen/ops/fmha_bwd.py +++ b/example/ck_tile/01_fmha/codegen/ops/fmha_bwd.py @@ -732,451 +732,7 @@ float fmha_bwd_v3_hdp_xqa_(const ck_tile::stream_config& s, fmha_bwd_args a) float fmha_bwd(fmha_bwd_traits t, fmha_bwd_args a, const ck_tile::stream_config& s){{ float r = -1; - - if (t.uses_bwd_v3 == true){{ - if ((t.is_group_mode == false) && (t.bias_type == bias_enum::no_bias) && (t.has_dbias == false) && (t.has_dropout == false) && (t.is_deterministic == false) && (a.hdim_q == a.hdim_v) && - (a.seqlen_q == a.seqlen_k) && (a.nhead_q % a.nhead_k == 0) && (a.stride_q == a.stride_do) && (a.nhead_stride_q == a.nhead_stride_do) && (a.batch_stride_q == a.batch_stride_do) && - (a.stride_k == a.stride_v) && (a.nhead_stride_k == a.nhead_stride_v) && (a.batch_stride_k == a.batch_stride_v) && (a.nhead_stride_k == a.nhead_stride_dk) && (a.nhead_stride_v == a.nhead_stride_dv) && - ((a.batch_stride_dk / a.batch_stride_k) == (a.nhead_q / a.nhead_k)) && ((a.batch_stride_dv / a.batch_stride_v) == (a.nhead_q / a.nhead_k))) {{ - if((a.hdim_q > 64) && (a.hdim_q <= 128) && (a.hdim_q % 8 == 0) && (a.seqlen_k % 64 == 0)){{ - if(t.data_type.compare("fp16") == 0){{ - if(t.mask_type == mask_enum::no_mask){{ - if((t.is_v3_atomic_fp32 == true) && (a.nhead_stride_dq_acc >= a.stride_dq_acc /*dq_acc only support BHSD*/)){{ - if(a.hdim_q == 128){{ - using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<128, FmhaBwdFp16, false, false, false>; - using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<128, FmhaBwdFp16, false, true, 0, false>; - using convert_dq_trait_ = fmha_bwd_convert_dq_traits_<128, FmhaBwdFp16, false, false, false, false>; - // const std::string bwd_v3_name = "bwd_v3_fp16_a32"; - r = fmha_bwd_v3_xqa_(s, a); - return r; - }} - else{{ - using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<128, FmhaBwdFp16, false, false, true>; - using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<128, FmhaBwdFp16, false, true, 0, true>; - using convert_dq_trait_ = fmha_bwd_convert_dq_traits_<128, FmhaBwdFp16, false, false, true, false>; - // const std::string bwd_v3_name = "bwd_v3_fp16_a32_pddv"; - r = fmha_bwd_v3_hdp_xqa_(s, a); - return r; - }} - }} - else if(t.is_v3_atomic_fp32 == false){{ - if(a.hdim_q == 128){{ - using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<128, FmhaBwdFp16, false, false, false>; - using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<128, FmhaBwdFp16, false, false, 0, false>; - // const std::string bwd_v3_name = "bwd_v3_fp16_a16"; - r = fmha_bwd_v3_xqa_(s, a); - return r; - }} - else{{ - using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<128, FmhaBwdFp16, false, false, true>; - using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<128, FmhaBwdFp16, false, false, 0, true>; - // const std::string bwd_v3_name = "bwd_v3_fp16_a16_pddv"; - r = fmha_bwd_v3_hdp_xqa_(s, a); - return r; - }} - }} - }} - else if((t.mask_type != mask_enum::no_mask) && ((a.window_size_left == -1) && (a.window_size_right == 0))){{ - if((t.is_v3_atomic_fp32 == true) && (a.nhead_stride_dq_acc >= a.stride_dq_acc /*dq_acc only support BHSD*/)){{ - if(a.hdim_q == 128){{ - using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<128, FmhaBwdFp16, false, false, false>; - using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<128, FmhaBwdFp16, true, true, 0, false>; - using convert_dq_trait_ = fmha_bwd_convert_dq_traits_<128, FmhaBwdFp16, false, false, false, false>; - // const std::string bwd_v3_name = "bwd_v3_fp16_causal_a32"; - r = fmha_bwd_v3_xqa_(s, a); - return r; - }} - else{{ - using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<128, FmhaBwdFp16, false, false, true>; - using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<128, FmhaBwdFp16, true, true, 0, true>; - using convert_dq_trait_ = fmha_bwd_convert_dq_traits_<128, FmhaBwdFp16, false, false, true, false>; - // const std::string bwd_v3_name = "bwd_v3_fp16_causal_a32_pddv"; - r = fmha_bwd_v3_hdp_xqa_(s, a); - return r; - }} - }} - else if(t.is_v3_atomic_fp32 == false){{ - if(a.hdim_q == 128){{ - using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<128, FmhaBwdFp16, false, false, false>; - using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<128, FmhaBwdFp16, true, false, 0, false>; - // const std::string bwd_v3_name = "bwd_v3_fp16_causal_a16"; - r = fmha_bwd_v3_xqa_(s, a); - return r; - }} - else{{ - using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<128, FmhaBwdFp16, false, false, true>; - using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<128, FmhaBwdFp16, true, false, 0, true>; - // const std::string bwd_v3_name = "bwd_v3_fp16_causal_a16_pddv"; - r = fmha_bwd_v3_hdp_xqa_(s, a); - return r; - }} - }} - }} - }} - else if(t.data_type.compare("bf16") == 0){{ - if(t.mask_type == mask_enum::no_mask){{ - if((t.is_v3_atomic_fp32 == true) && (a.nhead_stride_dq_acc >= a.stride_dq_acc /*dq_acc only support BHSD*/)){{ - if(t.how_v3_bf16_cvt == 0){{ - if(a.hdim_q == 128){{ - using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<128, FmhaBwdBf16, false, false, false>; - using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<128, FmhaBwdBf16, false, true, 0, false>; - using convert_dq_trait_ = fmha_bwd_convert_dq_traits_<128, FmhaBwdBf16, false, false, false, false>; - // const std::string bwd_v3_name = "bwd_v3_bf16_a32_rtne"; - r = fmha_bwd_v3_xqa_(s, a); - return r; - }} - else{{ - using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<128, FmhaBwdBf16, false, false, true>; - using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<128, FmhaBwdBf16, false, true, 0, true>; - using convert_dq_trait_ = fmha_bwd_convert_dq_traits_<128, FmhaBwdBf16, false, false, true, false>; - // const std::string bwd_v3_name = "bwd_v3_bf16_a32_rtne_pddv"; - r = fmha_bwd_v3_hdp_xqa_(s, a); - return r; - }} - }} - else if(t.how_v3_bf16_cvt == 1){{ - if(a.hdim_q == 128){{ - using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<128, FmhaBwdBf16, false, false, false>; - using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<128, FmhaBwdBf16, false, true, 1, false>; - using convert_dq_trait_ = fmha_bwd_convert_dq_traits_<128, FmhaBwdBf16, false, false, false, false>; - // const std::string bwd_v3_name = "bwd_v3_bf16_a32_rtna"; - r = fmha_bwd_v3_xqa_(s, a); - return r; - }} - else{{ - using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<128, FmhaBwdBf16, false, false, true>; - using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<128, FmhaBwdBf16, false, true, 1, true>; - using convert_dq_trait_ = fmha_bwd_convert_dq_traits_<128, FmhaBwdBf16, false, false, true, false>; - // const std::string bwd_v3_name = "bwd_v3_bf16_a32_rtna_pddv"; - r = fmha_bwd_v3_hdp_xqa_(s, a); - return r; - }} - }} - else if(t.how_v3_bf16_cvt == 2){{ - if(a.hdim_q == 128){{ - using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<128, FmhaBwdBf16, false, false, false>; - using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<128, FmhaBwdBf16, false, true, 2, false>; - using convert_dq_trait_ = fmha_bwd_convert_dq_traits_<128, FmhaBwdBf16, false, false, false, false>; - // const std::string bwd_v3_name = "bwd_v3_bf16_a32_rtz"; - r = fmha_bwd_v3_xqa_(s, a); - return r; - }} - else{{ - using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<128, FmhaBwdBf16, false, false, true>; - using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<128, FmhaBwdBf16, false, true, 2, true>; - using convert_dq_trait_ = fmha_bwd_convert_dq_traits_<128, FmhaBwdBf16, false, false, true, false>; - // const std::string bwd_v3_name = "bwd_v3_bf16_a32_rtz_pddv"; - r = fmha_bwd_v3_hdp_xqa_(s, a); - return r; - }} - }} - }} - else if(t.is_v3_atomic_fp32 == false){{ - if(t.how_v3_bf16_cvt == 0){{ - if(a.hdim_q == 128){{ - using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<128, FmhaBwdBf16, false, false, false>; - using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<128, FmhaBwdBf16, false, false, 0, false>; - // const std::string bwd_v3_name = "bwd_v3_bf16_a16_rtne"; - r = fmha_bwd_v3_xqa_(s, a); - return r; - }} - else{{ - using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<128, FmhaBwdBf16, false, false, true>; - using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<128, FmhaBwdBf16, false, false, 0, true>; - // const std::string bwd_v3_name = "bwd_v3_bf16_a16_rtne_pddv"; - r = fmha_bwd_v3_hdp_xqa_(s, a); - return r; - }} - }} - else if(t.how_v3_bf16_cvt == 1){{ - if(a.hdim_q == 128){{ - using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<128, FmhaBwdBf16, false, false, false>; - using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<128, FmhaBwdBf16, false, false, 1, false>; - // const std::string bwd_v3_name = "bwd_v3_bf16_a16_rtna"; - r = fmha_bwd_v3_xqa_(s, a); - return r; - }} - else{{ - using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<128, FmhaBwdBf16, false, false, true>; - using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<128, FmhaBwdBf16, false, false, 1, true>; - // const std::string bwd_v3_name = "bwd_v3_bf16_a16_rtna_pddv"; - r = fmha_bwd_v3_hdp_xqa_(s, a); - return r; - }} - }} - else if(t.how_v3_bf16_cvt == 2){{ - if(a.hdim_q == 128){{ - using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<128, FmhaBwdBf16, false, false, false>; - using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<128, FmhaBwdBf16, false, false, 2, false>; - // const std::string bwd_v3_name = "bwd_v3_bf16_a16_rtz"; - r = fmha_bwd_v3_xqa_(s, a); - return r; - }} - else{{ - using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<128, FmhaBwdBf16, false, false, true>; - using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<128, FmhaBwdBf16, false, false, 2, true>; - // const std::string bwd_v3_name = "bwd_v3_bf16_a16_rtz_pddv"; - r = fmha_bwd_v3_hdp_xqa_(s, a); - return r; - }} - }} - }} - }} - else if((t.mask_type != mask_enum::no_mask) && ((a.window_size_left == -1) && (a.window_size_right == 0))){{ - if((t.is_v3_atomic_fp32 == true) && (a.nhead_stride_dq_acc >= a.stride_dq_acc /*dq_acc only support BHSD*/)){{ - if(t.how_v3_bf16_cvt == 0){{ - if(a.hdim_q == 128){{ - using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<128, FmhaBwdBf16, false, false, false>; - using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<128, FmhaBwdBf16, true, true, 0, false>; - using convert_dq_trait_ = fmha_bwd_convert_dq_traits_<128, FmhaBwdBf16, false, false, false, false>; - // const std::string bwd_v3_name = "bwd_v3_bf16_causal_a32_rtne"; - r = fmha_bwd_v3_xqa_(s, a); - return r; - }} - else{{ - using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<128, FmhaBwdBf16, false, false, true>; - using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<128, FmhaBwdBf16, true, true, 0, true>; - using convert_dq_trait_ = fmha_bwd_convert_dq_traits_<128, FmhaBwdBf16, false, false, true, false>; - // const std::string bwd_v3_name = "bwd_v3_bf16_causal_a32_rtne_pddv"; - r = fmha_bwd_v3_hdp_xqa_(s, a); - return r; - }} - }} - else if(t.how_v3_bf16_cvt == 1){{ - if(a.hdim_q == 128){{ - using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<128, FmhaBwdBf16, false, false, false>; - using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<128, FmhaBwdBf16, true, true, 1, false>; - using convert_dq_trait_ = fmha_bwd_convert_dq_traits_<128, FmhaBwdBf16, false, false, false, false>; - // const std::string bwd_v3_name = "bwd_v3_bf16_causal_a32_rtna"; - r = fmha_bwd_v3_xqa_(s, a); - return r; - }} - else{{ - using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<128, FmhaBwdBf16, false, false, true>; - using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<128, FmhaBwdBf16, true, true, 1, true>; - using convert_dq_trait_ = fmha_bwd_convert_dq_traits_<128, FmhaBwdBf16, false, false, true, false>; - // const std::string bwd_v3_name = "bwd_v3_bf16_causal_a32_rtna_pddv"; - r = fmha_bwd_v3_hdp_xqa_(s, a); - return r; - }} - }} - else if(t.how_v3_bf16_cvt == 2){{ - if(a.hdim_q == 128){{ - using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<128, FmhaBwdBf16, false, false, false>; - using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<128, FmhaBwdBf16, true, true, 2, false>; - using convert_dq_trait_ = fmha_bwd_convert_dq_traits_<128, FmhaBwdBf16, false, false, false, false>; - // const std::string bwd_v3_name = "bwd_v3_bf16_causal_a32_rtz"; - r = fmha_bwd_v3_xqa_(s, a); - return r; - }} - else{{ - using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<128, FmhaBwdBf16, false, false, true>; - using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<128, FmhaBwdBf16, true, true, 2, true>; - using convert_dq_trait_ = fmha_bwd_convert_dq_traits_<128, FmhaBwdBf16, false, false, true, false>; - // const std::string bwd_v3_name = "bwd_v3_bf16_causal_a32_rtz_pddv"; - r = fmha_bwd_v3_hdp_xqa_(s, a); - return r; - }} - }} - }} - else if(t.is_v3_atomic_fp32 == false){{ - if(t.how_v3_bf16_cvt == 0){{ - if(a.hdim_q == 128){{ - using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<128, FmhaBwdBf16, false, false, false>; - using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<128, FmhaBwdBf16, true, false, 0, false>; - // const std::string bwd_v3_name = "bwd_v3_bf16_causal_a16_rtne"; - r = fmha_bwd_v3_xqa_(s, a); - return r; - }} - else{{ - using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<128, FmhaBwdBf16, false, false, true>; - using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<128, FmhaBwdBf16, true, false, 0, true>; - // const std::string bwd_v3_name = "bwd_v3_bf16_causal_a16_rtne_pddv"; - r = fmha_bwd_v3_hdp_xqa_(s, a); - return r; - }} - }} - else if(t.how_v3_bf16_cvt == 1){{ - if(a.hdim_q == 128){{ - using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<128, FmhaBwdBf16, false, false, false>; - using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<128, FmhaBwdBf16, true, false, 1, false>; - // const std::string bwd_v3_name = "bwd_v3_bf16_causal_a16_rtna"; - r = fmha_bwd_v3_xqa_(s, a); - return r; - }} - else{{ - using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<128, FmhaBwdBf16, false, false, true>; - using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<128, FmhaBwdBf16, true, false, 1, true>; - // const std::string bwd_v3_name = "bwd_v3_bf16_causal_a16_rtna_pddv"; - r = fmha_bwd_v3_hdp_xqa_(s, a); - return r; - }} - }} - else if(t.how_v3_bf16_cvt == 2){{ - if(a.hdim_q == 128){{ - using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<128, FmhaBwdBf16, false, false, false>; - using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<128, FmhaBwdBf16, true, false, 2, false>; - // const std::string bwd_v3_name = "bwd_v3_bf16_causal_a16_rtz"; - r = fmha_bwd_v3_xqa_(s, a); - return r; - }} - else{{ - using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<128, FmhaBwdBf16, false, false, true>; - using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<128, FmhaBwdBf16, true, false, 2, true>; - // const std::string bwd_v3_name = "bwd_v3_bf16_causal_a16_rtz_pddv"; - r = fmha_bwd_v3_hdp_xqa_(s, a); - return r; - }} - }} - }} - }} - }} - }} - else if((a.hdim_q == 64) && (a.seqlen_k % 64 == 0)){{ - if(t.data_type.compare("fp16") == 0){{ - if(t.mask_type == mask_enum::no_mask){{ - if((t.is_v3_atomic_fp32 == true) && (a.nhead_stride_dq_acc >= a.stride_dq_acc /*dq_acc only support BHSD*/)){{ - using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<64, FmhaBwdFp16, false, false, false>; - using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<64, FmhaBwdFp16, false, true, 0, false>; - using convert_dq_trait_ = fmha_bwd_convert_dq_traits_<64, FmhaBwdFp16, false, false, false, false>; - // const std::string bwd_v3_name = "bwd_v3_hd64_fp16_a32"; - r = fmha_bwd_v3_xqa_(s, a); - return r; - }} - else if(t.is_v3_atomic_fp32 == false){{ - using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<64, FmhaBwdFp16, false, false, false>; - using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<64, FmhaBwdFp16, false, false, 0, false>; - // const std::string bwd_v3_name = "bwd_v3_hd64_fp16_a16"; - r = fmha_bwd_v3_xqa_(s, a); - return r; - }} - }} - else if((t.mask_type != mask_enum::no_mask) && ((a.window_size_left == -1) && (a.window_size_right == 0))){{ - if((t.is_v3_atomic_fp32 == true) && (a.nhead_stride_dq_acc >= a.stride_dq_acc /*dq_acc only support BHSD*/)){{ - using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<64, FmhaBwdFp16, false, false, false>; - using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<64, FmhaBwdFp16, true, true, 0, false>; - using convert_dq_trait_ = fmha_bwd_convert_dq_traits_<64, FmhaBwdFp16, false, false, false, false>; - // const std::string bwd_v3_name = "bwd_v3_hd64_fp16_causal_a32"; - r = fmha_bwd_v3_xqa_(s, a); - return r; - }} - else if(t.is_v3_atomic_fp32 == false){{ - using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<64, FmhaBwdFp16, false, false, false>; - using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<64, FmhaBwdFp16, true, false, 0, false>; - // const std::string bwd_v3_name = "bwd_v3_hd64_fp16_causal_a16"; - r = fmha_bwd_v3_xqa_(s, a); - return r; - }} - }} - }} - else if(t.data_type.compare("bf16") == 0){{ - if(t.mask_type == mask_enum::no_mask){{ - if((t.is_v3_atomic_fp32 == true) && (a.nhead_stride_dq_acc >= a.stride_dq_acc /*dq_acc only support BHSD*/)){{ - if(t.how_v3_bf16_cvt == 0){{ - using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<64, FmhaBwdBf16, false, false, false>; - using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<64, FmhaBwdBf16, false, true, 0, false>; - using convert_dq_trait_ = fmha_bwd_convert_dq_traits_<64, FmhaBwdBf16, false, false, false, false>; - // const std::string bwd_v3_name = "bwd_v3_hd64_bf16_a32_rtne"; - r = fmha_bwd_v3_xqa_(s, a); - return r; - }} - else if(t.how_v3_bf16_cvt == 1){{ - using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<64, FmhaBwdBf16, false, false, false>; - using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<64, FmhaBwdBf16, false, true, 1, false>; - using convert_dq_trait_ = fmha_bwd_convert_dq_traits_<64, FmhaBwdBf16, false, false, false, false>; - // const std::string bwd_v3_name = "bwd_v3_hd64_bf16_a32_rtna"; - r = fmha_bwd_v3_xqa_(s, a); - return r; - }} - else if(t.how_v3_bf16_cvt == 2){{ - using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<64, FmhaBwdBf16, false, false, false>; - using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<64, FmhaBwdBf16, false, true, 2, false>; - using convert_dq_trait_ = fmha_bwd_convert_dq_traits_<64, FmhaBwdBf16, false, false, false, false>; - // const std::string bwd_v3_name = "bwd_v3_hd64_bf16_a32_rtz"; - r = fmha_bwd_v3_xqa_(s, a); - return r; - }} - }} - else if(t.is_v3_atomic_fp32 == false){{ - if(t.how_v3_bf16_cvt == 0){{ - using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<64, FmhaBwdBf16, false, false, false>; - using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<64, FmhaBwdBf16, false, false, 0, false>; - // const std::string bwd_v3_name = "bwd_v3_hd64_bf16_a16_rtne"; - r = fmha_bwd_v3_xqa_(s, a); - return r; - }} - else if(t.how_v3_bf16_cvt == 1){{ - using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<64, FmhaBwdBf16, false, false, false>; - using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<64, FmhaBwdBf16, false, false, 1, false>; - // const std::string bwd_v3_name = "bwd_v3_hd64_bf16_a16_rtna"; - r = fmha_bwd_v3_xqa_(s, a); - return r; - }} - else if(t.how_v3_bf16_cvt == 2){{ - using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<64, FmhaBwdBf16, false, false, false>; - using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<64, FmhaBwdBf16, false, false, 2, false>; - // const std::string bwd_v3_name = "bwd_v3_hd64_bf16_a16_rtz"; - r = fmha_bwd_v3_xqa_(s, a); - return r; - }} - }} - }} - else if((t.mask_type != mask_enum::no_mask) && ((a.window_size_left == -1) && (a.window_size_right == 0))){{ - if((t.is_v3_atomic_fp32 == true) && (a.nhead_stride_dq_acc >= a.stride_dq_acc /*dq_acc only support BHSD*/)){{ - if(t.how_v3_bf16_cvt == 0){{ - using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<64, FmhaBwdBf16, false, false, false>; - using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<64, FmhaBwdBf16, true, true, 0, false>; - using convert_dq_trait_ = fmha_bwd_convert_dq_traits_<64, FmhaBwdBf16, false, false, false, false>; - // const std::string bwd_v3_name = "bwd_v3_hd64_bf16_causal_a32_rtne"; - r = fmha_bwd_v3_xqa_(s, a); - return r; - }} - else if(t.how_v3_bf16_cvt == 1){{ - using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<64, FmhaBwdBf16, false, false, false>; - using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<64, FmhaBwdBf16, true, true, 1, false>; - using convert_dq_trait_ = fmha_bwd_convert_dq_traits_<64, FmhaBwdBf16, false, false, false, false>; - // const std::string bwd_v3_name = "bwd_v3_hd64_bf16_causal_a32_rtna"; - r = fmha_bwd_v3_xqa_(s, a); - return r; - }} - else if(t.how_v3_bf16_cvt == 2){{ - using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<64, FmhaBwdBf16, false, false, false>; - using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<64, FmhaBwdBf16, true, true, 2, false>; - using convert_dq_trait_ = fmha_bwd_convert_dq_traits_<64, FmhaBwdBf16, false, false, false, false>; - // const std::string bwd_v3_name = "bwd_v3_hd64_bf16_causal_a32_rtz"; - r = fmha_bwd_v3_xqa_(s, a); - return r; - }} - }} - else if(t.is_v3_atomic_fp32 == false){{ - if(t.how_v3_bf16_cvt == 0){{ - using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<64, FmhaBwdBf16, false, false, false>; - using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<64, FmhaBwdBf16, true, false, 0, false>; - const std::string bwd_v3_name = "bwd_v3_hd64_bf16_causal_a16_rtne"; - r = fmha_bwd_v3_xqa_(s, a); - return r; - }} - else if(t.how_v3_bf16_cvt == 1){{ - using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<64, FmhaBwdBf16, false, false, false>; - using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<64, FmhaBwdBf16, true, false, 1, false>; - // const std::string bwd_v3_name = "bwd_v3_hd64_bf16_causal_a16_rtna"; - r = fmha_bwd_v3_xqa_(s, a); - return r; - }} - else if(t.how_v3_bf16_cvt == 2){{ - using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<64, FmhaBwdBf16, false, false, false>; - using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<64, FmhaBwdBf16, true, false, 2, false>; - // const std::string bwd_v3_name = "bwd_v3_hd64_bf16_causal_a16_rtz"; - r = fmha_bwd_v3_xqa_(s, a); - return r; - }} - }} - }} - }} - }} - }} - }} - +{F_v3_dispatch} {F_dispatch} return r; }} @@ -1201,26 +757,70 @@ FMHA_BWD_API_INNER_DISPATCH=""" {F_if}((t.is_group_mode == {F_mode}) }} """ +FMHA_V3_DISPATCH=""" if ((t.uses_bwd_v3 == {F_use_v3}){{ + if ((t.is_group_mode == false) && (t.bias_type == bias_enum::no_bias) && (t.has_dbias == false) && (t.has_dropout == false) && (t.is_deterministic == false) && (a.hdim_q == a.hdim_v) && + (a.seqlen_q == a.seqlen_k) && (a.nhead_q % a.nhead_k == 0) && (a.stride_q == a.stride_do) && (a.nhead_stride_q == a.nhead_stride_do) && (a.batch_stride_q == a.batch_stride_do) && + (a.stride_k == a.stride_v) && (a.nhead_stride_k == a.nhead_stride_v) && (a.batch_stride_k == a.batch_stride_v) && (a.nhead_stride_k == a.nhead_stride_dk) && (a.nhead_stride_v == a.nhead_stride_dv) && + ((a.batch_stride_dk / a.batch_stride_k) == (a.nhead_q / a.nhead_k)) && ((a.batch_stride_dv / a.batch_stride_v) == (a.nhead_q / a.nhead_k))) {{ + if (((a.hdim_q >= 64) && (a.hdim_q <= 128) && (a.hdim_q % 8 == 0) && (a.seqlen_k % 64 == 0))) {{ + if (t.mask_type == mask_enum::no_mask){{ + if ((t.is_v3_atomic_fp32 == true) && (a.nhead_stride_dq_acc >= a.stride_dq_acc /*dq_acc only support BHSD*/)){{ + using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<{F_hdim}, {F_dtype}, false, false, {F_padding}>; + using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<{F_hdim}, {F_dtype}, false, {F_is_atomic32}, {F_how_v3_bf16_cvt}, {F_padding}>; + using convert_dq_trait_ = fmha_bwd_convert_dq_traits_<{F_hdim}, {F_dtype}, false, false, {F_padding}, false>; + r = fmha_bwd_v3{F_padding_suffix}_xqa_(s, a); + return r; + }} + else if (t.is_v3_atomic_fp32 == false){{ + using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<{F_hdim}, {F_dtype}, false, false, {F_padding}>; + using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<{F_hdim}, {F_dtype}, false, {F_is_atomic32}, {F_how_v3_bf16_cvt}, {F_padding}>; + r = fmha_bwd_v3{F_padding_suffix}_xqa_(s, a); + return r; + }} + }} + else if ((t.mask_type != mask_enum::no_mask) && ((a.window_size_left == -1) && (a.window_size_right == 0))){{ + if ((t.is_v3_atomic_fp32 == true) && (a.nhead_stride_dq_acc >= a.stride_dq_acc /*dq_acc only support BHSD*/)){{ + using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<{F_hdim}, {F_dtype}, false, false, {F_padding}>; + using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<{F_hdim}, {F_dtype}, true, {F_is_atomic32}, {F_how_v3_bf16_cvt}, {F_padding}>; + using convert_dq_trait_ = fmha_bwd_convert_dq_traits_<{F_hdim}, {F_dtype}, false, false, {F_padding}, false>; + r = fmha_bwd_v3{F_padding_suffix}_xqa_(s, a); + return r; + }} + else if (t.is_v3_atomic_fp32 == false){{ + using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<{F_hdim}, {F_dtype}, false, false, {F_padding}>; + using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<{F_hdim}, {F_dtype}, true, {F_is_atomic32}, {F_how_v3_bf16_cvt}, {F_padding}>; + r = fmha_bwd_v3{F_padding_suffix}_xqa_(s, a); + return r; + }} + }} + }} + }} + }} +""" + @dataclass class FmhaBwdDQDKDVApiTrait: - pipeline : str + pipeline : str # sync with fmha_bwd_traits<>, to generate fallback calls - hdim : str - dtype : str # data type - mode : str # value from MODE_MAP - bm0 : int # tile size along q seqlen (block size) - bn0 : int # tile size along k seqlen - bhdq : int # q head_dim - bhdv : int # v head_dim - mask : str - bias : str - dbias : str - dropout : str - spad : str - skpad : str - dpad : str - dvpad : str - deterministic : str + hdim : str + dtype : str # data type + mode : str # value from MODE_MAP + bm0 : int # tile size along q seqlen (block size) + bn0 : int # tile size along k seqlen + bhdq : int # q head_dim + bhdv : int # v head_dim + mask : str + bias : str + dbias : str + dropout : str + spad : str + skpad : str + dpad : str + dvpad : str + deterministic : str + uses_bwd_v3 : str + v3_atomic_fp32 : str + how_v3_bf16_cvt : int def scheck(self, spad1 : str) -> str: if self.mode == 'group': @@ -1293,7 +893,20 @@ class FmhaBwdApiPool: if not per_dtypes: # empty string we add some ignore to suppress warning in api per_dtypes += ' (void)t ; (void)s ; (void)a;' - return FMHA_BWD_KERNEL_HEADER + FMHA_BWD_API.format(F_dispatch = per_dtypes) + # v3 codegen: + v3_code = str() + for i, dtype in enumerate(self.dq_dk_dv_pool.keys()): + per_hdim_case=str() + for j, hdim in enumerate(self.dq_dk_dv_pool[dtype].keys()): + traits = self.dq_dk_dv_pool[dtype][hdim] + hdim_int = int(hdim) + hdim_int = (hdim_int + 64 - 1) / 64 * 64 + for k, trait in enumerate(traits): + padding = "t" if hdim_int % 64 == 0 else "f" + padding_suffix = "_hdp" if padding == "t" else "" + v3_code = v3_code + FMHA_V3_DISPATCH.format(F_use_v3=BOOL_MAP[trait.use_bwd_v3], F_hdim=hdim_int, F_dtype=BWD_DTYPE_MAP[dtype], F_padding=BOOL_MAP[padding], + F_is_atomic32=BOOL_MAP[trait.v3_atomic_fp32], F_how_v3_bf16_cvt=trait.how_v3_bf16_cvt, F_padding_suffix=padding_suffix) + return FMHA_BWD_KERNEL_HEADER + FMHA_BWD_API.format(F_dispatch = per_dtypes, F_v3_dispatch = v3_code) # GEMM0: Q@K=S^T # GEMM1: P^T@dO^T=dV(This was chosen as G1 to match fwd, but N1 must be equal to headdim_v) @@ -1425,6 +1038,7 @@ class FmhaBwdDQDKDVKernel: def filename(self) -> str: return self.name + ".cpp" + # TODO: Jim: update this def api_trait(self) -> FmhaBwdDQDKDVApiTrait: return FmhaBwdDQDKDVApiTrait(pipeline=self.F_pipeline, hdim=str(self.F_hdim), @@ -1442,7 +1056,10 @@ class FmhaBwdDQDKDVKernel: skpad=self.F_skpad, dpad=self.F_dpad, dvpad=self.F_dvpad, - deterministic=self.F_deterministic + deterministic=self.F_deterministic, + use_bwd_v3=self.use_bwd_v3, + v3_atomic_fp32=self.v3_atomic_fp32, + how_v3_bf16_cvt=self.how_v3_bf16_cvt ) # TODO: design a more practical way to do it diff --git a/example/ck_tile/01_fmha/generate.py b/example/ck_tile/01_fmha/generate.py index 5b1b6664cc..3ddfe77a19 100644 --- a/example/ck_tile/01_fmha/generate.py +++ b/example/ck_tile/01_fmha/generate.py @@ -64,7 +64,7 @@ if __name__ == "__main__": "--direction", # we keep 'direction' option for backward compatibility "-a", "--api", - default='fwd', + default='bwd', required=False, help="supply API(s) to generate (default: fwd). separated by comma." ) From 8b745f2c54bc0cd948a0d1009a614e9a2d1b817e Mon Sep 17 00:00:00 2001 From: Jim Date: Sat, 8 Feb 2025 17:16:24 +0800 Subject: [PATCH 02/10] codegen template --- .../ck_tile/01_fmha/codegen/cpp_symbol_map.py | 6 + .../ck_tile/01_fmha/codegen/ops/fmha_bwd.py | 226 ++++-------------- 2 files changed, 54 insertions(+), 178 deletions(-) diff --git a/example/ck_tile/01_fmha/codegen/cpp_symbol_map.py b/example/ck_tile/01_fmha/codegen/cpp_symbol_map.py index 332707eafd..ac02f8d8d6 100644 --- a/example/ck_tile/01_fmha/codegen/cpp_symbol_map.py +++ b/example/ck_tile/01_fmha/codegen/cpp_symbol_map.py @@ -126,3 +126,9 @@ BOOL_MAP = { "t" : "true", "f" : "false" } + +BF16_CVT_MAP = { + 0 : "rtne", + 1 : "rtna", + 2 : "rtz", +} \ No newline at end of file diff --git a/example/ck_tile/01_fmha/codegen/ops/fmha_bwd.py b/example/ck_tile/01_fmha/codegen/ops/fmha_bwd.py index 3d0a2dd52a..6f97a65013 100644 --- a/example/ck_tile/01_fmha/codegen/ops/fmha_bwd.py +++ b/example/ck_tile/01_fmha/codegen/ops/fmha_bwd.py @@ -161,6 +161,13 @@ std::string fmha_bwd_dq_dk_dv_get_name_() """ FMHA_BWD_API_FILENAME="fmha_bwd_api.cpp" + +FMHA_BWD_V3_TEMPLATE=""" +template<> struct FmhaBwdV3Name> {{ static constexpr const char * bwd_v3_name = "bwd_v3_{F_hdim_name}_{F_dtype_name}_{F_causal_name}_{F_atomic_name}_{F_bf16_cvt_name}"; }}; +template<> struct FmhaBwdV3Buf> {{ static constexpr unsigned char * bwd_v3_buf = bwd_{F_hdim_name}_{F_dtype_name}_{F_causal_name}_{F_atomic_name}_{F_bf16_cvt_name}; }}; +template<> struct FmhaBwdV3Ts> {{ static constexpr int ts_qo = {F_Ts_qo}; static constexpr int ts_kv = 192; }}; +""" + FMHA_BWD_API=""" #include #include "hsaco/fmha_hsaco.hpp" @@ -311,158 +318,7 @@ struct fmha_bwd_dq_dk_dv_v3_traits_ static constexpr bool kIsHDPad = kIsHDPad_; }}; -template struct FmhaBwdV3Name; -// ########################################################|HDim| DataType|kIsCausal|kIsAtomic32|BF16Cvt|kIsHDPad| -template<> struct FmhaBwdV3Name> {{ static constexpr const char * bwd_v3_name = "bwd_v3_bf16_a16_rtne"; }}; -template<> struct FmhaBwdV3Name> {{ static constexpr const char * bwd_v3_name = "bwd_v3_bf16_a16_rtna"; }}; -template<> struct FmhaBwdV3Name> {{ static constexpr const char * bwd_v3_name = "bwd_v3_bf16_a16_rtz"; }}; -template<> struct FmhaBwdV3Name> {{ static constexpr const char * bwd_v3_name = "bwd_v3_bf16_a32_rtne"; }}; -template<> struct FmhaBwdV3Name> {{ static constexpr const char * bwd_v3_name = "bwd_v3_bf16_a32_rtna"; }}; -template<> struct FmhaBwdV3Name> {{ static constexpr const char * bwd_v3_name = "bwd_v3_bf16_a32_rtz"; }}; -template<> struct FmhaBwdV3Name> {{ static constexpr const char * bwd_v3_name = "bwd_v3_bf16_causal_a16_rtne"; }}; -template<> struct FmhaBwdV3Name> {{ static constexpr const char * bwd_v3_name = "bwd_v3_bf16_causal_a16_rtna"; }}; -template<> struct FmhaBwdV3Name> {{ static constexpr const char * bwd_v3_name = "bwd_v3_bf16_causal_a16_rtz"; }}; -template<> struct FmhaBwdV3Name> {{ static constexpr const char * bwd_v3_name = "bwd_v3_bf16_causal_a32_rtne"; }}; -template<> struct FmhaBwdV3Name> {{ static constexpr const char * bwd_v3_name = "bwd_v3_bf16_causal_a32_rtna"; }}; -template<> struct FmhaBwdV3Name> {{ static constexpr const char * bwd_v3_name = "bwd_v3_bf16_causal_a32_rtz"; }}; -template<> struct FmhaBwdV3Name> {{ static constexpr const char * bwd_v3_name = "bwd_v3_fp16_a16"; }}; -template<> struct FmhaBwdV3Name> {{ static constexpr const char * bwd_v3_name = "bwd_v3_fp16_a32"; }}; -template<> struct FmhaBwdV3Name> {{ static constexpr const char * bwd_v3_name = "bwd_v3_fp16_causal_a16"; }}; -template<> struct FmhaBwdV3Name> {{ static constexpr const char * bwd_v3_name = "bwd_v3_fp16_causal_a32"; }}; -template<> struct FmhaBwdV3Name> {{ static constexpr const char * bwd_v3_name = "bwd_v3_bf16_a16_rtne_pddv"; }}; -template<> struct FmhaBwdV3Name> {{ static constexpr const char * bwd_v3_name = "bwd_v3_bf16_a16_rtna_pddv"; }}; -template<> struct FmhaBwdV3Name> {{ static constexpr const char * bwd_v3_name = "bwd_v3_bf16_a16_rtz_pddv"; }}; -template<> struct FmhaBwdV3Name> {{ static constexpr const char * bwd_v3_name = "bwd_v3_bf16_a32_rtne_pddv"; }}; -template<> struct FmhaBwdV3Name> {{ static constexpr const char * bwd_v3_name = "bwd_v3_bf16_a32_rtna_pddv"; }}; -template<> struct FmhaBwdV3Name> {{ static constexpr const char * bwd_v3_name = "bwd_v3_bf16_a32_rtz_pddv"; }}; -template<> struct FmhaBwdV3Name> {{ static constexpr const char * bwd_v3_name = "bwd_v3_bf16_causal_a16_rtne_pddv"; }}; -template<> struct FmhaBwdV3Name> {{ static constexpr const char * bwd_v3_name = "bwd_v3_bf16_causal_a16_rtna_pddv"; }}; -template<> struct FmhaBwdV3Name> {{ static constexpr const char * bwd_v3_name = "bwd_v3_bf16_causal_a16_rtz_pddv"; }}; -template<> struct FmhaBwdV3Name> {{ static constexpr const char * bwd_v3_name = "bwd_v3_bf16_causal_a32_rtne_pddv"; }}; -template<> struct FmhaBwdV3Name> {{ static constexpr const char * bwd_v3_name = "bwd_v3_bf16_causal_a32_rtna_pddv"; }}; -template<> struct FmhaBwdV3Name> {{ static constexpr const char * bwd_v3_name = "bwd_v3_bf16_causal_a32_rtz_pddv"; }}; -template<> struct FmhaBwdV3Name> {{ static constexpr const char * bwd_v3_name = "bwd_v3_fp16_a16_pddv"; }}; -template<> struct FmhaBwdV3Name> {{ static constexpr const char * bwd_v3_name = "bwd_v3_fp16_a32_pddv"; }}; -template<> struct FmhaBwdV3Name> {{ static constexpr const char * bwd_v3_name = "bwd_v3_fp16_causal_a16_pddv"; }}; -template<> struct FmhaBwdV3Name> {{ static constexpr const char * bwd_v3_name = "bwd_v3_fp16_causal_a32_pddv"; }}; -template<> struct FmhaBwdV3Name> {{ static constexpr const char * bwd_v3_name = "bwd_v3_hd64_bf16_a16_rtne"; }}; -template<> struct FmhaBwdV3Name> {{ static constexpr const char * bwd_v3_name = "bwd_v3_hd64_bf16_a16_rtna"; }}; -template<> struct FmhaBwdV3Name> {{ static constexpr const char * bwd_v3_name = "bwd_v3_hd64_bf16_a16_rtz"; }}; -template<> struct FmhaBwdV3Name> {{ static constexpr const char * bwd_v3_name = "bwd_v3_hd64_bf16_a32_rtne"; }}; -template<> struct FmhaBwdV3Name> {{ static constexpr const char * bwd_v3_name = "bwd_v3_hd64_bf16_a32_rtna"; }}; -template<> struct FmhaBwdV3Name> {{ static constexpr const char * bwd_v3_name = "bwd_v3_hd64_bf16_a32_rtz"; }}; -template<> struct FmhaBwdV3Name> {{ static constexpr const char * bwd_v3_name = "bwd_v3_hd64_bf16_causal_a16_rtne"; }}; -template<> struct FmhaBwdV3Name> {{ static constexpr const char * bwd_v3_name = "bwd_v3_hd64_bf16_causal_a16_rtna"; }}; -template<> struct FmhaBwdV3Name> {{ static constexpr const char * bwd_v3_name = "bwd_v3_hd64_bf16_causal_a16_rtz"; }}; -template<> struct FmhaBwdV3Name> {{ static constexpr const char * bwd_v3_name = "bwd_v3_hd64_bf16_causal_a32_rtne"; }}; -template<> struct FmhaBwdV3Name> {{ static constexpr const char * bwd_v3_name = "bwd_v3_hd64_bf16_causal_a32_rtna"; }}; -template<> struct FmhaBwdV3Name> {{ static constexpr const char * bwd_v3_name = "bwd_v3_hd64_bf16_causal_a32_rtz"; }}; -template<> struct FmhaBwdV3Name> {{ static constexpr const char * bwd_v3_name = "bwd_v3_hd64_fp16_a16"; }}; -template<> struct FmhaBwdV3Name> {{ static constexpr const char * bwd_v3_name = "bwd_v3_hd64_fp16_a32"; }}; -template<> struct FmhaBwdV3Name> {{ static constexpr const char * bwd_v3_name = "bwd_v3_hd64_fp16_causal_a16"; }}; -template<> struct FmhaBwdV3Name> {{ static constexpr const char * bwd_v3_name = "bwd_v3_hd64_fp16_causal_a32"; }}; - -template struct FmhaBwdV3Buf; -// #######################################################|HDim| DataType|kIsCausal|kIsAtomic32|BF16Cvt|kIsHDPad| -template<> struct FmhaBwdV3Buf> {{ static constexpr unsigned char * bwd_v3_buf = bwd_bf16_a16_rtne; }}; -template<> struct FmhaBwdV3Buf> {{ static constexpr unsigned char * bwd_v3_buf = bwd_bf16_a16_rtna; }}; -template<> struct FmhaBwdV3Buf> {{ static constexpr unsigned char * bwd_v3_buf = bwd_bf16_a16_rtz; }}; -template<> struct FmhaBwdV3Buf> {{ static constexpr unsigned char * bwd_v3_buf = bwd_bf16_a32_rtne; }}; -template<> struct FmhaBwdV3Buf> {{ static constexpr unsigned char * bwd_v3_buf = bwd_bf16_a32_rtna; }}; -template<> struct FmhaBwdV3Buf> {{ static constexpr unsigned char * bwd_v3_buf = bwd_bf16_a32_rtz; }}; -template<> struct FmhaBwdV3Buf> {{ static constexpr unsigned char * bwd_v3_buf = bwd_bf16_causal_a16_rtne; }}; -template<> struct FmhaBwdV3Buf> {{ static constexpr unsigned char * bwd_v3_buf = bwd_bf16_causal_a16_rtna; }}; -template<> struct FmhaBwdV3Buf> {{ static constexpr unsigned char * bwd_v3_buf = bwd_bf16_causal_a16_rtz; }}; -template<> struct FmhaBwdV3Buf> {{ static constexpr unsigned char * bwd_v3_buf = bwd_bf16_causal_a32_rtne; }}; -template<> struct FmhaBwdV3Buf> {{ static constexpr unsigned char * bwd_v3_buf = bwd_bf16_causal_a32_rtna; }}; -template<> struct FmhaBwdV3Buf> {{ static constexpr unsigned char * bwd_v3_buf = bwd_bf16_causal_a32_rtz; }}; -template<> struct FmhaBwdV3Buf> {{ static constexpr unsigned char * bwd_v3_buf = bwd_fp16_a16; }}; -template<> struct FmhaBwdV3Buf> {{ static constexpr unsigned char * bwd_v3_buf = bwd_fp16_a32; }}; -template<> struct FmhaBwdV3Buf> {{ static constexpr unsigned char * bwd_v3_buf = bwd_fp16_causal_a16; }}; -template<> struct FmhaBwdV3Buf> {{ static constexpr unsigned char * bwd_v3_buf = bwd_fp16_causal_a32; }}; -template<> struct FmhaBwdV3Buf> {{ static constexpr unsigned char * bwd_v3_buf = bwd_bf16_a16_rtne_pddv; }}; -template<> struct FmhaBwdV3Buf> {{ static constexpr unsigned char * bwd_v3_buf = bwd_bf16_a16_rtna_pddv; }}; -template<> struct FmhaBwdV3Buf> {{ static constexpr unsigned char * bwd_v3_buf = bwd_bf16_a16_rtz_pddv; }}; -template<> struct FmhaBwdV3Buf> {{ static constexpr unsigned char * bwd_v3_buf = bwd_bf16_a32_rtne_pddv; }}; -template<> struct FmhaBwdV3Buf> {{ static constexpr unsigned char * bwd_v3_buf = bwd_bf16_a32_rtna_pddv; }}; -template<> struct FmhaBwdV3Buf> {{ static constexpr unsigned char * bwd_v3_buf = bwd_bf16_a32_rtz_pddv; }}; -template<> struct FmhaBwdV3Buf> {{ static constexpr unsigned char * bwd_v3_buf = bwd_bf16_causal_a16_rtne_pddv; }}; -template<> struct FmhaBwdV3Buf> {{ static constexpr unsigned char * bwd_v3_buf = bwd_bf16_causal_a16_rtna_pddv; }}; -template<> struct FmhaBwdV3Buf> {{ static constexpr unsigned char * bwd_v3_buf = bwd_bf16_causal_a16_rtz_pddv; }}; -template<> struct FmhaBwdV3Buf> {{ static constexpr unsigned char * bwd_v3_buf = bwd_bf16_causal_a32_rtne_pddv; }}; -template<> struct FmhaBwdV3Buf> {{ static constexpr unsigned char * bwd_v3_buf = bwd_bf16_causal_a32_rtna_pddv; }}; -template<> struct FmhaBwdV3Buf> {{ static constexpr unsigned char * bwd_v3_buf = bwd_bf16_causal_a32_rtz_pddv; }}; -template<> struct FmhaBwdV3Buf> {{ static constexpr unsigned char * bwd_v3_buf = bwd_fp16_a16_pddv; }}; -template<> struct FmhaBwdV3Buf> {{ static constexpr unsigned char * bwd_v3_buf = bwd_fp16_a32_pddv; }}; -template<> struct FmhaBwdV3Buf> {{ static constexpr unsigned char * bwd_v3_buf = bwd_fp16_causal_a16_pddv; }}; -template<> struct FmhaBwdV3Buf> {{ static constexpr unsigned char * bwd_v3_buf = bwd_fp16_causal_a32_pddv; }}; -template<> struct FmhaBwdV3Buf> {{ static constexpr unsigned char * bwd_v3_buf = bwd_hd64_bf16_a16_rtne; }}; -template<> struct FmhaBwdV3Buf> {{ static constexpr unsigned char * bwd_v3_buf = bwd_hd64_bf16_a16_rtna; }}; -template<> struct FmhaBwdV3Buf> {{ static constexpr unsigned char * bwd_v3_buf = bwd_hd64_bf16_a16_rtz; }}; -template<> struct FmhaBwdV3Buf> {{ static constexpr unsigned char * bwd_v3_buf = bwd_hd64_bf16_a32_rtne; }}; -template<> struct FmhaBwdV3Buf> {{ static constexpr unsigned char * bwd_v3_buf = bwd_hd64_bf16_a32_rtna; }}; -template<> struct FmhaBwdV3Buf> {{ static constexpr unsigned char * bwd_v3_buf = bwd_hd64_bf16_a32_rtz; }}; -template<> struct FmhaBwdV3Buf> {{ static constexpr unsigned char * bwd_v3_buf = bwd_hd64_bf16_causal_a16_rtne; }}; -template<> struct FmhaBwdV3Buf> {{ static constexpr unsigned char * bwd_v3_buf = bwd_hd64_bf16_causal_a16_rtna; }}; -template<> struct FmhaBwdV3Buf> {{ static constexpr unsigned char * bwd_v3_buf = bwd_hd64_bf16_causal_a16_rtz; }}; -template<> struct FmhaBwdV3Buf> {{ static constexpr unsigned char * bwd_v3_buf = bwd_hd64_bf16_causal_a32_rtne; }}; -template<> struct FmhaBwdV3Buf> {{ static constexpr unsigned char * bwd_v3_buf = bwd_hd64_bf16_causal_a32_rtna; }}; -template<> struct FmhaBwdV3Buf> {{ static constexpr unsigned char * bwd_v3_buf = bwd_hd64_bf16_causal_a32_rtz; }}; -template<> struct FmhaBwdV3Buf> {{ static constexpr unsigned char * bwd_v3_buf = bwd_hd64_fp16_a16; }}; -template<> struct FmhaBwdV3Buf> {{ static constexpr unsigned char * bwd_v3_buf = bwd_hd64_fp16_a32; }}; -template<> struct FmhaBwdV3Buf> {{ static constexpr unsigned char * bwd_v3_buf = bwd_hd64_fp16_causal_a16; }}; -template<> struct FmhaBwdV3Buf> {{ static constexpr unsigned char * bwd_v3_buf = bwd_hd64_fp16_causal_a32; }}; - -template struct FmhaBwdV3Ts; -// ######################################################|HDim| DataType|kIsCausal|kIsAtomic32|BF16Cvt|kIsHDPad| -template<> struct FmhaBwdV3Ts> {{ static constexpr int ts_qo = 16; static constexpr int ts_kv = 192; }}; -template<> struct FmhaBwdV3Ts> {{ static constexpr int ts_qo = 16; static constexpr int ts_kv = 192; }}; -template<> struct FmhaBwdV3Ts> {{ static constexpr int ts_qo = 16; static constexpr int ts_kv = 192; }}; -template<> struct FmhaBwdV3Ts> {{ static constexpr int ts_qo = 16; static constexpr int ts_kv = 192; }}; -template<> struct FmhaBwdV3Ts> {{ static constexpr int ts_qo = 16; static constexpr int ts_kv = 192; }}; -template<> struct FmhaBwdV3Ts> {{ static constexpr int ts_qo = 16; static constexpr int ts_kv = 192; }}; -template<> struct FmhaBwdV3Ts> {{ static constexpr int ts_qo = 16; static constexpr int ts_kv = 192; }}; -template<> struct FmhaBwdV3Ts> {{ static constexpr int ts_qo = 16; static constexpr int ts_kv = 192; }}; -template<> struct FmhaBwdV3Ts> {{ static constexpr int ts_qo = 16; static constexpr int ts_kv = 192; }}; -template<> struct FmhaBwdV3Ts> {{ static constexpr int ts_qo = 16; static constexpr int ts_kv = 192; }}; -template<> struct FmhaBwdV3Ts> {{ static constexpr int ts_qo = 16; static constexpr int ts_kv = 192; }}; -template<> struct FmhaBwdV3Ts> {{ static constexpr int ts_qo = 16; static constexpr int ts_kv = 192; }}; -template<> struct FmhaBwdV3Ts> {{ static constexpr int ts_qo = 16; static constexpr int ts_kv = 192; }}; -template<> struct FmhaBwdV3Ts> {{ static constexpr int ts_qo = 16; static constexpr int ts_kv = 192; }}; -template<> struct FmhaBwdV3Ts> {{ static constexpr int ts_qo = 16; static constexpr int ts_kv = 192; }}; -template<> struct FmhaBwdV3Ts> {{ static constexpr int ts_qo = 16; static constexpr int ts_kv = 192; }}; -template<> struct FmhaBwdV3Ts> {{ static constexpr int ts_qo = 16; static constexpr int ts_kv = 192; }}; -template<> struct FmhaBwdV3Ts> {{ static constexpr int ts_qo = 16; static constexpr int ts_kv = 192; }}; -template<> struct FmhaBwdV3Ts> {{ static constexpr int ts_qo = 16; static constexpr int ts_kv = 192; }}; -template<> struct FmhaBwdV3Ts> {{ static constexpr int ts_qo = 16; static constexpr int ts_kv = 192; }}; -template<> struct FmhaBwdV3Ts> {{ static constexpr int ts_qo = 16; static constexpr int ts_kv = 192; }}; -template<> struct FmhaBwdV3Ts> {{ static constexpr int ts_qo = 16; static constexpr int ts_kv = 192; }}; -template<> struct FmhaBwdV3Ts> {{ static constexpr int ts_qo = 16; static constexpr int ts_kv = 192; }}; -template<> struct FmhaBwdV3Ts> {{ static constexpr int ts_qo = 16; static constexpr int ts_kv = 192; }}; -template<> struct FmhaBwdV3Ts> {{ static constexpr int ts_qo = 16; static constexpr int ts_kv = 192; }}; -template<> struct FmhaBwdV3Ts> {{ static constexpr int ts_qo = 16; static constexpr int ts_kv = 192; }}; -template<> struct FmhaBwdV3Ts> {{ static constexpr int ts_qo = 16; static constexpr int ts_kv = 192; }}; -template<> struct FmhaBwdV3Ts> {{ static constexpr int ts_qo = 16; static constexpr int ts_kv = 192; }}; -template<> struct FmhaBwdV3Ts> {{ static constexpr int ts_qo = 16; static constexpr int ts_kv = 192; }}; -template<> struct FmhaBwdV3Ts> {{ static constexpr int ts_qo = 16; static constexpr int ts_kv = 192; }}; -template<> struct FmhaBwdV3Ts> {{ static constexpr int ts_qo = 16; static constexpr int ts_kv = 192; }}; -template<> struct FmhaBwdV3Ts> {{ static constexpr int ts_qo = 16; static constexpr int ts_kv = 192; }}; -template<> struct FmhaBwdV3Ts> {{ static constexpr int ts_qo = 32; static constexpr int ts_kv = 192; }}; -template<> struct FmhaBwdV3Ts> {{ static constexpr int ts_qo = 32; static constexpr int ts_kv = 192; }}; -template<> struct FmhaBwdV3Ts> {{ static constexpr int ts_qo = 32; static constexpr int ts_kv = 192; }}; -template<> struct FmhaBwdV3Ts> {{ static constexpr int ts_qo = 32; static constexpr int ts_kv = 192; }}; -template<> struct FmhaBwdV3Ts> {{ static constexpr int ts_qo = 32; static constexpr int ts_kv = 192; }}; -template<> struct FmhaBwdV3Ts> {{ static constexpr int ts_qo = 32; static constexpr int ts_kv = 192; }}; -template<> struct FmhaBwdV3Ts> {{ static constexpr int ts_qo = 32; static constexpr int ts_kv = 192; }}; -template<> struct FmhaBwdV3Ts> {{ static constexpr int ts_qo = 32; static constexpr int ts_kv = 192; }}; -template<> struct FmhaBwdV3Ts> {{ static constexpr int ts_qo = 32; static constexpr int ts_kv = 192; }}; -template<> struct FmhaBwdV3Ts> {{ static constexpr int ts_qo = 32; static constexpr int ts_kv = 192; }}; -template<> struct FmhaBwdV3Ts> {{ static constexpr int ts_qo = 32; static constexpr int ts_kv = 192; }}; -template<> struct FmhaBwdV3Ts> {{ static constexpr int ts_qo = 32; static constexpr int ts_kv = 192; }}; -template<> struct FmhaBwdV3Ts> {{ static constexpr int ts_qo = 32; static constexpr int ts_kv = 192; }}; -template<> struct FmhaBwdV3Ts> {{ static constexpr int ts_qo = 32; static constexpr int ts_kv = 192; }}; -template<> struct FmhaBwdV3Ts> {{ static constexpr int ts_qo = 32; static constexpr int ts_kv = 192; }}; -template<> struct FmhaBwdV3Ts> {{ static constexpr int ts_qo = 32; static constexpr int ts_kv = 192; }}; +{F_template} class fmha_bwd_v3_kernel {{ @@ -732,7 +588,17 @@ float fmha_bwd_v3_hdp_xqa_(const ck_tile::stream_config& s, fmha_bwd_args a) float fmha_bwd(fmha_bwd_traits t, fmha_bwd_args a, const ck_tile::stream_config& s){{ float r = -1; + + if ((t.uses_bwd_v3 == true){{ + if ((t.is_group_mode == false) && (t.bias_type == bias_enum::no_bias) && (t.has_dbias == false) && (t.has_dropout == false) && (t.is_deterministic == false) && (a.hdim_q == a.hdim_v) && + (a.seqlen_q == a.seqlen_k) && (a.nhead_q % a.nhead_k == 0) && (a.stride_q == a.stride_do) && (a.nhead_stride_q == a.nhead_stride_do) && (a.batch_stride_q == a.batch_stride_do) && + (a.stride_k == a.stride_v) && (a.nhead_stride_k == a.nhead_stride_v) && (a.batch_stride_k == a.batch_stride_v) && (a.nhead_stride_k == a.nhead_stride_dk) && (a.nhead_stride_v == a.nhead_stride_dv) && + ((a.batch_stride_dk / a.batch_stride_k) == (a.nhead_q / a.nhead_k)) && ((a.batch_stride_dv / a.batch_stride_v) == (a.nhead_q / a.nhead_k))) {{ + if (((a.hdim_q >= 64) && (a.hdim_q <= 128) && (a.hdim_q % 8 == 0) && (a.seqlen_k % 64 == 0))) {{ {F_v3_dispatch} + }} + }} + }} {F_dispatch} return r; }} @@ -757,23 +623,18 @@ FMHA_BWD_API_INNER_DISPATCH=""" {F_if}((t.is_group_mode == {F_mode}) }} """ -FMHA_V3_DISPATCH=""" if ((t.uses_bwd_v3 == {F_use_v3}){{ - if ((t.is_group_mode == false) && (t.bias_type == bias_enum::no_bias) && (t.has_dbias == false) && (t.has_dropout == false) && (t.is_deterministic == false) && (a.hdim_q == a.hdim_v) && - (a.seqlen_q == a.seqlen_k) && (a.nhead_q % a.nhead_k == 0) && (a.stride_q == a.stride_do) && (a.nhead_stride_q == a.nhead_stride_do) && (a.batch_stride_q == a.batch_stride_do) && - (a.stride_k == a.stride_v) && (a.nhead_stride_k == a.nhead_stride_v) && (a.batch_stride_k == a.batch_stride_v) && (a.nhead_stride_k == a.nhead_stride_dk) && (a.nhead_stride_v == a.nhead_stride_dv) && - ((a.batch_stride_dk / a.batch_stride_k) == (a.nhead_q / a.nhead_k)) && ((a.batch_stride_dv / a.batch_stride_v) == (a.nhead_q / a.nhead_k))) {{ - if (((a.hdim_q >= 64) && (a.hdim_q <= 128) && (a.hdim_q % 8 == 0) && (a.seqlen_k % 64 == 0))) {{ +FMHA_V3_DISPATCH=""" if (t.mask_type == mask_enum::no_mask){{ if ((t.is_v3_atomic_fp32 == true) && (a.nhead_stride_dq_acc >= a.stride_dq_acc /*dq_acc only support BHSD*/)){{ using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<{F_hdim}, {F_dtype}, false, false, {F_padding}>; - using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<{F_hdim}, {F_dtype}, false, {F_is_atomic32}, {F_how_v3_bf16_cvt}, {F_padding}>; + using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<{F_hdim}, {F_dtype}, {F_is_causal}, {F_is_atomic32}, {F_how_v3_bf16_cvt}, {F_padding}>; using convert_dq_trait_ = fmha_bwd_convert_dq_traits_<{F_hdim}, {F_dtype}, false, false, {F_padding}, false>; r = fmha_bwd_v3{F_padding_suffix}_xqa_(s, a); return r; }} else if (t.is_v3_atomic_fp32 == false){{ using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<{F_hdim}, {F_dtype}, false, false, {F_padding}>; - using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<{F_hdim}, {F_dtype}, false, {F_is_atomic32}, {F_how_v3_bf16_cvt}, {F_padding}>; + using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<{F_hdim}, {F_dtype}, {F_is_causal}, {F_is_atomic32}, {F_how_v3_bf16_cvt}, {F_padding}>; r = fmha_bwd_v3{F_padding_suffix}_xqa_(s, a); return r; }} @@ -781,21 +642,19 @@ FMHA_V3_DISPATCH=""" if ((t.uses_bwd_v3 == {F_use_v3}){{ else if ((t.mask_type != mask_enum::no_mask) && ((a.window_size_left == -1) && (a.window_size_right == 0))){{ if ((t.is_v3_atomic_fp32 == true) && (a.nhead_stride_dq_acc >= a.stride_dq_acc /*dq_acc only support BHSD*/)){{ using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<{F_hdim}, {F_dtype}, false, false, {F_padding}>; - using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<{F_hdim}, {F_dtype}, true, {F_is_atomic32}, {F_how_v3_bf16_cvt}, {F_padding}>; + using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<{F_hdim}, {F_dtype}, {F_is_causal}, {F_is_atomic32}, {F_how_v3_bf16_cvt}, {F_padding}>; using convert_dq_trait_ = fmha_bwd_convert_dq_traits_<{F_hdim}, {F_dtype}, false, false, {F_padding}, false>; r = fmha_bwd_v3{F_padding_suffix}_xqa_(s, a); return r; }} else if (t.is_v3_atomic_fp32 == false){{ using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<{F_hdim}, {F_dtype}, false, false, {F_padding}>; - using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<{F_hdim}, {F_dtype}, true, {F_is_atomic32}, {F_how_v3_bf16_cvt}, {F_padding}>; + using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<{F_hdim}, {F_dtype}, {F_is_causal}, {F_is_atomic32}, {F_how_v3_bf16_cvt}, {F_padding}>; r = fmha_bwd_v3{F_padding_suffix}_xqa_(s, a); return r; }} }} }} - }} - }} """ @dataclass @@ -818,9 +677,6 @@ class FmhaBwdDQDKDVApiTrait: dpad : str dvpad : str deterministic : str - uses_bwd_v3 : str - v3_atomic_fp32 : str - how_v3_bf16_cvt : int def scheck(self, spad1 : str) -> str: if self.mode == 'group': @@ -851,6 +707,15 @@ class FmhaBwdDQDKDVApiTrait: if self.dvpad == 't': return f'a.hdim_v % {self.bhdv} != 0' else : return f'a.hdim_v % {self.bhdv} == 0' +@dataclass +class FmhaBwdV3DQDKDVApiTrait: + hdim : str + dtype : str # data type + is_causal : str + is_atomic : str + bf16_cvt : int + hd_pad : str + class FmhaBwdApiPool: def __init__(self, mask_impl): self.dq_dk_dv_pool = dict() @@ -885,7 +750,6 @@ class FmhaBwdApiPool: F_scheck=trait.scheck(spad1=spad1), F_skcheck=trait.skcheck, F_dcheck=trait.dcheck, F_dvcheck=trait.dvcheck, F_hdim=hdim, F_dtype=BWD_DTYPE_MAP[dtype], F_spad0=BOOL_MAP[trait.spad], F_spad1=BOOL_MAP[spad1], F_skpad=BOOL_MAP[trait.skpad], F_dpad=BOOL_MAP[trait.dpad], F_dvpad=BOOL_MAP[trait.dvpad], F_deterministic=BOOL_MAP[trait.deterministic]) - if_j = 'if' if j == 0 else 'else if' per_hdim_case = per_hdim_case + FMHA_BWD_API_PER_HDIM_CASE.format(F_if=if_j, F_hdim=hdim, F_inner_dispatch=inners) if_i = 'if' if i == 0 else 'else if' @@ -893,20 +757,30 @@ class FmhaBwdApiPool: if not per_dtypes: # empty string we add some ignore to suppress warning in api per_dtypes += ' (void)t ; (void)s ; (void)a;' - # v3 codegen: + v3_code = str() + gen_template = str() for i, dtype in enumerate(self.dq_dk_dv_pool.keys()): - per_hdim_case=str() for j, hdim in enumerate(self.dq_dk_dv_pool[dtype].keys()): traits = self.dq_dk_dv_pool[dtype][hdim] hdim_int = int(hdim) hdim_int = (hdim_int + 64 - 1) / 64 * 64 + Ts_qo = 32 if hdim == 64 else 16 for k, trait in enumerate(traits): padding = "t" if hdim_int % 64 == 0 else "f" padding_suffix = "_hdp" if padding == "t" else "" - v3_code = v3_code + FMHA_V3_DISPATCH.format(F_use_v3=BOOL_MAP[trait.use_bwd_v3], F_hdim=hdim_int, F_dtype=BWD_DTYPE_MAP[dtype], F_padding=BOOL_MAP[padding], - F_is_atomic32=BOOL_MAP[trait.v3_atomic_fp32], F_how_v3_bf16_cvt=trait.how_v3_bf16_cvt, F_padding_suffix=padding_suffix) - return FMHA_BWD_KERNEL_HEADER + FMHA_BWD_API.format(F_dispatch = per_dtypes, F_v3_dispatch = v3_code) + v3_code = v3_code + FMHA_V3_DISPATCH.format(F_hdim=hdim_int, F_dtype=BWD_DTYPE_MAP[dtype], F_padding=BOOL_MAP[padding], + F_is_atomic32=BOOL_MAP[trait.is_atomic], F_how_v3_bf16_cvt=trait.bf16_cvt, F_padding_suffix=padding_suffix) + + hdim_name = "hd64" if hdim_int == 64 else "" + dtype_name = dtype + causal_name = "causal" if trait.is_causal == "t" else "" + atomic_name = "a32" if trait.is_atomic == "t" else "a16" + bf16_cvt_name = BF16_CVT_MAP[trait.bf16_cvt] + gen_template = gen_template + FMHA_BWD_V3_TEMPLATE.format(F_hdim=hdim_int, F_dtype=BWD_DTYPE_MAP[dtype], F_is_atomic=BOOL_MAP[trait.is_atomic], + F_is_causal=BOOL_MAP[trait.is_causal], F_bf16_cvt=trait.bf16_cvt, F_hdpad=BOOL_MAP[padding], F_Ts_qo = Ts_qo, F_hdim_name=hdim_name, + F_dtype_name=dtype_name, F_causal_name=causal_name, F_atomic_name=atomic_name, F_bf16_cvt_name=bf16_cvt_name) + return FMHA_BWD_KERNEL_HEADER + FMHA_BWD_API.format(F_dispatch = per_dtypes, F_template = gen_template, F_v3_dispatch = v3_code) # GEMM0: Q@K=S^T # GEMM1: P^T@dO^T=dV(This was chosen as G1 to match fwd, but N1 must be equal to headdim_v) @@ -1038,7 +912,6 @@ class FmhaBwdDQDKDVKernel: def filename(self) -> str: return self.name + ".cpp" - # TODO: Jim: update this def api_trait(self) -> FmhaBwdDQDKDVApiTrait: return FmhaBwdDQDKDVApiTrait(pipeline=self.F_pipeline, hdim=str(self.F_hdim), @@ -1056,10 +929,7 @@ class FmhaBwdDQDKDVKernel: skpad=self.F_skpad, dpad=self.F_dpad, dvpad=self.F_dvpad, - deterministic=self.F_deterministic, - use_bwd_v3=self.use_bwd_v3, - v3_atomic_fp32=self.v3_atomic_fp32, - how_v3_bf16_cvt=self.how_v3_bf16_cvt + deterministic=self.F_deterministic ) # TODO: design a more practical way to do it From 86923c19f79b755eb6f0499f310cde218eb00f08 Mon Sep 17 00:00:00 2001 From: Jim Date: Tue, 11 Feb 2025 03:06:07 +0800 Subject: [PATCH 03/10] update --- .../ck_tile/01_fmha/codegen/cpp_symbol_map.py | 33 + .../ck_tile/01_fmha/codegen/ops/fmha_bwd.py | 191 +- example/ck_tile/01_fmha/fmha_fwd_api.cpp | 5903 +++++++++++++++++ example/ck_tile/01_fmha/generate.py | 2 +- 4 files changed, 6075 insertions(+), 54 deletions(-) create mode 100644 example/ck_tile/01_fmha/fmha_fwd_api.cpp diff --git a/example/ck_tile/01_fmha/codegen/cpp_symbol_map.py b/example/ck_tile/01_fmha/codegen/cpp_symbol_map.py index ac02f8d8d6..d5556bcb83 100644 --- a/example/ck_tile/01_fmha/codegen/cpp_symbol_map.py +++ b/example/ck_tile/01_fmha/codegen/cpp_symbol_map.py @@ -127,8 +127,41 @@ BOOL_MAP = { "f" : "false" } +BWD_V3_HDIM_MAP = { + "64": "64", + "128": "128" +} + BF16_CVT_MAP = { 0 : "rtne", 1 : "rtna", 2 : "rtz", +} + +BWD_V3_MASK_MAP = { + "t": "((t.mask_type != mask_enum::no_mask) && ((a.window_size_left == -1) && (a.window_size_right == 0)))", + "f": "(t.mask_type == mask_enum::no_mask)" +} + +BWD_V3_ATOMIC32_MAP = { + "t": "((t.is_v3_atomic_fp32 == true) && (a.nhead_stride_dq_acc >= a.stride_dq_acc /*dq_acc only support BHSD*/))", + "f": "(t.is_v3_atomic_fp32 == false)" +} + +BWD_V3_HDIM_CASE_MAP = { + 0: "(a.hdim_q == 128)", + 1: "(a.hdim_q == 64)", + 2: "((a.hdim_q > 64) && (a.hdim_q < 128))" +} + +BWD_V3_HDIM_CASE_CHECK_MAP = { + 0: 128, + 1: 64, + 2: 128 +} + +BWD_V3_PADDING_CHECK_MAP = { + 0: "false", + 1: "false", + 2: "true" } \ No newline at end of file diff --git a/example/ck_tile/01_fmha/codegen/ops/fmha_bwd.py b/example/ck_tile/01_fmha/codegen/ops/fmha_bwd.py index 6f97a65013..5806bf29d4 100644 --- a/example/ck_tile/01_fmha/codegen/ops/fmha_bwd.py +++ b/example/ck_tile/01_fmha/codegen/ops/fmha_bwd.py @@ -162,9 +162,8 @@ std::string fmha_bwd_dq_dk_dv_get_name_() FMHA_BWD_API_FILENAME="fmha_bwd_api.cpp" -FMHA_BWD_V3_TEMPLATE=""" -template<> struct FmhaBwdV3Name> {{ static constexpr const char * bwd_v3_name = "bwd_v3_{F_hdim_name}_{F_dtype_name}_{F_causal_name}_{F_atomic_name}_{F_bf16_cvt_name}"; }}; -template<> struct FmhaBwdV3Buf> {{ static constexpr unsigned char * bwd_v3_buf = bwd_{F_hdim_name}_{F_dtype_name}_{F_causal_name}_{F_atomic_name}_{F_bf16_cvt_name}; }}; +FMHA_BWD_V3_TEMPLATE="""template<> struct FmhaBwdV3Name> {{ static constexpr const char * bwd_v3_name = "bwd_v3{F_hdim_name}{F_dtype_name}{F_causal_name}{F_atomic_name}{F_bf16_cvt_name}"; }}; +template<> struct FmhaBwdV3Buf> {{ static constexpr unsigned char * bwd_v3_buf = bwd{F_hdim_name}{F_dtype_name}{F_causal_name}{F_atomic_name}{F_bf16_cvt_name}; }}; template<> struct FmhaBwdV3Ts> {{ static constexpr int ts_qo = {F_Ts_qo}; static constexpr int ts_kv = 192; }}; """ @@ -589,7 +588,7 @@ float fmha_bwd_v3_hdp_xqa_(const ck_tile::stream_config& s, fmha_bwd_args a) float fmha_bwd(fmha_bwd_traits t, fmha_bwd_args a, const ck_tile::stream_config& s){{ float r = -1; - if ((t.uses_bwd_v3 == true){{ + if (t.uses_bwd_v3 == true){{ if ((t.is_group_mode == false) && (t.bias_type == bias_enum::no_bias) && (t.has_dbias == false) && (t.has_dropout == false) && (t.is_deterministic == false) && (a.hdim_q == a.hdim_v) && (a.seqlen_q == a.seqlen_k) && (a.nhead_q % a.nhead_k == 0) && (a.stride_q == a.stride_do) && (a.nhead_stride_q == a.nhead_stride_do) && (a.batch_stride_q == a.batch_stride_do) && (a.stride_k == a.stride_v) && (a.nhead_stride_k == a.nhead_stride_v) && (a.batch_stride_k == a.batch_stride_v) && (a.nhead_stride_k == a.nhead_stride_dk) && (a.nhead_stride_v == a.nhead_stride_dv) && @@ -623,38 +622,40 @@ FMHA_BWD_API_INNER_DISPATCH=""" {F_if}((t.is_group_mode == {F_mode}) }} """ -FMHA_V3_DISPATCH=""" - if (t.mask_type == mask_enum::no_mask){{ - if ((t.is_v3_atomic_fp32 == true) && (a.nhead_stride_dq_acc >= a.stride_dq_acc /*dq_acc only support BHSD*/)){{ - using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<{F_hdim}, {F_dtype}, false, false, {F_padding}>; - using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<{F_hdim}, {F_dtype}, {F_is_causal}, {F_is_atomic32}, {F_how_v3_bf16_cvt}, {F_padding}>; - using convert_dq_trait_ = fmha_bwd_convert_dq_traits_<{F_hdim}, {F_dtype}, false, false, {F_padding}, false>; - r = fmha_bwd_v3{F_padding_suffix}_xqa_(s, a); - return r; - }} - else if (t.is_v3_atomic_fp32 == false){{ - using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<{F_hdim}, {F_dtype}, false, false, {F_padding}>; - using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<{F_hdim}, {F_dtype}, {F_is_causal}, {F_is_atomic32}, {F_how_v3_bf16_cvt}, {F_padding}>; - r = fmha_bwd_v3{F_padding_suffix}_xqa_(s, a); - return r; - }} +FMHA_BWD_V3_ATOMIC32_INNER_DISPATCH=""" using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<{F_hdim}, {F_dtype}, false, false, {F_padding}>; + using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<{F_hdim}, {F_dtype}, {F_is_causal}, {F_is_atomic32}, {F_how_v3_bf16_cvt}, {F_padding}>; + using convert_dq_trait_ = fmha_bwd_convert_dq_traits_<{F_hdim}, {F_dtype}, false, false, {F_padding}, false>; + r = fmha_bwd_v3_hdp_xqa_(s, a); + return r;""" + +FMHA_BWD_V3_ATOMIC16_INNER_DISPATCH=""" using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<{F_hdim}, {F_dtype}, false, false, {F_padding}>; + using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<{F_hdim}, {F_dtype}, {F_is_causal}, {F_is_atomic32}, {F_how_v3_bf16_cvt}, {F_padding}>; + r = fmha_bwd_v3_xqa_(s, a); + return r;""" + +FMHA_BWD_V3_PER_DTYPE_CASE=""" {F_if} (t.data_type.compare(\"{F_dtype}\") == 0) {{ +{per_bf16_cvt_dispatch} }} - else if ((t.mask_type != mask_enum::no_mask) && ((a.window_size_left == -1) && (a.window_size_right == 0))){{ - if ((t.is_v3_atomic_fp32 == true) && (a.nhead_stride_dq_acc >= a.stride_dq_acc /*dq_acc only support BHSD*/)){{ - using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<{F_hdim}, {F_dtype}, false, false, {F_padding}>; - using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<{F_hdim}, {F_dtype}, {F_is_causal}, {F_is_atomic32}, {F_how_v3_bf16_cvt}, {F_padding}>; - using convert_dq_trait_ = fmha_bwd_convert_dq_traits_<{F_hdim}, {F_dtype}, false, false, {F_padding}, false>; - r = fmha_bwd_v3{F_padding_suffix}_xqa_(s, a); - return r; +""" + +FMHA_BWD_V3_PER_BF16_CVT_CASE=""" {F_if} (t.how_v3_bf16_cvt == {F_bf16_cvt}) {{ +{per_mask_dispatch} }} - else if (t.is_v3_atomic_fp32 == false){{ - using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<{F_hdim}, {F_dtype}, false, false, {F_padding}>; - using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<{F_hdim}, {F_dtype}, {F_is_causal}, {F_is_atomic32}, {F_how_v3_bf16_cvt}, {F_padding}>; - r = fmha_bwd_v3{F_padding_suffix}_xqa_(s, a); - return r; - }} - }} - }} +""" + +FMHA_BWD_V3_PER_MASK_CASE=""" {F_if} {F_mask_expression}{{ +{per_atomic_dispatch} + }} +""" + +FMHA_BWD_V3_PER_ATOMIC_CASE=""" {F_if} {F_atomic_expression}{{ +{per_hdim_dispatch} + }} +""" + +FMHA_BWD_V3_PER_HDIM_CASE=""" {F_if} {F_hdim_expression}{{ +{inner_dispatch} + }} """ @dataclass @@ -714,11 +715,18 @@ class FmhaBwdV3DQDKDVApiTrait: is_causal : str is_atomic : str bf16_cvt : int - hd_pad : str + is_hdpad : str + + def remap_hdim(self): + hdim_int = int(self.hdim) + if hdim_int > 64: + self.hdim = 128 + hdim_int = (hdim_int + 64 - 1) / 64 * 64 class FmhaBwdApiPool: def __init__(self, mask_impl): self.dq_dk_dv_pool = dict() + self.dq_dk_dv_v3_pool = dict() self.mask_impl = mask_impl def register_dq_dk_dv_traits(self, trait : FmhaBwdDQDKDVApiTrait) -> None: @@ -730,6 +738,15 @@ class FmhaBwdApiPool: self.dq_dk_dv_pool[trait.dtype][trait.hdim].append(copy.copy(trait)) + def register_dq_dk_dv_v3_traits(self, trait : FmhaBwdV3DQDKDVApiTrait) -> None: + # TODO: do we need to check duplication? + if trait.dtype not in self.dq_dk_dv_v3_pool.keys(): + self.dq_dk_dv_v3_pool[trait.dtype] = dict() + if trait.hdim not in self.dq_dk_dv_v3_pool[trait.dtype].keys(): + self.dq_dk_dv_v3_pool[trait.dtype][trait.hdim] = list() + + self.dq_dk_dv_v3_pool[trait.dtype][trait.hdim].append(copy.copy(trait)) + @property def api(self) -> str: per_dtypes=str() @@ -758,28 +775,51 @@ class FmhaBwdApiPool: # empty string we add some ignore to suppress warning in api per_dtypes += ' (void)t ; (void)s ; (void)a;' - v3_code = str() gen_template = str() - for i, dtype in enumerate(self.dq_dk_dv_pool.keys()): - for j, hdim in enumerate(self.dq_dk_dv_pool[dtype].keys()): - traits = self.dq_dk_dv_pool[dtype][hdim] - hdim_int = int(hdim) - hdim_int = (hdim_int + 64 - 1) / 64 * 64 + for i, dtype in enumerate(self.dq_dk_dv_v3_pool.keys()): + for j, hdim in enumerate(BWD_V3_HDIM_MAP.keys()): + traits = self.dq_dk_dv_v3_pool[dtype][hdim] + hdim = int(hdim) Ts_qo = 32 if hdim == 64 else 16 for k, trait in enumerate(traits): - padding = "t" if hdim_int % 64 == 0 else "f" - padding_suffix = "_hdp" if padding == "t" else "" - v3_code = v3_code + FMHA_V3_DISPATCH.format(F_hdim=hdim_int, F_dtype=BWD_DTYPE_MAP[dtype], F_padding=BOOL_MAP[padding], - F_is_atomic32=BOOL_MAP[trait.is_atomic], F_how_v3_bf16_cvt=trait.bf16_cvt, F_padding_suffix=padding_suffix) - - hdim_name = "hd64" if hdim_int == 64 else "" - dtype_name = dtype - causal_name = "causal" if trait.is_causal == "t" else "" - atomic_name = "a32" if trait.is_atomic == "t" else "a16" - bf16_cvt_name = BF16_CVT_MAP[trait.bf16_cvt] - gen_template = gen_template + FMHA_BWD_V3_TEMPLATE.format(F_hdim=hdim_int, F_dtype=BWD_DTYPE_MAP[dtype], F_is_atomic=BOOL_MAP[trait.is_atomic], - F_is_causal=BOOL_MAP[trait.is_causal], F_bf16_cvt=trait.bf16_cvt, F_hdpad=BOOL_MAP[padding], F_Ts_qo = Ts_qo, F_hdim_name=hdim_name, + hdim_name = "_hd64" if hdim == 64 else "" + dtype_name = "_{}".format(dtype) + causal_name = "_causal" if trait.is_causal == "t" else "" + atomic_name = "_a32" if trait.is_atomic == "t" else "_a16" + bf16_cvt_name = "_{}".format(BF16_CVT_MAP[trait.bf16_cvt]) + gen_template = gen_template + FMHA_BWD_V3_TEMPLATE.format(F_hdim=hdim, F_dtype=BWD_DTYPE_MAP[dtype], F_is_atomic=BOOL_MAP[trait.is_atomic], + F_is_causal=BOOL_MAP[trait.is_causal], F_bf16_cvt=trait.bf16_cvt, F_hdpad=BOOL_MAP[trait.is_hdpad], F_Ts_qo = Ts_qo, F_hdim_name=hdim_name, F_dtype_name=dtype_name, F_causal_name=causal_name, F_atomic_name=atomic_name, F_bf16_cvt_name=bf16_cvt_name) + + v3_code = str() + for i, dtype in enumerate(self.dq_dk_dv_v3_pool.keys()): + per_bf16 = str() + for j, bf16_cvt in enumerate([0, 1, 2]): + per_mask = str() + if (dtype == "fp16") and (bf16_cvt in [1, 2]): + continue + for k, is_causal in enumerate(["t", "f"]): + per_atomic = str() + for l, is_atomic in enumerate(["t", "f"]): + per_hdim = str() + for m, hdim in enumerate(BWD_V3_HDIM_CASE_CHECK_MAP.values()): + if_m = 'if' if m == 0 else 'else if' + inners = str() + if is_atomic == "t": + inners = FMHA_BWD_V3_ATOMIC32_INNER_DISPATCH.format(F_hdim=hdim, F_dtype=BWD_DTYPE_MAP[dtype], F_is_causal=BOOL_MAP[is_causal], F_is_atomic32=BOOL_MAP[is_atomic], F_how_v3_bf16_cvt=bf16_cvt, F_padding=BWD_V3_PADDING_CHECK_MAP[m]) + else: + inners = FMHA_BWD_V3_ATOMIC16_INNER_DISPATCH.format(F_hdim=hdim, F_dtype=BWD_DTYPE_MAP[dtype], F_is_causal=BOOL_MAP[is_causal], F_is_atomic32=BOOL_MAP[is_atomic], F_how_v3_bf16_cvt=bf16_cvt, F_padding=BWD_V3_PADDING_CHECK_MAP[m]) + per_hdim = per_hdim + FMHA_BWD_V3_PER_HDIM_CASE.format(F_if=if_m, F_hdim_expression=BWD_V3_HDIM_CASE_MAP[m], inner_dispatch=inners) + + if_l = 'if' if l == 0 else 'else if' + per_atomic = per_atomic + FMHA_BWD_V3_PER_ATOMIC_CASE.format(F_if=if_l, F_atomic_expression=BWD_V3_ATOMIC32_MAP[is_atomic], per_hdim_dispatch=per_hdim) + if_k = 'if' if k == 0 else 'else if' + per_mask = per_mask + FMHA_BWD_V3_PER_MASK_CASE.format(F_if=if_k, F_mask_expression=BWD_V3_MASK_MAP[is_causal], per_atomic_dispatch=per_atomic) + if_j = 'if' if j == 0 else 'else if' + per_bf16 = per_bf16 + FMHA_BWD_V3_PER_BF16_CVT_CASE.format(F_if=if_j, F_bf16_cvt=bf16_cvt, per_mask_dispatch=per_mask) + if_i = 'if' if i == 0 else 'else if' + v3_code = v3_code + FMHA_BWD_V3_PER_DTYPE_CASE.format(F_if=if_i, F_dtype=dtype, per_bf16_cvt_dispatch=per_bf16) + return FMHA_BWD_KERNEL_HEADER + FMHA_BWD_API.format(F_dispatch = per_dtypes, F_template = gen_template, F_v3_dispatch = v3_code) # GEMM0: Q@K=S^T @@ -931,6 +971,40 @@ class FmhaBwdDQDKDVKernel: dvpad=self.F_dvpad, deterministic=self.F_deterministic ) + + +@dataclass +class FmhaBwdV3DQDKDVKernel: + F_hdim : int # hdim + F_dtype : str # data type + F_is_causal : str + F_is_atomic : str + F_bf16_cvt : int + F_is_hdpad : str + + # @property + # def gen_bwd_v3_template(self) -> str: + # hdim_template = 64 if self.F_hdim == 64 else 128 + # Ts_qo = 32 if hdim_template == 64 else 16 + # padding = "t" if self.F_hdim % 64 == 0 else "f" + # padding_suffix = "_hdp" if padding == "t" else "" + # hdim_name = "_hd64" if hdim_template == 64 else "" + # dtype_name = "_{}".format(self.F_dtype) + # causal_name = "_causal" if self.F_is_causal == "t" else "" + # atomic_name = "_a32" if self.F_is_atomic == "t" else "_a16" + # bf16_cvt_name = "_{}".format(BF16_CVT_MAP[self.F_bf16_cvt]) + # return FMHA_BWD_V3_TEMPLATE.format(F_hdim=hdim_template, F_dtype=BWD_DTYPE_MAP[self.F_dtype], F_is_atomic=BOOL_MAP[self.F_is_atomic], + # F_is_causal=BOOL_MAP[self.F_is_causal], F_bf16_cvt=self.F_bf16_cvt, F_hdpad=BOOL_MAP[padding], F_Ts_qo = Ts_qo, F_hdim_name=hdim_name, + # F_dtype_name=dtype_name, F_causal_name=causal_name, F_atomic_name=atomic_name, F_bf16_cvt_name=bf16_cvt_name) + + def v3_api_trait(self) -> FmhaBwdV3DQDKDVApiTrait: + return FmhaBwdV3DQDKDVApiTrait(hdim=str(self.F_hdim), + dtype=self.F_dtype, + is_causal=self.F_is_causal, + is_atomic=self.F_is_atomic, + bf16_cvt=self.F_bf16_cvt, + is_hdpad=self.F_is_hdpad + ) # TODO: design a more practical way to do it # this is current supported tile size & pipeline. @@ -995,6 +1069,17 @@ def get_bwd_dq_dk_dv_blobs(kernel_filter : Optional[str], receipt, mask_impl) -> api_pool.register_dq_dk_dv_traits(k.api_trait()) gen.append(k) + for hdim_str, is_causal, is_atomic, bf16_cvt, is_hdpad in itertools.product(d.keys(), ["t", "f"], ["t", "f"], [0, 1, 2], ["t", "f"]): + hdim = int(hdim_str) + k = FmhaBwdV3DQDKDVKernel(F_hdim=hdim, F_dtype=dtype, F_is_causal=is_causal, F_is_atomic=is_atomic, F_bf16_cvt=bf16_cvt, F_is_hdpad=is_hdpad) + if receipt == 3: + cond = (dtype == 'fp16') and (bf16_cvt in [1, 2]) + if cond: + # print(dtype, bf16_cvt) + continue + api_pool.register_dq_dk_dv_v3_traits(k.v3_api_trait()) + # gen.append(k) + return (api_pool, gen) FMHA_BWD_DOT_DO_O_KERNEL_BODY=""" diff --git a/example/ck_tile/01_fmha/fmha_fwd_api.cpp b/example/ck_tile/01_fmha/fmha_fwd_api.cpp new file mode 100644 index 0000000000..9f75eb03c1 --- /dev/null +++ b/example/ck_tile/01_fmha/fmha_fwd_api.cpp @@ -0,0 +1,5903 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2018-2024, Advanced Micro Devices, Inc. All rights reserved. + +// auto generated by generate.py +#include "fmha_fwd.hpp" + +float fmha_fwd(fmha_fwd_traits t, fmha_fwd_args a, const ck_tile::stream_config& s){ + float r = -1; + if(t.data_type.compare("fp16") == 0){ + if (t.hdim_q <= 32 && t.hdim_v <= 32) { + if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 32 == 0) && (a.hdim_v % 32 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 32 == 0) && (a.hdim_v % 32 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 32 == 0) && (a.hdim_v % 32 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 32 == 0) && (a.hdim_v % 32 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 32 == 0) && (a.hdim_v % 32 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 32 == 0) && (a.hdim_v % 32 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 32 == 0) && (a.hdim_v % 32 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 32 == 0) && (a.hdim_v % 32 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 32 == 0) && (a.hdim_v % 32 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 32 == 0) && (a.hdim_v % 32 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 32 == 0) && (a.hdim_v % 32 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 32 == 0) && (a.hdim_v % 32 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 32 == 0) && (a.hdim_v % 32 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 32 == 0) && (a.hdim_v % 32 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 32 == 0) && (a.hdim_v % 32 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 32 == 0) && (a.hdim_v % 32 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + + } + else if (t.hdim_q <= 64 && t.hdim_v <= 64) { + if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + + } + else if (t.hdim_q <= 128 && t.hdim_v <= 128) { + if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + + } + else if (t.hdim_q <= 256 && t.hdim_v <= 256) { + if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + + } + + } + else if(t.data_type.compare("bf16") == 0){ + if (t.hdim_q <= 32 && t.hdim_v <= 32) { + if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 32 == 0) && (a.hdim_v % 32 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 32 == 0) && (a.hdim_v % 32 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 32 == 0) && (a.hdim_v % 32 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 32 == 0) && (a.hdim_v % 32 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 32 == 0) && (a.hdim_v % 32 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 32 == 0) && (a.hdim_v % 32 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 32 == 0) && (a.hdim_v % 32 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 32 == 0) && (a.hdim_v % 32 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 32 == 0) && (a.hdim_v % 32 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 32 == 0) && (a.hdim_v % 32 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 32 == 0) && (a.hdim_v % 32 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 32 == 0) && (a.hdim_v % 32 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 32 == 0) && (a.hdim_v % 32 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 32 == 0) && (a.hdim_v % 32 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 32 == 0) && (a.hdim_v % 32 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 32 == 0) && (a.hdim_v % 32 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + + } + else if (t.hdim_q <= 64 && t.hdim_v <= 64) { + if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + + } + else if (t.hdim_q <= 128 && t.hdim_v <= 128) { + if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, false, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + + } + else if (t.hdim_q <= 256 && t.hdim_v <= 256) { + if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && + (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; + return fmha_fwd_(s, a); + } + + } + + } + else if(t.data_type.compare("fp8") == 0){ + if (t.hdim_q <= 64 && t.hdim_v <= 64) { + if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == true) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp8, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, true, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == true) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp8, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, true, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == true) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp8, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, true, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == true) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp8, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, true, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == true) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp8, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, true, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == true) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { + using trait_ = fmha_fwd_traits_<64, FmhaFwdFp8, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, true, false, false, false, false>; + return fmha_fwd_(s, a); + } + + } + else if (t.hdim_q <= 128 && t.hdim_v <= 128) { + if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == true) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp8, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, true, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == true) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp8, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, true, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == true) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp8, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, true, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == true) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp8, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, true, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == true) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp8, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, true, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == true) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { + using trait_ = fmha_fwd_traits_<128, FmhaFwdFp8, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, true, false, false, false, false>; + return fmha_fwd_(s, a); + } + + } + else if (t.hdim_q <= 256 && t.hdim_v <= 256) { + if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == true) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp8, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, true, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == true) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp8, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, true, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == true) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp8, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, true, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == true) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp8, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, true, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == true) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp8, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, true, false, false, false, false>; + return fmha_fwd_(s, a); + } + else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == true) && + (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { + using trait_ = fmha_fwd_traits_<256, FmhaFwdFp8, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, true, false, false, false, false>; + return fmha_fwd_(s, a); + } + + } + + } + + return r; +} diff --git a/example/ck_tile/01_fmha/generate.py b/example/ck_tile/01_fmha/generate.py index 3ddfe77a19..567b22bfd0 100644 --- a/example/ck_tile/01_fmha/generate.py +++ b/example/ck_tile/01_fmha/generate.py @@ -99,7 +99,7 @@ if __name__ == "__main__": parser.add_argument( "-r", "--receipt", - default=0, + default=3, required=False, help="codegen receipt. 0: generate only 8xhdim coverage\n" + \ " 1: generate more instance to cover all hdim\n" + \ From 7857f6217574e4d4571f0a9b89fc862e75f78fa4 Mon Sep 17 00:00:00 2001 From: Jim Date: Tue, 11 Feb 2025 14:04:18 +0800 Subject: [PATCH 04/10] remove generated files --- example/ck_tile/01_fmha/fmha_fwd_api.cpp | 5903 ---------------------- 1 file changed, 5903 deletions(-) delete mode 100644 example/ck_tile/01_fmha/fmha_fwd_api.cpp diff --git a/example/ck_tile/01_fmha/fmha_fwd_api.cpp b/example/ck_tile/01_fmha/fmha_fwd_api.cpp deleted file mode 100644 index 9f75eb03c1..0000000000 --- a/example/ck_tile/01_fmha/fmha_fwd_api.cpp +++ /dev/null @@ -1,5903 +0,0 @@ -// SPDX-License-Identifier: MIT -// Copyright (c) 2018-2024, Advanced Micro Devices, Inc. All rights reserved. - -// auto generated by generate.py -#include "fmha_fwd.hpp" - -float fmha_fwd(fmha_fwd_traits t, fmha_fwd_args a, const ck_tile::stream_config& s){ - float r = -1; - if(t.data_type.compare("fp16") == 0){ - if (t.hdim_q <= 32 && t.hdim_v <= 32) { - if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 32 == 0) && (a.hdim_v % 32 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 32 == 0) && (a.hdim_v % 32 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 32 == 0) && (a.hdim_v % 32 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 32 == 0) && (a.hdim_v % 32 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 32 == 0) && (a.hdim_v % 32 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 32 == 0) && (a.hdim_v % 32 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 32 == 0) && (a.hdim_v % 32 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 32 == 0) && (a.hdim_v % 32 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 32 == 0) && (a.hdim_v % 32 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 32 == 0) && (a.hdim_v % 32 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 32 == 0) && (a.hdim_v % 32 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 32 == 0) && (a.hdim_v % 32 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 32 == 0) && (a.hdim_v % 32 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 32 == 0) && (a.hdim_v % 32 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 32 == 0) && (a.hdim_v % 32 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 32 == 0) && (a.hdim_v % 32 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdFp16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - - } - else if (t.hdim_q <= 64 && t.hdim_v <= 64) { - if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - - } - else if (t.hdim_q <= 128 && t.hdim_v <= 128) { - if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - - } - else if (t.hdim_q <= 256 && t.hdim_v <= 256) { - if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - - } - - } - else if(t.data_type.compare("bf16") == 0){ - if (t.hdim_q <= 32 && t.hdim_v <= 32) { - if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 32 == 0) && (a.hdim_v % 32 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 32 == 0) && (a.hdim_v % 32 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 32 == 0) && (a.hdim_v % 32 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 32 == 0) && (a.hdim_v % 32 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 32 == 0) && (a.hdim_v % 32 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 32 == 0) && (a.hdim_v % 32 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 32 == 0) && (a.hdim_v % 32 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 32 == 0) && (a.hdim_v % 32 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 32 == 0) && (a.hdim_v % 32 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 32 == 0) && (a.hdim_v % 32 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 32 == 0) && (a.hdim_v % 32 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 32 == 0) && (a.hdim_v % 32 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 32 == 0) && (a.hdim_v % 32 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 32 == 0) && (a.hdim_v % 32 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 32 == 0) && (a.hdim_v % 32 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 32 == 0) && (a.hdim_v % 32 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, false, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 32 != 0*/) && (true /*a.hdim_v % 32 != 0*/)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<32, FmhaFwdBf16, true, 128, 64, 16, 32, 32, 32, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - - } - else if (t.hdim_q <= 64 && t.hdim_v <= 64) { - if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 64 != 0*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 64 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 64 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 64 != 0*/) && (true /*a.hdim_v % 64 != 0*/)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdBf16, true, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - - } - else if (t.hdim_q <= 128 && t.hdim_v <= 128) { - if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k != 0 && a.seqlen_k % 128 == 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, false, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true) && (a.seqlen_k == 0 || a.seqlen_k % 128 != 0) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 128 != 0*/) && (true /*a.hdim_v % 128 != 0*/)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (a.hdim_q % 8 == 0) && (a.hdim_v % 8 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdBf16, true, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS_ASYNC, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - - } - else if (t.hdim_q <= 256 && t.hdim_v <= 256) { - if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true /*a.seqlen_q % 128 != 0*/) && (true /*a.seqlen_k % 128 != 0*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == true) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, true, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == true) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, true, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == true) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, true, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == true) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == false) && - (true/*group mode spad always true*/) && (true/*group mode skpad always true*/) && (true /*a.hdim_q % 256 != 0*/) && (true /*a.hdim_v % 256 != 0*/)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdBf16, true, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, false, true, true, true, true>; - return fmha_fwd_(s, a); - } - - } - - } - else if(t.data_type.compare("fp8") == 0){ - if (t.hdim_q <= 64 && t.hdim_v <= 64) { - if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == true) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp8, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, true, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == true) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp8, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, true, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == true) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp8, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, true, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == true) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp8, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, true, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == true) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp8, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, true, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == true) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 64 == 0) && (a.hdim_q % 64 == 0) && (a.hdim_v % 64 == 0)) { - using trait_ = fmha_fwd_traits_<64, FmhaFwdFp8, false, 128, 64, 32, 64, 32, 64, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, true, false, false, false, false>; - return fmha_fwd_(s, a); - } - - } - else if (t.hdim_q <= 128 && t.hdim_v <= 128) { - if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == true) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp8, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, true, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == true) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp8, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, true, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == true) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp8, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, true, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == true) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp8, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, true, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == true) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp8, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, true, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == true) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 128 == 0) && (a.hdim_v % 128 == 0)) { - using trait_ = fmha_fwd_traits_<128, FmhaFwdFp8, false, 128, 128, 32, 128, 32, 128, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, true, false, false, false, false>; - return fmha_fwd_(s, a); - } - - } - else if (t.hdim_q <= 256 && t.hdim_v <= 256) { - if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == true) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp8, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, true, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == true) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp8, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, true, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type == mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == true) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp8, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, true, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::no_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == true) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp8, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::NO_BIAS, false, false, true, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::elementwise_bias) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == true) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp8, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS, false, false, true, false, false, false, false>; - return fmha_fwd_(s, a); - } - else if((t.is_group_mode == false) && (t.is_v_rowmajor == false) && (t.mask_type != mask_enum::no_mask) && (t.bias_type == bias_enum::alibi) && (t.has_lse == false) && (t.has_dropout == false) && (t.do_fp8_static_quant == true) && - (a.seqlen_q % 128 == 0) && (a.seqlen_k % 128 == 0) && (a.hdim_q % 256 == 0) && (a.hdim_v % 256 == 0)) { - using trait_ = fmha_fwd_traits_<256, FmhaFwdFp8, false, 128, 128, 32, 256, 32, 256, false, ck_tile::BlockFmhaPipelineEnum::QRKSVS, ck_tile::SimplifiedGenericAttentionMask, ck_tile::BlockAttentionBiasEnum::ALIBI, false, false, true, false, false, false, false>; - return fmha_fwd_(s, a); - } - - } - - } - - return r; -} From 7d45045c0036ae4dcce4cf5968355991956c7091 Mon Sep 17 00:00:00 2001 From: Jim Date: Wed, 12 Feb 2025 17:28:06 +0800 Subject: [PATCH 05/10] update: add restricts --- .../ck_tile/01_fmha/codegen/ops/fmha_bwd.py | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/example/ck_tile/01_fmha/codegen/ops/fmha_bwd.py b/example/ck_tile/01_fmha/codegen/ops/fmha_bwd.py index 5806bf29d4..0b9bff426a 100644 --- a/example/ck_tile/01_fmha/codegen/ops/fmha_bwd.py +++ b/example/ck_tile/01_fmha/codegen/ops/fmha_bwd.py @@ -162,8 +162,8 @@ std::string fmha_bwd_dq_dk_dv_get_name_() FMHA_BWD_API_FILENAME="fmha_bwd_api.cpp" -FMHA_BWD_V3_TEMPLATE="""template<> struct FmhaBwdV3Name> {{ static constexpr const char * bwd_v3_name = "bwd_v3{F_hdim_name}{F_dtype_name}{F_causal_name}{F_atomic_name}{F_bf16_cvt_name}"; }}; -template<> struct FmhaBwdV3Buf> {{ static constexpr unsigned char * bwd_v3_buf = bwd{F_hdim_name}{F_dtype_name}{F_causal_name}{F_atomic_name}{F_bf16_cvt_name}; }}; +FMHA_BWD_V3_TEMPLATE="""template<> struct FmhaBwdV3Name> {{ static constexpr const char * bwd_v3_name = "bwd_v3{F_hdim_name}{F_dtype_name}{F_causal_name}{F_atomic_name}{F_bf16_cvt_name}{F_hdpad_name}"; }}; +template<> struct FmhaBwdV3Buf> {{ static constexpr unsigned char * bwd_v3_buf = bwd{F_hdim_name}{F_dtype_name}{F_causal_name}{F_atomic_name}{F_bf16_cvt_name}{F_hdpad_name}; }}; template<> struct FmhaBwdV3Ts> {{ static constexpr int ts_qo = {F_Ts_qo}; static constexpr int ts_kv = 192; }}; """ @@ -782,14 +782,18 @@ class FmhaBwdApiPool: hdim = int(hdim) Ts_qo = 32 if hdim == 64 else 16 for k, trait in enumerate(traits): + if hdim == 64 and trait.is_hdpad == "t": + continue hdim_name = "_hd64" if hdim == 64 else "" dtype_name = "_{}".format(dtype) causal_name = "_causal" if trait.is_causal == "t" else "" atomic_name = "_a32" if trait.is_atomic == "t" else "_a16" bf16_cvt_name = "_{}".format(BF16_CVT_MAP[trait.bf16_cvt]) + bf16_cvt_name = bf16_cvt_name if dtype == "bf16" else "" + hdpad_name = "_pddv" if trait.is_hdpad == "t" else "" gen_template = gen_template + FMHA_BWD_V3_TEMPLATE.format(F_hdim=hdim, F_dtype=BWD_DTYPE_MAP[dtype], F_is_atomic=BOOL_MAP[trait.is_atomic], F_is_causal=BOOL_MAP[trait.is_causal], F_bf16_cvt=trait.bf16_cvt, F_hdpad=BOOL_MAP[trait.is_hdpad], F_Ts_qo = Ts_qo, F_hdim_name=hdim_name, - F_dtype_name=dtype_name, F_causal_name=causal_name, F_atomic_name=atomic_name, F_bf16_cvt_name=bf16_cvt_name) + F_dtype_name=dtype_name, F_causal_name=causal_name, F_atomic_name=atomic_name, F_bf16_cvt_name=bf16_cvt_name, F_hdpad_name=hdpad_name) v3_code = str() for i, dtype in enumerate(self.dq_dk_dv_v3_pool.keys()): @@ -1011,14 +1015,14 @@ class FmhaBwdV3DQDKDVKernel: def get_fmha_bwd_dq_dk_dv_tile_ppl_dict_from_dtype(dtype : str) -> Optional[dict]: if dtype == 'fp16' or dtype == 'bf16': return { - '32' : [FmhaBwdDQDKDVTileSize( 32, 128, 32, 32, 32, 32, 64, 32, 32, 1, 4, 1, 4, 1, 1, 2, 2, 1, 16, 16, 32, 16, 16, 16, 1), - "kr_ktr_vr_iglp", "kr_ktr_vr"], + # '32' : [FmhaBwdDQDKDVTileSize( 32, 128, 32, 32, 32, 32, 64, 32, 32, 1, 4, 1, 4, 1, 1, 2, 2, 1, 16, 16, 32, 16, 16, 16, 1), + # "kr_ktr_vr_iglp", "kr_ktr_vr"], '64' : [FmhaBwdDQDKDVTileSize( 32, 128, 64, 32, 64, 32, 32, 64, 64, 1, 4, 1, 4, 1, 1, 1, 4, 1, 16, 16, 32, 16, 16, 16, 1), "kr_ktr_vr_iglp", "kr_ktr_vr"], '128' : [FmhaBwdDQDKDVTileSize( 16, 128, 128, 16, 128, 16, 32, 128, 128, 1, 4, 1, 4, 1, 1, 1, 4, 1, 16, 16, 32, 16, 16, 16, 1), "kr_ktr_vr_iglp", "kr_ktr_vr"], - '256' : [FmhaBwdDQDKDVTileSize( 16, 64, 256, 16, 256, 16, 32, 256, 256, 1, 4, 1, 4, 1, 1, 1, 4, 1, 16, 16, 32, 16, 16, 16, 1), - "kr_ktr_vr_iglp", "kr_ktr_vr"] + # '256' : [FmhaBwdDQDKDVTileSize( 16, 64, 256, 16, 256, 16, 32, 256, 256, 1, 4, 1, 4, 1, 1, 1, 4, 1, 16, 16, 32, 16, 16, 16, 1), + # "kr_ktr_vr_iglp", "kr_ktr_vr"] } else: return None @@ -1061,8 +1065,11 @@ def get_bwd_dq_dk_dv_blobs(kernel_filter : Optional[str], receipt, mask_impl) -> continue if receipt == 3: cond = dtype in ['fp16', 'bf16'] - cond &= bias in ['no', 'alibi'] + cond &= bias in ['no'] + cond &= dropout in ['no'] cond &= dpad == dvpad + cond &= spad == skpad + cond &= spad == "f" cond &= deterministic == "f" if not cond: continue From 9f24a7ed410e18a283a1e7680bab89287478a34b Mon Sep 17 00:00:00 2001 From: Jim Date: Wed, 12 Feb 2025 17:34:55 +0800 Subject: [PATCH 06/10] fix: add template defination --- example/ck_tile/01_fmha/codegen/ops/fmha_bwd.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/example/ck_tile/01_fmha/codegen/ops/fmha_bwd.py b/example/ck_tile/01_fmha/codegen/ops/fmha_bwd.py index 0b9bff426a..b867bcc720 100644 --- a/example/ck_tile/01_fmha/codegen/ops/fmha_bwd.py +++ b/example/ck_tile/01_fmha/codegen/ops/fmha_bwd.py @@ -317,6 +317,9 @@ struct fmha_bwd_dq_dk_dv_v3_traits_ static constexpr bool kIsHDPad = kIsHDPad_; }}; +template struct FmhaBwdV3Name; +template struct FmhaBwdV3Buf; +template struct FmhaBwdV3Ts; {F_template} class fmha_bwd_v3_kernel From 14b4d6bb9143229b22826e5bcb8a1e98892a367a Mon Sep 17 00:00:00 2001 From: Jim Date: Wed, 12 Feb 2025 18:17:00 +0800 Subject: [PATCH 07/10] fix fp16 case --- example/ck_tile/01_fmha/codegen/ops/fmha_bwd.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/example/ck_tile/01_fmha/codegen/ops/fmha_bwd.py b/example/ck_tile/01_fmha/codegen/ops/fmha_bwd.py index b867bcc720..9997956192 100644 --- a/example/ck_tile/01_fmha/codegen/ops/fmha_bwd.py +++ b/example/ck_tile/01_fmha/codegen/ops/fmha_bwd.py @@ -800,11 +800,9 @@ class FmhaBwdApiPool: v3_code = str() for i, dtype in enumerate(self.dq_dk_dv_v3_pool.keys()): - per_bf16 = str() + per_bf16_cvt = str() for j, bf16_cvt in enumerate([0, 1, 2]): per_mask = str() - if (dtype == "fp16") and (bf16_cvt in [1, 2]): - continue for k, is_causal in enumerate(["t", "f"]): per_atomic = str() for l, is_atomic in enumerate(["t", "f"]): @@ -812,10 +810,11 @@ class FmhaBwdApiPool: for m, hdim in enumerate(BWD_V3_HDIM_CASE_CHECK_MAP.values()): if_m = 'if' if m == 0 else 'else if' inners = str() + bf16_cvt_tmp = 0 if dtype == "fp16" else bf16_cvt if is_atomic == "t": - inners = FMHA_BWD_V3_ATOMIC32_INNER_DISPATCH.format(F_hdim=hdim, F_dtype=BWD_DTYPE_MAP[dtype], F_is_causal=BOOL_MAP[is_causal], F_is_atomic32=BOOL_MAP[is_atomic], F_how_v3_bf16_cvt=bf16_cvt, F_padding=BWD_V3_PADDING_CHECK_MAP[m]) + inners = FMHA_BWD_V3_ATOMIC32_INNER_DISPATCH.format(F_hdim=hdim, F_dtype=BWD_DTYPE_MAP[dtype], F_is_causal=BOOL_MAP[is_causal], F_is_atomic32=BOOL_MAP[is_atomic], F_how_v3_bf16_cvt=bf16_cvt_tmp, F_padding=BWD_V3_PADDING_CHECK_MAP[m]) else: - inners = FMHA_BWD_V3_ATOMIC16_INNER_DISPATCH.format(F_hdim=hdim, F_dtype=BWD_DTYPE_MAP[dtype], F_is_causal=BOOL_MAP[is_causal], F_is_atomic32=BOOL_MAP[is_atomic], F_how_v3_bf16_cvt=bf16_cvt, F_padding=BWD_V3_PADDING_CHECK_MAP[m]) + inners = FMHA_BWD_V3_ATOMIC16_INNER_DISPATCH.format(F_hdim=hdim, F_dtype=BWD_DTYPE_MAP[dtype], F_is_causal=BOOL_MAP[is_causal], F_is_atomic32=BOOL_MAP[is_atomic], F_how_v3_bf16_cvt=bf16_cvt_tmp, F_padding=BWD_V3_PADDING_CHECK_MAP[m]) per_hdim = per_hdim + FMHA_BWD_V3_PER_HDIM_CASE.format(F_if=if_m, F_hdim_expression=BWD_V3_HDIM_CASE_MAP[m], inner_dispatch=inners) if_l = 'if' if l == 0 else 'else if' @@ -823,9 +822,9 @@ class FmhaBwdApiPool: if_k = 'if' if k == 0 else 'else if' per_mask = per_mask + FMHA_BWD_V3_PER_MASK_CASE.format(F_if=if_k, F_mask_expression=BWD_V3_MASK_MAP[is_causal], per_atomic_dispatch=per_atomic) if_j = 'if' if j == 0 else 'else if' - per_bf16 = per_bf16 + FMHA_BWD_V3_PER_BF16_CVT_CASE.format(F_if=if_j, F_bf16_cvt=bf16_cvt, per_mask_dispatch=per_mask) + per_bf16_cvt = per_bf16_cvt + FMHA_BWD_V3_PER_BF16_CVT_CASE.format(F_if=if_j, F_bf16_cvt=bf16_cvt, per_mask_dispatch=per_mask) if_i = 'if' if i == 0 else 'else if' - v3_code = v3_code + FMHA_BWD_V3_PER_DTYPE_CASE.format(F_if=if_i, F_dtype=dtype, per_bf16_cvt_dispatch=per_bf16) + v3_code = v3_code + FMHA_BWD_V3_PER_DTYPE_CASE.format(F_if=if_i, F_dtype=dtype, per_bf16_cvt_dispatch=per_bf16_cvt) return FMHA_BWD_KERNEL_HEADER + FMHA_BWD_API.format(F_dispatch = per_dtypes, F_template = gen_template, F_v3_dispatch = v3_code) From 9d772b9a4da98f7ad5053e9d860bd37bbef6e808 Mon Sep 17 00:00:00 2001 From: Jim Date: Wed, 12 Feb 2025 20:14:39 +0800 Subject: [PATCH 08/10] fix padding case --- example/ck_tile/01_fmha/codegen/ops/fmha_bwd.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/example/ck_tile/01_fmha/codegen/ops/fmha_bwd.py b/example/ck_tile/01_fmha/codegen/ops/fmha_bwd.py index 9997956192..a3c9109915 100644 --- a/example/ck_tile/01_fmha/codegen/ops/fmha_bwd.py +++ b/example/ck_tile/01_fmha/codegen/ops/fmha_bwd.py @@ -628,12 +628,12 @@ FMHA_BWD_API_INNER_DISPATCH=""" {F_if}((t.is_group_mode == {F_mode}) FMHA_BWD_V3_ATOMIC32_INNER_DISPATCH=""" using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<{F_hdim}, {F_dtype}, false, false, {F_padding}>; using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<{F_hdim}, {F_dtype}, {F_is_causal}, {F_is_atomic32}, {F_how_v3_bf16_cvt}, {F_padding}>; using convert_dq_trait_ = fmha_bwd_convert_dq_traits_<{F_hdim}, {F_dtype}, false, false, {F_padding}, false>; - r = fmha_bwd_v3_hdp_xqa_(s, a); + r = fmha_bwd_v3{F_padding_suffix}_xqa_(s, a); return r;""" FMHA_BWD_V3_ATOMIC16_INNER_DISPATCH=""" using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<{F_hdim}, {F_dtype}, false, false, {F_padding}>; using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<{F_hdim}, {F_dtype}, {F_is_causal}, {F_is_atomic32}, {F_how_v3_bf16_cvt}, {F_padding}>; - r = fmha_bwd_v3_xqa_(s, a); + r = fmha_bwd_v3{F_padding_suffix}_xqa_(s, a); return r;""" FMHA_BWD_V3_PER_DTYPE_CASE=""" {F_if} (t.data_type.compare(\"{F_dtype}\") == 0) {{ @@ -811,10 +811,11 @@ class FmhaBwdApiPool: if_m = 'if' if m == 0 else 'else if' inners = str() bf16_cvt_tmp = 0 if dtype == "fp16" else bf16_cvt + padding_suffix = "_hdp" if BWD_V3_PADDING_CHECK_MAP[m] == "true" else "" if is_atomic == "t": - inners = FMHA_BWD_V3_ATOMIC32_INNER_DISPATCH.format(F_hdim=hdim, F_dtype=BWD_DTYPE_MAP[dtype], F_is_causal=BOOL_MAP[is_causal], F_is_atomic32=BOOL_MAP[is_atomic], F_how_v3_bf16_cvt=bf16_cvt_tmp, F_padding=BWD_V3_PADDING_CHECK_MAP[m]) + inners = FMHA_BWD_V3_ATOMIC32_INNER_DISPATCH.format(F_hdim=hdim, F_dtype=BWD_DTYPE_MAP[dtype], F_is_causal=BOOL_MAP[is_causal], F_is_atomic32=BOOL_MAP[is_atomic], F_how_v3_bf16_cvt=bf16_cvt_tmp, F_padding=BWD_V3_PADDING_CHECK_MAP[m], F_padding_suffix=padding_suffix) else: - inners = FMHA_BWD_V3_ATOMIC16_INNER_DISPATCH.format(F_hdim=hdim, F_dtype=BWD_DTYPE_MAP[dtype], F_is_causal=BOOL_MAP[is_causal], F_is_atomic32=BOOL_MAP[is_atomic], F_how_v3_bf16_cvt=bf16_cvt_tmp, F_padding=BWD_V3_PADDING_CHECK_MAP[m]) + inners = FMHA_BWD_V3_ATOMIC16_INNER_DISPATCH.format(F_hdim=hdim, F_dtype=BWD_DTYPE_MAP[dtype], F_is_causal=BOOL_MAP[is_causal], F_is_atomic32=BOOL_MAP[is_atomic], F_how_v3_bf16_cvt=bf16_cvt_tmp, F_padding=BWD_V3_PADDING_CHECK_MAP[m], F_padding_suffix=padding_suffix) per_hdim = per_hdim + FMHA_BWD_V3_PER_HDIM_CASE.format(F_if=if_m, F_hdim_expression=BWD_V3_HDIM_CASE_MAP[m], inner_dispatch=inners) if_l = 'if' if l == 0 else 'else if' From a651e77b3087af25a95a3eabfd691de451ef27ea Mon Sep 17 00:00:00 2001 From: Jim Date: Thu, 13 Feb 2025 14:11:55 +0800 Subject: [PATCH 09/10] remove test code --- example/ck_tile/01_fmha/generate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/example/ck_tile/01_fmha/generate.py b/example/ck_tile/01_fmha/generate.py index 567b22bfd0..5b1b6664cc 100644 --- a/example/ck_tile/01_fmha/generate.py +++ b/example/ck_tile/01_fmha/generate.py @@ -64,7 +64,7 @@ if __name__ == "__main__": "--direction", # we keep 'direction' option for backward compatibility "-a", "--api", - default='bwd', + default='fwd', required=False, help="supply API(s) to generate (default: fwd). separated by comma." ) @@ -99,7 +99,7 @@ if __name__ == "__main__": parser.add_argument( "-r", "--receipt", - default=3, + default=0, required=False, help="codegen receipt. 0: generate only 8xhdim coverage\n" + \ " 1: generate more instance to cover all hdim\n" + \ From 41459498dcb5d797b30d2190a2ef2b520c2b4658 Mon Sep 17 00:00:00 2001 From: Jim Date: Thu, 13 Feb 2025 14:41:31 +0800 Subject: [PATCH 10/10] remove code --- example/ck_tile/01_fmha/codegen/ops/fmha_bwd.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/example/ck_tile/01_fmha/codegen/ops/fmha_bwd.py b/example/ck_tile/01_fmha/codegen/ops/fmha_bwd.py index a3c9109915..2661831782 100644 --- a/example/ck_tile/01_fmha/codegen/ops/fmha_bwd.py +++ b/example/ck_tile/01_fmha/codegen/ops/fmha_bwd.py @@ -989,21 +989,6 @@ class FmhaBwdV3DQDKDVKernel: F_bf16_cvt : int F_is_hdpad : str - # @property - # def gen_bwd_v3_template(self) -> str: - # hdim_template = 64 if self.F_hdim == 64 else 128 - # Ts_qo = 32 if hdim_template == 64 else 16 - # padding = "t" if self.F_hdim % 64 == 0 else "f" - # padding_suffix = "_hdp" if padding == "t" else "" - # hdim_name = "_hd64" if hdim_template == 64 else "" - # dtype_name = "_{}".format(self.F_dtype) - # causal_name = "_causal" if self.F_is_causal == "t" else "" - # atomic_name = "_a32" if self.F_is_atomic == "t" else "_a16" - # bf16_cvt_name = "_{}".format(BF16_CVT_MAP[self.F_bf16_cvt]) - # return FMHA_BWD_V3_TEMPLATE.format(F_hdim=hdim_template, F_dtype=BWD_DTYPE_MAP[self.F_dtype], F_is_atomic=BOOL_MAP[self.F_is_atomic], - # F_is_causal=BOOL_MAP[self.F_is_causal], F_bf16_cvt=self.F_bf16_cvt, F_hdpad=BOOL_MAP[padding], F_Ts_qo = Ts_qo, F_hdim_name=hdim_name, - # F_dtype_name=dtype_name, F_causal_name=causal_name, F_atomic_name=atomic_name, F_bf16_cvt_name=bf16_cvt_name) - def v3_api_trait(self) -> FmhaBwdV3DQDKDVApiTrait: return FmhaBwdV3DQDKDVApiTrait(hdim=str(self.F_hdim), dtype=self.F_dtype,