diff --git a/example/ck_tile/01_fmha/codegen/ops/fmha_bwd.py b/example/ck_tile/01_fmha/codegen/ops/fmha_bwd.py index 883853b79f..e286fc5b6a 100644 --- a/example/ck_tile/01_fmha/codegen/ops/fmha_bwd.py +++ b/example/ck_tile/01_fmha/codegen/ops/fmha_bwd.py @@ -1030,7 +1030,7 @@ float fmha_bwd_v3_swa_genl_(const ck_tile::stream_config& s, fmha_bwd_args a) args.Seqs_dv = a.stride_dv * 2; // TODO: convert l/r to x/y HERE - auto generic_mask = make_generic_attention_mask_coordinates_from_lr_window(a.window_size_left, a.window_size_right, a.seqlen_q, a.seqlen_k, a.mask_type == static_cast(mask_enum::mask_top_left)); + auto generic_mask = ck_tile::make_generic_attention_mask_coordinates_from_lr_window(a.window_size_left, a.window_size_right, a.seqlen_q, a.seqlen_k, a.mask_type == static_cast(mask_enum::mask_top_left)); args.mask_y = generic_mask.at(ck_tile::number<0>{{}}); args.mask_x = generic_mask.at(ck_tile::number<1>{{}}); @@ -1584,33 +1584,33 @@ float fmha_bwd<2>(fmha_bwd_traits t, fmha_bwd_args a, const ck_tile::stream_conf if((t.is_v3_atomic_fp32 == true) && (a.nhead_stride_dq_acc >= a.stride_dq_acc /*dq_acc only support BHSD*/)){{ if(t.how_v3_bf16_cvt == 0){{ if((a.seqlen_q % 64 == 0) && (a.hdim_q == 128)){{ - using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<128, FmhaBwdFp16, false, false, false>; - using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<128, FmhaBwdFp16, 2, true, 0, true, true>; - using convert_dq_trait_ = fmha_bwd_convert_dq_traits_<128, FmhaBwdFp16, false, false, false, false>; + using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<128, FmhaBwdBf16, false, false, false>; + using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<128, FmhaBwdBf16, 2, true, 0, true, true>; + using convert_dq_trait_ = fmha_bwd_convert_dq_traits_<128, FmhaBwdBf16, false, false, false, false>; // const std::string bwd_v3_name = "bwd_v3_bf16_swa_a32_rtne_psskddv"; r = fmha_bwd_v3_swa_genl_(s, a); return r; }} else if((a.seqlen_q % 64 != 0) && (a.hdim_q == 128)){{ - using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<128, FmhaBwdFp16, false, true, false>; - using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<128, FmhaBwdFp16, 2, true, 0, true, true>; - using convert_dq_trait_ = fmha_bwd_convert_dq_traits_<128, FmhaBwdFp16, false, true, false, false>; + using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<128, FmhaBwdBf16, false, true, false>; + using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<128, FmhaBwdBf16, 2, true, 0, true, true>; + using convert_dq_trait_ = fmha_bwd_convert_dq_traits_<128, FmhaBwdBf16, false, true, false, false>; // const std::string bwd_v3_name = "bwd_v3_bf16_swa_a32_rtne_psskddv"; r = fmha_bwd_v3_swa_genl_(s, a); return r; }} else if((a.seqlen_q % 64 == 0) && (a.hdim_q != 128)){{ - using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<128, FmhaBwdFp16, false, false, true>; - using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<128, FmhaBwdFp16, 2, true, 0, true, true>; - using convert_dq_trait_ = fmha_bwd_convert_dq_traits_<128, FmhaBwdFp16, false, false, true, false>; + using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<128, FmhaBwdBf16, false, false, true>; + using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<128, FmhaBwdBf16, 2, true, 0, true, true>; + using convert_dq_trait_ = fmha_bwd_convert_dq_traits_<128, FmhaBwdBf16, false, false, true, false>; // const std::string bwd_v3_name = "bwd_v3_bf16_swa_a32_rtne_psskddv; r = fmha_bwd_v3_swa_genl_(s, a); return r; }} else if((a.seqlen_q % 64 != 0) && (a.hdim_q != 128)){{ - using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<128, FmhaBwdFp16, false, true, true>; - using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<128, FmhaBwdFp16, 2, true, 0, true, true>; - using convert_dq_trait_ = fmha_bwd_convert_dq_traits_<128, FmhaBwdFp16, false, true, true, false>; + using dot_do_o_trait_ = fmha_bwd_dot_do_o_traits_<128, FmhaBwdBf16, false, true, true>; + using dq_dk_dv_v3_traits_ = fmha_bwd_dq_dk_dv_v3_traits_<128, FmhaBwdBf16, 2, true, 0, true, true>; + using convert_dq_trait_ = fmha_bwd_convert_dq_traits_<128, FmhaBwdBf16, false, true, true, false>; // const std::string bwd_v3_name = "bwd_v3_bf16_swa_a32_rtne_psskddv"; r = fmha_bwd_v3_swa_genl_(s, a); return r; diff --git a/example/ck_tile/01_fmha/hsaco/fmha_hsaco.hpp b/example/ck_tile/01_fmha/hsaco/fmha_hsaco.hpp index 4e51dd980e..ab97b67e58 100644 --- a/example/ck_tile/01_fmha/hsaco/fmha_hsaco.hpp +++ b/example/ck_tile/01_fmha/hsaco/fmha_hsaco.hpp @@ -32,8 +32,8 @@ extern unsigned char bwd_bf16_causal_a32_rtna_psskddv[]; extern unsigned char bwd_bf16_causal_a32_rtne_psskddv[]; extern unsigned char bwd_bf16_causal_a32_rtz_psskddv[]; extern unsigned char bwd_bf16_swa_a32_rtne_psskddv[]; -// extern unsigned char bwd_bf16_swa_a32_rtna_psskddv[]; -// extern unsigned char bwd_bf16_swa_a32_rtz_psskddv[]; +extern unsigned char bwd_bf16_swa_a32_rtna_psskddv[]; +extern unsigned char bwd_bf16_swa_a32_rtz_psskddv[]; extern unsigned char bwd_fp16_a16_pddv[]; extern unsigned char bwd_fp16_a32_psskddv[]; extern unsigned char bwd_fp16_causal_a16_pddv[];