diff --git a/include/ck_tile/ops/gemm/block/block_gemm_areg_breg_creg_eight_waves_v1.hpp b/include/ck_tile/ops/gemm/block/block_gemm_areg_breg_creg_eight_waves_v1.hpp index 7392263887..9f91c06e8e 100644 --- a/include/ck_tile/ops/gemm/block/block_gemm_areg_breg_creg_eight_waves_v1.hpp +++ b/include/ck_tile/ops/gemm/block/block_gemm_areg_breg_creg_eight_waves_v1.hpp @@ -103,8 +103,6 @@ struct BlockGemmARegBRegCRegEightWavesV1 using BWarpTensor = typename WarpGemm::BWarpTensor; using CWarpTensor = typename WarpGemm::CWarpTensor; - static_assert(std::is_same_v); - static constexpr auto a_warp_y_lengths = to_sequence(AWarpDstr{}.get_ys_to_d_descriptor().get_lengths()); static constexpr auto b_warp_y_lengths = diff --git a/include/ck_tile/ops/gemm/pipeline/gemm_pipeline_ag_bg_cr_comp_async_eight_waves_policy.hpp b/include/ck_tile/ops/gemm/pipeline/gemm_pipeline_ag_bg_cr_comp_async_eight_waves_policy.hpp index 26a8c9ed44..56709be910 100644 --- a/include/ck_tile/ops/gemm/pipeline/gemm_pipeline_ag_bg_cr_comp_async_eight_waves_policy.hpp +++ b/include/ck_tile/ops/gemm/pipeline/gemm_pipeline_ag_bg_cr_comp_async_eight_waves_policy.hpp @@ -36,11 +36,12 @@ struct GemmPipelineAgBgCrCompAsyncEightWavesPolicy static_assert(std::is_same_v, "BLayout must be ColumnMajor!"); static_assert( - is_any_of::value); + is_any_of:: + value); static_assert( - is_any_of::value); + is_any_of:: + value); static_assert(std::is_same_v); - static_assert(std::is_same_v); static constexpr auto WGAccess = std::is_same_v || std::is_same_v diff --git a/include/ck_tile/ops/gemm/warp/warp_gemm.hpp b/include/ck_tile/ops/gemm/warp/warp_gemm.hpp index cfd29c13f6..1d1951e389 100644 --- a/include/ck_tile/ops/gemm/warp/warp_gemm.hpp +++ b/include/ck_tile/ops/gemm/warp/warp_gemm.hpp @@ -511,6 +511,19 @@ using WarpGemmMfma_i32_32x32x16_i8_i8_CTransposed = using WarpGemmMfma_i32_16x16x32_i8_i8 = WarpGemmImpl< WarpGemmAttributeMfma>>; +template +using WarpGemmMfma_i32_16x16x64_i8_i8 = WarpGemmImpl< + WarpGemmAttributeMfma, + AttrNumAccessA, + AttrNumAccessB>>; + +template +using WarpGemmMfma_i32_16x16x64_i8_i8_CTransposed = + WarpGemmImpl, + AttrNumAccess>>; + using WarpGemmMfma_i32_16x16x32_i8_i8_CTransposed = WarpGemmImpl>>; diff --git a/include/ck_tile/ops/gemm/warp/warp_gemm_attribute_mfma_impl.hpp b/include/ck_tile/ops/gemm/warp/warp_gemm_attribute_mfma_impl.hpp index 0f64c2fdb5..78b339887d 100644 --- a/include/ck_tile/ops/gemm/warp/warp_gemm_attribute_mfma_impl.hpp +++ b/include/ck_tile/ops/gemm/warp/warp_gemm_attribute_mfma_impl.hpp @@ -2112,9 +2112,8 @@ struct WarpGemmAttributeMfmaImpl_i32_16x16x64_i8 DISPATCH_MFMA_CTRL_("v_mfma_i32_16x16x64_i8", Ctrl) else { -#if defined(__gfx95__) - c_vec = __builtin_amdgcn_mfma_i32_16x16x64_i8( - bit_cast(a_vec), bit_cast(b_vec), c_vec, 0, 0, 0); +#if defined(__gfx950__) + c_vec = __builtin_amdgcn_mfma_i32_16x16x64_i8(a_vec, b_vec, c_vec, 0, 0, 0); #else ck_tile::ignore = c_vec; ck_tile::ignore = a_vec; diff --git a/include/ck_tile/ops/gemm/warp/warp_gemm_dispatcher.hpp b/include/ck_tile/ops/gemm/warp/warp_gemm_dispatcher.hpp index 6180ef3001..ce32eea840 100644 --- a/include/ck_tile/ops/gemm/warp/warp_gemm_dispatcher.hpp +++ b/include/ck_tile/ops/gemm/warp/warp_gemm_dispatcher.hpp @@ -288,7 +288,20 @@ template<> struct Dispatcher { using template<> struct Dispatcher { using Type = WarpGemmMfma_i32_16x16x32_i8_i8_CTransposed; }; // WMMA cases template struct Dispatcher : WmmaTag { using Type = WarpGemmWmma_i32_16x16x16_i8_i8; }; +#if defined(__gfx125__) template struct Dispatcher : WmmaTag { using Type = WarpGemmWmma_i32_16x16x64_i8_i8; }; +#else +template +struct Dispatcher> +{ using Type = WarpGemmMfma_i32_16x16x64_i8_i8; }; +template +struct Dispatcher> +{ using Type = WarpGemmMfma_i32_16x16x64_i8_i8; }; + +template +struct Dispatcher> +{ using Type = WarpGemmMfma_i32_16x16x64_i8_i8_CTransposed; }; +#endif template struct Dispatcher : WmmaTag { using Type = WarpGemmWmma_i32_16x16x64_u8_u8; }; template ; template -using CompAsyncEightWavesConfig8Bit = std::tuple; +using CompAsyncEightWavesConfig8BitFP = std::tuple; + +template +using CompAsyncEightWavesConfig8BitINT = std::tuple; template using CompAsyncEightWavesConfig16Bit = std::tuple>; using KernelTypesCompAsyncEightWaves = - ::testing::Types, - CompAsyncEightWavesConfig8Bit, + ::testing::Types, + CompAsyncEightWavesConfig8BitFP, + CompAsyncEightWavesConfig8BitFP, CompAsyncEightWavesConfig4Bit, CompAsyncEightWavesConfig16Bit, CompAsyncEightWavesConfig16Bit>; diff --git a/test/ck_tile/gemm/test_gemm_pipeline_util.hpp b/test/ck_tile/gemm/test_gemm_pipeline_util.hpp index d1dc7a1813..536878118f 100644 --- a/test/ck_tile/gemm/test_gemm_pipeline_util.hpp +++ b/test/ck_tile/gemm/test_gemm_pipeline_util.hpp @@ -63,11 +63,18 @@ constexpr ck_tile::index_t get_k_warp_tile() return 16; #endif #else - if constexpr(PipelineType == GemmPipelineType::CompAsyncEightWaves && sizeof(PrecType) == 1) - return 128; - else if constexpr(PipelineType == GemmPipelineType::CompAsyncEightWaves && - sizeof(PrecType) == 2) - return 32; + if constexpr(PipelineType == GemmPipelineType::CompAsyncEightWaves) + if constexpr(std::is_same_v) + return 64; + else if constexpr(ck_tile::is_any_of::value) + return 128; + else if constexpr(ck_tile::is_any_of::value) + return 32; + else + static_assert(false, "CompAsyncEightWaves: unsupported type"); // CompAsyncConfig16x16x128 else if constexpr(PipelineType == GemmPipelineType::CompAsync && M_Warp_Tile == 16) return 128;