diff --git a/example/ck_tile/11_add_rmsnorm2d_rdquant/example_add_rmsnorm2d_rdquant_fwd.cpp b/example/ck_tile/11_add_rmsnorm2d_rdquant/example_add_rmsnorm2d_rdquant_fwd.cpp index ada4c6f2da..e1d3069f46 100644 --- a/example/ck_tile/11_add_rmsnorm2d_rdquant/example_add_rmsnorm2d_rdquant_fwd.cpp +++ b/example/ck_tile/11_add_rmsnorm2d_rdquant/example_add_rmsnorm2d_rdquant_fwd.cpp @@ -186,7 +186,7 @@ bool run(const ck_tile::ArgParser& arg_parser) // Rmsnorm2d { ck_tile::HostTensor invRms_host_ref({m}); - + ck_tile::HostTensor unquant_y_host_ref({m, n}, {stride, 1}); // CAUSION: kernel use ComputeDataType version of x, but we use XDataType here for // simplicity ck_tile::reference_rmsnorm2d_fwd( - x_host_ref, gamma_host, y_host, invRms_host_ref, epsilon); + x_host_ref, gamma_host, y_host, invRms_host_ref, unquant_y_host_ref, epsilon); + } // yscale diff --git a/include/ck_tile/ops/flatmm/pipeline/flatmm_pipeline_agmem_bgmem_creg_v1.hpp b/include/ck_tile/ops/flatmm/pipeline/flatmm_pipeline_agmem_bgmem_creg_v1.hpp index b767c53c26..df74c5628a 100644 --- a/include/ck_tile/ops/flatmm/pipeline/flatmm_pipeline_agmem_bgmem_creg_v1.hpp +++ b/include/ck_tile/ops/flatmm/pipeline/flatmm_pipeline_agmem_bgmem_creg_v1.hpp @@ -91,8 +91,14 @@ struct FlatmmPipelineAGmemBGmemCRegV1 constexpr index_t A_Buffer_Load_Inst_Num = kMPerBlock * kKPerBlock / BlockSize / KPerLoad; constexpr index_t A_LDS_Read_Inst_Num = MIterPerWarp * KIterPerWarp; constexpr index_t B_Buffer_Load_Inst_Num = NIterPerWarp * KIterPerWarp; -#endif -#if defined(USING_MFMA_16x16x32) && defined(ENABLE_FP8) + // constexpr index_t A_LDS_Read_Inst_Remain = A_LDS_Read_Inst_Num - A_Buffer_Load_Inst_Num; + ignore = A_Buffer_Load_Inst_Num; + ignore = A_LDS_Read_Inst_Num; + ignore = B_Buffer_Load_Inst_Num; + + if constexpr (WG::kM == 16 && WG::kN == 16) + { +//#if defined(USING_MFMA_16x16x32) && defined(ENABLE_FP8) static_for<0, A_Buffer_Load_Inst_Num, 1>{}([&](auto i) { ignore = i; __builtin_amdgcn_sched_group_barrier(0x100, 1, 0); // DS read @@ -114,11 +120,11 @@ struct FlatmmPipelineAGmemBGmemCRegV1 __builtin_amdgcn_sched_group_barrier(0x200, 1, 0); // DS write __builtin_amdgcn_sched_group_barrier(0x008, 4, 0); // MFMA }); + } + else if constexpr (WG::kM == 32 && WG::kN == 32) + { -#elif defined(USING_MFMA_32x32x16) - ignore = A_Buffer_Load_Inst_Num; - ignore = A_LDS_Read_Inst_Num; - ignore = B_Buffer_Load_Inst_Num; +//#elif defined(USING_MFMA_32x32x16) #if 0 static_for<0, A_LDS_Read_Inst_Num / 2 - A_Buffer_Load_Inst_Num - B_Buffer_Load_Inst_Num, @@ -153,7 +159,8 @@ struct FlatmmPipelineAGmemBGmemCRegV1 }); __builtin_amdgcn_sched_group_barrier(0x008, 4, 0); // MFMA #endif -#endif + } +//#endif } template diff --git a/include/ck_tile/ops/flatmm/pipeline/flatmm_pipeline_agmem_bgmem_creg_v1_policy.hpp b/include/ck_tile/ops/flatmm/pipeline/flatmm_pipeline_agmem_bgmem_creg_v1_policy.hpp index 8e0f056fe8..4ae7c480fe 100644 --- a/include/ck_tile/ops/flatmm/pipeline/flatmm_pipeline_agmem_bgmem_creg_v1_policy.hpp +++ b/include/ck_tile/ops/flatmm/pipeline/flatmm_pipeline_agmem_bgmem_creg_v1_policy.hpp @@ -19,7 +19,14 @@ struct UniversalFlatmmPipelineAgBgCrPolicy CK_TILE_HOST_DEVICE static constexpr auto MakeALdsBlockDescriptor() { using namespace ck_tile; -#if defined(USING_MFMA_16x16x32) && defined(ENABLE_FP8) + // constexpr auto config = BlockFlatmm::BlockPolicy::template GetWarpGemmMWarpNWarp(); + + //using WG = remove_cvref_t())>; + constexpr index_t MPerXdl = Problem::BlockGemmShape::WarpTile::at(I0); + constexpr index_t NPerXdl = Problem::BlockGemmShape::WarpTile::at(I1); + if constexpr (MPerXdl == 16 && NPerXdl == 16) + { +//#if defined(USING_MFMA_16x16x32) && defined(ENABLE_FP8) /*reduce transform layers,compare with old ck*/ constexpr index_t MPerBlock = Problem::BlockGemmShape::kM; constexpr index_t KPerBlock = Problem::BlockGemmShape::kK; @@ -48,7 +55,10 @@ struct UniversalFlatmmPipelineAgBgCrPolicy make_tuple(sequence<0>{}, sequence<1>{})); return a_lds_block_desc; -#elif defined(USING_MFMA_32x32x16) + } + else + { +//#elif defined(USING_MFMA_32x32x16) constexpr index_t kMPerBlock = Problem::BlockGemmShape::kM; constexpr index_t kKPerBlock = Problem::BlockGemmShape::kK; constexpr index_t kKPack = GetSmemPackA(); @@ -67,7 +77,8 @@ struct UniversalFlatmmPipelineAgBgCrPolicy make_tuple(sequence<0>{}, sequence<1>{})); return a_lds_block_desc; -#endif +//#endif + } /*xor*/ #if 0 constexpr index_t kMPerBlock = Problem::BlockGemmShape::kM;