diff --git a/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdl_cshuffle_v3.hpp b/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdl_cshuffle_v3.hpp index 4ce2976524..e4f485971a 100644 --- a/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdl_cshuffle_v3.hpp +++ b/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdl_cshuffle_v3.hpp @@ -1647,7 +1647,7 @@ struct GridwiseGemm_xdl_cshuffle_v3 using ThreadwiseTransfer = std::conditional_t< is_gfx650_and_bf16_output(), - ThreadwiseTensorSliceTransfer_v1r3_pass_through< + ThreadwiseTensorSliceTransfer_v1r3_packed_cast< AccDataType, CShuffleDataType, decltype(c_thread_desc_m0_n0_m1_n1_m2_m3_m4_n2), @@ -1761,20 +1761,20 @@ struct GridwiseGemm_xdl_cshuffle_v3 // make sure it's safe to write to LDS block_sync_lds(); - if constexpr (is_gfx650_and_bf16_output()) - { - auto c_thread_packed_cast = PackedCastV2< - M2, - M4, - CShuffleMXdlPerWavePerShuffle, - CShuffleNXdlPerWavePerShuffle - >{}; - c_thread_packed_cast.Run( - c_thread_desc_m0_n0_m1_n1_m2_m3_m4_n2, // source desc (TensorDescriptor struct) - sfc_c_vgpr.GetIndexTupleOfNumber(access_id), // source slice origin - c_thread_buf // source buffer - ); - } + // if constexpr (is_gfx650_and_bf16_output()) + // { + // auto c_thread_packed_cast = PackedCastV2< + // M2, + // M4, + // CShuffleMXdlPerWavePerShuffle, + // CShuffleNXdlPerWavePerShuffle + // >{}; + // c_thread_packed_cast.Run( + // c_thread_desc_m0_n0_m1_n1_m2_m3_m4_n2, // source desc (TensorDescriptor struct) + // sfc_c_vgpr.GetIndexTupleOfNumber(access_id), // source slice origin + // c_thread_buf // source buffer + // ); + // } // each thread write its data from VGPR to LDS c_thread_copy_vgpr_to_lds.Run(c_thread_desc_m0_n0_m1_n1_m2_m3_m4_n2, @@ -2091,7 +2091,7 @@ struct GridwiseGemm_xdl_cshuffle_v3 using ThreadwiseTransfer = std::conditional_t< is_gfx650_and_bf16_output(), - ThreadwiseTensorSliceTransfer_v1r3_pass_through< + ThreadwiseTensorSliceTransfer_v1r3_packed_cast< AccDataType, CShuffleDataType, decltype(c_thread_desc_m0_n0_m1_n1_m2_m3_m4_n2), @@ -2203,20 +2203,20 @@ struct GridwiseGemm_xdl_cshuffle_v3 // make sure it's safe to write to LDS block_sync_lds(); - if constexpr (is_gfx650_and_bf16_output()) - { - auto c_thread_packed_cast = PackedCastV2< - M2, - M4, - CShuffleMXdlPerWavePerShuffle, - CShuffleNXdlPerWavePerShuffle - >{}; - c_thread_packed_cast.Run( - c_thread_desc_m0_n0_m1_n1_m2_m3_m4_n2, // source desc - sfc_c_vgpr.GetIndexTupleOfNumber(access_id), // source slice origin - c_thread_buf // source buffer - ); - } + // if constexpr (is_gfx650_and_bf16_output()) + // { + // auto c_thread_packed_cast = PackedCastV2< + // M2, + // M4, + // CShuffleMXdlPerWavePerShuffle, + // CShuffleNXdlPerWavePerShuffle + // >{}; + // c_thread_packed_cast.Run( + // c_thread_desc_m0_n0_m1_n1_m2_m3_m4_n2, // source desc + // sfc_c_vgpr.GetIndexTupleOfNumber(access_id), // source slice origin + // c_thread_buf // source buffer + // ); + // } // each thread write its data from VGPR to LDS c_thread_copy_vgpr_to_lds.Run(c_thread_desc_m0_n0_m1_n1_m2_m3_m4_n2, diff --git a/include/ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer.hpp b/include/ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer.hpp index 094482092a..e857d8d65e 100644 --- a/include/ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer.hpp +++ b/include/ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer.hpp @@ -279,10 +279,10 @@ struct ThreadwiseTensorSliceTransfer_v1r3_packed_cast // Currently these must match, either true or false for both. // Otherwise we increase the register usage and end-up using the scratch memory. - // The linear pattern seems to be better tahn serpentine. + // The linear pattern seems to be better than serpentine. // Howeverm for LDS writes, serpentine pattern could be better. - static constexpr bool SerpentineAccessPatternDst = true; - static constexpr bool SerpentineAccessPatternSrc = true; + static constexpr bool SerpentineAccessPatternDst = false; + static constexpr bool SerpentineAccessPatternSrc = false; static constexpr index_t nDim = SliceLengths::Size();