From 2d50ecbeb5f3bdf1ea1e8fd6296dc92b7010477b Mon Sep 17 00:00:00 2001 From: mtgu0705 Date: Thu, 2 Jan 2025 19:41:04 +0800 Subject: [PATCH] fixed --- .../threadwise_tensor_slice_transfer.hpp | 64 +------------- .../profiler/profile_gemm_b_scale_impl.hpp | 87 +++++++++---------- 2 files changed, 44 insertions(+), 107 deletions(-) diff --git a/include/ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer.hpp b/include/ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer.hpp index 36cbe22d77..8c65ef32ae 100644 --- a/include/ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer.hpp +++ b/include/ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer.hpp @@ -1146,36 +1146,7 @@ struct ThreadwiseTensorSliceTransfer_v4 }); } - if constexpr(is_same, pk_i4_t>::value && - is_same, half_t>::value) - { - // copy data from src_tmp_vector to dst_tmp_vector (data cast data from SrcData to - // DstData) - vector_type_maker_t dst_tmp_vector; - - constexpr index_t pack_size = 8; - - static_assert(SrcScalarPerVector % pack_size == 0, ""); - - using src_v_t = typename vector_type_maker_t::type; - using dst_v_t = typename vector_type_maker_t::type; - - static_for<0, SrcScalarPerVector / pack_size, 1>{}([&](auto i) { - ck::tensor_operation::element_wise::PassThroughPack8{}( - dst_tmp_vector.template AsType()(i), - src_tmp_vector.template AsType()[i]); - }); - - // copy data from dst_tmp_vector into dst_buf - static_for<0, SrcScalarPerVector, 1>{}([&](auto i) { - constexpr index_t dst_offset = dst_desc.CalculateOffset( - dst_origin_idx + data_to_origin_disp_idx + i * src_scalar_step_in_vector); - - dst_buf(Number{}) = dst_tmp_vector.template AsType()[i]; - }); - } - else if constexpr(is_same, pk_i4_t>::value && - is_same, f8_t>::value) + if constexpr(is_same, pk_i4_t>::value) { // copy data from src_tmp_vector to dst_tmp_vector (data cast data from SrcData to // DstData) @@ -1361,7 +1332,6 @@ struct ThreadwiseTensorSliceTransfer_v4 } else if constexpr(SrcBuffer::IsStaticBuffer()) { - static_assert(false, ""); static_for<0, SrcScalarPerVector, 1>{}([&](auto i) { constexpr index_t src_offset = src_desc.CalculateOffset( src_ref_to_origin_disp_idx + data_to_origin_disp_idx + @@ -1371,10 +1341,8 @@ struct ThreadwiseTensorSliceTransfer_v4 }); } - if constexpr(is_same, pk_i4_t>::value && - is_same, half_t>::value) + if constexpr(is_same, pk_i4_t>::value) { - // copy data from src_tmp_vector to dst_tmp_vector (data cast data from SrcData to // DstData) vector_type_maker_t dst_tmp_vector; @@ -1405,34 +1373,6 @@ struct ThreadwiseTensorSliceTransfer_v4 dst_buf(Number{}) = dst_tmp_vector.template AsType()[i]; }); } - else if constexpr(is_same, pk_i4_t>::value && - is_same, f8_t>::value) - { - // copy data from src_tmp_vector to dst_tmp_vector (data cast data from SrcData to - // DstData) - vector_type_maker_t dst_tmp_vector; - - constexpr index_t pack_size = 8; - - static_assert(SrcScalarPerVector % pack_size == 0, ""); - - using src_v_t = typename vector_type_maker_t::type; - using dst_v_t = typename vector_type_maker_t::type; - - static_for<0, SrcScalarPerVector / pack_size, 1>{}([&](auto i) { - ck::tensor_operation::element_wise::PassThroughPack8{}( - dst_tmp_vector.template AsType()(i), - src_tmp_vector.template AsType()[i]); - }); - - // copy data from dst_tmp_vector into dst_buf - static_for<0, SrcScalarPerVector, 1>{}([&](auto i) { - constexpr index_t dst_offset = dst_desc.CalculateOffset( - dst_origin_idx + data_to_origin_disp_idx + i * src_scalar_step_in_vector); - - dst_buf(Number{}) = dst_tmp_vector.template AsType()[i]; - }); - } else if constexpr(is_same, f8_t>::value && is_same, half_t>::value && SrcScalarPerVector % 2 == 0) diff --git a/profiler/include/profiler/profile_gemm_b_scale_impl.hpp b/profiler/include/profiler/profile_gemm_b_scale_impl.hpp index 60c875c129..fe977e766e 100644 --- a/profiler/include/profiler/profile_gemm_b_scale_impl.hpp +++ b/profiler/include/profiler/profile_gemm_b_scale_impl.hpp @@ -8,18 +8,18 @@ #include #include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/impl/device_gemm_xdl_cshuffle_v3_b_scale.hpp" #include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/impl/device_gemm_xdl_cshuffle_v3_b_scale.hpp" #include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" #include "ck/library/tensor_operation_instance/gpu/gemm_b_scale.hpp" -#include "ck/library/reference_tensor_operation/cpu/reference_gemm.hpp" #include "ck/library/utility/check_err.hpp" #include "ck/library/utility/device_memory.hpp" #include "ck/library/utility/host_tensor.hpp" #include "ck/library/utility/host_tensor_generator.hpp" #include "ck/library/utility/literals.hpp" +#include "ck/library/reference_tensor_operation/cpu/reference_gemm.hpp" namespace ck { namespace profiler { @@ -162,9 +162,9 @@ bool profile_gemm_b_scale_impl(int do_verification, ck::pk_i4_t i4x2 = b_k_n(k, n).data; int8_t i4 = 0; if(k % 2 == 1) - i4 = (i4x2 >> 0) & 0xf; + i4 = (i4x2.data >> 0) & 0xf; else - i4 = (i4x2 >> 4) & 0xf; + i4 = (i4x2.data >> 4) & 0xf; i4 = i4 - 8; v_b = ck::type_convert(i4); @@ -218,64 +218,61 @@ bool profile_gemm_b_scale_impl(int do_verification, } } - // vector pk_i4x4 permute - for(int i = 0; i < N; i++) + if(is_same_v && is_same_v) { - for(int j = 0; j < K; j += 8) + // vector pk_i4x4 permute + for(int i = 0; i < N; i++) { - int input[8]; - - for(int k = 0; k < 4; k++) + for(int j = 0; j < K; j += 8) { - int i4x2 = b_k_n_permute(j + k * 2, i).data; - input[k * 2 + 0] = (i4x2 >> 4) & 0xf; - input[k * 2 + 1] = (i4x2 >> 0) & 0xf; - } + int input[8]; - // permute 01234567->20643175 - { - int hi = input[2]; - int lo = input[0]; - int i4x2 = (hi << 4) | lo; + for(int k = 0; k < 4; k++) + { + int i4x2 = b_k_n_permute(j + k * 2, i).data; + input[k * 2 + 0] = (i4x2 >> 4) & 0xf; + input[k * 2 + 1] = (i4x2 >> 0) & 0xf; + } - b_k_n_permute(j + 0, i) = i4x2; - } + // permute 01234567->20643175 + { + int hi = input[2]; + int lo = input[0]; + int i4x2 = (hi << 4) | lo; - { - int hi = input[6]; - int lo = input[4]; - int i4x2 = (hi << 4) | lo; + b_k_n_permute(j + 0, i) = i4x2; + } - b_k_n_permute(j + 2, i) = i4x2; - } + { + int hi = input[6]; + int lo = input[4]; + int i4x2 = (hi << 4) | lo; - { - int hi = input[3]; - int lo = input[1]; - int i4x2 = (hi << 4) | lo; + b_k_n_permute(j + 2, i) = i4x2; + } - b_k_n_permute(j + 4, i) = i4x2; - } + { + int hi = input[3]; + int lo = input[1]; + int i4x2 = (hi << 4) | lo; - { - int hi = input[7]; - int lo = input[5]; - int i4x2 = (hi << 4) | lo; + b_k_n_permute(j + 4, i) = i4x2; + } - b_k_n_permute(j + 6, i) = i4x2; + { + int hi = input[7]; + int lo = input[5]; + int i4x2 = (hi << 4) | lo; + + b_k_n_permute(j + 6, i) = i4x2; + } } } } } else { - for(int i = 0; i < N; i++) - { - for(int j = 0; j < K; j++) - { - b_k_n_permute(i * K + j) = b_k_n(i * K + j); - } - } + b_k_n_permute = b_k_n; } b_device_buf.ToDevice(b_k_n_permute.mData.data());