From eb158d9b86290dc7f1ec67487a9b61b48e06706b Mon Sep 17 00:00:00 2001 From: Qun Lin Date: Mon, 2 Jun 2025 15:05:10 +0800 Subject: [PATCH] add device factory and fix build error. --- .../device_grouped_conv_bwd_weight_dl_v4.hpp | 88 ++++++++---- .../grouped_conv_bwd_weight_dl_v4_fp16.cpp | 16 +++ .../run_grouped_conv_bwd_weight_example.inc | 125 +++++++++++++----- 3 files changed, 167 insertions(+), 62 deletions(-) diff --git a/example/20_grouped_conv_bwd_weight/device_grouped_conv_bwd_weight_dl_v4.hpp b/example/20_grouped_conv_bwd_weight/device_grouped_conv_bwd_weight_dl_v4.hpp index e1c1cc52bf..b668f5dc90 100644 --- a/example/20_grouped_conv_bwd_weight/device_grouped_conv_bwd_weight_dl_v4.hpp +++ b/example/20_grouped_conv_bwd_weight/device_grouped_conv_bwd_weight_dl_v4.hpp @@ -152,7 +152,7 @@ struct GridwiseGroupedConv2DBwdWeightDlV4 static constexpr index_t ShareMemOutSize = TileOut_Align_H * TileOut_Align_W * sizeof(OutDataType) * NBatch; - static constexpr index_t NumTilePerBlock = BlockSize / WaveSize; + static constexpr index_t NumTilePerBlock = BlockSize / WaveSize / NumWavePerTile; using InDataVector = typename vector_type::type; using OutDataVector = typename vector_type::type; @@ -204,21 +204,37 @@ struct GridwiseGroupedConv2DBwdWeightDlV4 // interleave data auto* p_scratch_base = p_scratch + i * NumVectorPerPixel * ScalarPerVector; - static_for<0, ScalarPerVector, 1>{}([&](auto j) { + if constexpr(DstScalarPerVector == 1) + { + static_assert(NBatch == 1); + static_for<0, ScalarPerVector, 1>{}( + [&](auto j) { p_scratch_base[j * NumVectorPerPixel] = tmp[0][j.value]; }); + } + else if constexpr(ScalarPerVector == 1) + { + static_assert(DstScalarPerVector > 1); static_for<0, NBatch, 1>{}([&](auto n) { - p_scratch_base[j * NumVectorPerPixel + n / DstScalarPerVector] - [n / DstScalarPerVector] = tmp[n][j.value]; + p_scratch_base[n / DstScalarPerVector][n % DstScalarPerVector] = tmp[n]; }); - }); + } + else + { + static_for<0, ScalarPerVector, 1>{}([&](auto j) { + static_for<0, NBatch, 1>{}([&](auto n) { + p_scratch_base[j * NumVectorPerPixel + n / DstScalarPerVector] + [n % DstScalarPerVector] = tmp[n][j.value]; + }); + }); + } }); if constexpr(AlignedPackH != PackH) { - static_assert(NumWavePerTile == 1); - if (y_offset < (TileH - NumGroup * PackH)) + //static_assert(NumWavePerTile == 1); + if(y_offset < (TileH - NumGroup * PackH)) { constexpr auto i = PackH; - const index_t y = y_offset + i * NumGroup; + const index_t y = y_offset + i * NumGroup; // load data SrcVector tmp[NBatch]; static_for<0, NBatch, 1>{}([&](auto n) { @@ -228,12 +244,28 @@ struct GridwiseGroupedConv2DBwdWeightDlV4 // interleave data auto* p_scratch_base = p_scratch + i * NumVectorPerPixel * ScalarPerVector; - static_for<0, ScalarPerVector, 1>{}([&](auto j) { - static_for<0, NBatch, 1>{}([&](auto n) { - p_scratch_base[j * NumVectorPerPixel + n / DstScalarPerVector] - [n / DstScalarPerVector] = tmp[n][j.value]; + if constexpr(DstScalarPerVector == 1) + { + static_assert(NBatch == 1); + static_for<0, ScalarPerVector, 1>{}([&](auto j) { + p_scratch_base[j * NumVectorPerPixel] = tmp[0][j.value]; }); - }); + } + else if constexpr(ScalarPerVector == 1) + { + static_for<0, NBatch, 1>{}([&](auto n) { + p_scratch_base[n / DstScalarPerVector][n % DstScalarPerVector] = tmp[n]; + }); + } + else + { + static_for<0, ScalarPerVector, 1>{}([&](auto j) { + static_for<0, NBatch, 1>{}([&](auto n) { + p_scratch_base[j * NumVectorPerPixel + n / DstScalarPerVector] + [n % DstScalarPerVector] = tmp[n][j.value]; + }); + }); + } } } } @@ -245,7 +277,7 @@ struct GridwiseGroupedConv2DBwdWeightDlV4 const DestVector* p_scratch, DestVector* p_sharemem) { - //constexpr index_t PackW = TileW / ScalarPerVector; + constexpr index_t PackW = TileW / ScalarPerVector; constexpr index_t AlignedPackW = GetAlignedPackW(); static_assert(AlignedPackW < WaveSize); @@ -260,7 +292,7 @@ struct GridwiseGroupedConv2DBwdWeightDlV4 return (y_ * TileW_Stride + x_) * NumVectorPerPixel; }; - //if(x < PackW) + if(x < PackW) { static_for<0, PackH, 1>{}([&](auto i) { const index_t y = y_offset + i * AlignedPackW; @@ -271,7 +303,7 @@ struct GridwiseGroupedConv2DBwdWeightDlV4 }); if constexpr(AlignedPackH != PackH) { - static_assert(NumWavePerTile == 1); + //static_assert(NumWavePerTile == 1); if (y_offset < (TileH - NumGroup * PackH)) { constexpr auto i = PackH; @@ -342,11 +374,11 @@ struct GridwiseGroupedConv2DBwdWeightDlV4 const index_t lane_id = __lane_id(); constexpr index_t ThreadPerBatch = WaveSize / BatchPerWave; - - static_assert(TileIn_Pack_H % NumWavePerTile == 0); - static_assert(TileOut_Pack_H % NumWavePerTile == 0); - InDataVector tmp_in[TileIn_Pack_H / NumWavePerTile * NumVectorPerPixel * InScalarPerVector] = {}; - OutDataVector tmp_out[TileOut_Pack_H / NumWavePerTile * NumVectorPerPixel * OutScalarPerVector] = {}; + //Debug> xx3; + static_assert(Tile_H % NumWavePerTile == 0); + static_assert(TileOut_H % NumWavePerTile == 0); + InDataVector tmp_in[math::integer_divide_ceil(TileIn_Pack_H, NumWavePerTile) * NumVectorPerPixel * InScalarPerVector] = {}; + OutDataVector tmp_out[math::integer_divide_ceil(TileOut_Pack_H, NumWavePerTile) * NumVectorPerPixel * OutScalarPerVector] = {}; static_assert(NumTilePerBlock == 1 || NumWavePerTile == 1); @@ -383,12 +415,12 @@ struct GridwiseGroupedConv2DBwdWeightDlV4 auto* p_in = arg.p_in_grid_ + g_idx * in_g_stride + n_idx * in_n_stride; auto* p_out = arg.p_out_grid_ + g_idx * out_g_stride + n_idx * out_n_stride; - constexpr index_t Copy_TileIn_H = TileIn_H / NumWavePerTile; + constexpr index_t Copy_Tile_H = Tile_H / NumWavePerTile; constexpr index_t Copy_TileOut_H = TileOut_H / NumWavePerTile; if constexpr (NumWavePerTile > 1) { static_assert(RequirePadding == false); - p_in += Copy_TileIn_H * hi_stride * wave_id; + p_in += Copy_Tile_H * hi_stride * wave_id; p_out += Copy_TileOut_H * ho_stride * wave_id; } @@ -447,7 +479,7 @@ struct GridwiseGroupedConv2DBwdWeightDlV4 } // prefetch 0 - load_data_from_global( + load_data_from_global( p_in, lane_id, in_n_stride, hi, wi, hi_stride, wi_stride, tmp_in); load_data_from_global( p_out, lane_id, out_n_stride, ho, wo, ho_stride, wo_stride, tmp_out); @@ -459,11 +491,11 @@ struct GridwiseGroupedConv2DBwdWeightDlV4 if constexpr (NumWavePerTile > 1) { static_assert(RequirePadding == false); - share_in += Copy_TileIn_H * TileIn_Align_W * NumVectorPerPixel * wave_id; + share_in += Copy_Tile_H * TileIn_Align_W * NumVectorPerPixel * wave_id; share_out += Copy_TileOut_H * TileOut_Align_W * NumVectorPerPixel * wave_id; } - write_data_to_lds( + write_data_to_lds( lane_id, tmp_in, share_in); write_data_to_lds( lane_id, tmp_out, share_out); @@ -481,7 +513,7 @@ struct GridwiseGroupedConv2DBwdWeightDlV4 while(num_loop > 0) { - load_data_from_global( + load_data_from_global( p_in, lane_id, in_n_stride, hi, wi, hi_stride, wi_stride, tmp_in); load_data_from_global( p_out, lane_id, out_n_stride, ho, wo, ho_stride, wo_stride, tmp_out); @@ -493,7 +525,7 @@ struct GridwiseGroupedConv2DBwdWeightDlV4 x, y, ho, wo, hout_base, share_in, share_out, acc); // write 0 - write_data_to_lds( + write_data_to_lds( lane_id, tmp_in, share_in); write_data_to_lds( lane_id, tmp_out, share_out); diff --git a/example/20_grouped_conv_bwd_weight/grouped_conv_bwd_weight_dl_v4_fp16.cpp b/example/20_grouped_conv_bwd_weight/grouped_conv_bwd_weight_dl_v4_fp16.cpp index 469bd4b97d..a271493b4a 100644 --- a/example/20_grouped_conv_bwd_weight/grouped_conv_bwd_weight_dl_v4_fp16.cpp +++ b/example/20_grouped_conv_bwd_weight/grouped_conv_bwd_weight_dl_v4_fp16.cpp @@ -6,6 +6,7 @@ #include "ck/utility/blkgemmpipe_scheduler.hpp" #include "device_grouped_conv_bwd_weight_dl_v4.hpp" +#define ENABLE_CONV_FACTORY 1 using InDataType = F16; using WeiDataType = F16; @@ -43,6 +44,21 @@ using DeviceConvBwdWeightInstance = 2, // DstScalarPerVector false>; +using DeviceConvBwdWeightFactory = std::tuple< + // NDimSpatial BlockSize InLayout WeiLayout OutLayout InDataType WeiDataType OutDataType BlockTileSize FilterSize FilterParam(dilation, stride, pad) NBatch NumWavePerTile InScalarPerVector OutScalarPerVector DstScalarPerVector RequirePadding + ck::tensor_operation::device::DeviceGroupedConvBwdWeightDlV4<2, 256, ALayout, BLayout, ELayout, InDataType, WeiDataType, OutDataType, S<28, 28>, 5, ck::Tuple, S<1,1>, S<2,2>>, InElementOp, WeiElementOp, OutElementOp, 2, 1, 4, 4, 2, false> + , ck::tensor_operation::device::DeviceGroupedConvBwdWeightDlV4<2, 256, ALayout, BLayout, ELayout, InDataType, WeiDataType, OutDataType, S<14, 14>, 5, ck::Tuple, S<1,1>, S<2,2>>, InElementOp, WeiElementOp, OutElementOp, 8, 1, 2, 2, 8, false> + , ck::tensor_operation::device::DeviceGroupedConvBwdWeightDlV4<2, 64, ALayout, BLayout, ELayout, InDataType, WeiDataType, OutDataType, S<7, 7>, 5, ck::Tuple, S<1,1>, S<2,2>>, InElementOp, WeiElementOp, OutElementOp, 16, 1, 1, 1, 8, false> + , ck::tensor_operation::device::DeviceGroupedConvBwdWeightDlV4<2, 128, ALayout, BLayout, ELayout, InDataType, WeiDataType, OutDataType, S<56, 56>, 5, ck::Tuple, S<2,2>, S<2,2>>, InElementOp, WeiElementOp, OutElementOp, 2, 2, 4, 2, 2, false> + , ck::tensor_operation::device::DeviceGroupedConvBwdWeightDlV4<2, 256, ALayout, BLayout, ELayout, InDataType, WeiDataType, OutDataType, S<14, 14>, 5, ck::Tuple, S<2,2>, S<2,2>>, InElementOp, WeiElementOp, OutElementOp, 8, 1, 2, 1, 8, false> + , ck::tensor_operation::device::DeviceGroupedConvBwdWeightDlV4<2, 256, ALayout, BLayout, ELayout, InDataType, WeiDataType, OutDataType, S<112, 112>, 3, ck::Tuple, S<1,1>, S<1,1>>, InElementOp, WeiElementOp, OutElementOp, 1, 4, 8, 8, 1, false> + , ck::tensor_operation::device::DeviceGroupedConvBwdWeightDlV4<2, 128, ALayout, BLayout, ELayout, InDataType, WeiDataType, OutDataType, S<56, 56>, 3, ck::Tuple, S<1,1>, S<1,1>>, InElementOp, WeiElementOp, OutElementOp, 2, 2, 4, 4, 2, false> + , ck::tensor_operation::device::DeviceGroupedConvBwdWeightDlV4<2, 256, ALayout, BLayout, ELayout, InDataType, WeiDataType, OutDataType, S<28, 28>, 3, ck::Tuple, S<1,1>, S<1,1>>, InElementOp, WeiElementOp, OutElementOp, 2, 1, 4, 4, 2, false> + , ck::tensor_operation::device::DeviceGroupedConvBwdWeightDlV4<2, 64, ALayout, BLayout, ELayout, InDataType, WeiDataType, OutDataType, S<14, 14>, 3, ck::Tuple, S<1,1>, S<1,1>>, InElementOp, WeiElementOp, OutElementOp, 8, 1, 2, 2, 8, false> + , ck::tensor_operation::device::DeviceGroupedConvBwdWeightDlV4<2, 256, ALayout, BLayout, ELayout, InDataType, WeiDataType, OutDataType, S<112, 112>, 3, ck::Tuple, S<2,2>, S<1,1>>, InElementOp, WeiElementOp, OutElementOp, 1, 4, 8, 4, 1, false> + , ck::tensor_operation::device::DeviceGroupedConvBwdWeightDlV4<2, 256, ALayout, BLayout, ELayout, InDataType, WeiDataType, OutDataType, S<28, 28>, 3, ck::Tuple, S<2,2>, S<1,1>>, InElementOp, WeiElementOp, OutElementOp, 2, 1, 4, 2, 2, false> +>; + template using HostConvBwdWeightInstance = ck::tensor_operation::host::ReferenceConvBwdWeight{}; + auto ref_invoker = ref_conv.MakeInvoker(); + auto ref_argument = ref_conv.MakeArgument(in, + wei_host_result, + out, + conv_param.conv_filter_strides_, + conv_param.conv_filter_dilations_, + conv_param.input_left_pads_, + conv_param.input_right_pads_, + InElementOp{}, + WeiElementOp{}, + OutElementOp{}, + {}, + {}, + {}); + + ref_invoker.Run(ref_argument); + } + +#if ENABLE_CONV_FACTORY + ck::static_for<0, std::tuple_size_v, 1>{}([&](auto i) -> void { + const auto device_conv_bwd_weight_instance = std::get(DeviceConvBwdWeightFactory{}); + using DeviceConvBwdWeightInstance = ck::remove_cvref_t; + auto conv = DeviceConvBwdWeightInstance{}; + auto invoker = conv.MakeInvoker(); + auto argument = conv.MakeArgument(static_cast(in_device_buf.GetDeviceBuffer()), + static_cast(wei_device_buf.GetDeviceBuffer()), + static_cast(out_device_buf.GetDeviceBuffer()), + input_lengths, + input_strides, + filter_lengths, + weights_strides, + output_lengths, + output_strides, + conv_filter_strides, + conv_filter_dilations, + input_left_pads, + input_right_pads, + InElementOp{}, + WeiElementOp{}, + OutElementOp{}, + split_k); + + DeviceMem gemm_workspace_dev(conv.GetWorkSpaceSize(&argument)); + conv.SetWorkSpacePointer(&argument, gemm_workspace_dev.GetDeviceBuffer()); + + if(conv.IsSupportedArgument(argument)) + { + std::cout << "Run conv :" << conv.GetTypeString() << std::endl; + invoker.ShowInfo(argument); + float avg_time = invoker.Run(argument, StreamConfig{nullptr, config.time_kernel}); + if(config.time_kernel) + { + std::size_t flop = conv_param.GetFlops(); + std::size_t num_btype = conv_param.GetByte(); + + float tflops = static_cast(flop) / 1.E9 / avg_time; + + float gb_per_sec = num_btype / 1.E6 / avg_time; + + std::cerr << "Perf: " << avg_time << " ms, " << tflops << " TFlops, " << gb_per_sec + << " GB/s" << std::endl + << "DeviceOp: " << conv.GetTypeString() << std::endl; + } + if(config.do_verification) + { + wei_device_buf.FromDevice(wei_device_result.mData.data()); + + ck::utils::check_err(wei_device_result.mData, wei_host_result.mData); + } + } + }); +#else auto conv = DeviceConvBwdWeightInstance{}; auto invoker = conv.MakeInvoker(); auto argument = conv.MakeArgument(static_cast(in_device_buf.GetDeviceBuffer()), @@ -107,45 +182,27 @@ bool run_grouped_conv_bwd_weight(const ExecutionConfig& config, return true; } - invoker.Run(argument, StreamConfig{nullptr, config.time_kernel}); + float avg_time = invoker.Run(argument, StreamConfig{nullptr, config.time_kernel}); + if(config.time_kernel) + { + std::size_t flop = conv_param.GetFlops(); + std::size_t num_btype = conv_param.GetByte(); + + float tflops = static_cast(flop) / 1.E9 / avg_time; + + float gb_per_sec = num_btype / 1.E6 / avg_time; + + std::cerr << "Perf: " << avg_time << " ms, " << tflops << " TFlops, " << gb_per_sec + << " GB/s" << std::endl + << "DeviceOp: " << conv.GetTypeString() << std::endl; + } if(config.do_verification) { - auto ref_conv = HostConvBwdWeightInstance{}; - auto ref_invoker = ref_conv.MakeInvoker(); - auto ref_argument = ref_conv.MakeArgument(in, - wei_host_result, - out, - conv_param.conv_filter_strides_, - conv_param.conv_filter_dilations_, - conv_param.input_left_pads_, - conv_param.input_right_pads_, - InElementOp{}, - WeiElementOp{}, - OutElementOp{}, - {}, - {}, - {}); - - ref_invoker.Run(ref_argument); - wei_device_buf.FromDevice(wei_device_result.mData.data()); return ck::utils::check_err(wei_device_result.mData, wei_host_result.mData); } - - float avg_time = invoker.Run(argument, StreamConfig{nullptr, config.time_kernel}); - - std::size_t flop = conv_param.GetFlops(); - std::size_t num_btype = conv_param.GetByte(); - - float tflops = static_cast(flop) / 1.E9 / avg_time; - - float gb_per_sec = num_btype / 1.E6 / avg_time; - - std::cerr << "Perf: " << avg_time << " ms, " << tflops << " TFlops, " << gb_per_sec << " GB/s" - << std::endl - << "DeviceOp: " << conv.GetTypeString() << std::endl; - +#endif return true; }