diff --git a/include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_bwd_weight_explicit_xdl.hpp b/include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_bwd_weight_explicit_xdl.hpp index b01052a966..f6e2a383ab 100644 --- a/include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_bwd_weight_explicit_xdl.hpp +++ b/include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_bwd_weight_explicit_xdl.hpp @@ -303,11 +303,11 @@ struct DeviceGroupedConvBwdWeight_Explicit_Xdl static bool IsSupportedArgument(const Argument& arg) { - if (arg.split_k_ < 0) + if(arg.split_k_ < 0) { - // TODO: Add split-K autodeduction. - // This will probably require adding interface to the GEMM operation for - // querying the optimal split-K value, as we cannot easily access the actual GEMM kernel + // TODO: Add split-K autodeduction. + // This will probably require adding interface to the GEMM operation for + // querying the optimal split-K value, as we cannot easily access the actual GEMM kernel // from here. return false; } diff --git a/include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_bwd_weight_multiple_d_xdl_cshuffle.hpp b/include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_bwd_weight_multiple_d_xdl_cshuffle.hpp index f685d80a04..b22d1a83cc 100644 --- a/include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_bwd_weight_multiple_d_xdl_cshuffle.hpp +++ b/include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_bwd_weight_multiple_d_xdl_cshuffle.hpp @@ -549,26 +549,26 @@ struct DeviceGroupedConvBwdWeightMultipleD_Xdl_CShuffle { MaximumActiveBlocksPerMultiprocessor() { - constexpr int dynamic_smem_size = 0; - int max_occupancy = 0; + constexpr int dynamic_smem_size = 0; + int max_occupancy = 0; hip_check_error(hipOccupancyMaxActiveBlocksPerMultiprocessor( - &max_occupancy, - kernel_batched_gemm_xdlops_bwd_weight< - GridwiseGemm, - ADataType, - BDataType, - AccDataType, - OutElementwiseOperation, - InElementwiseOperation, - element_wise::PassThrough, - remove_reference_t, - remove_reference_t, - remove_reference_t, - remove_reference_t, - ComputePtrOffsetOfStridedBatch, - true>, - BlockSize, - dynamic_smem_size)); + &max_occupancy, + kernel_batched_gemm_xdlops_bwd_weight< + GridwiseGemm, + ADataType, + BDataType, + AccDataType, + OutElementwiseOperation, + InElementwiseOperation, + element_wise::PassThrough, + remove_reference_t, + remove_reference_t, + remove_reference_t, + remove_reference_t, + ComputePtrOffsetOfStridedBatch, + true>, + BlockSize, + dynamic_smem_size)); value_ = std::max(1, max_occupancy); } int value_; @@ -643,32 +643,33 @@ struct DeviceGroupedConvBwdWeightMultipleD_Xdl_CShuffle end(a_g_n_k_wos_lengths), begin(output_spatial_lengths_)); - if (split_k < 0) + if(split_k < 0) { constexpr int k_batch_initial = 1; const auto descs_initial = - conv_to_gemm_transformer - .template MakeABCGridDescriptor_A_K0_M_K1_B_K0_N_K1_C_M_N( - Conv_N_, - Conv_K_, - Conv_C_, - input_spatial_lengths_, - filter_spatial_lengths_, - output_spatial_lengths_, - b_g_n_c_wis_strides, - e_g_k_c_xs_strides, - a_g_n_k_wos_strides, - conv_filter_strides, - conv_filter_dilations, - input_left_pads, - input_right_pads, - k_batch_initial); + conv_to_gemm_transformer + .template MakeABCGridDescriptor_A_K0_M_K1_B_K0_N_K1_C_M_N( + Conv_N_, + Conv_K_, + Conv_C_, + input_spatial_lengths_, + filter_spatial_lengths_, + output_spatial_lengths_, + b_g_n_c_wis_strides, + e_g_k_c_xs_strides, + a_g_n_k_wos_strides, + conv_filter_strides, + conv_filter_dilations, + input_left_pads, + input_right_pads, + k_batch_initial); - const auto& ce_grid_desc_m_n = descs_initial[I2]; - const auto& block_2_ctile_map = - GridwiseGemm::MakeCBlockClusterAdaptor(ce_grid_desc_m_n, M01, N01, k_batch_initial); + const auto& ce_grid_desc_m_n = descs_initial[I2]; + const auto& block_2_ctile_map = GridwiseGemm::MakeCBlockClusterAdaptor( + ce_grid_desc_m_n, M01, N01, k_batch_initial); - const auto grid_size = block_2_ctile_map.CalculateGridSize(ce_grid_desc_m_n) * Conv_G_; + const auto grid_size = + block_2_ctile_map.CalculateGridSize(ce_grid_desc_m_n) * Conv_G_; k_batch_ = get_best_occupancy_k_batch_value(max_occupancy.value_, grid_size); } else diff --git a/include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_bwd_weight_two_stage_xdl_cshuffle.hpp b/include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_bwd_weight_two_stage_xdl_cshuffle.hpp index 3803c2ac85..9cf43ae0a9 100644 --- a/include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_bwd_weight_two_stage_xdl_cshuffle.hpp +++ b/include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_bwd_weight_two_stage_xdl_cshuffle.hpp @@ -512,44 +512,42 @@ struct DeviceGroupedConvBwdWeightTwoStage_Xdl_CShuffle { constexpr int dynamic_smem_size = 0; constexpr index_t minimum_occupancy = - BlkGemmPipeSched == BlockGemmPipelineScheduler::Intrawave ? 1 : 2; + BlkGemmPipeSched == BlockGemmPipelineScheduler::Intrawave ? 1 : 2; int max_occupancy = 0; if constexpr(BlkGemmPipelineVer == BlockGemmPipelineVersion::v4) { hip_check_error(hipOccupancyMaxActiveBlocksPerMultiprocessor( - &max_occupancy, - kernel_grouped_conv_bwd_weight_xdl_cshuffle_v3_2lds< - GridwiseGemm, - remove_reference_t, - remove_reference_t, - remove_reference_t< - DeviceOp::CGridDesc_MBlock_MPerBlock_NBlock_NPerBlock>, - ComputePtrOffsetOfStridedBatch, - NumGroupsToMerge, - true, - InMemoryDataOperationEnum::AtomicAdd, - minimum_occupancy>, - BlockSize, - dynamic_smem_size)); + &max_occupancy, + kernel_grouped_conv_bwd_weight_xdl_cshuffle_v3_2lds< + GridwiseGemm, + remove_reference_t, + remove_reference_t, + remove_reference_t, + ComputePtrOffsetOfStridedBatch, + NumGroupsToMerge, + true, + InMemoryDataOperationEnum::AtomicAdd, + minimum_occupancy>, + BlockSize, + dynamic_smem_size)); } - else + else { hip_check_error(hipOccupancyMaxActiveBlocksPerMultiprocessor( - &max_occupancy, - kernel_grouped_conv_bwd_weight_xdl_cshuffle_v3< - GridwiseGemm, - remove_reference_t, - remove_reference_t, - remove_reference_t< - DeviceOp::CGridDesc_MBlock_MPerBlock_NBlock_NPerBlock>, - ComputePtrOffsetOfStridedBatch, - NumGroupsToMerge, - true, - InMemoryDataOperationEnum::AtomicAdd, - minimum_occupancy>, - BlockSize, - dynamic_smem_size)); + &max_occupancy, + kernel_grouped_conv_bwd_weight_xdl_cshuffle_v3< + GridwiseGemm, + remove_reference_t, + remove_reference_t, + remove_reference_t, + ComputePtrOffsetOfStridedBatch, + NumGroupsToMerge, + true, + InMemoryDataOperationEnum::AtomicAdd, + minimum_occupancy>, + BlockSize, + dynamic_smem_size)); } value_ = std::max(1, max_occupancy); } @@ -629,49 +627,51 @@ struct DeviceGroupedConvBwdWeightTwoStage_Xdl_CShuffle conv_ngchw_to_nhwgc_transformer.TransposeWeiStrides(e_g_k_c_xs_lengths, e_g_k_c_xs_strides); - if (split_k < 0) + if(split_k < 0) { constexpr int k_batch_initial = 1; const auto descs_initial = - conv_to_gemm_transformer_v2 - .template MakeABCGridDescriptor_A_K0_M_K1_B_K0_N_K1_C_M_N( - Conv_N_, - Conv_K_, - Conv_C_, - input_spatial_lengths_, - filter_spatial_lengths_, - output_spatial_lengths_, - b_g_n_c_wis_strides, - e_g_k_c_xs_strides, - a_g_n_k_wos_strides, - conv_filter_strides, - conv_filter_dilations, - input_left_pads, - input_right_pads, - k_batch_initial); + conv_to_gemm_transformer_v2 + .template MakeABCGridDescriptor_A_K0_M_K1_B_K0_N_K1_C_M_N( + Conv_N_, + Conv_K_, + Conv_C_, + input_spatial_lengths_, + filter_spatial_lengths_, + output_spatial_lengths_, + b_g_n_c_wis_strides, + e_g_k_c_xs_strides, + a_g_n_k_wos_strides, + conv_filter_strides, + conv_filter_dilations, + input_left_pads, + input_right_pads, + k_batch_initial); const auto& a_grid_desc_kbatch_k0_m_k1 = descs_initial[I0]; const auto& b_grid_desc_kbatch_k0_n_k1 = descs_initial[I1]; - const auto gemmM = a_grid_desc_kbatch_k0_m_k1.GetLength(I1); - const auto gemmN = b_grid_desc_kbatch_k0_n_k1.GetLength(I1); + const auto gemmM = a_grid_desc_kbatch_k0_m_k1.GetLength(I1); + const auto gemmN = b_grid_desc_kbatch_k0_n_k1.GetLength(I1); - const auto grid_size = GridwiseGemm::Block2CTileMap::CalculateGridSize(gemmM, gemmN) * Conv_G_ / NumGroupsToMerge; + const auto grid_size = + GridwiseGemm::Block2CTileMap::CalculateGridSize(gemmM, gemmN) * Conv_G_ / + NumGroupsToMerge; k_batch_ = get_best_occupancy_k_batch_value(max_occupancy.value_, grid_size); // Ensure that k_batch_ does not exceed the maximum value // for the GEMM pipeline ck::index_t gemmK; - std::tie(std::ignore, std::ignore, gemmK) = + std::tie(std::ignore, std::ignore, gemmK) = get_bwd_weight_gemm_sizes(a_g_n_k_wos_lengths, e_g_k_c_xs_lengths); const auto k_batch_max = static_cast((gemmK - 1) / KPerBlock); - k_batch_ = std::min(k_batch_, k_batch_max); - - if (ck::EnvIsEnabled(CK_ENV(CK_LOGGING))) + k_batch_ = std::min(k_batch_, k_batch_max); + + if(ck::EnvIsEnabled(CK_ENV(CK_LOGGING))) { - std::cout << "[SPLIT-K AUTODEDUCE] k_batch max value: " - << k_batch_max << std::endl; - std::cout << "[SPLIT-K AUTODEDUCE] Final k_batch value: " - << k_batch_ << std::endl; + std::cout << "[SPLIT-K AUTODEDUCE] k_batch max value: " << k_batch_max + << std::endl; + std::cout << "[SPLIT-K AUTODEDUCE] Final k_batch value: " << k_batch_ + << std::endl; } } else diff --git a/include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_bwd_weight_xdl_cshuffle.hpp b/include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_bwd_weight_xdl_cshuffle.hpp index da98cd6f71..72a2f73cb1 100644 --- a/include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_bwd_weight_xdl_cshuffle.hpp +++ b/include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_bwd_weight_xdl_cshuffle.hpp @@ -428,30 +428,29 @@ struct DeviceGroupedConvBwdWeight_Xdl_CShuffle MaximumActiveBlocksPerMultiprocessor() { constexpr int dynamic_smem_size = 0; - int max_occupancy = 0; + int max_occupancy = 0; hip_check_error(hipOccupancyMaxActiveBlocksPerMultiprocessor( - &max_occupancy, - kernel_batched_gemm_xdlops_bwd_weight< - GridwiseGemm, - ADataType, - BDataType, - CDataType, - OutElementwiseOperation, - InElementwiseOperation, - WeiElementwiseOperation, - remove_reference_t, - remove_reference_t, - remove_reference_t, - remove_reference_t, - ComputePtrOffsetOfStridedBatch<>, - false>, // Both true/false give the same occupancy. - BlockSize, - dynamic_smem_size)); + &max_occupancy, + kernel_batched_gemm_xdlops_bwd_weight< + GridwiseGemm, + ADataType, + BDataType, + CDataType, + OutElementwiseOperation, + InElementwiseOperation, + WeiElementwiseOperation, + remove_reference_t, + remove_reference_t, + remove_reference_t, + remove_reference_t, + ComputePtrOffsetOfStridedBatch<>, + false>, // Both true/false give the same occupancy. + BlockSize, + dynamic_smem_size)); value_ = std::max(1, max_occupancy); } int value_; }; - struct Argument : public BaseArgument, public ArgumentSplitK { @@ -527,34 +526,36 @@ struct DeviceGroupedConvBwdWeight_Xdl_CShuffle conv_ngchw_to_nhwgc_transformer.TransposeWeiStrides(e_g_k_c_xs_lengths, e_g_k_c_xs_strides); - if (split_k < 0) + if(split_k < 0) { constexpr int k_batch_initial = 1; const auto descs_initial = - conv_to_gemm_transformer - .template MakeABCGridDescriptor_A_K0_M_K1_B_K0_N_K1_C_M_N( - Conv_N_, - Conv_K_, - Conv_C_, - input_spatial_lengths_, - filter_spatial_lengths_, - output_spatial_lengths_, - b_g_n_c_wis_strides_transposed, - e_g_k_c_xs_strides_transposed, - a_g_n_k_wos_strides_transposed, - conv_filter_strides, - conv_filter_dilations, - input_left_pads, - input_right_pads, - k_batch_initial); + conv_to_gemm_transformer + .template MakeABCGridDescriptor_A_K0_M_K1_B_K0_N_K1_C_M_N( + Conv_N_, + Conv_K_, + Conv_C_, + input_spatial_lengths_, + filter_spatial_lengths_, + output_spatial_lengths_, + b_g_n_c_wis_strides_transposed, + e_g_k_c_xs_strides_transposed, + a_g_n_k_wos_strides_transposed, + conv_filter_strides, + conv_filter_dilations, + input_left_pads, + input_right_pads, + k_batch_initial); const auto& c_grid_desc_m_n = descs_initial[I2]; - const auto& block_2_ctile_map = GridwiseGemm::MakeCBlockClusterAdaptor(c_grid_desc_m_n, M01, N01, k_batch_initial); + const auto& block_2_ctile_map = GridwiseGemm::MakeCBlockClusterAdaptor( + c_grid_desc_m_n, M01, N01, k_batch_initial); - const auto grid_size = block_2_ctile_map.CalculateGridSize(c_grid_desc_m_n) * Conv_G_; + const auto grid_size = + block_2_ctile_map.CalculateGridSize(c_grid_desc_m_n) * Conv_G_; k_batch_ = get_best_occupancy_k_batch_value(max_occupancy.value_, grid_size); } - else + else { k_batch_ = split_k; } diff --git a/include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_bwd_weight_xdl_cshuffle_v3.hpp b/include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_bwd_weight_xdl_cshuffle_v3.hpp index 9bbfafd72b..7aca470395 100644 --- a/include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_bwd_weight_xdl_cshuffle_v3.hpp +++ b/include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_bwd_weight_xdl_cshuffle_v3.hpp @@ -389,42 +389,40 @@ struct DeviceGroupedConvBwdWeight_Xdl_CShuffleV3 { constexpr int dynamic_smem_size = 0; constexpr index_t minimum_occupancy = - BlkGemmPipeSched == BlockGemmPipelineScheduler::Intrawave ? 1 : 2; + BlkGemmPipeSched == BlockGemmPipelineScheduler::Intrawave ? 1 : 2; int max_occupancy = 0; if constexpr(BlkGemmPipelineVer == BlockGemmPipelineVersion::v4) { hip_check_error(hipOccupancyMaxActiveBlocksPerMultiprocessor( - &max_occupancy, - kernel_grouped_conv_bwd_weight_xdl_cshuffle_v3_2lds< - GridwiseGemm, - remove_reference_t, - remove_reference_t, - remove_reference_t< - DeviceOp::CGridDesc_MBlock_MPerBlock_NBlock_NPerBlock>, - ComputePtrOffsetOfStridedBatch, - true, - InMemoryDataOperationEnum::AtomicAdd, - minimum_occupancy>, - BlockSize, - dynamic_smem_size)); + &max_occupancy, + kernel_grouped_conv_bwd_weight_xdl_cshuffle_v3_2lds< + GridwiseGemm, + remove_reference_t, + remove_reference_t, + remove_reference_t, + ComputePtrOffsetOfStridedBatch, + true, + InMemoryDataOperationEnum::AtomicAdd, + minimum_occupancy>, + BlockSize, + dynamic_smem_size)); } - else + else { hip_check_error(hipOccupancyMaxActiveBlocksPerMultiprocessor( - &max_occupancy, - kernel_grouped_conv_bwd_weight_xdl_cshuffle_v3< - GridwiseGemm, - remove_reference_t, - remove_reference_t, - remove_reference_t< - DeviceOp::CGridDesc_MBlock_MPerBlock_NBlock_NPerBlock>, - ComputePtrOffsetOfStridedBatch, - true, - InMemoryDataOperationEnum::AtomicAdd, - minimum_occupancy>, - BlockSize, - dynamic_smem_size)); + &max_occupancy, + kernel_grouped_conv_bwd_weight_xdl_cshuffle_v3< + GridwiseGemm, + remove_reference_t, + remove_reference_t, + remove_reference_t, + ComputePtrOffsetOfStridedBatch, + true, + InMemoryDataOperationEnum::AtomicAdd, + minimum_occupancy>, + BlockSize, + dynamic_smem_size)); } value_ = std::max(1, max_occupancy); } @@ -494,49 +492,50 @@ struct DeviceGroupedConvBwdWeight_Xdl_CShuffleV3 end(a_g_n_k_wos_lengths), begin(output_spatial_lengths_)); - if (split_k < 0) + if(split_k < 0) { constexpr int k_batch_initial = 1; const auto descs_initial = - conv_to_gemm_transformer - .template MakeABCGridDescriptor_A_K0_M_K1_B_K0_N_K1_C_M_N( - Conv_N_, - Conv_K_, - Conv_C_, - input_spatial_lengths_, - filter_spatial_lengths_, - output_spatial_lengths_, - b_g_n_c_wis_strides, - e_g_k_c_xs_strides, - a_g_n_k_wos_strides, - conv_filter_strides, - conv_filter_dilations, - input_left_pads, - input_right_pads, - k_batch_initial); + conv_to_gemm_transformer + .template MakeABCGridDescriptor_A_K0_M_K1_B_K0_N_K1_C_M_N( + Conv_N_, + Conv_K_, + Conv_C_, + input_spatial_lengths_, + filter_spatial_lengths_, + output_spatial_lengths_, + b_g_n_c_wis_strides, + e_g_k_c_xs_strides, + a_g_n_k_wos_strides, + conv_filter_strides, + conv_filter_dilations, + input_left_pads, + input_right_pads, + k_batch_initial); const auto& a_grid_desc_kbatch_k0_m_k1 = descs_initial[I0]; const auto& b_grid_desc_kbatch_k0_n_k1 = descs_initial[I1]; - const auto gemmM = a_grid_desc_kbatch_k0_m_k1.GetLength(I1); - const auto gemmN = b_grid_desc_kbatch_k0_n_k1.GetLength(I1); + const auto gemmM = a_grid_desc_kbatch_k0_m_k1.GetLength(I1); + const auto gemmN = b_grid_desc_kbatch_k0_n_k1.GetLength(I1); - const auto grid_size = GridwiseGemm::Block2CTileMap::CalculateGridSize(gemmM, gemmN) * Conv_G_; + const auto grid_size = + GridwiseGemm::Block2CTileMap::CalculateGridSize(gemmM, gemmN) * Conv_G_; k_batch_ = get_best_occupancy_k_batch_value(max_occupancy.value_, grid_size); // Ensure that k_batch_ does not exceed the maximum value // for the GEMM pipeline ck::index_t gemmK; - std::tie(std::ignore, std::ignore, gemmK) = + std::tie(std::ignore, std::ignore, gemmK) = get_bwd_weight_gemm_sizes(a_g_n_k_wos_lengths, e_g_k_c_xs_lengths); const auto k_batch_max = static_cast((gemmK - 1) / K0PerBlock); - k_batch_ = std::min(k_batch_, k_batch_max); - - if (ck::EnvIsEnabled(CK_ENV(CK_LOGGING))) + k_batch_ = std::min(k_batch_, k_batch_max); + + if(ck::EnvIsEnabled(CK_ENV(CK_LOGGING))) { - std::cout << "[SPLIT-K AUTODEDUCE] k_batch max value: " - << k_batch_max << std::endl; - std::cout << "[SPLIT-K AUTODEDUCE] Final k_batch value: " - << k_batch_ << std::endl; + std::cout << "[SPLIT-K AUTODEDUCE] k_batch max value: " << k_batch_max + << std::endl; + std::cout << "[SPLIT-K AUTODEDUCE] Final k_batch value: " << k_batch_ + << std::endl; } } else diff --git a/include/ck/tensor_operation/gpu/device/impl/split_k_arg.hpp b/include/ck/tensor_operation/gpu/device/impl/split_k_arg.hpp index 3d4b270d61..de683f3282 100644 --- a/include/ck/tensor_operation/gpu/device/impl/split_k_arg.hpp +++ b/include/ck/tensor_operation/gpu/device/impl/split_k_arg.hpp @@ -3,14 +3,13 @@ #pragma once - namespace ck { namespace tensor_operation { namespace device { struct ArgumentSplitK { - index_t k_batch_{1}; + index_t k_batch_{1}; }; } // namespace device diff --git a/include/ck/tensor_operation/gpu/device/impl/split_k_utils.hpp b/include/ck/tensor_operation/gpu/device/impl/split_k_utils.hpp index 8bd3056d2b..99e0635bec 100644 --- a/include/ck/tensor_operation/gpu/device/impl/split_k_utils.hpp +++ b/include/ck/tensor_operation/gpu/device/impl/split_k_utils.hpp @@ -15,16 +15,16 @@ namespace device { struct DeviceProperties { - DeviceProperties() - { - hipDeviceProp_t dev_prop; - hipDevice_t dev; - hip_check_error(hipGetDevice(&dev)); - hip_check_error(hipGetDeviceProperties(&dev_prop, dev)); + DeviceProperties() + { + hipDeviceProp_t dev_prop; + hipDevice_t dev; + hip_check_error(hipGetDevice(&dev)); + hip_check_error(hipGetDeviceProperties(&dev_prop, dev)); - num_cu_ = dev_prop.multiProcessorCount; - }; - int num_cu_; + num_cu_ = dev_prop.multiProcessorCount; + }; + int num_cu_; }; inline ck::index_t get_best_occupancy_k_batch_value(int max_occupancy, ck::index_t grid_size) @@ -33,49 +33,51 @@ inline ck::index_t get_best_occupancy_k_batch_value(int max_occupancy, ck::index const int max_capacity = max_occupancy * device_properties.num_cu_; ck::index_t k_batch = 1; - const auto optimal_split = static_cast(std::floor((1.0 * max_capacity) / grid_size)); - if (optimal_split > 1) + const auto optimal_split = + static_cast(std::floor((1.0 * max_capacity) / grid_size)); + if(optimal_split > 1) { - k_batch = optimal_split; - } - - if (ck::EnvIsEnabled(CK_ENV(CK_LOGGING))) + k_batch = optimal_split; + } + + if(ck::EnvIsEnabled(CK_ENV(CK_LOGGING))) { - std::cout << "[SPLIT-K AUTODEDUCE] Max active thread blocks per CU for GEMM kernel: " << max_occupancy << std::endl; - std::cout << "[SPLIT-K AUTODEDUCE] Output grid size: " << grid_size << std::endl; - std::cout << "[SPLIT-K AUTODEDUCE] Optimal split-k value " << k_batch << std::endl; + std::cout << "[SPLIT-K AUTODEDUCE] Max active thread blocks per CU for GEMM kernel: " + << max_occupancy << std::endl; + std::cout << "[SPLIT-K AUTODEDUCE] Output grid size: " << grid_size << std::endl; + std::cout << "[SPLIT-K AUTODEDUCE] Optimal split-k value " << k_batch << std::endl; } return k_batch; } template -inline auto get_bwd_weight_gemm_sizes( - const std::array& a_g_n_k_wos_lengths, - const std::array& e_g_k_c_xs_lengths) +inline auto +get_bwd_weight_gemm_sizes(const std::array& a_g_n_k_wos_lengths, + const std::array& e_g_k_c_xs_lengths) { - static constexpr auto I1 = Number<1>{}; - static constexpr auto I2 = Number<2>{}; + static constexpr auto I1 = Number<1>{}; + static constexpr auto I2 = Number<2>{}; - // The input array has elements in the order: G, N, K, Do, Ho, Wo - // GemmK = N * Do * Ho * Wo for the BWD weight pass. - constexpr index_t spatial_offset = 3; - const index_t DoHoWo = std::accumulate(begin(a_g_n_k_wos_lengths) + spatial_offset, - end(a_g_n_k_wos_lengths), - index_t{1}, - std::multiplies<>{}); - const auto gemmK = a_g_n_k_wos_lengths[I1] * DoHoWo; + // The input array has elements in the order: G, N, K, Do, Ho, Wo + // GemmK = N * Do * Ho * Wo for the BWD weight pass. + constexpr index_t spatial_offset = 3; + const index_t DoHoWo = std::accumulate(begin(a_g_n_k_wos_lengths) + spatial_offset, + end(a_g_n_k_wos_lengths), + index_t{1}, + std::multiplies<>{}); + const auto gemmK = a_g_n_k_wos_lengths[I1] * DoHoWo; - // The GEMM M dimension is the number of output channels. - const auto gemmM = e_g_k_c_xs_lengths[I1]; + // The GEMM M dimension is the number of output channels. + const auto gemmM = e_g_k_c_xs_lengths[I1]; - // The output array has elements in the order: G, K, C, X, Y, Z - // GemmN = C * X * Y * Z for the BWD weight pass. - const index_t XYZ = std::accumulate(begin(e_g_k_c_xs_lengths) + spatial_offset, - end(e_g_k_c_xs_lengths), - index_t{1}, - std::multiplies<>{}); - const auto gemmN = e_g_k_c_xs_lengths[I2] * XYZ; - return std::make_tuple(gemmM, gemmN, gemmK); + // The output array has elements in the order: G, K, C, X, Y, Z + // GemmN = C * X * Y * Z for the BWD weight pass. + const index_t XYZ = std::accumulate(begin(e_g_k_c_xs_lengths) + spatial_offset, + end(e_g_k_c_xs_lengths), + index_t{1}, + std::multiplies<>{}); + const auto gemmN = e_g_k_c_xs_lengths[I2] * XYZ; + return std::make_tuple(gemmM, gemmN, gemmK); } } // namespace device diff --git a/profiler/include/profiler/profile_grouped_conv_bwd_weight_impl.hpp b/profiler/include/profiler/profile_grouped_conv_bwd_weight_impl.hpp index 3f7852e5dc..53f45f18e1 100644 --- a/profiler/include/profiler/profile_grouped_conv_bwd_weight_impl.hpp +++ b/profiler/include/profiler/profile_grouped_conv_bwd_weight_impl.hpp @@ -139,9 +139,9 @@ bool profile_grouped_conv_bwd_weight_impl(int do_verification, std::cout << "found " << op_ptrs.size() << " instances" << std::endl; std::string best_op_name; - float best_avg_time = 0; - float best_tflops = 0; - float best_gb_per_sec = 0; + float best_avg_time = 0; + float best_tflops = 0; + float best_gb_per_sec = 0; std::string best_split_k("1"); // profile device Conv instances @@ -171,14 +171,14 @@ bool profile_grouped_conv_bwd_weight_impl(int do_verification, range_copy(conv_param.input_left_pads_, begin(input_left_pads)); range_copy(conv_param.input_right_pads_, begin(input_right_pads)); - std::vector split_k_list = {/*auto deduce value*/-1, 1, 2, 4, 8, 16, 32, 64, 128}; + std::vector split_k_list = {/*auto deduce value*/ -1, 1, 2, 4, 8, 16, 32, 64, 128}; - if (split_k != "all") + if(split_k != "all") { try { ck::index_t split_k_value = std::stoi(split_k); - split_k_list = {split_k_value}; + split_k_list = {split_k_value}; } catch(const std::exception& e) { @@ -209,14 +209,15 @@ bool profile_grouped_conv_bwd_weight_impl(int do_verification, out_element_op, split_k_list[split_k_id]); - auto split_k_value = split_k_list[split_k_id]; + auto split_k_value = split_k_list[split_k_id]; auto split_k_param_str = std::to_string(split_k_value); - auto* split_k_arg = dynamic_cast(argument_ptr.get()); - if (split_k_arg && split_k_value < 0) + auto* split_k_arg = + dynamic_cast(argument_ptr.get()); + if(split_k_arg && split_k_value < 0) { - split_k_value = split_k_arg->k_batch_; + split_k_value = split_k_arg->k_batch_; split_k_param_str = std::to_string(split_k_value) + " (best occupancy)"; - } + } const std::size_t workspace_sz = op_ptr->GetWorkSpaceSize(argument_ptr.get()); DeviceMem workspace_dev(workspace_sz); diff --git a/test/grouped_convnd_bwd_weight/test_grouped_convnd_bwd_weight_interface_xdl.cpp b/test/grouped_convnd_bwd_weight/test_grouped_convnd_bwd_weight_interface_xdl.cpp index 9e822c5765..2ad1cd11f0 100644 --- a/test/grouped_convnd_bwd_weight/test_grouped_convnd_bwd_weight_interface_xdl.cpp +++ b/test/grouped_convnd_bwd_weight/test_grouped_convnd_bwd_weight_interface_xdl.cpp @@ -98,26 +98,26 @@ class TestGroupedConvndBwdWeight : public ::testing::Test bool is_supported = true; - for (const auto split_k : split_ks) + for(const auto split_k : split_ks) { auto argument = conv.MakeArgument(nullptr, - nullptr, - nullptr, - input_lengths, - input_strides, - filter_lengths, - weights_strides, - output_lengths, - output_strides, - conv_filter_strides, - conv_filter_dilations, - input_left_pads, - input_right_pads, - PassThrough{}, - PassThrough{}, - PassThrough{}, - split_k); - is_supported &=conv.IsSupportedArgument(argument); + nullptr, + nullptr, + input_lengths, + input_strides, + filter_lengths, + weights_strides, + output_lengths, + output_strides, + conv_filter_strides, + conv_filter_dilations, + input_left_pads, + input_right_pads, + PassThrough{}, + PassThrough{}, + PassThrough{}, + split_k); + is_supported &= conv.IsSupportedArgument(argument); } return is_supported; } diff --git a/test/grouped_convnd_bwd_weight/test_grouped_convnd_bwd_weight_v3_interface_xdl.cpp b/test/grouped_convnd_bwd_weight/test_grouped_convnd_bwd_weight_v3_interface_xdl.cpp index 4e668c6d7c..bfd55a7c55 100644 --- a/test/grouped_convnd_bwd_weight/test_grouped_convnd_bwd_weight_v3_interface_xdl.cpp +++ b/test/grouped_convnd_bwd_weight/test_grouped_convnd_bwd_weight_v3_interface_xdl.cpp @@ -98,26 +98,26 @@ class TestGroupedConvndBwdWeight : public ::testing::Test bool is_supported = true; - for (const auto split_k : split_ks) + for(const auto split_k : split_ks) { auto argument = conv.MakeArgument(nullptr, - nullptr, - nullptr, - input_lengths, - input_strides, - filter_lengths, - weights_strides, - output_lengths, - output_strides, - conv_filter_strides, - conv_filter_dilations, - input_left_pads, - input_right_pads, - PassThrough{}, - PassThrough{}, - PassThrough{}, - split_k); - is_supported &=conv.IsSupportedArgument(argument); + nullptr, + nullptr, + input_lengths, + input_strides, + filter_lengths, + weights_strides, + output_lengths, + output_strides, + conv_filter_strides, + conv_filter_dilations, + input_left_pads, + input_right_pads, + PassThrough{}, + PassThrough{}, + PassThrough{}, + split_k); + is_supported &= conv.IsSupportedArgument(argument); } return is_supported; }