diff --git a/CHANGELOG.md b/CHANGELOG.md index 9001c2f4e7..31f129b581 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,8 +25,8 @@ Full documentation for Composable Kernel is not yet available. - Added multi-embeddings support (#542). - Added Navi3x blockwise GEMM and real GEMM support (#541). - Added Navi grouped ConvBwdWeight support (#505). -- Added pool3d forward (#697). -- Added maxpool backward (#750). +- Added MaxPool, AvgPool forward (#815). +- Added MaxPool backward (#750). ### Changed - Changed ... diff --git a/client_example/19_pool_fwd/avg_pool3d_fwd.cpp b/client_example/19_pool_fwd/avg_pool3d_fwd.cpp index 2edaf474b5..db8e0569d7 100644 --- a/client_example/19_pool_fwd/avg_pool3d_fwd.cpp +++ b/client_example/19_pool_fwd/avg_pool3d_fwd.cpp @@ -16,6 +16,9 @@ using InDataType = ck::half_t; using OutDataType = ck::half_t; using IndexDataType = int32_t; +using InLayout = ck::tensor_layout::convolution::NDHWC; +using OutLayout = ck::tensor_layout::convolution::NDHWC; + constexpr ck::index_t InOutRank = 5; constexpr ck::index_t WindowRank = 3; #if 0 @@ -44,33 +47,41 @@ struct SimpleDeviceMem int main(int argc, char* argv[]) { - ck::index_t N = 2; - ck::index_t C = 32; - ck::index_t Z = 2; - ck::index_t Y = 2; - ck::index_t X = 2; - ck::index_t Di = 30; - ck::index_t Hi = 30; - ck::index_t Wi = 30; - ck::index_t window_stride_d = 2; - ck::index_t window_stride_h = 2; - ck::index_t window_stride_w = 2; - ck::index_t in_left_pad_d = 1; - ck::index_t in_left_pad_h = 1; - ck::index_t in_left_pad_w = 1; - ck::index_t in_right_pad_d = 1; - ck::index_t in_right_pad_h = 1; - ck::index_t in_right_pad_w = 1; + ck::index_t N = 2; + ck::index_t C = 32; + ck::index_t Z = 2; + ck::index_t Y = 2; + ck::index_t X = 2; + ck::index_t Di = 30; + ck::index_t Hi = 30; + ck::index_t Wi = 30; + ck::index_t window_stride_d = 2; + ck::index_t window_stride_h = 2; + ck::index_t window_stride_w = 2; + ck::index_t window_dilation_d = 1; + ck::index_t window_dilation_h = 1; + ck::index_t window_dilation_w = 1; + ck::index_t in_left_pad_d = 1; + ck::index_t in_left_pad_h = 1; + ck::index_t in_left_pad_w = 1; + ck::index_t in_right_pad_d = 1; + ck::index_t in_right_pad_h = 1; + ck::index_t in_right_pad_w = 1; - ck::index_t Do = (Di + in_left_pad_d + in_right_pad_d - Z) / window_stride_d + 1; - ck::index_t Ho = (Hi + in_left_pad_h + in_right_pad_h - Y) / window_stride_h + 1; - ck::index_t Wo = (Wi + in_left_pad_w + in_right_pad_w - X) / window_stride_w + 1; + const ck::index_t Zs = (Z - 1) * window_dilation_d + 1; + const ck::index_t Ys = (Y - 1) * window_dilation_h + 1; + const ck::index_t Xs = (X - 1) * window_dilation_w + 1; + ck::index_t Do = (Di + in_left_pad_d + in_right_pad_d - Zs) / window_stride_d + 1; + ck::index_t Ho = (Hi + in_left_pad_h + in_right_pad_h - Ys) / window_stride_h + 1; + ck::index_t Wo = (Wi + in_left_pad_w + in_right_pad_w - Xs) / window_stride_w + 1; // Pool API only support the order of NCDHW std::vector in_length = {N, C, Di, Hi, Wi}; std::vector out_length = {N, C, Do, Ho, Wo}; std::vector window_spatial_lengths = {Z, Y, X}; - std::vector window_strides = {window_stride_d, window_stride_h, window_stride_w}; + std::vector window_strides = {window_stride_d, window_stride_h, window_stride_w}; + std::vector window_dilations{ + window_dilation_d, window_dilation_h, window_dilation_w}; std::vector input_left_pads = {in_left_pad_d, in_left_pad_h, in_left_pad_w}; std::vector input_right_pads = {in_right_pad_d, in_right_pad_h, in_right_pad_w}; @@ -90,6 +101,8 @@ int main(int argc, char* argv[]) InDataType, OutDataType, IndexDataType, + InLayout, + OutLayout, ReduceOpId, OutputIndex>; @@ -122,6 +135,7 @@ int main(int argc, char* argv[]) out_tensor_stride, out_tensor_stride, window_strides, + window_dilations, input_left_pads, input_right_pads, {2, 3, 4}); @@ -181,6 +195,7 @@ int main(int argc, char* argv[]) out_tensor_stride, out_tensor_stride, window_strides, + window_dilations, input_left_pads, input_right_pads, {2, 3, 4}); diff --git a/client_example/19_pool_fwd/max_pool2d_fwd.cpp b/client_example/19_pool_fwd/max_pool2d_fwd.cpp index c776dc12da..84b818a60f 100644 --- a/client_example/19_pool_fwd/max_pool2d_fwd.cpp +++ b/client_example/19_pool_fwd/max_pool2d_fwd.cpp @@ -10,14 +10,18 @@ #include "ck/tensor_operation/gpu/device/device_pool_fwd.hpp" #include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/pool2d_fwd.hpp" +#include "ck/library/tensor_operation_instance/gpu/pool3d_fwd.hpp" using InDataType = ck::half_t; using OutDataType = ck::half_t; using IndexDataType = int32_t; -constexpr ck::index_t InOutRank = 4; -constexpr ck::index_t WindowRank = 2; +// We use pool3d to implement pool2d in this example +using InLayout = ck::tensor_layout::convolution::NDHWC; +using OutLayout = ck::tensor_layout::convolution::NDHWC; + +constexpr ck::index_t InOutRank = 5; +constexpr ck::index_t WindowRank = 3; #if 1 constexpr auto ReduceOpId = ck::ReduceTensorOp::MAX; constexpr bool OutputIndex = true; @@ -42,31 +46,66 @@ struct SimpleDeviceMem void* p_mem_; }; +void TransformPool2dparamToPool3d(std::vector& input_lengths, + std::vector& window_lengths, + std::vector& output_lengths, + std::vector& input_stride, + std::vector& output_stride, + std::vector& indices_stride, + std::vector& window_strides, + std::vector& window_dilations, + std::vector& input_left_pads, + std::vector& input_right_pads, + std::vector& pooling_dims) +{ + // NCHW to NCDHW + input_lengths.insert(input_lengths.begin() + 2, 1); + output_lengths.insert(output_lengths.begin() + 2, 1); + input_stride.insert(input_stride.begin() + 2, 0); + output_stride.insert(output_stride.begin() + 2, 0); + indices_stride.insert(indices_stride.begin() + 2, 0); + + // YX to ZYX + window_lengths.insert(window_lengths.begin(), 1); + window_strides.insert(window_strides.begin(), 0); + window_dilations.insert(window_dilations.begin(), 0); + input_left_pads.insert(input_left_pads.begin(), 0); + input_right_pads.insert(input_right_pads.begin(), 0); + + pooling_dims = {2, 3, 4}; +} + int main(int argc, char* argv[]) { - ck::index_t N = 2; - ck::index_t C = 32; - ck::index_t Y = 2; - ck::index_t X = 2; - ck::index_t Hi = 30; - ck::index_t Wi = 30; - ck::index_t window_stride_h = 2; - ck::index_t window_stride_w = 2; - ck::index_t in_left_pad_h = 1; - ck::index_t in_left_pad_w = 1; - ck::index_t in_right_pad_h = 1; - ck::index_t in_right_pad_w = 1; + ck::index_t N = 2; + ck::index_t C = 32; + ck::index_t Y = 2; + ck::index_t X = 2; + ck::index_t Hi = 30; + ck::index_t Wi = 30; + ck::index_t window_stride_h = 2; + ck::index_t window_stride_w = 2; + ck::index_t window_dilation_h = 1; + ck::index_t window_dilation_w = 1; + ck::index_t in_left_pad_h = 1; + ck::index_t in_left_pad_w = 1; + ck::index_t in_right_pad_h = 1; + ck::index_t in_right_pad_w = 1; - ck::index_t Ho = (Hi + in_left_pad_h + in_right_pad_h - Y) / window_stride_h + 1; - ck::index_t Wo = (Wi + in_left_pad_w + in_right_pad_w - X) / window_stride_w + 1; + const ck::index_t Ys = (Y - 1) * window_dilation_h + 1; + const ck::index_t Xs = (X - 1) * window_dilation_w + 1; + ck::index_t Ho = (Hi + in_left_pad_h + in_right_pad_h - Ys) / window_stride_h + 1; + ck::index_t Wo = (Wi + in_left_pad_w + in_right_pad_w - Xs) / window_stride_w + 1; // Pool API only support the order of NCHW std::vector in_length = {N, C, Hi, Wi}; std::vector out_length = {N, C, Ho, Wo}; std::vector window_spatial_lengths = {Y, X}; std::vector window_strides = {window_stride_h, window_stride_w}; + std::vector window_dilations = {window_dilation_h, window_dilation_w}; std::vector input_left_pads = {in_left_pad_h, in_left_pad_w}; std::vector input_right_pads = {in_right_pad_h, in_right_pad_w}; + std::vector pooling_dims = {2, 3}; std::size_t in_tensor_size = N * C * Hi * Wi; std::size_t out_tensor_size = N * C * Ho * Wo; @@ -75,6 +114,18 @@ int main(int argc, char* argv[]) std::vector in_tensor_stride = {C * Hi * Wi, 1, Wi * C, C}; std::vector out_tensor_stride = {C * Ho * Wo, 1, Wo * C, C}; + TransformPool2dparamToPool3d(in_length, + window_spatial_lengths, + out_length, + in_tensor_stride, + out_tensor_stride, + out_tensor_stride, + window_strides, + window_dilations, + input_left_pads, + input_right_pads, + pooling_dims); + SimpleDeviceMem in_device_buf(sizeof(InDataType) * in_tensor_size); SimpleDeviceMem out_device_buf(sizeof(OutDataType) * out_tensor_size); SimpleDeviceMem out_indices_device_buf(sizeof(IndexDataType) * out_tensor_size); @@ -84,6 +135,8 @@ int main(int argc, char* argv[]) InDataType, OutDataType, IndexDataType, + InLayout, + OutLayout, ReduceOpId, OutputIndex>; @@ -116,9 +169,10 @@ int main(int argc, char* argv[]) out_tensor_stride, out_tensor_stride, window_strides, + window_dilations, input_left_pads, input_right_pads, - {2, 3}); + pooling_dims); auto invoker_ptr = op_ptr->MakeInvokerPointer(); @@ -175,9 +229,10 @@ int main(int argc, char* argv[]) out_tensor_stride, out_tensor_stride, window_strides, + window_dilations, input_left_pads, input_right_pads, - {2, 3}); + pooling_dims); auto invoker_ptr = op_ptr->MakeInvokerPointer(); diff --git a/example/13_pool2d_fwd/pool2d_fwd_common.hpp b/example/13_pool2d_fwd/pool2d_fwd_common.hpp index 1157ccd387..3ce08fd2af 100644 --- a/example/13_pool2d_fwd/pool2d_fwd_common.hpp +++ b/example/13_pool2d_fwd/pool2d_fwd_common.hpp @@ -39,31 +39,35 @@ bool pool_test(bool do_verification, ck::index_t Wi, ck::index_t window_stride_h, ck::index_t window_stride_w, + ck::index_t window_dilation_h, + ck::index_t window_dilation_w, ck::index_t in_left_pad_h, ck::index_t in_left_pad_w, ck::index_t in_right_pad_h, ck::index_t in_right_pad_w) { using DevicePoolFwdInstance = - ck::tensor_operation::device::DevicePool2dFwd_Input_N_Hi_Wi_C_Output_N_Ho_Wo_C< - InDataType, // InDataType - OutDataType, // OutDataType - IndexDataType, // IndexDataType - ComputeDataType, // ComputeDataType - ReduceOpId, - OutputIndex, - 64, // BlockSize - 64, // ReduceMThreadClusterSize - 1, // ReduceKThreadClusterSize - 4, // ReduceMThreadSliceSize - 1, // ReduceKThreadSliceSize - 4>; // InSrcOutDstVectorSize + ck::tensor_operation::device::DevicePool2dFwd_NHWC_NHWC; // InSrcOutDstVectorSize - const ck::index_t Ho = (Hi + in_left_pad_h + in_right_pad_h - Y) / window_stride_h + 1; - const ck::index_t Wo = (Wi + in_left_pad_w + in_right_pad_w - X) / window_stride_w + 1; + const ck::index_t Ys = (Y - 1) * window_dilation_h + 1; + const ck::index_t Xs = (X - 1) * window_dilation_w + 1; + const ck::index_t Ho = (Hi + in_left_pad_h + in_right_pad_h - Ys) / window_stride_h + 1; + const ck::index_t Wo = (Wi + in_left_pad_w + in_right_pad_w - Xs) / window_stride_w + 1; const std::vector window_spatial_lengths{Y, X}; const std::vector window_strides{window_stride_h, window_stride_w}; + const std::vector window_dilations{window_dilation_h, window_dilation_w}; const std::vector input_left_pads{in_left_pad_h, in_left_pad_w}; const std::vector input_right_pads{in_right_pad_h, in_right_pad_w}; @@ -123,6 +127,7 @@ bool pool_test(bool do_verification, {C * Ho * Wo, 1, Wo * C, C}, {C * Ho * Wo, 1, Wo * C, C}, window_strides, + window_dilations, input_left_pads, input_right_pads, {2, 3}); @@ -144,8 +149,8 @@ bool pool_test(bool do_verification, float gb_per_sec = num_btype / 1.E6 / ave_time; - std::cout << "Perf: " << ave_time << " ms, " << tflops << " TFlops, " << gb_per_sec << " GB/s" - << std::endl; + std::cout << "Perf: " << ave_time << " ms, " << tflops << " TFlops, " << gb_per_sec + << " GB / s " << std::endl; bool pass = true; @@ -169,6 +174,7 @@ bool pool_test(bool do_verification, out_indices_n_c_ho_wo_host, window_spatial_lengths, window_strides, + window_dilations, input_left_pads, input_right_pads); diff --git a/example/13_pool2d_fwd/pool2d_fwd_fp16.cpp b/example/13_pool2d_fwd/pool2d_fwd_fp16.cpp index daf8540d49..d767e92248 100644 --- a/example/13_pool2d_fwd/pool2d_fwd_fp16.cpp +++ b/example/13_pool2d_fwd/pool2d_fwd_fp16.cpp @@ -34,18 +34,20 @@ int main(int argc, char* argv[]) bool time_kernel; // Pool shape - ck::index_t N = 128; - ck::index_t C = 192; - ck::index_t Y = 3; - ck::index_t X = 3; - ck::index_t Hi = 71; - ck::index_t Wi = 71; - ck::index_t window_stride_h = 2; - ck::index_t window_stride_w = 2; - ck::index_t in_left_pad_h = 1; - ck::index_t in_left_pad_w = 1; - ck::index_t in_right_pad_h = 1; - ck::index_t in_right_pad_w = 1; + ck::index_t N = 128; + ck::index_t C = 192; + ck::index_t Y = 3; + ck::index_t X = 3; + ck::index_t Hi = 71; + ck::index_t Wi = 71; + ck::index_t window_stride_h = 2; + ck::index_t window_stride_w = 2; + ck::index_t window_dilation_h = 1; + ck::index_t window_dilation_w = 1; + ck::index_t in_left_pad_h = 1; + ck::index_t in_left_pad_w = 1; + ck::index_t in_right_pad_h = 1; + ck::index_t in_right_pad_w = 1; if(argc == 1) { @@ -59,31 +61,33 @@ int main(int argc, char* argv[]) init_method = std::stoi(argv[2]); time_kernel = static_cast(std::stoi(argv[3])); } - else if(argc == 16) + else if(argc == 18) { do_verification = std::stoi(argv[1]); init_method = std::stoi(argv[2]); time_kernel = static_cast(std::stoi(argv[3])); - N = std::stoi(argv[4]); - C = std::stoi(argv[5]); - Y = std::stoi(argv[6]); - X = std::stoi(argv[7]); - Hi = std::stoi(argv[8]); - Wi = std::stoi(argv[9]); - window_stride_h = std::stoi(argv[10]); - window_stride_w = std::stoi(argv[11]); - in_left_pad_h = std::stoi(argv[12]); - in_left_pad_w = std::stoi(argv[13]); - in_right_pad_h = std::stoi(argv[14]); - in_right_pad_w = std::stoi(argv[15]); + N = std::stoi(argv[4]); + C = std::stoi(argv[5]); + Y = std::stoi(argv[6]); + X = std::stoi(argv[7]); + Hi = std::stoi(argv[8]); + Wi = std::stoi(argv[9]); + window_stride_h = std::stoi(argv[10]); + window_stride_w = std::stoi(argv[11]); + window_dilation_h = std::stoi(argv[12]); + window_dilation_w = std::stoi(argv[13]); + in_left_pad_h = std::stoi(argv[14]); + in_left_pad_w = std::stoi(argv[15]); + in_right_pad_h = std::stoi(argv[16]); + in_right_pad_w = std::stoi(argv[17]); } else { printf("arg1: verification (0=no, 1=yes)\n"); printf("arg2: initialization (0=no init, 1=integer value, 2=decimal value)\n"); printf("arg3: time kernel (0=no, 1=yes)\n"); - printf("arg4 to 15: N, C, Y, X, Hi, Wi, Sy, Sx, LeftPy, LeftPx, RightPy, " + printf("arg4 to 15: N, C, Y, X, Hi, Wi, Sy, Sx, Dy, Dx, LeftPy, LeftPx, RightPy, " "RightPx\n"); exit(0); } @@ -107,6 +111,8 @@ int main(int argc, char* argv[]) Wi, window_stride_h, window_stride_w, + window_dilation_h, + window_dilation_w, in_left_pad_h, in_left_pad_w, in_right_pad_h, diff --git a/example/13_pool2d_fwd/pool2d_fwd_fp32.cpp b/example/13_pool2d_fwd/pool2d_fwd_fp32.cpp index 323e3f61f8..2621500ef1 100644 --- a/example/13_pool2d_fwd/pool2d_fwd_fp32.cpp +++ b/example/13_pool2d_fwd/pool2d_fwd_fp32.cpp @@ -34,18 +34,20 @@ int main(int argc, char* argv[]) bool time_kernel; // Pool shape - ck::index_t N = 128; - ck::index_t C = 192; - ck::index_t Y = 3; - ck::index_t X = 3; - ck::index_t Hi = 71; - ck::index_t Wi = 71; - ck::index_t window_stride_h = 2; - ck::index_t window_stride_w = 2; - ck::index_t in_left_pad_h = 1; - ck::index_t in_left_pad_w = 1; - ck::index_t in_right_pad_h = 1; - ck::index_t in_right_pad_w = 1; + ck::index_t N = 128; + ck::index_t C = 192; + ck::index_t Y = 3; + ck::index_t X = 3; + ck::index_t Hi = 71; + ck::index_t Wi = 71; + ck::index_t window_stride_h = 2; + ck::index_t window_stride_w = 2; + ck::index_t window_dilation_h = 1; + ck::index_t window_dilation_w = 1; + ck::index_t in_left_pad_h = 1; + ck::index_t in_left_pad_w = 1; + ck::index_t in_right_pad_h = 1; + ck::index_t in_right_pad_w = 1; if(argc == 1) { @@ -59,31 +61,33 @@ int main(int argc, char* argv[]) init_method = std::stoi(argv[2]); time_kernel = static_cast(std::stoi(argv[3])); } - else if(argc == 16) + else if(argc == 18) { do_verification = std::stoi(argv[1]); init_method = std::stoi(argv[2]); time_kernel = static_cast(std::stoi(argv[3])); - N = std::stoi(argv[4]); - C = std::stoi(argv[5]); - Y = std::stoi(argv[6]); - X = std::stoi(argv[7]); - Hi = std::stoi(argv[8]); - Wi = std::stoi(argv[9]); - window_stride_h = std::stoi(argv[10]); - window_stride_w = std::stoi(argv[11]); - in_left_pad_h = std::stoi(argv[12]); - in_left_pad_w = std::stoi(argv[13]); - in_right_pad_h = std::stoi(argv[14]); - in_right_pad_w = std::stoi(argv[15]); + N = std::stoi(argv[4]); + C = std::stoi(argv[5]); + Y = std::stoi(argv[6]); + X = std::stoi(argv[7]); + Hi = std::stoi(argv[8]); + Wi = std::stoi(argv[9]); + window_stride_h = std::stoi(argv[10]); + window_stride_w = std::stoi(argv[11]); + window_dilation_h = std::stoi(argv[12]); + window_dilation_w = std::stoi(argv[13]); + in_left_pad_h = std::stoi(argv[14]); + in_left_pad_w = std::stoi(argv[15]); + in_right_pad_h = std::stoi(argv[16]); + in_right_pad_w = std::stoi(argv[17]); } else { printf("arg1: verification (0=no, 1=yes)\n"); printf("arg2: initialization (0=no init, 1=integer value, 2=decimal value)\n"); printf("arg3: time kernel (0=no, 1=yes)\n"); - printf("arg4 to 15: N, C, Y, X, Hi, Wi, Sy, Sx, LeftPy, LeftPx, RightPy, " + printf("arg4 to 15: N, C, Y, X, Hi, Wi, Sy, Sx, Dy, Dx, LeftPy, LeftPx, RightPy, " "RightPx\n"); exit(0); } @@ -107,6 +111,8 @@ int main(int argc, char* argv[]) Wi, window_stride_h, window_stride_w, + window_dilation_h, + window_dilation_w, in_left_pad_h, in_left_pad_w, in_right_pad_h, diff --git a/example/48_pool3d_fwd/pool3d_fwd_common.hpp b/example/48_pool3d_fwd/pool3d_fwd_common.hpp index 565bb94e47..39032fa123 100644 --- a/example/48_pool3d_fwd/pool3d_fwd_common.hpp +++ b/example/48_pool3d_fwd/pool3d_fwd_common.hpp @@ -18,7 +18,45 @@ #include "ck/library/utility/literals.hpp" #include "ck/library/reference_tensor_operation/cpu/reference_pool_fwd.hpp" -template +std::vector f_tensor_strides_ncdhw(ck::index_t N_, + ck::index_t C_, + ck::index_t D, + ck::index_t H, + ck::index_t W, + TensorLayout layout) +{ + using namespace ck::literals; + (void)N_; + if constexpr(ck::is_same::value) + return {C_ * D * H * W, D * H * W, H * W, W, 1_uz}; + else if constexpr(ck::is_same::value) + return {D * C_ * H * W, 1_uz, C_ * H * W, W * C_, C_}; +}; + +template +HostTensorDescriptor f_host_tensor_descriptor(std::size_t N_, + std::size_t C_, + std::size_t D, + std::size_t H, + std::size_t W, + TensorLayout layout) +{ + using namespace ck::literals; + + if constexpr(ck::is_same::value) + { + return HostTensorDescriptor({N_, C_, D, H, W}, {C_ * D * H * W, D * H * W, H * W, W, 1_uz}); + } + else if constexpr(ck::is_same::value) + { + return HostTensorDescriptor({N_, C_, D, H, W}, + {D * C_ * H * W, 1_uz, C_ * H * W, W * C_, C_}); + } +}; + +template ; // InSrcOutDstVectorSize - - const ck::index_t Do = (Di + in_left_pad_d + in_right_pad_d - Z) / window_stride_d + 1; - const ck::index_t Ho = (Hi + in_left_pad_h + in_right_pad_h - Y) / window_stride_h + 1; - const ck::index_t Wo = (Wi + in_left_pad_w + in_right_pad_w - X) / window_stride_w + 1; + const ck::index_t Zs = (Z - 1) * window_dilation_d + 1; + const ck::index_t Ys = (Y - 1) * window_dilation_h + 1; + const ck::index_t Xs = (X - 1) * window_dilation_w + 1; + const ck::index_t Do = (Di + in_left_pad_d + in_right_pad_d - Zs) / window_stride_d + 1; + const ck::index_t Ho = (Hi + in_left_pad_h + in_right_pad_h - Ys) / window_stride_h + 1; + const ck::index_t Wo = (Wi + in_left_pad_w + in_right_pad_w - Xs) / window_stride_w + 1; const std::vector window_spatial_lengths{Z, Y, X}; const std::vector window_strides{ window_stride_d, window_stride_h, window_stride_w}; + const std::vector window_dilations{ + window_dilation_d, window_dilation_h, window_dilation_w}; const std::vector input_left_pads{in_left_pad_d, in_left_pad_h, in_left_pad_w}; const std::vector input_right_pads{in_right_pad_d, in_right_pad_h, in_right_pad_w}; - // tensor layout - auto f_host_tensor_descriptor = [](std::size_t N_, - std::size_t C_, - std::size_t D, - std::size_t H, - std::size_t W, - auto layout) { - using namespace ck::literals; - - if constexpr(ck::is_same::value) - { - return HostTensorDescriptor({N_, C_, D, H, W}, - {C_ * D * H * W, D * H * W, H * W, W, 1_uz}); - } - else if constexpr(ck::is_same::value) - { - return HostTensorDescriptor({N_, C_, D, H, W}, - {D * C_ * H * W, 1_uz, C_ * H * W, W * C_, C_}); - } - }; - Tensor in_n_c_di_hi_wi(f_host_tensor_descriptor(N, C, Di, Hi, Wi, InLayout{})); Tensor out_n_c_do_ho_wo_host( f_host_tensor_descriptor(N, C, Do, Ho, Wo, OutLayout{})); @@ -126,10 +135,11 @@ bool pool3d_test(bool do_verification, {N, C, Di, Hi, Wi}, {Z, Y, X}, {N, C, Do, Ho, Wo}, - {Di * C * Hi * Wi, 1, C * Hi * Wi, Wi * C, C}, - {Do * C * Ho * Wo, 1, C * Ho * Wo, Wo * C, C}, - {Do * C * Ho * Wo, 1, C * Ho * Wo, Wo * C, C}, + f_tensor_strides_ncdhw(N, C, Di, Hi, Wi, InLayout{}), + f_tensor_strides_ncdhw(N, C, Do, Ho, Wo, OutLayout{}), + f_tensor_strides_ncdhw(N, C, Do, Ho, Wo, OutLayout{}), window_strides, + window_dilations, input_left_pads, input_right_pads, {2, 3, 4}); @@ -165,6 +175,7 @@ bool pool3d_test(bool do_verification, out_indices_n_c_do_ho_wo_host, window_spatial_lengths, window_strides, + window_dilations, input_left_pads, input_right_pads); diff --git a/example/48_pool3d_fwd/pool3d_fwd_fp16.cpp b/example/48_pool3d_fwd/pool3d_fwd_fp16.cpp index 9afb51201d..b9ac61033d 100644 --- a/example/48_pool3d_fwd/pool3d_fwd_fp16.cpp +++ b/example/48_pool3d_fwd/pool3d_fwd_fp16.cpp @@ -27,31 +27,49 @@ static constexpr auto ReduceOpId = ck::ReduceTensorOp::AVG; static constexpr bool OutputIndex = false; static constexpr bool PropagateNan = false; +using DevicePoolFwdInstance = + ck::tensor_operation::device::DevicePool3dFwd_NDHWC_NDHWC; // InSrcOutDstVectorSize + int main() { bool do_verification = true; bool time_kernel = false; // Pool shape - ck::index_t N = 2; - ck::index_t C = 32; - ck::index_t Z = 2; - ck::index_t Y = 2; - ck::index_t X = 2; - ck::index_t Di = 30; - ck::index_t Hi = 30; - ck::index_t Wi = 30; - ck::index_t window_stride_d = 2; - ck::index_t window_stride_h = 2; - ck::index_t window_stride_w = 2; - ck::index_t in_left_pad_d = 1; - ck::index_t in_left_pad_h = 1; - ck::index_t in_left_pad_w = 1; - ck::index_t in_right_pad_d = 1; - ck::index_t in_right_pad_h = 1; - ck::index_t in_right_pad_w = 1; + ck::index_t N = 2; + ck::index_t C = 32; + ck::index_t Z = 2; + ck::index_t Y = 2; + ck::index_t X = 2; + ck::index_t Di = 30; + ck::index_t Hi = 30; + ck::index_t Wi = 30; + ck::index_t window_stride_d = 2; + ck::index_t window_stride_h = 2; + ck::index_t window_stride_w = 2; + ck::index_t window_dilation_d = 1; + ck::index_t window_dilation_h = 1; + ck::index_t window_dilation_w = 1; + ck::index_t in_left_pad_d = 1; + ck::index_t in_left_pad_h = 1; + ck::index_t in_left_pad_w = 1; + ck::index_t in_right_pad_d = 1; + ck::index_t in_right_pad_h = 1; + ck::index_t in_right_pad_w = 1; - bool pass = pool3d_test; // InSrcOutDstVectorSize + ck::tensor_operation::device::DevicePool2dFwd_NHWC_NHWC; // InSrcOutDstVectorSize using DeviceMaxPoolBwdInstance = ck::tensor_operation::device:: DeviceIndexPoolBwdImpl; - const ck::index_t Ho = (Hi + in_left_pad_h + in_right_pad_h - Y) / window_stride_h + 1; - const ck::index_t Wo = (Wi + in_left_pad_w + in_right_pad_w - X) / window_stride_w + 1; + const ck::index_t Ys = (Y - 1) * window_dilation_h + 1; + const ck::index_t Xs = (X - 1) * window_dilation_w + 1; + const ck::index_t Ho = (Hi + in_left_pad_h + in_right_pad_h - Ys) / window_stride_h + 1; + const ck::index_t Wo = (Wi + in_left_pad_w + in_right_pad_w - Xs) / window_stride_w + 1; const std::vector window_spatial_lengths{Y, X}; const std::vector window_strides{window_stride_h, window_stride_w}; + const std::vector window_dilations{window_dilation_h, window_dilation_w}; const std::vector input_left_pads{in_left_pad_h, in_left_pad_w}; const std::vector input_right_pads{in_right_pad_h, in_right_pad_w}; @@ -128,6 +132,7 @@ bool maxpool_bwd_test(bool do_verification, {C * Ho * Wo, 1, Wo * C, C}, {C * Ho * Wo, 1, Wo * C, C}, window_strides, + window_dilations, input_left_pads, input_right_pads, {2, 3}); @@ -191,6 +196,7 @@ bool maxpool_bwd_test(bool do_verification, indices_n_c_ho_wo_host, window_spatial_lengths, window_strides, + window_dilations, input_left_pads, input_right_pads); ref_pooling_fwd_invoker.Run(ref_pooling_fwd_argument); diff --git a/example/49_maxpool2d_bwd/maxpool2d_bwd_fp16.cpp b/example/49_maxpool2d_bwd/maxpool2d_bwd_fp16.cpp index c6b0b2c860..a4f982d855 100644 --- a/example/49_maxpool2d_bwd/maxpool2d_bwd_fp16.cpp +++ b/example/49_maxpool2d_bwd/maxpool2d_bwd_fp16.cpp @@ -24,18 +24,20 @@ int main() bool time_kernel = false; // Pool shape - ck::index_t N = 1; - ck::index_t C = 1; - ck::index_t Y = 3; - ck::index_t X = 3; - ck::index_t Hi = 32; - ck::index_t Wi = 32; - ck::index_t window_stride_h = 1; - ck::index_t window_stride_w = 1; - ck::index_t in_left_pad_h = 0; - ck::index_t in_left_pad_w = 0; - ck::index_t in_right_pad_h = 0; - ck::index_t in_right_pad_w = 0; + ck::index_t N = 1; + ck::index_t C = 1; + ck::index_t Y = 3; + ck::index_t X = 3; + ck::index_t Hi = 32; + ck::index_t Wi = 32; + ck::index_t window_stride_h = 1; + ck::index_t window_stride_w = 1; + ck::index_t window_dilation_h = 1; + ck::index_t window_dilation_w = 1; + ck::index_t in_left_pad_h = 0; + ck::index_t in_left_pad_w = 0; + ck::index_t in_right_pad_h = 0; + ck::index_t in_right_pad_w = 0; bool pass = maxpool_bwd_test struct DevicePoolFwd : public BaseOperator @@ -25,13 +27,14 @@ struct DevicePoolFwd : public BaseOperator MakeArgumentPointer(const void* p_in_dev, void* p_out_dev, void* p_out_indices_dev, - std::vector input_lengths, - std::vector window_lengths, - std::vector output_lengths, - std::vector input_stride, - std::vector output_stride, - std::vector indices_stride, - std::vector window_strides, + std::vector input_n_c_wis_lengths, + std::vector window_xs_lengths, + std::vector output_n_c_wos_lengths, + std::vector input_n_c_wis_stride, + std::vector output_n_c_wis_stride, + std::vector indices_n_c_wis_stride, + std::vector window_xs_strides, + std::vector window_xs_dilations, std::vector input_left_pads, std::vector input_right_pads, std::vector pooling_dims) = 0; diff --git a/include/ck/tensor_operation/gpu/device/impl/device_pool2d_fwd_nhwc_nhwc.hpp b/include/ck/tensor_operation/gpu/device/impl/device_pool2d_fwd_nhwc_nhwc.hpp index 3f27c629dd..c94c568c49 100644 --- a/include/ck/tensor_operation/gpu/device/impl/device_pool2d_fwd_nhwc_nhwc.hpp +++ b/include/ck/tensor_operation/gpu/device/impl/device_pool2d_fwd_nhwc_nhwc.hpp @@ -3,16 +3,7 @@ #pragma once -#include -#include - -#include "ck/tensor_description/tensor_descriptor.hpp" -#include "ck/tensor_description/tensor_descriptor_helper.hpp" -#include "ck/tensor_operation/gpu/device/reduction_operator_mapping.hpp" -#include "ck/tensor_operation/gpu/device/device_pool_fwd.hpp" -#include "ck/tensor_operation/gpu/grid/gridwise_2d_reduction_threadwise.hpp" -#include "ck/host_utility/device_prop.hpp" -#include "ck/host_utility/kernel_launch.hpp" +#include "ck/tensor_operation/gpu/device/impl/device_pool3d_fwd_ndhwc_ndhwc.hpp" namespace ck { namespace tensor_operation { @@ -30,255 +21,32 @@ template -struct DevicePool2dFwd_Input_N_Hi_Wi_C_Output_N_Ho_Wo_C - : public DevicePoolFwd<4, 2, InDataType, OutDataType, IndexDataType, ReduceOpId, OutputIndex> +struct DevicePool2dFwd_NHWC_NHWC : public DevicePool3dFwd_NDHWC_NDHWC { - static constexpr auto I0 = Number<0>{}; - static constexpr auto I1 = Number<1>{}; - static constexpr auto I2 = Number<2>{}; - static constexpr auto I3 = Number<3>{}; - static constexpr auto I4 = Number<4>{}; - static constexpr auto I5 = Number<5>{}; - - static constexpr index_t InOutRank = 4; - static constexpr index_t WindowRank = 2; - - using ReduceOperation = typename reduce_binary_operator::opType; - - using InElementwiseOperation = - typename reduce_unary_operator::InElementwiseOperation; - - using AccElementwiseOperation = - typename reduce_unary_operator::AccElementwiseOperation; - - static constexpr index_t InSrcOutDstVectorDim = - 0; // for NHWC, the dim C is the vector Dim for both input and output in memory, which is - // not reduced. - - static constexpr ck::index_t ReduceM_BlockTileSize = - ReduceMThreadClusterSize * ReduceMThreadSliceSize; - static constexpr ck::index_t ReduceK_BlockTileSize = - ReduceKThreadClusterSize * ReduceKThreadSliceSize; - - static auto MakeABGridDescriptor_A_M_K_B_M(ck::index_t N, - ck::index_t C, - std::vector input_spatial_lengths, - std::vector window_spatial_lengths, - std::vector output_spatial_lengths, - std::vector window_strides, - std::vector input_left_pads, - std::vector input_right_pads) - { - const index_t Hi = input_spatial_lengths[0]; - const index_t Wi = input_spatial_lengths[1]; - - const index_t Ho = output_spatial_lengths[0]; - const index_t Wo = output_spatial_lengths[1]; - - const index_t Y = window_spatial_lengths[0]; - const index_t X = window_spatial_lengths[1]; - - const index_t ConvStrideH = window_strides[0]; - const index_t ConvStrideW = window_strides[1]; - - const index_t InLeftPadH = input_left_pads[0]; - const index_t InLeftPadW = input_left_pads[1]; - - const index_t InRightPadH = input_right_pads[0]; - const index_t InRightPadW = input_right_pads[1]; - - const index_t ReduceMRaw = N * Ho * Wo * C; - const index_t ReduceMPad = - math::integer_least_multiple(ReduceMRaw, ReduceM_BlockTileSize) - ReduceMRaw; - - const index_t ReduceKRaw = Y * X; - const index_t ReduceKPad = - math::integer_least_multiple(ReduceKRaw, ReduceK_BlockTileSize) - ReduceKRaw; - - // A[ReduceM, ReduceK] - const auto in_grid_desc_n_hi_wi_c = - make_naive_tensor_descriptor_packed(make_tuple(N, Hi, Wi, C)); - - const auto in_grid_desc_n_hip_wip_c = transform_tensor_descriptor( - in_grid_desc_n_hi_wi_c, - make_tuple(make_pass_through_transform(N), - make_pad_transform(Hi, InLeftPadH, InRightPadH), - make_pad_transform(Wi, InLeftPadW, InRightPadW), - make_pass_through_transform(C)), - make_tuple(Sequence<0>{}, Sequence<1>{}, Sequence<2>{}, Sequence<3>{}), - make_tuple(Sequence<0>{}, Sequence<1>{}, Sequence<2>{}, Sequence<3>{})); - - const auto in_grid_desc_n_y_ho_x_wo_c = transform_tensor_descriptor( - in_grid_desc_n_hip_wip_c, - make_tuple(make_pass_through_transform(N), - make_embed_transform(make_tuple(Y, Ho), make_tuple(I1, ConvStrideH)), - make_embed_transform(make_tuple(X, Wo), make_tuple(I1, ConvStrideW)), - make_pass_through_transform(C)), - make_tuple(Sequence<0>{}, Sequence<1>{}, Sequence<2>{}, Sequence<3>{}), - make_tuple(Sequence<0>{}, Sequence<1, 2>{}, Sequence<3, 4>{}, Sequence<5>{})); - - const auto in_grid_desc_reducemraw_reducekraw = - transform_tensor_descriptor(in_grid_desc_n_y_ho_x_wo_c, - make_tuple(make_merge_transform(make_tuple(N, Ho, Wo, C)), - make_merge_transform(make_tuple(Y, X))), - make_tuple(Sequence<0, 2, 4, 5>{}, Sequence<1, 3>{}), - make_tuple(Sequence<0>{}, Sequence<1>{})); - - const auto in_grid_desc_reducem_reducek = transform_tensor_descriptor( - in_grid_desc_reducemraw_reducekraw, - make_tuple(make_right_pad_transform(ReduceMRaw, ReduceMPad), - make_right_pad_transform(ReduceKRaw, ReduceKPad)), - make_tuple(Sequence<0>{}, Sequence<1>{}), - make_tuple(Sequence<0>{}, Sequence<1>{})); - - // B[ReduceM] - const auto out_grid_desc_reducemraw = - make_naive_tensor_descriptor_packed(make_tuple(N * Ho * Wo * C)); - - const auto out_grid_desc_reducem = transform_tensor_descriptor( - out_grid_desc_reducemraw, - make_tuple(make_right_pad_transform(ReduceMRaw, ReduceMPad)), - make_tuple(Sequence<0>{}), - make_tuple(Sequence<0>{})); - - return make_tuple(in_grid_desc_reducem_reducek, out_grid_desc_reducem); - } - - using ABGridDescs = decltype(MakeABGridDescriptor_A_M_K_B_M(1, 1, {}, {}, {}, {}, {}, {})); - using AGridDesc_M_K = remove_cvref_t; - using BGridDesc_M = remove_cvref_t; - - // TODO - struct Argument : public BaseArgument - { - Argument(const InDataType* p_in_dev, - OutDataType* p_out_dev, - IndexDataType* p_out_indices_dev, - ck::index_t N, - ck::index_t C, - std::vector& input_spatial_lengths, - std::vector& window_spatial_lengths, - std::vector& output_spatial_lengths, - std::vector& window_strides, - std::vector& input_left_pads, - std::vector& input_right_pads) - : p_in_dev_{p_in_dev}, - p_out_dev_{p_out_dev}, - p_out_indices_dev_{p_out_indices_dev}, - a_grid_desc_m_k_{}, - b_grid_desc_m_{} - { - const auto descs = MakeABGridDescriptor_A_M_K_B_M(N, - C, - input_spatial_lengths, - window_spatial_lengths, - output_spatial_lengths, - window_strides, - input_left_pads, - input_right_pads); - - a_grid_desc_m_k_ = descs[I0]; - b_grid_desc_m_ = descs[I1]; - - invariant_lowest_length_ = C; - reduce_lowest_length_ = window_spatial_lengths[1]; - - int32_t reduceLength = window_spatial_lengths[0] * window_spatial_lengths[1]; - - std::tie(in_element_op_, acc_element_op_) = - reduce_unary_operator::GetElementwiseOperator(reduceLength); - } - - const InDataType* p_in_dev_; - OutDataType* p_out_dev_; - IndexDataType* p_out_indices_dev_; - AGridDesc_M_K a_grid_desc_m_k_; - BGridDesc_M b_grid_desc_m_; - InElementwiseOperation in_element_op_; - AccElementwiseOperation acc_element_op_; - - // for checking vector load/store - ck::index_t invariant_lowest_length_; - ck::index_t reduce_lowest_length_; - }; - - struct Invoker : public BaseInvoker - { - float Run(const Argument& arg, const StreamConfig& stream_config = StreamConfig{}) - { - using gridwise_reduce = - GridwiseReduction_mk_to_m_threadwise; - const auto kernel = - kernel_reduce_threadwise; - - ck::index_t ReduceM = arg.a_grid_desc_m_k_.GetLength(I0); - - const index_t grid_size = (ReduceM / ReduceM_BlockTileSize); - - return launch_and_time_kernel(stream_config, - kernel, - dim3(grid_size), - dim3(BlockSize), - 0, - arg.a_grid_desc_m_k_, - arg.b_grid_desc_m_, - arg.in_element_op_, - arg.acc_element_op_, - float(1), - arg.p_in_dev_, - nullptr, - float(0), - arg.p_out_dev_, - arg.p_out_indices_dev_); - } - - float Run(const BaseArgument* p_arg, - const StreamConfig& stream_config = StreamConfig{}) override - { - return Run(*dynamic_cast(p_arg), stream_config); - } - }; - - bool IsSupportedArgument(const BaseArgument* p_arg) override - { - const Argument* pArg = dynamic_cast(p_arg); - - if(pArg->invariant_lowest_length_ % InSrcOutDstVectorSize != 0) - { - return (false); - } - - return (true); - } - std::unique_ptr MakeArgumentPointer(const void* p_in_dev, void* p_out_dev, @@ -286,62 +54,57 @@ struct DevicePool2dFwd_Input_N_Hi_Wi_C_Output_N_Ho_Wo_C std::vector input_lengths, std::vector window_lengths, std::vector output_lengths, - std::vector, // Suppose tensor layout = NHWC - std::vector, // Suppose tensor layout = NHWC - std::vector, // Suppose tensor layout = NHWC + std::vector input_stride, + std::vector output_stride, + std::vector indices_stride, std::vector window_strides, + std::vector window_dilations, std::vector input_left_pads, std::vector input_right_pads, std::vector pooling_dims) override { + static constexpr index_t InOutRank = 4; + static constexpr index_t WindowRank = 2; + if(input_lengths.size() != InOutRank || window_lengths.size() != WindowRank || input_lengths.size() != InOutRank || window_strides.size() != WindowRank || - input_left_pads.size() != WindowRank || input_right_pads.size() != WindowRank) + window_dilations.size() != WindowRank || input_left_pads.size() != WindowRank || + input_right_pads.size() != WindowRank) throw std::runtime_error("dimension is incorrect"); if(pooling_dims != std::vector{2, 3}) throw std::runtime_error("pooling_dims only support {2, 3} in pool2d so far"); - index_t N = input_lengths[0]; - index_t C = input_lengths[1]; - index_t Hi = input_lengths[2]; - index_t Wi = input_lengths[3]; - index_t Ho = output_lengths[2]; - index_t Wo = output_lengths[3]; + // NCHW to NCDHW + input_lengths.insert(input_lengths.begin() + 2, 1); + output_lengths.insert(output_lengths.begin() + 2, 1); + input_stride.insert(input_stride.begin() + 2, 0); + output_stride.insert(output_stride.begin() + 2, 0); + indices_stride.insert(indices_stride.begin() + 2, 0); - std::vector input_spatial_lengths = {Hi, Wi}; - std::vector output_spatial_lengths = {Ho, Wo}; + // YX to ZYX + window_lengths.insert(window_lengths.begin(), 1); + window_strides.insert(window_strides.begin(), 0); + window_dilations.insert(window_dilations.begin(), 0); + input_left_pads.insert(input_left_pads.begin(), 0); + input_right_pads.insert(input_right_pads.begin(), 0); - return std::make_unique(static_cast(p_in_dev), - static_cast(p_out_dev), - static_cast(p_out_indices_dev), - N, - C, - input_spatial_lengths, - window_lengths, - output_spatial_lengths, - window_strides, - input_left_pads, - input_right_pads); - } + pooling_dims = {2, 3, 4}; - std::unique_ptr MakeInvokerPointer() override - { - return std::make_unique(Invoker{}); - } - - std::string GetTypeString() const override - { - auto str = std::stringstream(); - - // clang-format off - str << "DevicePool2dFwd_Input_N_Hi_Wi_C_Output_N_Ho_Wo_C<" << BlockSize << ","; - str << "M_C" << ReduceMThreadClusterSize << "_S" << ReduceMThreadSliceSize << ","; - str << "K_C" << ReduceKThreadClusterSize << "_S" << ReduceKThreadSliceSize << ","; - str <<"InSrcOutDstVectorSize_" << InSrcOutDstVectorSize << ">"; - // clang-format on - - return str.str(); + return DevicePool3D::MakeArgumentPointer(p_in_dev, + p_out_dev, + p_out_indices_dev, + input_lengths, + window_lengths, + output_lengths, + input_stride, + output_stride, + indices_stride, + window_strides, + window_dilations, + input_left_pads, + input_right_pads, + pooling_dims); } }; diff --git a/include/ck/tensor_operation/gpu/device/impl/device_pool3d_fwd_ndhwc_ndhwc.hpp b/include/ck/tensor_operation/gpu/device/impl/device_pool3d_fwd_ndhwc_ndhwc.hpp index 0ab6c24758..384805a0a3 100644 --- a/include/ck/tensor_operation/gpu/device/impl/device_pool3d_fwd_ndhwc_ndhwc.hpp +++ b/include/ck/tensor_operation/gpu/device/impl/device_pool3d_fwd_ndhwc_ndhwc.hpp @@ -8,8 +8,10 @@ #include "ck/tensor_description/tensor_descriptor.hpp" #include "ck/tensor_description/tensor_descriptor_helper.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" #include "ck/tensor_operation/gpu/device/reduction_operator_mapping.hpp" #include "ck/tensor_operation/gpu/device/device_pool_fwd.hpp" +#include "ck/tensor_operation/gpu/device/impl/device_reduce_common.hpp" #include "ck/tensor_operation/gpu/grid/gridwise_2d_reduction_threadwise.hpp" #include "ck/host_utility/device_prop.hpp" #include "ck/host_utility/kernel_launch.hpp" @@ -30,8 +32,15 @@ template -struct DevicePool3dFwd_Input_N_Di_Hi_Wi_C_Output_N_Do_Ho_Wo_C - : public DevicePoolFwd<5, 3, InDataType, OutDataType, IndexDataType, ReduceOpId, OutputIndex> +struct DevicePool3dFwd_NDHWC_NDHWC : public DevicePoolFwd<5, + 3, + InDataType, + OutDataType, + IndexDataType, + tensor_layout::convolution::NDHWC, + tensor_layout::convolution::NDHWC, + ReduceOpId, + OutputIndex> { static constexpr auto I0 = Number<0>{}; static constexpr auto I1 = Number<1>{}; @@ -51,45 +60,48 @@ struct DevicePool3dFwd_Input_N_Di_Hi_Wi_C_Output_N_Do_Ho_Wo_C using AccElementwiseOperation = typename reduce_unary_operator::AccElementwiseOperation; - // for NDHWC, the dim C is the vector Dim for both input and output in memory, which is not - // reduced. - static constexpr index_t InSrcOutDstVectorDim = 0; - static constexpr ck::index_t M_BlockTileSize = MThreadClusterSize * MThreadSliceSize; static constexpr ck::index_t K_BlockTileSize = KThreadClusterSize * KThreadSliceSize; - static auto MakeABGridDescriptor_A_M_K_B_M(ck::index_t N, - ck::index_t C, - std::vector input_spatial_lengths, - std::vector window_spatial_lengths, - std::vector output_spatial_lengths, - std::vector window_strides, - std::vector input_left_pads, - std::vector input_right_pads) + static auto MakeABGridDescriptor_A_M_K_B_M(std::vector input_ncdhw_lengths, + std::vector output_ncdhw_lengths, + std::vector input_ncdhw_stride, + std::vector output_ncdhw_stride, + std::vector window_spatial_zyx_lengths, + std::vector window_zyx_strides, + std::vector window_zyx_dilations, + std::vector input_left_dhw_pads, + std::vector input_right_dhw_pads) { - const index_t Di = input_spatial_lengths[0]; - const index_t Hi = input_spatial_lengths[1]; - const index_t Wi = input_spatial_lengths[2]; + const index_t N = input_ncdhw_lengths[0]; + const index_t C = input_ncdhw_lengths[1]; + const index_t Di = input_ncdhw_lengths[2]; + const index_t Hi = input_ncdhw_lengths[3]; + const index_t Wi = input_ncdhw_lengths[4]; - const index_t Do = output_spatial_lengths[0]; - const index_t Ho = output_spatial_lengths[1]; - const index_t Wo = output_spatial_lengths[2]; + const index_t Do = output_ncdhw_lengths[2]; + const index_t Ho = output_ncdhw_lengths[3]; + const index_t Wo = output_ncdhw_lengths[4]; - const index_t Z = window_spatial_lengths[0]; - const index_t Y = window_spatial_lengths[1]; - const index_t X = window_spatial_lengths[2]; + const index_t Z = window_spatial_zyx_lengths[0]; + const index_t Y = window_spatial_zyx_lengths[1]; + const index_t X = window_spatial_zyx_lengths[2]; - const index_t ConvStrideD = window_strides[0]; - const index_t ConvStrideH = window_strides[1]; - const index_t ConvStrideW = window_strides[2]; + const index_t WindowStrideD = window_zyx_strides[0]; + const index_t WindowStrideH = window_zyx_strides[1]; + const index_t WindowStrideW = window_zyx_strides[2]; - const index_t InLeftPadD = input_left_pads[0]; - const index_t InLeftPadH = input_left_pads[1]; - const index_t InLeftPadW = input_left_pads[2]; + const index_t WindowDilationD = window_zyx_dilations[0]; + const index_t WindowDilationH = window_zyx_dilations[1]; + const index_t WindowDilationW = window_zyx_dilations[2]; - const index_t InRightPadD = input_right_pads[0]; - const index_t InRightPadH = input_right_pads[1]; - const index_t InRightPadW = input_right_pads[2]; + const index_t InLeftPadD = input_left_dhw_pads[0]; + const index_t InLeftPadH = input_left_dhw_pads[1]; + const index_t InLeftPadW = input_left_dhw_pads[2]; + + const index_t InRightPadD = input_right_dhw_pads[0]; + const index_t InRightPadH = input_right_dhw_pads[1]; + const index_t InRightPadW = input_right_dhw_pads[2]; const index_t MRaw = N * Do * Ho * Wo * C; const index_t MPad = math::integer_least_multiple(MRaw, M_BlockTileSize) - MRaw; @@ -98,8 +110,15 @@ struct DevicePool3dFwd_Input_N_Di_Hi_Wi_C_Output_N_Do_Ho_Wo_C const index_t KPad = math::integer_least_multiple(KRaw, K_BlockTileSize) - KRaw; // A[ReduceM, ReduceK] - const auto in_grid_desc_n_di_hi_wi_c = - make_naive_tensor_descriptor_packed(make_tuple(N, Di, Hi, Wi, C)); + const index_t Ni_stride = input_ncdhw_stride[0]; + const index_t Ci_stride = input_ncdhw_stride[1]; + const index_t Di_stride = input_ncdhw_stride[2]; + const index_t Hi_stride = input_ncdhw_stride[3]; + const index_t Wi_stride = input_ncdhw_stride[4]; + + const auto in_grid_desc_n_di_hi_wi_c = make_naive_tensor_descriptor( + make_tuple(N, Di, Hi, Wi, C), + make_tuple(Ni_stride, Di_stride, Hi_stride, Wi_stride, Ci_stride)); const auto in_grid_desc_n_dip_hip_wip_c = transform_tensor_descriptor( in_grid_desc_n_di_hi_wi_c, @@ -113,11 +132,12 @@ struct DevicePool3dFwd_Input_N_Di_Hi_Wi_C_Output_N_Do_Ho_Wo_C const auto in_grid_desc_n_z_do_y_ho_x_wo_c = transform_tensor_descriptor( in_grid_desc_n_dip_hip_wip_c, - make_tuple(make_pass_through_transform(N), - make_embed_transform(make_tuple(Z, Do), make_tuple(I1, ConvStrideD)), - make_embed_transform(make_tuple(Y, Ho), make_tuple(I1, ConvStrideH)), - make_embed_transform(make_tuple(X, Wo), make_tuple(I1, ConvStrideW)), - make_pass_through_transform(C)), + make_tuple( + make_pass_through_transform(N), + make_embed_transform(make_tuple(Z, Do), make_tuple(WindowDilationD, WindowStrideD)), + make_embed_transform(make_tuple(Y, Ho), make_tuple(WindowDilationH, WindowStrideH)), + make_embed_transform(make_tuple(X, Wo), make_tuple(WindowDilationW, WindowStrideW)), + make_pass_through_transform(C)), make_tuple(Sequence<0>{}, Sequence<1>{}, Sequence<2>{}, Sequence<3>{}, Sequence<4>{}), make_tuple(Sequence<0>{}, Sequence<1, 2>{}, @@ -139,8 +159,21 @@ struct DevicePool3dFwd_Input_N_Di_Hi_Wi_C_Output_N_Do_Ho_Wo_C make_tuple(Sequence<0>{}, Sequence<1>{})); // B[ReduceM] - const auto out_grid_desc_reducemraw = - make_naive_tensor_descriptor_packed(make_tuple(N * Do * Ho * Wo * C)); + const index_t No_stride = output_ncdhw_stride[0]; + const index_t Co_stride = output_ncdhw_stride[1]; + const index_t Do_stride = output_ncdhw_stride[2]; + const index_t Ho_stride = output_ncdhw_stride[3]; + const index_t Wo_stride = output_ncdhw_stride[4]; + + const auto out_grid_desc_n_do_ho_wo_c = make_naive_tensor_descriptor( + make_tuple(N, Di, Hi, Wi, C), + make_tuple(No_stride, Do_stride, Ho_stride, Wo_stride, Co_stride)); + + const auto out_grid_desc_reducemraw = transform_tensor_descriptor( + out_grid_desc_n_do_ho_wo_c, + make_tuple(make_merge_transform(make_tuple(N, Do, Ho, Wo, C))), + make_tuple(Sequence<0, 1, 2, 3, 4>{}), + make_tuple(Sequence<0>{})); const auto out_grid_desc_reducem = transform_tensor_descriptor(out_grid_desc_reducemraw, @@ -151,7 +184,9 @@ struct DevicePool3dFwd_Input_N_Di_Hi_Wi_C_Output_N_Do_Ho_Wo_C return make_tuple(in_grid_desc_reducem_reducek, out_grid_desc_reducem); } - using ABGridDescs = decltype(MakeABGridDescriptor_A_M_K_B_M(1, 1, {}, {}, {}, {}, {}, {})); + using ABGridDescs = + decltype(MakeABGridDescriptor_A_M_K_B_M({}, {}, {}, {}, {}, {}, {}, {}, {})); + using AGridDesc_M_K = remove_cvref_t; using BGridDesc_M = remove_cvref_t; @@ -160,36 +195,41 @@ struct DevicePool3dFwd_Input_N_Di_Hi_Wi_C_Output_N_Do_Ho_Wo_C Argument(const InDataType* p_in_dev, OutDataType* p_out_dev, IndexDataType* p_out_indices_dev, - ck::index_t N, - ck::index_t C, - std::vector& input_spatial_lengths, - std::vector& window_spatial_lengths, - std::vector& output_spatial_lengths, - std::vector& window_strides, - std::vector& input_left_pads, - std::vector& input_right_pads) + std::vector& input_ncdhw_lengths, + std::vector& output_ncdhw_lengths, + std::vector& input_ncdhw_stride, + std::vector& output_ncdhw_stride, + std::vector&, // indices_ncdhw_stride + std::vector& window_spatial_zyx_lengths, + std::vector& window_zyx_strides, + std::vector& window_zyx_dilations, + std::vector& input_left_dhw_pads, + std::vector& input_right_dhw_pads) : p_in_dev_{p_in_dev}, p_out_dev_{p_out_dev}, p_out_indices_dev_{p_out_indices_dev}, a_grid_desc_m_k_{}, - b_grid_desc_m_{} + b_grid_desc_m_{}, + input_ncdhw_lengths_{input_ncdhw_lengths}, + output_ncdhw_lengths_{output_ncdhw_lengths}, + input_ncdhw_stride_{input_ncdhw_stride}, + output_ncdhw_stride_{output_ncdhw_stride} { - const auto descs = MakeABGridDescriptor_A_M_K_B_M(N, - C, - input_spatial_lengths, - window_spatial_lengths, - output_spatial_lengths, - window_strides, - input_left_pads, - input_right_pads); + const auto descs = MakeABGridDescriptor_A_M_K_B_M(input_ncdhw_lengths, + output_ncdhw_lengths, + input_ncdhw_stride, + output_ncdhw_stride, + window_spatial_zyx_lengths, + window_zyx_strides, + window_zyx_dilations, + input_left_dhw_pads, + input_right_dhw_pads); a_grid_desc_m_k_ = descs[I0]; b_grid_desc_m_ = descs[I1]; - invariant_lowest_length_ = C; - - int32_t reduceLength = - window_spatial_lengths[0] * window_spatial_lengths[1] * window_spatial_lengths[2]; + int32_t reduceLength = window_spatial_zyx_lengths[0] * window_spatial_zyx_lengths[1] * + window_spatial_zyx_lengths[2]; std::tie(in_element_op_, acc_element_op_) = reduce_unary_operator::GetElementwiseOperator(reduceLength); @@ -200,17 +240,25 @@ struct DevicePool3dFwd_Input_N_Di_Hi_Wi_C_Output_N_Do_Ho_Wo_C IndexDataType* p_out_indices_dev_; AGridDesc_M_K a_grid_desc_m_k_; BGridDesc_M b_grid_desc_m_; + InElementwiseOperation in_element_op_; AccElementwiseOperation acc_element_op_; // for checking vector load/store - ck::index_t invariant_lowest_length_; + std::vector input_ncdhw_lengths_; + std::vector output_ncdhw_lengths_; + std::vector input_ncdhw_stride_; + std::vector output_ncdhw_stride_; }; struct Invoker : public BaseInvoker { float Run(const Argument& arg, const StreamConfig& stream_config = StreamConfig{}) { + // for NDHWC, the dim C is the fastest dimension, and is not reduced. + // Hence, it is in M dimension for reduction kernel. + static constexpr index_t InSrcOutDstVectorDim = 0; // 0: M, 1: K + using gridwise_reduce = GridwiseReduction_mk_to_m_threadwise(p_arg); - if(pArg->invariant_lowest_length_ % InSrcOutDstVectorSize != 0) - { + // C should be fastest dimension + if(pArg->input_ncdhw_stride_[1] != 1) return false; + + for(int i = 0; i < InOutRank; ++i) + { + if(pArg->input_ncdhw_stride_[i] == 1 && + pArg->input_ncdhw_lengths_[i] % InSrcOutDstVectorSize != 0) + return false; + + if(pArg->output_ncdhw_stride_[i] == 1 && + pArg->output_ncdhw_lengths_[i] % InSrcOutDstVectorSize != 0) + return false; } return true; } - std::unique_ptr + virtual std::unique_ptr MakeArgumentPointer(const void* p_in_dev, void* p_out_dev, void* p_out_indices_dev, - std::vector input_lengths, - std::vector window_lengths, - std::vector output_lengths, - std::vector, // Suppose tensor layout = NDHWC - std::vector, // Suppose tensor layout = NDHWC - std::vector, // Suppose tensor layout = NDHWC - std::vector window_strides, - std::vector input_left_pads, - std::vector input_right_pads, + std::vector input_ncdhw_lengths, + std::vector window_zyx_lengths, + std::vector output_ncdhw_lengths, + std::vector input_ncdhw_stride, + std::vector output_ncdhw_stride, + std::vector indices_ncdhw_stride, + std::vector window_zyx_strides, + std::vector window_zyx_dilations, + std::vector input_left_dhw_pads, + std::vector input_right_dhw_pads, std::vector pooling_dims) override { - if(input_lengths.size() != InOutRank || window_lengths.size() != WindowRank || - input_lengths.size() != InOutRank || window_strides.size() != WindowRank || - input_left_pads.size() != WindowRank || input_right_pads.size() != WindowRank) + if(input_ncdhw_lengths.size() != InOutRank || window_zyx_lengths.size() != WindowRank || + input_ncdhw_lengths.size() != InOutRank || window_zyx_strides.size() != WindowRank || + window_zyx_dilations.size() != WindowRank || input_left_dhw_pads.size() != WindowRank || + input_right_dhw_pads.size() != WindowRank) throw std::runtime_error("dimension is incorrect"); if(pooling_dims != std::vector{2, 3, 4}) throw std::runtime_error("pooling_dims only support {2, 3, 4} in pool3d so far"); - index_t N = input_lengths[0]; - index_t C = input_lengths[1]; - index_t Di = input_lengths[2]; - index_t Hi = input_lengths[3]; - index_t Wi = input_lengths[4]; - index_t Do = output_lengths[2]; - index_t Ho = output_lengths[3]; - index_t Wo = output_lengths[4]; - - std::vector input_spatial_lengths = {Di, Hi, Wi}; - std::vector output_spatial_lengths = {Do, Ho, Wo}; + if(output_ncdhw_stride != indices_ncdhw_stride) + throw std::runtime_error( + "output_ncdhw_stride need to be equal to indices_ncdhw_stride for now"); return std::make_unique(static_cast(p_in_dev), static_cast(p_out_dev), static_cast(p_out_indices_dev), - N, - C, - input_spatial_lengths, - window_lengths, - output_spatial_lengths, - window_strides, - input_left_pads, - input_right_pads); + input_ncdhw_lengths, + output_ncdhw_lengths, + input_ncdhw_stride, + output_ncdhw_stride, + indices_ncdhw_stride, + window_zyx_lengths, + window_zyx_strides, + window_zyx_dilations, + input_left_dhw_pads, + input_right_dhw_pads); } std::unique_ptr MakeInvokerPointer() override @@ -342,7 +396,7 @@ struct DevicePool3dFwd_Input_N_Di_Hi_Wi_C_Output_N_Do_Ho_Wo_C auto str = std::stringstream(); // clang-format off - str << "DevicePool3dFwd_Input_N_Di_Hi_Wi_C_Output_N_Do_Ho_Wo_C<" << BlockSize << ","; + str << "DevicePool3dFwd_NDHWC_NDHWC<" << BlockSize << ","; str << "M_C" << MThreadClusterSize << "_S" << MThreadSliceSize << ","; str << "K_C" << KThreadClusterSize << "_S" << KThreadSliceSize << ","; str <<"InSrcOutDstVectorSize_" << InSrcOutDstVectorSize << ">"; diff --git a/library/include/ck/library/reference_tensor_operation/cpu/reference_pool_fwd.hpp b/library/include/ck/library/reference_tensor_operation/cpu/reference_pool_fwd.hpp index 696fb5eaf1..067e0b2eb9 100644 --- a/library/include/ck/library/reference_tensor_operation/cpu/reference_pool_fwd.hpp +++ b/library/include/ck/library/reference_tensor_operation/cpu/reference_pool_fwd.hpp @@ -39,6 +39,7 @@ struct ReferencePoolingFwd : public device::BaseOperator Tensor& out_indices, const std::vector& window_spatial_lengths, const std::vector& window_strides, + const std::vector& window_dilations, const std::vector& in_left_pads, const std::vector& /*in_right_pads*/) : in_(in), @@ -46,6 +47,7 @@ struct ReferencePoolingFwd : public device::BaseOperator out_indices_(out_indices), window_spatial_lengths_(window_spatial_lengths), window_strides_(window_strides), + window_dilations_(window_dilations), in_left_pads_(in_left_pads), reduceLength_(1) { @@ -58,6 +60,7 @@ struct ReferencePoolingFwd : public device::BaseOperator Tensor& out_indices_; const std::vector& window_spatial_lengths_; const std::vector& window_strides_; + const std::vector& window_dilations_; const std::vector& in_left_pads_; int reduceLength_; }; @@ -85,14 +88,17 @@ struct ReferencePoolingFwd : public device::BaseOperator for(ck::index_t z = 0; z < arg.window_spatial_lengths_[0]; ++z) { - ck::index_t di = do_ * arg.window_strides_[0] + z - arg.in_left_pads_[0]; + ck::index_t di = do_ * arg.window_strides_[0] + + z * arg.window_dilations_[0] - arg.in_left_pads_[0]; for(ck::index_t y = 0; y < arg.window_spatial_lengths_[1]; ++y) { - ck::index_t hi = ho * arg.window_strides_[1] + y - arg.in_left_pads_[1]; + ck::index_t hi = ho * arg.window_strides_[1] + + y * arg.window_dilations_[1] - arg.in_left_pads_[1]; for(ck::index_t x = 0; x < arg.window_spatial_lengths_[2]; ++x) { - ck::index_t wi = - wo * arg.window_strides_[2] + x - arg.in_left_pads_[2]; + ck::index_t wi = wo * arg.window_strides_[2] + + x * arg.window_dilations_[2] - + arg.in_left_pads_[2]; if(di >= 0 && di < static_cast(arg.in_.mDesc.GetLengths()[2]) && hi >= 0 && @@ -136,14 +142,17 @@ struct ReferencePoolingFwd : public device::BaseOperator for(ck::index_t z = 0; z < arg.window_spatial_lengths_[0]; ++z) { - ck::index_t di = do_ * arg.window_strides_[0] + z - arg.in_left_pads_[0]; + ck::index_t di = do_ * arg.window_strides_[0] + + z * arg.window_dilations_[0] - arg.in_left_pads_[0]; for(ck::index_t y = 0; y < arg.window_spatial_lengths_[1]; ++y) { - ck::index_t hi = ho * arg.window_strides_[1] + y - arg.in_left_pads_[1]; + ck::index_t hi = ho * arg.window_strides_[1] + + y * arg.window_dilations_[1] - arg.in_left_pads_[1]; for(ck::index_t x = 0; x < arg.window_spatial_lengths_[2]; ++x) { - ck::index_t wi = - wo * arg.window_strides_[2] + x - arg.in_left_pads_[2]; + ck::index_t wi = wo * arg.window_strides_[2] + + x * arg.window_dilations_[2] - + arg.in_left_pads_[2]; if(di >= 0 && di < static_cast(arg.in_.mDesc.GetLengths()[2]) && hi >= 0 && @@ -202,10 +211,12 @@ struct ReferencePoolingFwd : public device::BaseOperator for(ck::index_t y = 0; y < arg.window_spatial_lengths_[0]; ++y) { - ck::index_t hi = ho * arg.window_strides_[0] + y - arg.in_left_pads_[0]; + ck::index_t hi = ho * arg.window_strides_[0] + + y * arg.window_dilations_[0] - arg.in_left_pads_[0]; for(ck::index_t x = 0; x < arg.window_spatial_lengths_[1]; ++x) { - ck::index_t wi = wo * arg.window_strides_[1] + x - arg.in_left_pads_[1]; + ck::index_t wi = wo * arg.window_strides_[1] + + x * arg.window_dilations_[1] - arg.in_left_pads_[1]; if(hi >= 0 && hi < static_cast(arg.in_.mDesc.GetLengths()[2]) && wi >= 0 && @@ -308,6 +319,7 @@ struct ReferencePoolingFwd : public device::BaseOperator Tensor& out_indices, const std::vector& window_spatial_lengths, const std::vector& window_strides, + const std::vector& window_dilations, const std::vector& in_left_pads, const std::vector& in_right_pads) { @@ -316,6 +328,7 @@ struct ReferencePoolingFwd : public device::BaseOperator out_indices, window_spatial_lengths, window_strides, + window_dilations, in_left_pads, in_right_pads}; } diff --git a/library/include/ck/library/tensor_operation_instance/gpu/pool2d_fwd.hpp b/library/include/ck/library/tensor_operation_instance/gpu/pool2d_fwd.hpp deleted file mode 100644 index ebdc5f9308..0000000000 --- a/library/include/ck/library/tensor_operation_instance/gpu/pool2d_fwd.hpp +++ /dev/null @@ -1,114 +0,0 @@ -// SPDX-License-Identifier: MIT -// Copyright (c) 2018-2023, Advanced Micro Devices, Inc. All rights reserved. - -#pragma once - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" -#include "ck/tensor_operation/gpu/device/device_pool_fwd.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" - -#include "ck/library/tensor_operation_instance/device_operation_instance_factory.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -static constexpr auto InOutRank = 4; -static constexpr auto WindowRank = 2; - -static constexpr auto MaxOp = ck::ReduceTensorOp::MAX; -static constexpr auto AvgOp = ck::ReduceTensorOp::AVG; -#ifdef __fp16__ -// FP16 -void add_device_pool2d_fwd_nhwc_f16_instances( - std::vector< - std::unique_ptr>>&); - -void add_device_pool2d_fwd_nhwc_f16_instances( - std::vector< - std::unique_ptr>>&); - -// FP16 - return index -void add_device_pool2d_fwd_nhwc_index_f16_instances( - std::vector< - std::unique_ptr>>&); -#endif -#ifdef __fp32__ -// FP32 -void add_device_pool2d_fwd_nhwc_f32_instances( - std::vector< - std::unique_ptr>>&); - -void add_device_pool2d_fwd_nhwc_f32_instances( - std::vector< - std::unique_ptr>>&); - -// FP32 - return index -void add_device_pool2d_fwd_nhwc_index_f32_instances( - std::vector< - std::unique_ptr>>&); -#endif -template -struct DeviceOperationInstanceFactory> -{ - using DeviceOp = DevicePoolFwd; - - static auto GetInstances() - { - std::vector> op_ptrs; -#ifdef __fp16__ - if constexpr(is_same_v && is_same_v && - is_same_v) - { - if constexpr(OutputIndex && ReduceOpId == MaxOp) - { - add_device_pool2d_fwd_nhwc_index_f16_instances(op_ptrs); - } - else - { - add_device_pool2d_fwd_nhwc_f16_instances(op_ptrs); - } - } -#endif -#ifdef __fp32__ - if constexpr(is_same_v && is_same_v && - is_same_v) - { - if constexpr(OutputIndex && ReduceOpId == MaxOp) - { - add_device_pool2d_fwd_nhwc_index_f32_instances(op_ptrs); - } - else - { - add_device_pool2d_fwd_nhwc_f32_instances(op_ptrs); - } - } -#endif - return op_ptrs; - } -}; - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck diff --git a/library/include/ck/library/tensor_operation_instance/gpu/pool3d_fwd.hpp b/library/include/ck/library/tensor_operation_instance/gpu/pool3d_fwd.hpp index b4b4c140ec..ae0514faa0 100644 --- a/library/include/ck/library/tensor_operation_instance/gpu/pool3d_fwd.hpp +++ b/library/include/ck/library/tensor_operation_instance/gpu/pool3d_fwd.hpp @@ -25,36 +25,38 @@ static constexpr auto AvgOp = ck::ReduceTensorOp::AVG; #ifdef __fp16__ // FP16 void add_device_pool3d_fwd_ndhwc_f16_instances( - std::vector< - std::unique_ptr>>&); + std::vector>>&); void add_device_pool3d_fwd_ndhwc_f16_instances( - std::vector< - std::unique_ptr>>&); + std::vector>>&); // FP16 - return index void add_device_pool3d_fwd_ndhwc_index_f16_instances( - std::vector< - std::unique_ptr>>&); + std::vector>>&); #endif #ifdef __fp32__ // FP32 void add_device_pool3d_fwd_ndhwc_f32_instances( - std::vector< - std::unique_ptr>>&); + std::vector>>&); void add_device_pool3d_fwd_ndhwc_f32_instances( - std::vector< - std::unique_ptr>>&); + std::vector>>&); // FP32 - return index void add_device_pool3d_fwd_ndhwc_index_f32_instances( - std::vector< - std::unique_ptr>>&); + std::vector>>&); #endif template struct DeviceOperationInstanceFactory> { @@ -70,40 +74,46 @@ struct DeviceOperationInstanceFactory; static auto GetInstances() { std::vector> op_ptrs; -#ifdef __fp16__ - if constexpr(is_same_v && is_same_v && - is_same_v) + if constexpr(is_same_v && is_same_v) { - if constexpr(OutputIndex && ReduceOpId == MaxOp) +#ifdef __fp16__ + if constexpr(is_same_v && is_same_v && + is_same_v) { - add_device_pool3d_fwd_ndhwc_index_f16_instances(op_ptrs); + if constexpr(OutputIndex && ReduceOpId == MaxOp) + { + add_device_pool3d_fwd_ndhwc_index_f16_instances(op_ptrs); + } + else + { + add_device_pool3d_fwd_ndhwc_f16_instances(op_ptrs); + } } - else - { - add_device_pool3d_fwd_ndhwc_f16_instances(op_ptrs); - } - } #endif #ifdef __fp32__ - if constexpr(is_same_v && is_same_v && - is_same_v) - { - if constexpr(OutputIndex && ReduceOpId == MaxOp) + if constexpr(is_same_v && is_same_v && + is_same_v) { - add_device_pool3d_fwd_ndhwc_index_f32_instances(op_ptrs); + if constexpr(OutputIndex && ReduceOpId == MaxOp) + { + add_device_pool3d_fwd_ndhwc_index_f32_instances(op_ptrs); + } + else + { + add_device_pool3d_fwd_ndhwc_f32_instances(op_ptrs); + } } - else - { - add_device_pool3d_fwd_ndhwc_f32_instances(op_ptrs); - } - } #endif + } + return op_ptrs; } }; diff --git a/library/src/tensor_operation_instance/gpu/pool3d_fwd/CMakeLists.txt b/library/src/tensor_operation_instance/gpu/pool3d_fwd/CMakeLists.txt new file mode 100644 index 0000000000..dd843426b2 --- /dev/null +++ b/library/src/tensor_operation_instance/gpu/pool3d_fwd/CMakeLists.txt @@ -0,0 +1,10 @@ +set(DEVICE_POOL3D_FWD_INSTANCES) +if(DTYPES MATCHES "fp16" OR NOT DEFINED DTYPES) + list(APPEND DEVICE_POOL3D_FWD_INSTANCES device_avg_pool3d_fwd_ndhwc_f16_instance.cpp + device_max_pool3d_fwd_ndhwc_f16_instance.cpp) +endif() +if(DTYPES MATCHES "fp32" OR NOT DEFINED DTYPES) + list(APPEND DEVICE_POOL3D_FWD_INSTANCES device_avg_pool3d_fwd_ndhwc_f32_instance.cpp + device_max_pool3d_fwd_ndhwc_f32_instance.cpp) +endif() +add_instance_library(device_pool3d_fwd_instance ${DEVICE_POOL3D_FWD_INSTANCES}) diff --git a/library/src/tensor_operation_instance/gpu/pool_fwd/device_avg_pool3d_fwd_ndhwc_f16_instance.cpp b/library/src/tensor_operation_instance/gpu/pool3d_fwd/device_avg_pool3d_fwd_ndhwc_f16_instance.cpp similarity index 81% rename from library/src/tensor_operation_instance/gpu/pool_fwd/device_avg_pool3d_fwd_ndhwc_f16_instance.cpp rename to library/src/tensor_operation_instance/gpu/pool3d_fwd/device_avg_pool3d_fwd_ndhwc_f16_instance.cpp index 62bcad992a..4ebd50bae6 100644 --- a/library/src/tensor_operation_instance/gpu/pool_fwd/device_avg_pool3d_fwd_ndhwc_f16_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/pool3d_fwd/device_avg_pool3d_fwd_ndhwc_f16_instance.cpp @@ -11,7 +11,9 @@ namespace instance { static constexpr auto ReduceOpId = ck::ReduceTensorOp::AVG; void add_device_pool3d_fwd_ndhwc_f16_instances( - std::vector>>& instances) + std::vector< + std::unique_ptr>>& + instances) { add_device_operation_instances( instances, device_pool3d_fwd_ndhwc_instances{}); diff --git a/library/src/tensor_operation_instance/gpu/pool_fwd/device_avg_pool3d_fwd_ndhwc_f32_instance.cpp b/library/src/tensor_operation_instance/gpu/pool3d_fwd/device_avg_pool3d_fwd_ndhwc_f32_instance.cpp similarity index 81% rename from library/src/tensor_operation_instance/gpu/pool_fwd/device_avg_pool3d_fwd_ndhwc_f32_instance.cpp rename to library/src/tensor_operation_instance/gpu/pool3d_fwd/device_avg_pool3d_fwd_ndhwc_f32_instance.cpp index 47896be911..dcb19110b2 100644 --- a/library/src/tensor_operation_instance/gpu/pool_fwd/device_avg_pool3d_fwd_ndhwc_f32_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/pool3d_fwd/device_avg_pool3d_fwd_ndhwc_f32_instance.cpp @@ -11,7 +11,9 @@ namespace instance { static constexpr auto ReduceOpId = ck::ReduceTensorOp::AVG; void add_device_pool3d_fwd_ndhwc_f32_instances( - std::vector>>& instances) + std::vector< + std::unique_ptr>>& + instances) { add_device_operation_instances( instances, device_pool3d_fwd_ndhwc_instances{}); diff --git a/library/src/tensor_operation_instance/gpu/pool_fwd/device_max_pool3d_fwd_ndhwc_f16_instance.cpp b/library/src/tensor_operation_instance/gpu/pool3d_fwd/device_max_pool3d_fwd_ndhwc_f16_instance.cpp similarity index 74% rename from library/src/tensor_operation_instance/gpu/pool_fwd/device_max_pool3d_fwd_ndhwc_f16_instance.cpp rename to library/src/tensor_operation_instance/gpu/pool3d_fwd/device_max_pool3d_fwd_ndhwc_f16_instance.cpp index dbfc4acfdf..46b16bd005 100644 --- a/library/src/tensor_operation_instance/gpu/pool_fwd/device_max_pool3d_fwd_ndhwc_f16_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/pool3d_fwd/device_max_pool3d_fwd_ndhwc_f16_instance.cpp @@ -11,14 +11,18 @@ namespace instance { static constexpr auto ReduceOpId = ck::ReduceTensorOp::MAX; void add_device_pool3d_fwd_ndhwc_f16_instances( - std::vector>>& instances) + std::vector< + std::unique_ptr>>& + instances) { add_device_operation_instances( instances, device_pool3d_fwd_ndhwc_instances{}); } void add_device_pool3d_fwd_ndhwc_index_f16_instances( - std::vector>>& instances) + std::vector< + std::unique_ptr>>& + instances) { add_device_operation_instances( instances, device_pool3d_fwd_ndhwc_instances{}); diff --git a/library/src/tensor_operation_instance/gpu/pool_fwd/device_max_pool3d_fwd_ndhwc_f32_instance.cpp b/library/src/tensor_operation_instance/gpu/pool3d_fwd/device_max_pool3d_fwd_ndhwc_f32_instance.cpp similarity index 74% rename from library/src/tensor_operation_instance/gpu/pool_fwd/device_max_pool3d_fwd_ndhwc_f32_instance.cpp rename to library/src/tensor_operation_instance/gpu/pool3d_fwd/device_max_pool3d_fwd_ndhwc_f32_instance.cpp index 63b3e8df8e..b4b0e74d28 100644 --- a/library/src/tensor_operation_instance/gpu/pool_fwd/device_max_pool3d_fwd_ndhwc_f32_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/pool3d_fwd/device_max_pool3d_fwd_ndhwc_f32_instance.cpp @@ -11,14 +11,18 @@ namespace instance { static constexpr auto ReduceOpId = ck::ReduceTensorOp::MAX; void add_device_pool3d_fwd_ndhwc_f32_instances( - std::vector>>& instances) + std::vector< + std::unique_ptr>>& + instances) { add_device_operation_instances( instances, device_pool3d_fwd_ndhwc_instances{}); } void add_device_pool3d_fwd_ndhwc_index_f32_instances( - std::vector>>& instances) + std::vector< + std::unique_ptr>>& + instances) { add_device_operation_instances( instances, device_pool3d_fwd_ndhwc_instances{}); diff --git a/library/src/tensor_operation_instance/gpu/pool3d_fwd/pool_fwd_instance_common.hpp b/library/src/tensor_operation_instance/gpu/pool3d_fwd/pool_fwd_instance_common.hpp new file mode 100644 index 0000000000..4d23ceab27 --- /dev/null +++ b/library/src/tensor_operation_instance/gpu/pool3d_fwd/pool_fwd_instance_common.hpp @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2018-2023, Advanced Micro Devices, Inc. All rights reserved. + +#pragma once + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/impl/device_pool2d_fwd_nhwc_nhwc.hpp" +#include "ck/tensor_operation/gpu/device/impl/device_pool3d_fwd_ndhwc_ndhwc.hpp" +#include "ck/utility/data_type.hpp" + +#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" + +namespace ck { +namespace tensor_operation { +namespace device { +namespace instance { + +using I32 = int32_t; +using F16 = ck::half_t; +using F32 = float; +using NDHWC = ck::tensor_layout::convolution::NDHWC; + +template +using device_pool3d_fwd_ndhwc_instances = + // clang-format off + std::tuple < + DevicePool3dFwd_NDHWC_NDHWC, + DevicePool3dFwd_NDHWC_NDHWC, + DevicePool3dFwd_NDHWC_NDHWC + // clang-format on + >; + +} // namespace instance +} // namespace device +} // namespace tensor_operation +} // namespace ck diff --git a/library/src/tensor_operation_instance/gpu/pool_fwd/CMakeLists.txt b/library/src/tensor_operation_instance/gpu/pool_fwd/CMakeLists.txt deleted file mode 100644 index ba2a8991bd..0000000000 --- a/library/src/tensor_operation_instance/gpu/pool_fwd/CMakeLists.txt +++ /dev/null @@ -1,14 +0,0 @@ -set(DEVICE_POOL_FWD_INSTANCES) -if(DTYPES MATCHES "fp16" OR NOT DEFINED DTYPES) - list(APPEND DEVICE_POOL_FWD_INSTANCES device_avg_pool2d_fwd_nhwc_f16_instance.cpp - device_avg_pool3d_fwd_ndhwc_f16_instance.cpp - device_max_pool2d_fwd_nhwc_f16_instance.cpp - device_max_pool3d_fwd_ndhwc_f16_instance.cpp) -endif() -if(DTYPES MATCHES "fp32" OR NOT DEFINED DTYPES) - list(APPEND DEVICE_POOL_FWD_INSTANCES device_avg_pool2d_fwd_nhwc_f32_instance.cpp - device_avg_pool3d_fwd_ndhwc_f32_instance.cpp - device_max_pool2d_fwd_nhwc_f32_instance.cpp - device_max_pool3d_fwd_ndhwc_f32_instance.cpp) -endif() -add_instance_library(device_pool_fwd_instance ${DEVICE_POOL_FWD_INSTANCES}) diff --git a/library/src/tensor_operation_instance/gpu/pool_fwd/device_avg_pool2d_fwd_nhwc_f16_instance.cpp b/library/src/tensor_operation_instance/gpu/pool_fwd/device_avg_pool2d_fwd_nhwc_f16_instance.cpp deleted file mode 100644 index 508ad3873b..0000000000 --- a/library/src/tensor_operation_instance/gpu/pool_fwd/device_avg_pool2d_fwd_nhwc_f16_instance.cpp +++ /dev/null @@ -1,23 +0,0 @@ -// SPDX-License-Identifier: MIT -// Copyright (c) 2018-2023, Advanced Micro Devices, Inc. All rights reserved. - -#include "pool_fwd_instance_common.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -static constexpr auto ReduceOpId = ck::ReduceTensorOp::AVG; - -void add_device_pool2d_fwd_nhwc_f16_instances( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, device_pool2d_fwd_nhwc_instances{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck diff --git a/library/src/tensor_operation_instance/gpu/pool_fwd/device_avg_pool2d_fwd_nhwc_f32_instance.cpp b/library/src/tensor_operation_instance/gpu/pool_fwd/device_avg_pool2d_fwd_nhwc_f32_instance.cpp deleted file mode 100644 index ada96a93a2..0000000000 --- a/library/src/tensor_operation_instance/gpu/pool_fwd/device_avg_pool2d_fwd_nhwc_f32_instance.cpp +++ /dev/null @@ -1,23 +0,0 @@ -// SPDX-License-Identifier: MIT -// Copyright (c) 2018-2023, Advanced Micro Devices, Inc. All rights reserved. - -#include "pool_fwd_instance_common.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -static constexpr auto ReduceOpId = ck::ReduceTensorOp::AVG; - -void add_device_pool2d_fwd_nhwc_f32_instances( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, device_pool2d_fwd_nhwc_instances{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck diff --git a/library/src/tensor_operation_instance/gpu/pool_fwd/device_max_pool2d_fwd_nhwc_f16_instance.cpp b/library/src/tensor_operation_instance/gpu/pool_fwd/device_max_pool2d_fwd_nhwc_f16_instance.cpp deleted file mode 100644 index 35c8522d9f..0000000000 --- a/library/src/tensor_operation_instance/gpu/pool_fwd/device_max_pool2d_fwd_nhwc_f16_instance.cpp +++ /dev/null @@ -1,30 +0,0 @@ -// SPDX-License-Identifier: MIT -// Copyright (c) 2018-2023, Advanced Micro Devices, Inc. All rights reserved. - -#include "pool_fwd_instance_common.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -static constexpr auto ReduceOpId = ck::ReduceTensorOp::MAX; - -void add_device_pool2d_fwd_nhwc_f16_instances( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, device_pool2d_fwd_nhwc_instances{}); -} - -void add_device_pool2d_fwd_nhwc_index_f16_instances( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, device_pool2d_fwd_nhwc_instances{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck diff --git a/library/src/tensor_operation_instance/gpu/pool_fwd/device_max_pool2d_fwd_nhwc_f32_instance.cpp b/library/src/tensor_operation_instance/gpu/pool_fwd/device_max_pool2d_fwd_nhwc_f32_instance.cpp deleted file mode 100644 index 75b7629f24..0000000000 --- a/library/src/tensor_operation_instance/gpu/pool_fwd/device_max_pool2d_fwd_nhwc_f32_instance.cpp +++ /dev/null @@ -1,30 +0,0 @@ -// SPDX-License-Identifier: MIT -// Copyright (c) 2018-2023, Advanced Micro Devices, Inc. All rights reserved. - -#include "pool_fwd_instance_common.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -static constexpr auto ReduceOpId = ck::ReduceTensorOp::MAX; - -void add_device_pool2d_fwd_nhwc_f32_instances( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, device_pool2d_fwd_nhwc_instances{}); -} - -void add_device_pool2d_fwd_nhwc_index_f32_instances( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, device_pool2d_fwd_nhwc_instances{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck diff --git a/library/src/tensor_operation_instance/gpu/pool_fwd/pool_fwd_instance_common.hpp b/library/src/tensor_operation_instance/gpu/pool_fwd/pool_fwd_instance_common.hpp deleted file mode 100644 index 8aa707885b..0000000000 --- a/library/src/tensor_operation_instance/gpu/pool_fwd/pool_fwd_instance_common.hpp +++ /dev/null @@ -1,55 +0,0 @@ -// SPDX-License-Identifier: MIT -// Copyright (c) 2018-2023, Advanced Micro Devices, Inc. All rights reserved. - -#pragma once - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/impl/device_pool2d_fwd_nhwc_nhwc.hpp" -#include "ck/tensor_operation/gpu/device/impl/device_pool3d_fwd_ndhwc_ndhwc.hpp" -#include "ck/utility/data_type.hpp" - -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -using I32 = int32_t; -using F16 = ck::half_t; -using F32 = float; - -template -using device_pool2d_fwd_nhwc_instances = - // clang-format off - std::tuple < - DevicePool2dFwd_Input_N_Hi_Wi_C_Output_N_Ho_Wo_C, - DevicePool2dFwd_Input_N_Hi_Wi_C_Output_N_Ho_Wo_C, - DevicePool2dFwd_Input_N_Hi_Wi_C_Output_N_Ho_Wo_C - // clang-format on - >; - -template -using device_pool3d_fwd_ndhwc_instances = - // clang-format off - std::tuple < - DevicePool3dFwd_Input_N_Di_Hi_Wi_C_Output_N_Do_Ho_Wo_C, - DevicePool3dFwd_Input_N_Di_Hi_Wi_C_Output_N_Do_Ho_Wo_C, - DevicePool3dFwd_Input_N_Di_Hi_Wi_C_Output_N_Do_Ho_Wo_C - // clang-format on - >; - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck diff --git a/profiler/include/profiler/profile_pool2d_fwd_impl.hpp b/profiler/include/profiler/profile_pool2d_fwd_impl.hpp deleted file mode 100644 index 0c888db1f4..0000000000 --- a/profiler/include/profiler/profile_pool2d_fwd_impl.hpp +++ /dev/null @@ -1,264 +0,0 @@ -// SPDX-License-Identifier: MIT -// Copyright (c) 2018-2023, Advanced Micro Devices, Inc. All rights reserved. - -#pragma once - -#include - -#include "ck/ck.hpp" -#include "ck/library/tensor_operation_instance/gpu/pool2d_fwd.hpp" -#include "ck/library/utility/check_err.hpp" -#include "ck/library/utility/device_memory.hpp" -#include "ck/library/utility/host_tensor.hpp" -#include "ck/library/utility/host_tensor_generator.hpp" -#include "ck/library/utility/literals.hpp" -#include "ck/library/reference_tensor_operation/cpu/reference_pool_fwd.hpp" - -namespace ck { -namespace profiler { - -template -bool profile_pool2d_fwd_impl(int do_verification, - int init_method, - bool do_log, - bool time_kernel, - std::vector in_length, // NCHW - std::vector window_spatial_lengths, - std::vector window_strides, - std::vector input_left_pads, - std::vector input_right_pads) -{ - constexpr index_t InOutRank = 4; - constexpr index_t WindowRank = 2; - - if(in_length.size() != InOutRank || window_spatial_lengths.size() != WindowRank || - window_strides.size() != WindowRank || input_left_pads.size() != WindowRank || - input_right_pads.size() != WindowRank) - return false; - - std::vector out_length(InOutRank); - - int N = in_length[0]; - int C = in_length[1]; - - out_length[0] = N; - out_length[1] = C; - - // Calculate Ho, Wo - for(int i = 2; i < InOutRank; ++i) - { - auto pad1 = input_left_pads[i - 2]; - auto pad2 = input_right_pads[i - 2]; - auto windows_size = window_spatial_lengths[i - 2]; - auto windows_stride = window_strides[i - 2]; - out_length[i] = (in_length[i] + pad1 + pad2 - windows_size) / windows_stride + 1; - } - - int Hi = in_length[2]; - int Wi = in_length[3]; - int Ho = out_length[2]; - int Wo = out_length[3]; - - auto f_host_tensor_descriptor = - [](std::size_t N_, std::size_t C_, std::size_t H, std::size_t W) { - using namespace ck::literals; - return HostTensorDescriptor({N_, C_, H, W}, {C_ * H * W, 1_uz, W * C_, C_}); - }; - - Tensor in_n_c_hi_wi(f_host_tensor_descriptor(N, C, Hi, Wi)); - Tensor out_n_c_ho_wo_host(f_host_tensor_descriptor(N, C, Ho, Wo)); - Tensor out_indices_n_c_ho_wo_host(f_host_tensor_descriptor(N, C, Ho, Wo)); - - Tensor out_n_c_ho_wo_device(f_host_tensor_descriptor(N, C, Ho, Wo)); - Tensor out_indices_n_c_ho_wo_device(f_host_tensor_descriptor(N, C, Ho, Wo)); - - switch(init_method) - { - case 0: in_n_c_hi_wi.GenerateTensorValue(GeneratorTensor_1{}); break; - case 1: in_n_c_hi_wi.GenerateTensorValue(GeneratorTensor_2{-5, 5}); break; - default: in_n_c_hi_wi.GenerateTensorValue(GeneratorTensor_3{-0.5, 0.5}); - } - - DeviceMem in_device_buf(sizeof(InDataType) * in_n_c_hi_wi.mDesc.GetElementSpaceSize()); - DeviceMem out_device_buf(sizeof(OutDataType) * - out_n_c_ho_wo_device.mDesc.GetElementSpaceSize()); - DeviceMem out_indices_device_buf(sizeof(IndexDataType) * - out_indices_n_c_ho_wo_device.mDesc.GetElementSpaceSize()); - - in_device_buf.ToDevice(in_n_c_hi_wi.mData.data()); - - // add device normalization instances - using DeviceOp = ck::tensor_operation::device::DevicePoolFwd; - - // get device op instances - const auto instance_ptrs = - ck::tensor_operation::device::instance::DeviceOperationInstanceFactory< - DeviceOp>::GetInstances(); - - std::cout << "found " << instance_ptrs.size() << " instances" << std::endl; - - std::string best_instance_name; - float best_avg_time = std::numeric_limits::max(); - float best_gb_per_sec = 0; - - if(do_verification) - { - using ReferenceInstance = ck::tensor_operation::host::ReferencePoolingFwd; - - ReferenceInstance ref; - auto ref_argument = ref.MakeArgument(in_n_c_hi_wi, - out_n_c_ho_wo_host, - out_indices_n_c_ho_wo_host, - window_spatial_lengths, - window_strides, - input_left_pads, - input_right_pads); - auto ref_invoker = ref.MakeInvoker(); - ref_invoker.Run(ref_argument); - } - - int num_kernel = 0; - - for(auto& inst_ptr : instance_ptrs) - { - auto argument_ptr = inst_ptr->MakeArgumentPointer( - static_cast(in_device_buf.GetDeviceBuffer()), - static_cast(out_device_buf.GetDeviceBuffer()), - static_cast(out_indices_device_buf.GetDeviceBuffer()), - in_length, - window_spatial_lengths, - out_length, - {C * Hi * Wi, 1, Wi * C, C}, - {C * Ho * Wo, 1, Wo * C, C}, - {C * Ho * Wo, 1, Wo * C, C}, - window_strides, - input_left_pads, - input_right_pads, - {2, 3}); - - if(inst_ptr->IsSupportedArgument(argument_ptr.get())) - { - ++num_kernel; - } - else - { - if(time_kernel) - { - std::cout << inst_ptr->GetTypeString() << " skipped due to unsupported argument: "; - LogRange(std::cout << "input lengths = ", in_length, ", ") << std::endl; - } - - continue; - } - - auto invoker_ptr = inst_ptr->MakeInvokerPointer(); - - float avg_time = invoker_ptr->Run(argument_ptr.get(), StreamConfig{nullptr, time_kernel}); - - std::size_t num_bytes = in_n_c_hi_wi.mDesc.GetElementSize() * sizeof(InDataType) + - out_n_c_ho_wo_host.mDesc.GetElementSize() * sizeof(OutDataType); - - if constexpr(OutputIndex) - num_bytes += out_indices_n_c_ho_wo_host.mDesc.GetElementSize() * sizeof(IndexDataType); - - float gb_per_sec = num_bytes / 1.E6 / avg_time; - - if(time_kernel) - std::cout << "Perf: " << std::setw(10) << avg_time << " ms, " << gb_per_sec << " GB/s, " - << inst_ptr->GetTypeString() << std::endl; - - if(avg_time < best_avg_time) - { - best_instance_name = inst_ptr->GetTypeString(); - best_avg_time = avg_time; - best_gb_per_sec = gb_per_sec; - } - - if(do_verification) - { - out_device_buf.FromDevice(out_n_c_ho_wo_device.mData.data()); - - bool pass = ck::utils::check_err(out_n_c_ho_wo_device.mData, - out_n_c_ho_wo_host.mData, - "Error: Incorrect results", - 1e-3, - 1e-3); - - if constexpr(OutputIndex) - { - out_indices_device_buf.FromDevice(out_indices_n_c_ho_wo_device.mData.data()); - - pass = pass && ck::utils::check_err(out_indices_n_c_ho_wo_device, - out_indices_n_c_ho_wo_host); - } - - if(do_log) - { - LogRangeAsType(std::cout << "in_n_c_hi_wi : ", in_n_c_hi_wi.mData, ",") - << std::endl; - LogRangeAsType( - std::cout << "out_n_c_ho_wo_host : ", out_n_c_ho_wo_host.mData, ",") - << std::endl; - LogRangeAsType( - std::cout << "out_n_c_ho_wo_device : ", out_n_c_ho_wo_device.mData, ",") - << std::endl; - - if constexpr(OutputIndex) - LogRangeAsType(std::cout << "out_indices_n_c_ho_wo_device : ", - out_indices_n_c_ho_wo_device.mData, - ",") - << std::endl; - } - - if(!pass) - { - std::cout << inst_ptr->GetTypeString() << " failed verification: "; - LogRange(std::cout << "lengths = [", in_length, ", ") << "]." << std::endl; - return false; - } - else - { - if(time_kernel) - std::cout << "pass" << std::endl; - } - } - } - - if(time_kernel) - { - LogRange(std::cout << "length = ", in_length, ",") << std::endl; - std::cout << "best perf = " << best_avg_time << " ms, " << best_gb_per_sec << " GB/s, " - << best_instance_name << std::endl; - } - - if(num_kernel == 0) - { - std::cout << "Error: No kernel is applicable" << std::endl; - return false; - } - - return true; -} - -} // namespace profiler -} // namespace ck diff --git a/profiler/include/profiler/profile_pool3d_fwd_impl.hpp b/profiler/include/profiler/profile_pool3d_fwd_impl.hpp index 41b57fd853..02fde48d6e 100644 --- a/profiler/include/profiler/profile_pool3d_fwd_impl.hpp +++ b/profiler/include/profiler/profile_pool3d_fwd_impl.hpp @@ -21,6 +21,8 @@ template @@ -31,6 +33,7 @@ bool profile_pool3d_fwd_impl(int do_verification, std::vector in_length, // NCDHW std::vector window_spatial_lengths, std::vector window_strides, + std::vector window_dilations, std::vector input_left_pads, std::vector input_right_pads) { @@ -38,8 +41,8 @@ bool profile_pool3d_fwd_impl(int do_verification, constexpr index_t WindowRank = 3; if(in_length.size() != InOutRank || window_spatial_lengths.size() != WindowRank || - window_strides.size() != WindowRank || input_left_pads.size() != WindowRank || - input_right_pads.size() != WindowRank) + window_strides.size() != WindowRank || window_dilations.size() != WindowRank || + input_left_pads.size() != WindowRank || input_right_pads.size() != WindowRank) return false; std::vector out_length(InOutRank); @@ -53,11 +56,13 @@ bool profile_pool3d_fwd_impl(int do_verification, // Calculate Do, Ho, Wo for(int i = 2; i < InOutRank; ++i) { - auto pad1 = input_left_pads[i - 2]; - auto pad2 = input_right_pads[i - 2]; - auto windows_size = window_spatial_lengths[i - 2]; - auto windows_stride = window_strides[i - 2]; - out_length[i] = (in_length[i] + pad1 + pad2 - windows_size) / windows_stride + 1; + auto pad1 = input_left_pads[i - 2]; + auto pad2 = input_right_pads[i - 2]; + auto windows_size = window_spatial_lengths[i - 2]; + auto windows_stride = window_strides[i - 2]; + auto windows_dilation = window_dilations[i - 2]; + auto eff = (windows_size - 1) * windows_dilation + 1; + out_length[i] = (in_length[i] + pad1 + pad2 - eff) / windows_stride + 1; } int Di = in_length[2]; @@ -104,6 +109,8 @@ bool profile_pool3d_fwd_impl(int do_verification, InDataType, OutDataType, IndexDataType, + InLayout, + OutLayout, ReduceOpId, OutputIndex>; @@ -136,6 +143,7 @@ bool profile_pool3d_fwd_impl(int do_verification, out_indices_n_c_do_ho_wo_host, window_spatial_lengths, window_strides, + window_dilations, input_left_pads, input_right_pads); auto ref_invoker = ref.MakeInvoker(); @@ -157,6 +165,7 @@ bool profile_pool3d_fwd_impl(int do_verification, {Do * C * Ho * Wo, 1, C * Ho * Wo, Wo * C, C}, {Do * C * Ho * Wo, 1, C * Ho * Wo, Wo * C, C}, window_strides, + window_dilations, input_left_pads, input_right_pads, {2, 3, 4}); diff --git a/profiler/src/CMakeLists.txt b/profiler/src/CMakeLists.txt index a4ea5c399c..f98747c93a 100644 --- a/profiler/src/CMakeLists.txt +++ b/profiler/src/CMakeLists.txt @@ -17,7 +17,6 @@ set(PROFILER_SOURCES profile_reduce.cpp profile_groupnorm.cpp profile_layernorm.cpp - profile_avg_pool2d_fwd.cpp profile_max_pool3d_fwd.cpp profile_softmax.cpp profile_batchnorm_fwd.cpp @@ -74,7 +73,7 @@ target_link_libraries(${PROFILER_EXECUTABLE} PRIVATE device_reduce_instance) target_link_libraries(${PROFILER_EXECUTABLE} PRIVATE device_batchnorm_instance) target_link_libraries(${PROFILER_EXECUTABLE} PRIVATE device_contraction_bilinear_instance) target_link_libraries(${PROFILER_EXECUTABLE} PRIVATE device_contraction_scale_instance) -target_link_libraries(${PROFILER_EXECUTABLE} PRIVATE device_pool_fwd_instance) +target_link_libraries(${PROFILER_EXECUTABLE} PRIVATE device_pool3d_fwd_instance) target_link_libraries(${PROFILER_EXECUTABLE} PRIVATE device_grouped_conv2d_bwd_data_instance) target_link_libraries(${PROFILER_EXECUTABLE} PRIVATE device_grouped_conv3d_bwd_data_instance) if(DL_KERNELS) diff --git a/profiler/src/profile_avg_pool2d_fwd.cpp b/profiler/src/profile_avg_pool2d_fwd.cpp deleted file mode 100644 index c67897c044..0000000000 --- a/profiler/src/profile_avg_pool2d_fwd.cpp +++ /dev/null @@ -1,141 +0,0 @@ -// SPDX-License-Identifier: MIT -// Copyright (c) 2018-2023, Advanced Micro Devices, Inc. All rights reserved. - -#include -#include -#include - -#include "profiler/data_type_enum.hpp" -#include "profiler/profile_pool2d_fwd_impl.hpp" -#include "profiler_operation_registry.hpp" - -using ck::index_t; - -struct avgPoolFwdArgParser -{ - std::unordered_map> long_opts = { - {"length", {}}, {"wsize", {}}, {"wstride", {}}, {"pad1", {}}, {"pad2", {}}}; - - bool parse_opt(int argc, char* argv[], const std::string& key, int i) - { - if(std::string("--") + key == argv[i]) - { - int pos = i; - while(++i < argc && argv[i][0] != '-') {} - int end = i; - for(int j = pos + 1; j < end; j++) - { - long_opts[key].push_back(std::stoi(argv[j])); - } - return true; - } - return false; - } - - void operator()(int argc, char* argv[]) - { - for(auto& kv : long_opts) - { - for(int i = 1; i < argc; i++) - { - if(parse_opt(argc, argv, kv.first, i)) - break; - } - } - } -}; - -void print_help_avg_pool2d_fwd() -{ - std::cout << "arg1: data type (0: fp16; 1: fp32)\n" - << "arg2: verification (0: no; 1: yes)\n" - << "arg3: initialization (0: no init; 1: integer value; 2: decimal value)\n" - << "arg4: print tensor value (0: no; 1: yes)\n" - << "arg5: time kernel (0=no, 1=yes)\n" - << "--length: input tensor length for NDHW(e.g, --length 2 32 30 30) \n" - << "--wsize: window size for YX (e.g, --wsize 2 2) \n" - << "--wstride: window stride for HW (e.g, --wstride 2 2) \n" - << "--pad1: left side of padding in HW (e.g, --pad1 1 1) \n" - << "--pad2: right side of padding in HW (e.g, --pad2 1 1) \n" - << "eg: ckProfiler avg_pool2d_fwd 0 1 2 0 1 0 --length 2 32 30 30 --wsize 2 2 " - "--wstride 2 2 --pad1 1 1 --pad2 1 1" - << std::endl; -} - -int profile_avg_pool2d_fwd(int argc, char* argv[]) -{ - ck::DataTypeEnum data_type = ck::DataTypeEnum::Half; - bool do_verification = true; - int init_method = 0; - bool do_log = false; - bool time_kernel = true; - - std::vector in_length = {2, 32, 30, 30}; - std::vector wsize = {2, 2}; - std::vector wstride = {2, 2}; - std::vector pad1 = {1, 1}; - std::vector pad2 = {1, 1}; - - if(argc != 2 && argc != 25) - { - print_help_avg_pool2d_fwd(); - return 0; - } - else if(argc == 25) - { - data_type = static_cast(std::stoi(argv[2])); - do_verification = std::stoi(argv[3]); - init_method = std::stoi(argv[4]); - do_log = std::stoi(argv[5]); - time_kernel = std::stoi(argv[6]); - - // parse the long options - avgPoolFwdArgParser arg_parser; - arg_parser(argc, argv); - in_length = arg_parser.long_opts["length"]; - wsize = arg_parser.long_opts["wsize"]; - wstride = arg_parser.long_opts["wstride"]; - pad1 = arg_parser.long_opts["pad1"]; - pad2 = arg_parser.long_opts["pad2"]; - } - - using F16 = ck::half_t; - using F32 = float; - using I32 = int32_t; - constexpr auto ReduceOpId = ck::ReduceTensorOp::AVG; - - if(data_type == ck::DataTypeEnum::Half) - { - ck::profiler::profile_pool2d_fwd_impl( - do_verification, - init_method, - do_log, - time_kernel, - in_length, - wsize, - wstride, - pad1, - pad2); - } - else if(data_type == ck::DataTypeEnum::Float) - { - ck::profiler::profile_pool2d_fwd_impl( - do_verification, - init_method, - do_log, - time_kernel, - in_length, - wsize, - wstride, - pad1, - pad2); - } - else - { - throw std::runtime_error("not implemented yet"); - } - - return 0; -} - -REGISTER_PROFILER_OPERATION("avg_pool2d_fwd", "avg_pool2d fwd", profile_avg_pool2d_fwd); diff --git a/profiler/src/profile_max_pool3d_fwd.cpp b/profiler/src/profile_max_pool3d_fwd.cpp index cf6db2cfc9..da7ea9af4b 100644 --- a/profiler/src/profile_max_pool3d_fwd.cpp +++ b/profiler/src/profile_max_pool3d_fwd.cpp @@ -13,8 +13,12 @@ using ck::index_t; struct maxPoolFwdArgParser { - std::unordered_map> long_opts = { - {"length", {}}, {"wsize", {}}, {"wstride", {}}, {"pad1", {}}, {"pad2", {}}}; + std::unordered_map> long_opts = {{"length", {}}, + {"wsize", {}}, + {"wstride", {}}, + {"wdilation", {}}, + {"pad1", {}}, + {"pad2", {}}}; bool parse_opt(int argc, char* argv[], const std::string& key, int i) { @@ -56,10 +60,11 @@ void print_help_max_pool3d_fwd() << "--length: input tensor length for NCDHW(e.g, --length 2 32 30 30 30) \n" << "--wsize: window size for ZYX (e.g, --wsize 2 2 2) \n" << "--wstride: window stride for DHW (e.g, --wstride 2 2 2) \n" + << "--wdilation: window dilation for DHW (e.g, --wdilation 1 1 1) \n" << "--pad1: left side of padding in DHW (e.g, --pad1 1 1 1) \n" << "--pad2: right side of padding in DHW (e.g, --pad2 1 1 1) \n" << "eg: ckProfiler max_pool3d_fwd 0 1 2 0 1 0 --length 2 32 30 30 30 --wsize 2 2 2 " - "--wstride 2 2 2 --pad1 1 1 1 --pad2 1 1 1" + "--wstride 2 2 2 --wdilation 1 1 1 --pad1 1 1 1 --pad2 1 1 1" << std::endl; } @@ -75,15 +80,16 @@ int profile_max_pool3d_fwd(int argc, char* argv[]) std::vector in_length = {2, 32, 30, 30, 30}; std::vector wsize = {2, 2, 2}; std::vector wstride = {2, 2, 2}; + std::vector wdilation = {1, 1, 1}; std::vector pad1 = {1, 1, 1}; std::vector pad2 = {1, 1, 1}; - if(argc != 2 && argc != 30) + if(argc != 2 && argc != 34) { print_help_max_pool3d_fwd(); return 0; } - else if(argc == 30) + else if(argc == 34) { data_type = static_cast(std::stoi(argv[2])); do_verification = std::stoi(argv[3]); @@ -98,64 +104,79 @@ int profile_max_pool3d_fwd(int argc, char* argv[]) in_length = arg_parser.long_opts["length"]; wsize = arg_parser.long_opts["wsize"]; wstride = arg_parser.long_opts["wstride"]; + wdilation = arg_parser.long_opts["wdilation"]; pad1 = arg_parser.long_opts["pad1"]; pad2 = arg_parser.long_opts["pad2"]; } - using F16 = ck::half_t; - using F32 = float; - using I32 = int32_t; + using F16 = ck::half_t; + using F32 = float; + using I32 = int32_t; + using NDHWC = ck::tensor_layout::convolution::NDHWC; + +#if 1 constexpr auto ReduceOpId = ck::ReduceTensorOp::MAX; +#else + constexpr auto ReduceOpId = ck::ReduceTensorOp::AVG; +#endif if(data_type == ck::DataTypeEnum::Half) { if(return_index) - ck::profiler::profile_pool3d_fwd_impl( - do_verification, - init_method, - do_log, - time_kernel, - in_length, - wsize, - wstride, - pad1, - pad2); + ck::profiler:: + profile_pool3d_fwd_impl( + do_verification, + init_method, + do_log, + time_kernel, + in_length, + wsize, + wstride, + wdilation, + pad1, + pad2); else - ck::profiler::profile_pool3d_fwd_impl( - do_verification, - init_method, - do_log, - time_kernel, - in_length, - wsize, - wstride, - pad1, - pad2); + ck::profiler:: + profile_pool3d_fwd_impl( + do_verification, + init_method, + do_log, + time_kernel, + in_length, + wsize, + wstride, + wdilation, + pad1, + pad2); } else if(data_type == ck::DataTypeEnum::Float) { if(return_index) - ck::profiler::profile_pool3d_fwd_impl( - do_verification, - init_method, - do_log, - time_kernel, - in_length, - wsize, - wstride, - pad1, - pad2); + ck::profiler:: + profile_pool3d_fwd_impl( + do_verification, + init_method, + do_log, + time_kernel, + in_length, + wsize, + wstride, + wdilation, + pad1, + pad2); else - ck::profiler::profile_pool3d_fwd_impl( - do_verification, - init_method, - do_log, - time_kernel, - in_length, - wsize, - wstride, - pad1, - pad2); + ck::profiler:: + profile_pool3d_fwd_impl( + do_verification, + init_method, + do_log, + time_kernel, + in_length, + wsize, + wstride, + wdilation, + pad1, + pad2); } else { diff --git a/test/pool_fwd/CMakeLists.txt b/test/pool_fwd/CMakeLists.txt index 6f59b95f6f..e421a79fde 100644 --- a/test/pool_fwd/CMakeLists.txt +++ b/test/pool_fwd/CMakeLists.txt @@ -1,16 +1,10 @@ add_custom_target(test_pool_fwd) -add_gtest_executable(test_avg_pool2d_fwd test_avg_pool2d_fwd.cpp) add_gtest_executable(test_avg_pool3d_fwd test_avg_pool3d_fwd.cpp) -add_gtest_executable(test_max_pool2d_fwd test_max_pool2d_fwd.cpp) add_gtest_executable(test_max_pool3d_fwd test_max_pool3d_fwd.cpp) -target_link_libraries(test_avg_pool2d_fwd PRIVATE utility device_pool_fwd_instance) -target_link_libraries(test_avg_pool3d_fwd PRIVATE utility device_pool_fwd_instance) -target_link_libraries(test_max_pool2d_fwd PRIVATE utility device_pool_fwd_instance) -target_link_libraries(test_max_pool3d_fwd PRIVATE utility device_pool_fwd_instance) +target_link_libraries(test_avg_pool3d_fwd PRIVATE utility device_pool3d_fwd_instance) +target_link_libraries(test_max_pool3d_fwd PRIVATE utility device_pool3d_fwd_instance) -add_dependencies(test_pool_fwd test_avg_pool2d_fwd) add_dependencies(test_pool_fwd test_avg_pool3d_fwd) -add_dependencies(test_pool_fwd test_max_pool2d_fwd) add_dependencies(test_pool_fwd test_max_pool3d_fwd) diff --git a/test/pool_fwd/test_avg_pool2d_fwd.cpp b/test/pool_fwd/test_avg_pool2d_fwd.cpp deleted file mode 100644 index 31f4a166f2..0000000000 --- a/test/pool_fwd/test_avg_pool2d_fwd.cpp +++ /dev/null @@ -1,60 +0,0 @@ -// SPDX-License-Identifier: MIT -// Copyright (c) 2018-2023, Advanced Micro Devices, Inc. All rights reserved. - -#include "gtest/gtest.h" -#include "profiler/profile_pool2d_fwd_impl.hpp" -#include "test_pool_fwd_common.hpp" - -template -class TestAvgPool2dFwd : public ::testing::Test -{ - protected: - using InDataType = std::tuple_element_t<0, Tuple>; - using OutDataType = std::tuple_element_t<1, Tuple>; - using ComputeDataType = std::tuple_element_t<2, Tuple>; - using IndexDataType = std::tuple_element_t<3, Tuple>; - - std::vector params; - - void Run() - { - for(auto param : params) - { - bool success = - ck::profiler::profile_pool2d_fwd_impl(true, - 2, - false, - false, - param.length_, - param.window_spatial_lengths_, - param.window_strides_, - param.input_left_pads_, - param.input_right_pads_); - EXPECT_TRUE(success); - } - } -}; - -#ifdef __fp16__ -using KernelTypes = - ::testing::Types, std::tuple>; -#else -using KernelTypes = ::testing::Types>; -#endif - -TYPED_TEST_SUITE(TestAvgPool2dFwd, KernelTypes); -TYPED_TEST(TestAvgPool2dFwd, Test_Pool) -{ - // length, window_length, window_stride, left_pad, right_pad - this->params = {{{1, 1, 1, 1}, {1, 1}, {1, 1}, {0, 0}, {0, 0}}, - {{2, 16, 64, 64}, {64, 64}, {1, 1}, {0, 0}, {0, 0}}, - {{2, 32, 30, 30}, {2, 2}, {2, 2}, {1, 1}, {1, 1}}}; - - this->Run(); -} diff --git a/test/pool_fwd/test_avg_pool3d_fwd.cpp b/test/pool_fwd/test_avg_pool3d_fwd.cpp index a99a3c698e..81849ffa6a 100644 --- a/test/pool_fwd/test_avg_pool3d_fwd.cpp +++ b/test/pool_fwd/test_avg_pool3d_fwd.cpp @@ -25,6 +25,8 @@ class TestAvgPool3dFwd : public ::testing::Test OutDataType, ComputeDataType, IndexDataType, + ck::tensor_layout::convolution::NDHWC, + ck::tensor_layout::convolution::NDHWC, ck::ReduceTensorOp::AVG, false, false>(true, @@ -34,6 +36,7 @@ class TestAvgPool3dFwd : public ::testing::Test param.length_, param.window_spatial_lengths_, param.window_strides_, + param.window_dilations_, param.input_left_pads_, param.input_right_pads_); EXPECT_TRUE(success); @@ -49,10 +52,11 @@ using KernelTypes = ::testing::Types>; TYPED_TEST_SUITE(TestAvgPool3dFwd, KernelTypes); TYPED_TEST(TestAvgPool3dFwd, Test_Pool) { - // length, window_length, window_stride, left_pad, right_pad - this->params = {{{1, 1, 1, 1, 1}, {1, 1, 1}, {1, 1, 1}, {0, 0, 0}, {0, 0, 0}}, - {{2, 16, 64, 64, 64}, {64, 64, 64}, {1, 1, 1}, {0, 0, 0}, {0, 0, 0}}, - {{2, 32, 30, 30, 30}, {2, 2, 2}, {2, 2, 2}, {1, 1, 1}, {1, 1, 1}}}; + // length, window_length, window_stride, window_dilation, left_pad, right_pad + this->params = {{{1, 1, 1, 1, 1}, {1, 1, 1}, {1, 1, 1}, {1, 1, 1}, {0, 0, 0}, {0, 0, 0}}, + {{2, 16, 64, 64, 64}, {64, 64, 64}, {1, 1, 1}, {1, 1, 1}, {0, 0, 0}, {0, 0, 0}}, + {{2, 16, 64, 64, 64}, {4, 4, 4}, {4, 4, 4}, {2, 2, 2}, {0, 0, 0}, {0, 0, 0}}, + {{2, 32, 30, 30, 30}, {2, 2, 2}, {2, 2, 2}, {1, 1, 1}, {1, 1, 1}, {1, 1, 1}}}; this->Run(); } diff --git a/test/pool_fwd/test_max_pool2d_fwd.cpp b/test/pool_fwd/test_max_pool2d_fwd.cpp deleted file mode 100644 index c9050848d2..0000000000 --- a/test/pool_fwd/test_max_pool2d_fwd.cpp +++ /dev/null @@ -1,77 +0,0 @@ -// SPDX-License-Identifier: MIT -// Copyright (c) 2018-2023, Advanced Micro Devices, Inc. All rights reserved. - -#include "gtest/gtest.h" -#include "profiler/profile_pool2d_fwd_impl.hpp" -#include "test_pool_fwd_common.hpp" - -template -class TestMaxPool2dFwd : public ::testing::Test -{ - protected: - using InDataType = std::tuple_element_t<0, Tuple>; - using OutDataType = std::tuple_element_t<1, Tuple>; - using ComputeDataType = std::tuple_element_t<2, Tuple>; - using IndexDataType = std::tuple_element_t<3, Tuple>; - - std::vector params; - - void Run() - { - for(auto param : params) - { - // max pool - bool success = - ck::profiler::profile_pool2d_fwd_impl(true, - 2, - false, - false, - param.length_, - param.window_spatial_lengths_, - param.window_strides_, - param.input_left_pads_, - param.input_right_pads_); - EXPECT_TRUE(success); - - // max pool + index - success = ck::profiler::profile_pool2d_fwd_impl(true, - 2, - false, - false, - param.length_, - param.window_spatial_lengths_, - param.window_strides_, - param.input_left_pads_, - param.input_right_pads_); - EXPECT_TRUE(success); - } - } -}; -#ifdef __fp16__ -using KernelTypes = - ::testing::Types, std::tuple>; -#else -using KernelTypes = ::testing::Types>; -#endif -TYPED_TEST_SUITE(TestMaxPool2dFwd, KernelTypes); -TYPED_TEST(TestMaxPool2dFwd, Test_Pool) -{ - // length, window_length, window_stride, left_pad, right_pad - this->params = {{{1, 1, 1, 1}, {1, 1}, {1, 1}, {0, 0}, {0, 0}}, - {{2, 16, 64, 64}, {64, 64}, {1, 1}, {0, 0}, {0, 0}}, - {{2, 32, 30, 30}, {2, 2}, {2, 2}, {1, 1}, {1, 1}}}; - - this->Run(); -} diff --git a/test/pool_fwd/test_max_pool3d_fwd.cpp b/test/pool_fwd/test_max_pool3d_fwd.cpp index 7c0aada473..cd79c1ad8e 100644 --- a/test/pool_fwd/test_max_pool3d_fwd.cpp +++ b/test/pool_fwd/test_max_pool3d_fwd.cpp @@ -26,6 +26,8 @@ class TestMaxPool3dFwd : public ::testing::Test OutDataType, ComputeDataType, IndexDataType, + ck::tensor_layout::convolution::NDHWC, + ck::tensor_layout::convolution::NDHWC, ck::ReduceTensorOp::MAX, false, false>(true, @@ -35,6 +37,7 @@ class TestMaxPool3dFwd : public ::testing::Test param.length_, param.window_spatial_lengths_, param.window_strides_, + param.window_dilations_, param.input_left_pads_, param.input_right_pads_); EXPECT_TRUE(success); @@ -44,6 +47,8 @@ class TestMaxPool3dFwd : public ::testing::Test OutDataType, ComputeDataType, IndexDataType, + ck::tensor_layout::convolution::NDHWC, + ck::tensor_layout::convolution::NDHWC, ck::ReduceTensorOp::MAX, false, true>(true, @@ -53,6 +58,7 @@ class TestMaxPool3dFwd : public ::testing::Test param.length_, param.window_spatial_lengths_, param.window_strides_, + param.window_dilations_, param.input_left_pads_, param.input_right_pads_); EXPECT_TRUE(success); @@ -70,10 +76,11 @@ using KernelTypes = ::testing::Types>; TYPED_TEST_SUITE(TestMaxPool3dFwd, KernelTypes); TYPED_TEST(TestMaxPool3dFwd, Test_Pool) { - // length, window_length, window_stride, left_pad, right_pad - this->params = {{{1, 1, 1, 1, 1}, {1, 1, 1}, {1, 1, 1}, {0, 0, 0}, {0, 0, 0}}, - {{2, 16, 64, 64, 64}, {64, 64, 64}, {1, 1, 1}, {0, 0, 0}, {0, 0, 0}}, - {{2, 32, 30, 30, 30}, {2, 2, 2}, {2, 2, 2}, {1, 1, 1}, {1, 1, 1}}}; + // length, window_length, window_stride, window_dilation, left_pad, right_pad + this->params = {{{1, 1, 1, 1, 1}, {1, 1, 1}, {1, 1, 1}, {1, 1, 1}, {0, 0, 0}, {0, 0, 0}}, + {{2, 16, 64, 64, 64}, {64, 64, 64}, {1, 1, 1}, {1, 1, 1}, {0, 0, 0}, {0, 0, 0}}, + {{2, 16, 64, 64, 64}, {4, 4, 4}, {4, 4, 4}, {2, 2, 2}, {0, 0, 0}, {0, 0, 0}}, + {{2, 32, 30, 30, 30}, {2, 2, 2}, {2, 2, 2}, {1, 1, 1}, {1, 1, 1}, {1, 1, 1}}}; this->Run(); } diff --git a/test/pool_fwd/test_pool_fwd_common.hpp b/test/pool_fwd/test_pool_fwd_common.hpp index f018635170..6bfcb47bad 100644 --- a/test/pool_fwd/test_pool_fwd_common.hpp +++ b/test/pool_fwd/test_pool_fwd_common.hpp @@ -14,11 +14,13 @@ struct PoolingParam PoolingParam(const std::vector& length, const std::vector& window_spatial_lengths, const std::vector& window_strides, + const std::vector& window_dilations, const std::vector& input_left_pads, const std::vector& input_right_pads) : length_(length), window_spatial_lengths_(window_spatial_lengths), window_strides_(window_strides), + window_dilations_(window_dilations), input_left_pads_(input_left_pads), input_right_pads_(input_right_pads) { @@ -26,6 +28,7 @@ struct PoolingParam std::vector length_; std::vector window_spatial_lengths_; std::vector window_strides_; + std::vector window_dilations_; std::vector input_left_pads_; std::vector input_right_pads_; };