From 7b6451b68efada41ede3e1effb2c0aa9cf1ca314 Mon Sep 17 00:00:00 2001 From: Yashvardhan Agarwal Date: Thu, 9 Oct 2025 17:13:26 +0300 Subject: [PATCH 001/262] [CK_TILE] Pooling FWD (Lwpck 3683) (#2956) * Pooling 2D/3D with refernce * Tests & cleanup - added test for ppoling - cleanup - removed 2d example * Comment resolution - README added - example target name rectified - appropriate arg description and comments added * clang-format * appropriate blocksize calc * modifications for future indexing addition - instead of transforming views we now transform the descriptors, so that the same descriptor can be re-used for index tensor in the future * some basic fixes * comment resolutions * comment resolutions --------- Co-authored-by: Illia Silin <98187287+illsilin@users.noreply.github.com> --- CHANGELOG.md | 1 + example/ck_tile/36_pooling/CMakeLists.txt | 8 + example/ck_tile/36_pooling/README.md | 42 ++ example/ck_tile/36_pooling/pool3d.cpp | 188 +++++++ example/ck_tile/CMakeLists.txt | 1 + .../ck_tile/host/reference/reference_pool.hpp | 147 ++++++ include/ck_tile/ops/pool.hpp | 11 + .../ops/pooling/kernel/pool_kernel.hpp | 496 ++++++++++++++++++ .../pooling/pipeline/pool_default_policy.hpp | 80 +++ .../ops/pooling/pipeline/pool_problem.hpp | 33 ++ .../ops/pooling/pipeline/pool_shape.hpp | 57 ++ test/ck_tile/CMakeLists.txt | 1 + test/ck_tile/pooling/CMakeLists.txt | 3 + test/ck_tile/pooling/test_pooling.cpp | 249 +++++++++ 14 files changed, 1317 insertions(+) create mode 100644 example/ck_tile/36_pooling/CMakeLists.txt create mode 100644 example/ck_tile/36_pooling/README.md create mode 100644 example/ck_tile/36_pooling/pool3d.cpp create mode 100644 include/ck_tile/host/reference/reference_pool.hpp create mode 100644 include/ck_tile/ops/pool.hpp create mode 100644 include/ck_tile/ops/pooling/kernel/pool_kernel.hpp create mode 100644 include/ck_tile/ops/pooling/pipeline/pool_default_policy.hpp create mode 100644 include/ck_tile/ops/pooling/pipeline/pool_problem.hpp create mode 100644 include/ck_tile/ops/pooling/pipeline/pool_shape.hpp create mode 100644 test/ck_tile/pooling/CMakeLists.txt create mode 100644 test/ck_tile/pooling/test_pooling.cpp diff --git a/CHANGELOG.md b/CHANGELOG.md index 9aadc3dc54..a8fe7b4afb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,7 @@ Documentation for Composable Kernel available at [https://rocm.docs.amd.com/proj * Added the row-wise column-wise quantization for CK_TILE GEMM & CK_TILE Grouped GEMM. * Added support for f32 to FMHA (fwd/bwd). * Added tensor-wise quantization for CK_TILE GEMM. +* Added pooling kernel in CK_TILE ### Optimized diff --git a/example/ck_tile/36_pooling/CMakeLists.txt b/example/ck_tile/36_pooling/CMakeLists.txt new file mode 100644 index 0000000000..425a8c83ba --- /dev/null +++ b/example/ck_tile/36_pooling/CMakeLists.txt @@ -0,0 +1,8 @@ +set(EXAMPLE_POOL_3D "tile_example_pool3d") +message(DEBUG "adding example ${EXAMPLE_POOL_3D}") + +add_executable(${EXAMPLE_POOL_3D} EXCLUDE_FROM_ALL pool3d.cpp) +target_include_directories(${EXAMPLE_POOL_3D} PRIVATE ${CMAKE_CURRENT_LIST_DIR}) + +target_compile_options(${EXAMPLE_POOL_3D} PRIVATE ${EXAMPLE_POOL_COMPILE_OPTIONS}) + diff --git a/example/ck_tile/36_pooling/README.md b/example/ck_tile/36_pooling/README.md new file mode 100644 index 0000000000..ab49b57095 --- /dev/null +++ b/example/ck_tile/36_pooling/README.md @@ -0,0 +1,42 @@ +# Pooling Operator + +This folder contains example for the pooling operator using ck_tile tile-programming implementation. Currently the pooling kernel only supports 2D and 3D pooling. + +## build +``` +# in the root of ck_tile +mkdir build && cd build +# you can replace with the appropriate architecture (for example gfx90a or gfx942) or leave it blank +../script/cmake-ck-dev.sh ../ +# The 3D pooling example +make tile_example_pool3d -j`nproc` +``` +This will result in an executable `build/bin/tile_example_pool3d` + +## example +``` +args: + -N batch size (default:2) + -D depth dimension (default:30) + -H height dimension (default:30) + -W width dimension (default:30) + -C channel dimension (default:32) + -Z pooling window depth (default:2) + -Y pooling window height (default:2) + -X pooling window width (default:2) + -Sz window stride depth (default:2) + -Sy window stride height (default:2) + -Sx window stride width (default:2) + -Dz window dilation depth (default:1) + -Dy window dilation height (default:1) + -Dx window dilation width (default:1) + -LeftPz left padding depth (default:1) + -LeftPy left padding height (default:1) + -LeftPx left padding width (default:1) + -RightPz right padding depth (default:1) + -RightPy right padding height (default:1) + -RightPx right padding width (default:1) + -v 0: No validation, 1: CPU validation (default:1) + -warmup number of iterations before benchmark (default:0) + -repeat number of iterations to benchmark (default:1) +``` diff --git a/example/ck_tile/36_pooling/pool3d.cpp b/example/ck_tile/36_pooling/pool3d.cpp new file mode 100644 index 0000000000..bdfa1d99b3 --- /dev/null +++ b/example/ck_tile/36_pooling/pool3d.cpp @@ -0,0 +1,188 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2025, Advanced Micro Devices, Inc. All rights reserved. + +#include "ck_tile/host.hpp" +#include "ck_tile/ops/pool.hpp" +#include "ck_tile/host/reference/reference_pool.hpp" +#include + +// Parse command-line arguments for 3D pooling example +auto create_args(int argc, char* argv[]) +{ + ck_tile::ArgParser arg_parser; + arg_parser.insert("N", "2", "N dimension") + .insert("H", "30", "H dimension") + .insert("W", "30", "W dimension") + .insert("C", "32", "C dimension") + .insert("D", "30", "D dimension") + .insert("Z", "2", "Z dimension") + .insert("Y", "2", "Y dimension") + .insert("X", "2", "X dimension") + .insert("Sz", "2", "window stride d") + .insert("Sy", "2", "window stride h") + .insert("Sx", "2", "window stride w") + .insert("Dz", "1", "window dilation d") + .insert("Dy", "1", "window dilation h") + .insert("Dx", "1", "window dilation w") + .insert("LeftPz", "1", "left padding d") + .insert("LeftPy", "1", "left padding h") + .insert("LeftPx", "1", "left padding w") + .insert("RightPz", "1", "right padding d") + .insert("RightPy", "1", "right padding h") + .insert("RightPx", "1", "right padding w") + .insert("v", "1", "cpu validation or not") + .insert("warmup", "0", "cold iter") + .insert("repeat", "1", "hot iter"); + + bool result = arg_parser.parse(argc, argv); + return std::make_tuple(result, arg_parser); +} + +template +bool run(const ck_tile::ArgParser& arg_parser) +{ + + const ck_tile::index_t N = arg_parser.get_int("N"); + const ck_tile::index_t H = arg_parser.get_int("H"); + const ck_tile::index_t W = arg_parser.get_int("W"); + const ck_tile::index_t C = arg_parser.get_int("C"); + const ck_tile::index_t D = arg_parser.get_int("D"); + + const ck_tile::index_t Z = arg_parser.get_int("Z"); + const ck_tile::index_t Y = arg_parser.get_int("Y"); + const ck_tile::index_t X = arg_parser.get_int("X"); + + const ck_tile::index_t Sz = arg_parser.get_int("Sz"); + const ck_tile::index_t Sy = arg_parser.get_int("Sy"); + const ck_tile::index_t Sx = arg_parser.get_int("Sx"); + + const ck_tile::index_t Dz = arg_parser.get_int("Dz"); + const ck_tile::index_t Dy = arg_parser.get_int("Dy"); + const ck_tile::index_t Dx = arg_parser.get_int("Dx"); + + const ck_tile::index_t LeftPz = arg_parser.get_int("LeftPz"); + const ck_tile::index_t LeftPy = arg_parser.get_int("LeftPy"); + const ck_tile::index_t LeftPx = arg_parser.get_int("LeftPx"); + const ck_tile::index_t RightPz = arg_parser.get_int("RightPz"); + const ck_tile::index_t RightPy = arg_parser.get_int("RightPy"); + const ck_tile::index_t RightPx = arg_parser.get_int("RightPx"); + + const ck_tile::index_t Zs = (Z - 1) * Dz + 1; + const ck_tile::index_t Ys = (Y - 1) * Dy + 1; + const ck_tile::index_t Xs = (X - 1) * Dx + 1; + + const ck_tile::index_t Do = (D + LeftPz + RightPz - Zs) / Sz + 1; + const ck_tile::index_t Ho = (H + LeftPy + RightPy - Ys) / Sy + 1; + const ck_tile::index_t Wo = (W + LeftPx + RightPx - Xs) / Sx + 1; + + printf("Input parameters:\n"); + printf("N: %d, D: %d, H: %d, W: %d, C: %d\n", N, D, H, W, C); + printf("Window Z: %d, Y: %d, X: %d, Stride Z: %d, Y: %d, X: %d\n", Z, Y, X, Sz, Sy, Sx); + printf("Output Do: %d, Ho: %d, Wo: %d\n", Do, Ho, Wo); + + int do_validation = arg_parser.get_int("v"); + int warmup = arg_parser.get_int("warmup"); + int repeat = arg_parser.get_int("repeat"); + + // Shapes / strides / parameters (NDHWC) + const auto input_shape = ck_tile::make_tuple(N, D, H, W, C); + const auto output_shape = ck_tile::make_tuple(N, Do, Ho, Wo, C); + const auto input_strides = ck_tile::make_tuple(D * H * W * C, H * W * C, W * C, C, 1); + const auto output_strides = ck_tile::make_tuple(Do * Ho * Wo * C, Ho * Wo * C, Wo * C, C, 1); + const auto window_spatial_lengths = ck_tile::make_tuple(Z, Y, X); + const auto window_strides = ck_tile::make_tuple(Sz, Sy, Sx); + const auto window_dilations = ck_tile::make_tuple(Dz, Dy, Dx); + const auto input_left_pads = ck_tile::make_tuple(LeftPz, LeftPy, LeftPx); + const auto input_right_pads = ck_tile::make_tuple(RightPz, RightPy, RightPx); + + ck_tile::HostTensor in({N, D, H, W, C}, {D * H * W * C, H * W * C, W * C, C, 1}); + ck_tile::HostTensor out({N, Do, Ho, Wo, C}, + {Do * Ho * Wo * C, Ho * Wo * C, Wo * C, C, 1}); + ck_tile::HostTensor out_ref({N, Do, Ho, Wo, C}, + {Do * Ho * Wo * C, Ho * Wo * C, Wo * C, C, 1}); + + ck_tile::FillUniformDistribution{-5.f, 5.f}(in); + + ck_tile::DeviceMem in_buf(in.get_element_space_size_in_bytes()); + ck_tile::DeviceMem out_buf(out.get_element_space_size_in_bytes()); + + in_buf.ToDevice(in.data()); + + using ReduceOp = ck_tile::ReduceOp::Max; + using BlockWarps = ck_tile::sequence<4, 1>; + using BlockTile = ck_tile::sequence<128, 128>; + using WarpTile = ck_tile::sequence<32, 128>; + using ThreadTile = ck_tile::sequence<8, 8>; + + using Shape = ck_tile::PoolShape; + using Problem = ck_tile::PoolProblem; + using Kernel = ck_tile::PoolKernel; + + constexpr ck_tile::index_t kBlockPerCu = 1; + const ck_tile::index_t kBlockSize = Kernel::BlockSize(); + + auto host_args = ck_tile::PoolHostArgs{ + static_cast(in_buf.GetDeviceBuffer()), + static_cast(out_buf.GetDeviceBuffer()), + input_shape, + output_shape, + input_strides, + output_strides, + window_spatial_lengths, + window_strides, + window_dilations, + input_left_pads, + input_right_pads}; + + auto kernel_args = Kernel::MakeKernelArgs(host_args); + + const ck_tile::index_t kGridSize = Kernel::CalculateGridSize(kernel_args); + std::cout << "grid size " << kGridSize << std::endl; + + // Validate kernel can handle the given configuration + if(!Kernel::IsSupportedArgument(kernel_args)) + { + throw std::runtime_error("ERROR: Kernel arguments are not supported! \n"); + } + + float ave_time = launch_kernel( + ck_tile::stream_config{nullptr, true, 0, warmup, repeat}, + ck_tile::make_kernel(Kernel{}, kGridSize, kBlockSize, 0, kernel_args)); + + std::size_t num_btype = + sizeof(InDataType) * N * D * H * W * C + sizeof(OutDataType) * N * Do * Ho * Wo * C; + + float gb_per_sec = num_btype / 1.E6 / ave_time; + + std::cout << "Perf: " << ave_time << " ms, " << gb_per_sec << " GB/s" << std::endl; + + bool pass = true; + + if(do_validation) + { + ck_tile::reference_pool3d( + in, out_ref, kernel_args, ReduceOp{}); + out_buf.FromDevice(out.mData.data()); + pass = ck_tile::check_err(out, out_ref); + + std::cout << "valid:" << (pass ? "y" : "n") << std::flush << std::endl; + } + + return pass; +} + +int main(int argc, char* argv[]) +{ + auto [result, arg_parser] = create_args(argc, argv); + if(!result) + return -1; + + return run(arg_parser) ? 0 : -2; +} diff --git a/example/ck_tile/CMakeLists.txt b/example/ck_tile/CMakeLists.txt index 931f2d315f..7a8ae065db 100644 --- a/example/ck_tile/CMakeLists.txt +++ b/example/ck_tile/CMakeLists.txt @@ -23,6 +23,7 @@ add_subdirectory(20_grouped_convolution) add_subdirectory(21_elementwise) add_subdirectory(22_gemm_multi_abd) add_subdirectory(35_batched_transpose) +add_subdirectory(36_pooling) add_subdirectory(38_block_scale_gemm) add_subdirectory(39_copy) add_subdirectory(40_streamk_gemm) diff --git a/include/ck_tile/host/reference/reference_pool.hpp b/include/ck_tile/host/reference/reference_pool.hpp new file mode 100644 index 0000000000..1b3e45bce8 --- /dev/null +++ b/include/ck_tile/host/reference/reference_pool.hpp @@ -0,0 +1,147 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2025, Advanced Micro Devices, Inc. All rights reserved. + +#pragma once + +#include "ck_tile/core.hpp" +#include "ck_tile/host/host_tensor.hpp" +#include + +namespace ck_tile { + +template +CK_TILE_HOST void reference_pool2d(const HostTensor& input, + HostTensor& output, + PoolKernelArgs kargs, + ReduceOp reduce_op) +{ + const ck_tile::index_t N = kargs.input_shape.at(ck_tile::number<0>{}); + const ck_tile::index_t H = kargs.input_shape.at(ck_tile::number<1>{}); + const ck_tile::index_t W = kargs.input_shape.at(ck_tile::number<2>{}); + const ck_tile::index_t C = kargs.input_shape.at(ck_tile::number<3>{}); + + const ck_tile::index_t Ho = kargs.output_shape.at(ck_tile::number<1>{}); + const ck_tile::index_t Wo = kargs.output_shape.at(ck_tile::number<2>{}); + + const ck_tile::index_t Y = kargs.window_lengths.at(ck_tile::number<0>{}); + const ck_tile::index_t X = kargs.window_lengths.at(ck_tile::number<1>{}); + + const ck_tile::index_t Sy = kargs.window_strides.at(ck_tile::number<0>{}); + const ck_tile::index_t Sx = kargs.window_strides.at(ck_tile::number<1>{}); + + const ck_tile::index_t Dy = kargs.window_dilations.at(ck_tile::number<0>{}); + const ck_tile::index_t Dx = kargs.window_dilations.at(ck_tile::number<1>{}); + + const ck_tile::index_t LeftPy = kargs.input_left_pads.at(ck_tile::number<0>{}); + const ck_tile::index_t LeftPx = kargs.input_left_pads.at(ck_tile::number<1>{}); + // Right padding is handled implicitly by bounds checking + + auto f = [&](auto n, auto ho, auto wo, auto c) { + ComputeDataType v_acc = reduce_op.template GetIdentityValue(); + + for(ck_tile::index_t y = 0; y < Y; ++y) + { + // Calculate input height index with stride, dilation, and padding + ck_tile::index_t hi = ho * Sy + y * Dy - LeftPy; + + for(ck_tile::index_t x = 0; x < X; ++x) + { + // Calculate input width index with stride, dilation, and padding + ck_tile::index_t wi = wo * Sx + x * Dx - LeftPx; + + if(hi >= 0 && hi < H && wi >= 0 && wi < W) + { + const ComputeDataType v_in = type_convert(input(n, hi, wi, c)); + v_acc = reduce_op(v_acc, v_in); + } + // For positions outside bounds, we implicitly use identity value + } + } + + output(n, ho, wo, c) = ck_tile::type_convert(v_acc); + }; + + // Parallelize over all output dimensions + make_ParallelTensorFunctor(f, N, Ho, Wo, C)(std::thread::hardware_concurrency()); +} + +template +CK_TILE_HOST void reference_pool3d(const HostTensor& input, + HostTensor& output, + PoolKernelArgs kargs, + ReduceOp reduce_op) +{ + const ck_tile::index_t N = kargs.input_shape.at(ck_tile::number<0>{}); + const ck_tile::index_t D = kargs.input_shape.at(ck_tile::number<1>{}); + const ck_tile::index_t H = kargs.input_shape.at(ck_tile::number<2>{}); + const ck_tile::index_t W = kargs.input_shape.at(ck_tile::number<3>{}); + const ck_tile::index_t C = kargs.input_shape.at(ck_tile::number<4>{}); + + const ck_tile::index_t Do = kargs.output_shape.at(ck_tile::number<1>{}); + const ck_tile::index_t Ho = kargs.output_shape.at(ck_tile::number<2>{}); + const ck_tile::index_t Wo = kargs.output_shape.at(ck_tile::number<3>{}); + + const ck_tile::index_t Z = kargs.window_lengths.at(ck_tile::number<0>{}); + const ck_tile::index_t Y = kargs.window_lengths.at(ck_tile::number<1>{}); + const ck_tile::index_t X = kargs.window_lengths.at(ck_tile::number<2>{}); + + const ck_tile::index_t Sz = kargs.window_strides.at(ck_tile::number<0>{}); + const ck_tile::index_t Sy = kargs.window_strides.at(ck_tile::number<1>{}); + const ck_tile::index_t Sx = kargs.window_strides.at(ck_tile::number<2>{}); + + const ck_tile::index_t Dz = kargs.window_dilations.at(ck_tile::number<0>{}); + const ck_tile::index_t Dy = kargs.window_dilations.at(ck_tile::number<1>{}); + const ck_tile::index_t Dx = kargs.window_dilations.at(ck_tile::number<2>{}); + + const ck_tile::index_t LeftPz = kargs.input_left_pads.at(ck_tile::number<0>{}); + const ck_tile::index_t LeftPy = kargs.input_left_pads.at(ck_tile::number<1>{}); + const ck_tile::index_t LeftPx = kargs.input_left_pads.at(ck_tile::number<2>{}); + // Right padding is handled implicitly by bounds checking + + auto f = [&](auto n, auto do_, auto ho, auto wo, auto c) { + ComputeDataType v_acc = reduce_op.template GetIdentityValue(); + + for(ck_tile::index_t z = 0; z < Z; ++z) + { + // Calculate input depth index with stride, dilation, and padding + ck_tile::index_t di = do_ * Sz + z * Dz - LeftPz; + + for(ck_tile::index_t y = 0; y < Y; ++y) + { + // Calculate input height index with stride, dilation, and padding + ck_tile::index_t hi = ho * Sy + y * Dy - LeftPy; + + for(ck_tile::index_t x = 0; x < X; ++x) + { + // Calculate input width index with stride, dilation, and padding + ck_tile::index_t wi = wo * Sx + x * Dx - LeftPx; + + if(di >= 0 && di < D && hi >= 0 && hi < H && wi >= 0 && wi < W) + { + const ComputeDataType v_in = + type_convert(input(n, di, hi, wi, c)); + v_acc = reduce_op(v_acc, v_in); + } + // For positions outside bounds, we implicitly use identity value + } + } + } + + output(n, do_, ho, wo, c) = ck_tile::type_convert(v_acc); + }; + + // Parallelize over all output dimensions + make_ParallelTensorFunctor(f, N, Do, Ho, Wo, C)(std::thread::hardware_concurrency()); +} + +} // namespace ck_tile diff --git a/include/ck_tile/ops/pool.hpp b/include/ck_tile/ops/pool.hpp new file mode 100644 index 0000000000..350ef17dcb --- /dev/null +++ b/include/ck_tile/ops/pool.hpp @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2025, Advanced Micro Devices, Inc. All rights reserved. + +#pragma once + +#include "ck_tile/ops/pooling/kernel/pool_kernel.hpp" +#include "ck_tile/ops/pooling/pipeline/pool_problem.hpp" +#include "ck_tile/ops/pooling/pipeline/pool_shape.hpp" +#include "ck_tile/ops/common/generic_2d_block_shape.hpp" +#include "ck_tile/ops/common/tensor_layout.hpp" +#include "ck_tile/ops/common/utils.hpp" diff --git a/include/ck_tile/ops/pooling/kernel/pool_kernel.hpp b/include/ck_tile/ops/pooling/kernel/pool_kernel.hpp new file mode 100644 index 0000000000..93567e7161 --- /dev/null +++ b/include/ck_tile/ops/pooling/kernel/pool_kernel.hpp @@ -0,0 +1,496 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2025, Advanced Micro Devices, Inc. All rights reserved. + +#pragma once + +#include "ck_tile/core.hpp" +#include "ck_tile/ops/pooling/pipeline/pool_default_policy.hpp" +#include "ck_tile/ops/common.hpp" +#include + +namespace ck_tile { + +/// @brief Host arguments for pooling operations +template +struct PoolHostArgs +{ + + CK_TILE_HOST PoolHostArgs(const void* input_ptr_, + void* output_ptr_, + TensorShape input_shape_, + TensorShape output_shape_, + TensorShape input_strides_, + TensorShape output_strides_, + WindowShape window_lengths_, + WindowShape window_strides_, + WindowShape window_dilations_, + WindowShape input_left_pads_, + WindowShape input_right_pads_) + : input_ptr(input_ptr_), + output_ptr(output_ptr_), + input_shape(input_shape_), + output_shape(output_shape_), + input_strides(input_strides_), + output_strides(output_strides_), + window_lengths(window_lengths_), + window_strides(window_strides_), + window_dilations(window_dilations_), + input_left_pads(input_left_pads_), + input_right_pads(input_right_pads_) + { + } + + const void* input_ptr; + void* output_ptr; + + TensorShape input_shape; + TensorShape output_shape; + TensorShape input_strides; + TensorShape output_strides; + WindowShape window_lengths; + WindowShape window_strides; + WindowShape window_dilations; + WindowShape input_left_pads; + WindowShape input_right_pads; +}; + +/// @brief Kernel arguments for pooling operations +template +struct PoolKernelArgs +{ + const void* input_ptr; + void* output_ptr; + TensorShape input_shape; + TensorShape output_shape; + TensorShape input_strides; + TensorShape output_strides; + WindowShape window_lengths; + WindowShape window_strides; + WindowShape window_dilations; + WindowShape input_left_pads; + WindowShape input_right_pads; +}; + +template +struct PoolKernel +{ + using Problem = ck_tile::remove_cvref_t; + using Policy = ck_tile::remove_cvref_t; + + using InDataType = ck_tile::remove_cvref_t; + using ComputeDataType = ck_tile::remove_cvref_t; + using OutDataType = ck_tile::remove_cvref_t; + + static constexpr index_t kBlockSize = Problem::BlockShape::BlockSize; + + CK_TILE_HOST static constexpr auto BlockSize() + { + return is_wave32() ? kBlockSize / 2 : kBlockSize; + } + + template + static CK_TILE_DEVICE auto MakeTensorView2D(PoolKernelArgs kargs) + { + using S = typename Problem::BlockShape; + + // Compile-time validation for 2D pooling + static_assert(TensorShape::size() == 4, "2D pooling requires 4D input tensor (N,H,W,C)"); + static_assert(WindowShape::size() == 2, "2D pooling requires 2D window shape (Y,X)"); + + // Extract dimension values + const index_t N = kargs.input_shape.at(number<0>{}); + const index_t H = kargs.input_shape.at(number<1>{}); + const index_t W = kargs.input_shape.at(number<2>{}); + const index_t C = kargs.input_shape.at(number<3>{}); + + const index_t No = kargs.output_shape.at(number<0>{}); + const index_t Ho = kargs.output_shape.at(number<1>{}); + const index_t Wo = kargs.output_shape.at(number<2>{}); + const index_t Co = kargs.output_shape.at(number<3>{}); + + const index_t Y = kargs.window_lengths.at(number<0>{}); + const index_t X = kargs.window_lengths.at(number<1>{}); + + const index_t WindowStrideH = kargs.window_strides.at(number<0>{}); + const index_t WindowStrideW = kargs.window_strides.at(number<1>{}); + + const index_t WindowDilationH = kargs.window_dilations.at(number<0>{}); + const index_t WindowDilationW = kargs.window_dilations.at(number<1>{}); + + const index_t InLeftPadH = kargs.input_left_pads.at(number<0>{}); + const index_t InLeftPadW = kargs.input_left_pads.at(number<1>{}); + + const index_t InRightPadH = kargs.input_right_pads.at(number<0>{}); + const index_t InRightPadW = kargs.input_right_pads.at(number<1>{}); + + const index_t MRaw = N * Ho * Wo * C; + const index_t KRaw = Y * X; + const index_t MPad = integer_least_multiple(MRaw, S::Block_M) - MRaw; + const index_t KPad = integer_least_multiple(KRaw, S::Block_N) - KRaw; + + auto reduce_op = typename Problem::ReduceOp{}; + + // Create input descriptor with all transformations + auto in_desc = make_naive_tensor_descriptor(kargs.input_shape, kargs.input_strides); + + // Apply spatial padding to input descriptor + const auto padded_in_desc = transform_tensor_descriptor( + in_desc, + make_tuple(make_pass_through_transform(N), + make_pad_transform(H, InLeftPadH, InRightPadH), + make_pad_transform(W, InLeftPadW, InRightPadW), + make_pass_through_transform(C)), + make_tuple(sequence<0>{}, sequence<1>{}, sequence<2>{}, sequence<3>{}), + make_tuple(sequence<0>{}, sequence<1>{}, sequence<2>{}, sequence<3>{})); + + // Create sliding windows by embedding pooling windows into descriptor + const auto embed_in_desc = transform_tensor_descriptor( + padded_in_desc, + make_tuple( + make_pass_through_transform(N), + make_embed_transform(make_tuple(Y, Ho), make_tuple(WindowDilationH, WindowStrideH)), + make_embed_transform(make_tuple(X, Wo), make_tuple(WindowDilationW, WindowStrideW)), + make_pass_through_transform(C)), + make_tuple(sequence<0>{}, sequence<1>{}, sequence<2>{}, sequence<3>{}), + make_tuple(sequence<0>{}, sequence<1, 2>{}, sequence<3, 4>{}, sequence<5>{})); + + // Reshape into 2D matrix: output positions (M) x pooling window elements (K) + const auto merged_embed_in_desc = + transform_tensor_descriptor(embed_in_desc, + make_tuple(make_merge_transform(make_tuple(N, Ho, Wo, C)), + make_merge_transform(make_tuple(Y, X))), + make_tuple(sequence<0, 2, 4, 5>{}, sequence<1, 3>{}), + make_tuple(sequence<0>{}, sequence<1>{})); + + const auto in_desc_padded = transform_tensor_descriptor( + merged_embed_in_desc, + make_tuple(make_right_pad_transform(MRaw, MPad), make_right_pad_transform(KRaw, KPad)), + make_tuple(sequence<0>{}, sequence<1>{}), + make_tuple(sequence<0>{}, sequence<1>{})); + + // Create output descriptor with transformations + auto out_desc = make_naive_tensor_descriptor(kargs.output_shape, kargs.output_strides); + + const auto merged_out_desc = transform_tensor_descriptor( + out_desc, + make_tuple(make_merge_transform(make_tuple(No, Ho, Wo, Co))), + make_tuple(sequence<0, 1, 2, 3>{}), + make_tuple(sequence<0>{})); + + const auto out_desc_padded = + transform_tensor_descriptor(merged_out_desc, + make_tuple(make_right_pad_transform(MRaw, MPad)), + make_tuple(sequence<0>{}), + make_tuple(sequence<0>{})); + + // Now create buffer views and tensor views with the fully transformed descriptors + const InDataType in_identity = + type_convert(reduce_op.template GetIdentityValue()); + const OutDataType out_identity = + type_convert(reduce_op.template GetIdentityValue()); + + auto in_buffer_view = make_buffer_view( + static_cast(kargs.input_ptr), + in_desc.get_element_space_size(), + in_identity); + const auto in_tensor_padded = + tensor_view{in_buffer_view, + in_desc_padded}; + + auto out_buffer_view = make_buffer_view( + static_cast(kargs.output_ptr), + out_desc.get_element_space_size(), + out_identity); + const auto out_tensor_padded = + tensor_view{out_buffer_view, + out_desc_padded}; + + return make_tuple(in_tensor_padded, out_tensor_padded); + } + + template + static CK_TILE_DEVICE auto MakeTensorView3D(PoolKernelArgs kargs) + { + using S = typename Problem::BlockShape; + + // Compile-time validation for 3D pooling + static_assert(TensorShape::size() == 5, "3D pooling requires 5D input tensor (N,D,H,W,C)"); + static_assert(WindowShape::size() == 3, "3D pooling requires 3D window shape (Z,Y,X)"); + + // Extract dimension values + const index_t N = kargs.input_shape.at(number<0>{}); + const index_t D = kargs.input_shape.at(number<1>{}); + const index_t H = kargs.input_shape.at(number<2>{}); + const index_t W = kargs.input_shape.at(number<3>{}); + const index_t C = kargs.input_shape.at(number<4>{}); + + const index_t No = kargs.output_shape.at(number<0>{}); + const index_t Do = kargs.output_shape.at(number<1>{}); + const index_t Ho = kargs.output_shape.at(number<2>{}); + const index_t Wo = kargs.output_shape.at(number<3>{}); + const index_t Co = kargs.output_shape.at(number<4>{}); + + const index_t Z = kargs.window_lengths.at(number<0>{}); + const index_t Y = kargs.window_lengths.at(number<1>{}); + const index_t X = kargs.window_lengths.at(number<2>{}); + + const index_t WindowStrideD = kargs.window_strides.at(number<0>{}); + const index_t WindowStrideH = kargs.window_strides.at(number<1>{}); + const index_t WindowStrideW = kargs.window_strides.at(number<2>{}); + + const index_t WindowDilationD = kargs.window_dilations.at(number<0>{}); + const index_t WindowDilationH = kargs.window_dilations.at(number<1>{}); + const index_t WindowDilationW = kargs.window_dilations.at(number<2>{}); + + const index_t InLeftPadD = kargs.input_left_pads.at(number<0>{}); + const index_t InLeftPadH = kargs.input_left_pads.at(number<1>{}); + const index_t InLeftPadW = kargs.input_left_pads.at(number<2>{}); + + const index_t InRightPadD = kargs.input_right_pads.at(number<0>{}); + const index_t InRightPadH = kargs.input_right_pads.at(number<1>{}); + const index_t InRightPadW = kargs.input_right_pads.at(number<2>{}); + + const index_t MRaw = N * Do * Ho * Wo * C; + const index_t KRaw = Z * Y * X; + const index_t MPad = integer_least_multiple(MRaw, S::Block_M) - MRaw; + const index_t KPad = integer_least_multiple(KRaw, S::Block_N) - KRaw; + + auto reduce_op = typename Problem::ReduceOp{}; + + // Create input descriptor with all transformations + auto in_desc = make_naive_tensor_descriptor(kargs.input_shape, kargs.input_strides); + + // Apply spatial padding to input descriptor (all 3D dimensions) + const auto padded_in_desc = transform_tensor_descriptor( + in_desc, + make_tuple(make_pass_through_transform(N), + make_pad_transform(D, InLeftPadD, InRightPadD), + make_pad_transform(H, InLeftPadH, InRightPadH), + make_pad_transform(W, InLeftPadW, InRightPadW), + make_pass_through_transform(C)), + make_tuple(sequence<0>{}, sequence<1>{}, sequence<2>{}, sequence<3>{}, sequence<4>{}), + make_tuple(sequence<0>{}, sequence<1>{}, sequence<2>{}, sequence<3>{}, sequence<4>{})); + + // Create 3D sliding windows by embedding pooling windows into descriptor + const auto embed_in_desc = transform_tensor_descriptor( + padded_in_desc, + make_tuple( + make_pass_through_transform(N), + make_embed_transform(make_tuple(Z, Do), make_tuple(WindowDilationD, WindowStrideD)), + make_embed_transform(make_tuple(Y, Ho), make_tuple(WindowDilationH, WindowStrideH)), + make_embed_transform(make_tuple(X, Wo), make_tuple(WindowDilationW, WindowStrideW)), + make_pass_through_transform(C)), + make_tuple(sequence<0>{}, sequence<1>{}, sequence<2>{}, sequence<3>{}, sequence<4>{}), + make_tuple(sequence<0>{}, + sequence<1, 2>{}, + sequence<3, 4>{}, + sequence<5, 6>{}, + sequence<7>{})); + + // Reshape into 2D matrix: output positions (M) x pooling window elements (K) + const auto merged_embed_in_desc = transform_tensor_descriptor( + embed_in_desc, + make_tuple(make_merge_transform(make_tuple(N, Do, Ho, Wo, C)), + make_merge_transform(make_tuple(Z, Y, X))), + make_tuple(sequence<0, 2, 4, 6, 7>{}, sequence<1, 3, 5>{}), + make_tuple(sequence<0>{}, sequence<1>{})); + + const auto in_desc_padded = transform_tensor_descriptor( + merged_embed_in_desc, + make_tuple(make_right_pad_transform(MRaw, MPad), make_right_pad_transform(KRaw, KPad)), + make_tuple(sequence<0>{}, sequence<1>{}), + make_tuple(sequence<0>{}, sequence<1>{})); + + // Create output descriptor with transformations + auto out_desc = make_naive_tensor_descriptor(kargs.output_shape, kargs.output_strides); + + const auto merged_out_desc = transform_tensor_descriptor( + out_desc, + make_tuple(make_merge_transform(make_tuple(No, Do, Ho, Wo, Co))), + make_tuple(sequence<0, 1, 2, 3, 4>{}), + make_tuple(sequence<0>{})); + + const auto out_desc_padded = + transform_tensor_descriptor(merged_out_desc, + make_tuple(make_right_pad_transform(MRaw, MPad)), + make_tuple(sequence<0>{}), + make_tuple(sequence<0>{})); + + // Now create buffer views and tensor views with the fully transformed descriptors + const InDataType in_identity = + type_convert(reduce_op.template GetIdentityValue()); + const OutDataType out_identity = + type_convert(reduce_op.template GetIdentityValue()); + + auto in_buffer_view = make_buffer_view( + static_cast(kargs.input_ptr), + in_desc.get_element_space_size(), + in_identity); + const auto in_tensor_padded = + tensor_view{in_buffer_view, + in_desc_padded}; + + auto out_buffer_view = make_buffer_view( + static_cast(kargs.output_ptr), + out_desc.get_element_space_size(), + out_identity); + const auto out_tensor_padded = + tensor_view{out_buffer_view, + out_desc_padded}; + + return make_tuple(in_tensor_padded, out_tensor_padded); + } + + public: + template + CK_TILE_DEVICE void operator()(PoolKernelArgs kargs) const + { + using S = typename Problem::BlockShape; + + // Compile-time validation for supported window dimensions + static_assert(WindowShape::size() == 2 || WindowShape::size() == 3, + "Only 2D and 3D pooling operations are supported"); + + const auto iM = get_block_id() * S::Block_M; + + // Get tensors based on dimensionality + auto [in_tensor_padded, out_tensor_padded] = [&]() { + if constexpr(WindowShape::size() == 2) + return MakeTensorView2D(kargs); + else if constexpr(WindowShape::size() == 3) + return MakeTensorView3D(kargs); + else + static_assert(WindowShape::size() == 2 || WindowShape::size() == 3, + "Unsupported WindowShape rank: only 2D or 3D pooling is supported"); + }(); + + auto reduce_op = typename Problem::ReduceOp{}; + + auto x_window = make_tile_window(in_tensor_padded, + make_tuple(number{}, number{}), + {iM, 0}, + Policy::template MakeXBlockTileDistribution()); + auto y_window = make_tile_window(out_tensor_padded, make_tuple(number{}), {iM}); + + __shared__ char smem[Policy::template GetSmemSize()]; + + const auto reduce_len = + in_tensor_padded.get_tensor_descriptor().get_lengths().at(number<1>{}); + index_t num_k_tiles = + __builtin_amdgcn_readfirstlane(integer_divide_ceil(reduce_len, S::Block_N)); + + auto block_reduce2d = Policy::template GetBlockReduce2d(); + auto block_reduce2d_sync = Policy::template GetBlockReduce2dSync(); + auto block_reduce2d_cross_warp = Policy::template GetBlockReduce2dCrossWarpSync(); + + using XTensorTile = decltype(load_tile(x_window)); + auto y_tile = block_reduce2d.template MakeYBlockTile(); + set_tile(y_tile, reduce_op.template GetIdentityValue()); + + for(int k_tile = __builtin_amdgcn_readfirstlane(0); k_tile < num_k_tiles; ++k_tile) + { + const auto x_tile = load_tile(x_window); + block_reduce2d(x_tile, y_tile, reduce_op); + move_tile_window(x_window, {0, S::Block_N}); + } + + block_reduce2d_sync(y_tile, reduce_op); + block_reduce2d_cross_warp(y_tile, smem, reduce_op); + store_tile(y_window, cast_tile(y_tile)); + } + + /// @brief Validates if the given arguments are supported by the pooling kernel. + /// + /// @param kargs The pooling kernel arguments containing all necessary parameters. + /// + /// @return true if the arguments are supported, false otherwise. + /// + /// @note Requirements: + /// - Last dimension (C) must be contiguous (stride = 1) for vectorized access + /// - Window dimensions must be supported (2D or 3D) + /// - All dimension sizes must be consistent between input and output + template + CK_TILE_HOST static bool IsSupportedArgument(PoolKernelArgs kargs) + { + constexpr index_t InputRank = TensorShape::size(); + constexpr index_t OutputRank = TensorShape::size(); // Same as input rank + constexpr index_t WindowRank = WindowShape::size(); + + // Validate window dimensions (only 2D and 3D supported) + if constexpr(WindowRank != 2 && WindowRank != 3) + { + if(ck_tile::EnvIsEnabled(CK_TILE_ENV(CK_TILE_LOGGING))) + { + CK_TILE_ERROR("Only 2D and 3D pooling are supported!"); + } + return false; + } + + // Validate that input rank matches expected rank for window dimensions + if constexpr((WindowRank == 2 && InputRank != 4) || (WindowRank == 3 && InputRank != 5)) + { + if(ck_tile::EnvIsEnabled(CK_TILE_ENV(CK_TILE_LOGGING))) + { + CK_TILE_ERROR("Input tensor rank doesn't match window dimensions!"); + } + return false; + } + + // Check that channel dimension (last dimension) is contiguous for both input and output + if(kargs.input_strides.at(number{}) != 1) + { + if(ck_tile::EnvIsEnabled(CK_TILE_ENV(CK_TILE_LOGGING))) + { + CK_TILE_ERROR("Input tensor's channel dimension must have stride 1!"); + } + return false; + } + + if(kargs.output_strides.at(number{}) != 1) + { + if(ck_tile::EnvIsEnabled(CK_TILE_ENV(CK_TILE_LOGGING))) + { + CK_TILE_ERROR("Output tensor's channel dimension must have stride 1!"); + } + return false; + } + + return true; + } + + /// @param kargs The pooling kernel arguments + /// @return The calculated grid size + template + CK_TILE_HOST static constexpr index_t + CalculateGridSize(PoolKernelArgs kargs) + { + using S = typename Problem::BlockShape; + + // Calculate total output elements (M dimension) + index_t M = 1; + static_for<0, TensorShape::size(), 1>{}([&](auto i) { M *= kargs.output_shape.at(i); }); + + // Calculate grid size: ceil(M / Block_M) + return (M + S::Block_M - 1) / S::Block_M; + } + + /// @brief Create kernel arguments from host arguments + template + CK_TILE_HOST static constexpr auto + MakeKernelArgs(PoolHostArgs& host_args) + { + return PoolKernelArgs{host_args.input_ptr, + host_args.output_ptr, + host_args.input_shape, + host_args.output_shape, + host_args.input_strides, + host_args.output_strides, + host_args.window_lengths, + host_args.window_strides, + host_args.window_dilations, + host_args.input_left_pads, + host_args.input_right_pads}; + } +}; + +} // namespace ck_tile diff --git a/include/ck_tile/ops/pooling/pipeline/pool_default_policy.hpp b/include/ck_tile/ops/pooling/pipeline/pool_default_policy.hpp new file mode 100644 index 0000000000..a5b5fac63d --- /dev/null +++ b/include/ck_tile/ops/pooling/pipeline/pool_default_policy.hpp @@ -0,0 +1,80 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2025, Advanced Micro Devices, Inc. All rights reserved. + +#pragma once + +#include "ck_tile/core.hpp" +#include "ck_tile/ops/reduce/block/block_reduce2d_problem.hpp" +#include "ck_tile/ops/reduce/block/block_reduce2d.hpp" + +namespace ck_tile { + +struct PoolDefaultPolicy +{ + template + CK_TILE_DEVICE static constexpr auto MakeXBlockTileDistribution() + { + using S = typename Problem::BlockShape; + return make_static_tile_distribution( + tile_distribution_encoding< + sequence<>, + tuple< + sequence, + sequence>, + tuple, sequence<1, 2>>, + tuple, sequence<2, 2>>, + sequence<1, 1, 2, 2>, + sequence<0, 3, 0, 3>>{}); + } + + template + CK_TILE_HOST_DEVICE static constexpr auto GetBlockReduce2d() + { + using P_ = BlockReduce2dProblem; + return BlockReduce2d{}; + } + + template + CK_TILE_HOST_DEVICE static constexpr auto GetBlockReduce2dSync() + { + using P_ = BlockReduce2dProblem; + return BlockReduce2dSync{}; + } + + template + CK_TILE_HOST_DEVICE static constexpr auto GetBlockReduce2dCrossWarpSync() + { + using P_ = BlockReduce2dProblem; + return BlockReduce2dCrossWarpSync{}; + } + + template + CK_TILE_HOST_DEVICE static constexpr index_t GetSmemSize() + { + if constexpr(Problem::kNeedCrossWarpSync) + { + using P_ = BlockReduce2dProblem; + + using block_reduce2d = BlockReduce2d; + using x_block_tile = + decltype(make_static_distributed_tensor( + MakeXBlockTileDistribution())); + using y_block_tile = decltype(block_reduce2d::template MakeYBlockTile()); + + return GetBlockReduce2dCrossWarpSync().template GetSmemSize(); + } + else + { + return 1; // zero size arrays are an extension + } + } +}; +} // namespace ck_tile diff --git a/include/ck_tile/ops/pooling/pipeline/pool_problem.hpp b/include/ck_tile/ops/pooling/pipeline/pool_problem.hpp new file mode 100644 index 0000000000..83a43318bc --- /dev/null +++ b/include/ck_tile/ops/pooling/pipeline/pool_problem.hpp @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2025, Advanced Micro Devices, Inc. All rights reserved. + +#pragma once + +#include "ck_tile/core.hpp" + +namespace ck_tile { + +template +struct PoolProblem +{ + using InDataType = remove_cvref_t; + using OutDataType = remove_cvref_t; + using ComputeDataType = remove_cvref_t; + using IndexDataType = remove_cvref_t; + using BlockShape = remove_cvref_t; + using ReduceOp = ReduceOp_; + using OutputIndex = bool_constant; + using PropagateNan = bool_constant; + + static constexpr bool kNeedCrossLaneSync = BlockShape::ThreadPerWarp_N > 1; + static constexpr bool kNeedCrossWarpSync = BlockShape::WarpPerBlock_N > 1; +}; + +} // namespace ck_tile diff --git a/include/ck_tile/ops/pooling/pipeline/pool_shape.hpp b/include/ck_tile/ops/pooling/pipeline/pool_shape.hpp new file mode 100644 index 0000000000..5879fe593e --- /dev/null +++ b/include/ck_tile/ops/pooling/pipeline/pool_shape.hpp @@ -0,0 +1,57 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2025, Advanced Micro Devices, Inc. All rights reserved. + +#pragma once + +#include "ck_tile/core.hpp" + +namespace ck_tile { + +template + typename BlockTile, // block size, seq + typename WarpTile, // warp size, seq + typename ThreadTile> // contiguous pixels(vector size) along seq +struct PoolShape +{ + static constexpr index_t Block_M = BlockTile::at(number<0>{}); + static constexpr index_t Block_N = BlockTile::at(number<1>{}); + + static constexpr index_t Warp_M = WarpTile::at(number<0>{}); + static constexpr index_t Warp_N = WarpTile::at(number<1>{}); + + static constexpr index_t ThreadTile_M = ThreadTile::at(number<0>{}); + static constexpr index_t ThreadTile_N = ThreadTile::at(number<1>{}); + + static constexpr index_t WarpPerBlock_M = BlockWarps::at(number<0>{}); + static constexpr index_t WarpPerBlock_N = BlockWarps::at(number<1>{}); + + static_assert(Warp_M % ThreadTile_M == 0, "Warp_M must be divisible by ThreadTile_M"); + static_assert(Warp_N % ThreadTile_N == 0, "Warp_N must be divisible by ThreadTile_N"); + static_assert((Warp_M * Warp_N / ThreadTile_M / ThreadTile_N) % ck_tile::get_warp_size() == 0, + "Warp_M * Warp_N / ThreadTile_M / ThreadTile_N must be a multiple of warp size"); + + // Scale factor to account for warp size + // WarpSizeScaleFactor = warp tile/ thread tile / warp size + static constexpr index_t WarpSizeScaleFactor = + Warp_M * Warp_N / ThreadTile_M / ThreadTile_N / ck_tile::get_warp_size(); + + static constexpr index_t WarpSizeScaleFactor_M = + (Warp_M / ThreadTile_M > Warp_N / ThreadTile_N) ? WarpSizeScaleFactor : 1; + static constexpr index_t WarpSizeScaleFactor_N = + (Warp_M / ThreadTile_M > Warp_N / ThreadTile_N) ? 1 : WarpSizeScaleFactor; + + static constexpr index_t ThreadPerWarp_M = Warp_M / ThreadTile_M / WarpSizeScaleFactor_M; + static constexpr index_t ThreadPerWarp_N = Warp_N / ThreadTile_N / WarpSizeScaleFactor_N; + + static_assert((Block_M * WarpSizeScaleFactor_M) % (WarpPerBlock_M * Warp_M) == 0, + "Block_M * WarpSizeScaleFactor_M must be divisible by WarpPerBlock_M * Warp_M"); + static_assert((Block_N * WarpSizeScaleFactor_N) % (WarpPerBlock_N * Warp_N) == 0, + "Block_N * WarpSizeScaleFactor_N must be divisible by WarpPerBlock_N * Warp_N"); + + static constexpr index_t Repeat_M = Block_M * WarpSizeScaleFactor_M / (WarpPerBlock_M * Warp_M); + static constexpr index_t Repeat_N = Block_N * WarpSizeScaleFactor_N / (WarpPerBlock_N * Warp_N); + + static constexpr index_t BlockSize = + ck_tile::get_warp_size() * reduce_on_sequence(BlockWarps{}, multiplies{}, number<1>{}); +}; +} // namespace ck_tile diff --git a/test/ck_tile/CMakeLists.txt b/test/ck_tile/CMakeLists.txt index 04be25f30a..5fa6918c10 100644 --- a/test/ck_tile/CMakeLists.txt +++ b/test/ck_tile/CMakeLists.txt @@ -31,3 +31,4 @@ add_subdirectory(epilogue) add_subdirectory(atomic_add_op) add_subdirectory(fmha) add_subdirectory(gemm_tile_engine) +add_subdirectory(pooling) diff --git a/test/ck_tile/pooling/CMakeLists.txt b/test/ck_tile/pooling/CMakeLists.txt new file mode 100644 index 0000000000..83c36cb321 --- /dev/null +++ b/test/ck_tile/pooling/CMakeLists.txt @@ -0,0 +1,3 @@ +if(GPU_TARGETS MATCHES "gfx9|gfx11|gfx12") + add_gtest_executable(test_ck_tile_pooling test_pooling.cpp) +endif() diff --git a/test/ck_tile/pooling/test_pooling.cpp b/test/ck_tile/pooling/test_pooling.cpp new file mode 100644 index 0000000000..3cec19d2d6 --- /dev/null +++ b/test/ck_tile/pooling/test_pooling.cpp @@ -0,0 +1,249 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2025, Advanced Micro Devices, Inc. All rights reserved. + +#include +#include +#include +#include +#include + +#include "ck_tile/core.hpp" +#include "ck_tile/host.hpp" +#include "ck_tile/ops/pool.hpp" +#include "ck_tile/host/reference/reference_pool.hpp" +#include "ck_tile/host/kernel_launch.hpp" + +template +class TestCkTilePooling : public ::testing::Test +{ + protected: + using InDataType = std::tuple_element_t<0, Tuple>; + using OutDataType = std::tuple_element_t<1, Tuple>; + using ComputeDataType = std::tuple_element_t<2, Tuple>; + using ReduceOpType = std::tuple_element_t<3, Tuple>; + using BlockWarps_ = std::tuple_element_t<4, Tuple>; + using BlockTile_ = std::tuple_element_t<5, Tuple>; + using WarpTile_ = std::tuple_element_t<6, Tuple>; + using ThreadTile_ = std::tuple_element_t<7, Tuple>; + + using TestPoolShape = ck_tile::PoolShape; + + // 3D pooling configuration + struct Config3D + { + ck_tile::index_t N, D, H, W, C; + ck_tile::index_t Z, Y, X; + ck_tile::index_t Sz, Sy, Sx; + ck_tile::index_t Dz, Dy, Dx; + ck_tile::index_t LeftPz, LeftPy, LeftPx; + ck_tile::index_t RightPz, RightPy, RightPx; + std::string name; + }; + + bool RunPool3D(const Config3D& config) + { + std::cout << "Testing 3D: " << config.name << " ... "; + + const ck_tile::index_t Zs = (config.Z - 1) * config.Dz + 1; + const ck_tile::index_t Ys = (config.Y - 1) * config.Dy + 1; + const ck_tile::index_t Xs = (config.X - 1) * config.Dx + 1; + const ck_tile::index_t Do = + (config.D + config.LeftPz + config.RightPz - Zs) / config.Sz + 1; + const ck_tile::index_t Ho = + (config.H + config.LeftPy + config.RightPy - Ys) / config.Sy + 1; + const ck_tile::index_t Wo = + (config.W + config.LeftPx + config.RightPx - Xs) / config.Sx + 1; + + const auto input_shape = + ck_tile::make_tuple(config.N, config.D, config.H, config.W, config.C); + const auto output_shape = ck_tile::make_tuple(config.N, Do, Ho, Wo, config.C); + const auto input_strides = ck_tile::make_tuple(config.D * config.H * config.W * config.C, + config.H * config.W * config.C, + config.W * config.C, + config.C, + 1); + const auto output_strides = ck_tile::make_tuple( + Do * Ho * Wo * config.C, Ho * Wo * config.C, Wo * config.C, config.C, 1); + const auto window_spatial_lengths = ck_tile::make_tuple(config.Z, config.Y, config.X); + const auto window_strides = ck_tile::make_tuple(config.Sz, config.Sy, config.Sx); + const auto window_dilations = ck_tile::make_tuple(config.Dz, config.Dy, config.Dx); + const auto input_left_pads = + ck_tile::make_tuple(config.LeftPz, config.LeftPy, config.LeftPx); + const auto input_right_pads = + ck_tile::make_tuple(config.RightPz, config.RightPy, config.RightPx); + + ck_tile::HostTensor h_in({config.N, config.D, config.H, config.W, config.C}, + {config.D * config.H * config.W * config.C, + config.H * config.W * config.C, + config.W * config.C, + config.C, + 1}); + ck_tile::HostTensor h_out( + {config.N, Do, Ho, Wo, config.C}, + {Do * Ho * Wo * config.C, Ho * Wo * config.C, Wo * config.C, config.C, 1}); + ck_tile::HostTensor h_out_ref( + {config.N, Do, Ho, Wo, config.C}, + {Do * Ho * Wo * config.C, Ho * Wo * config.C, Wo * config.C, config.C, 1}); + + ck_tile::FillUniformDistribution{-5.f, 5.f}(h_in); + h_out.SetZero(); + h_out_ref.SetZero(); + + ck_tile::DeviceMem d_in_mem(h_in.get_element_space_size_in_bytes()); + ck_tile::DeviceMem d_out_mem(h_out.get_element_space_size_in_bytes()); + + d_in_mem.ToDevice(h_in.data()); + d_out_mem.ToDevice(h_out.data()); + + using Problem = ck_tile::PoolProblem; + using Kernel = ck_tile::PoolKernel; + + constexpr ck_tile::index_t kBlockPerCu = 1; + const ck_tile::index_t kBlockSize = Kernel::BlockSize(); + + auto host_args = + ck_tile::PoolHostArgs{ + static_cast(d_in_mem.GetDeviceBuffer()), + static_cast(d_out_mem.GetDeviceBuffer()), + input_shape, + output_shape, + input_strides, + output_strides, + window_spatial_lengths, + window_strides, + window_dilations, + input_left_pads, + input_right_pads}; + + auto kernel_args = Kernel::MakeKernelArgs(host_args); + + const ck_tile::index_t kGridSize = Kernel::CalculateGridSize(kernel_args); + + if(!Kernel::IsSupportedArgument(kernel_args)) + { + return true; + } + + // Run kernel + ck_tile::launch_kernel( + ck_tile::stream_config{nullptr, false, 0}, + ck_tile::make_kernel(Kernel{}, kGridSize, kBlockSize, 0, kernel_args)); + + // Run reference implementation + ck_tile::reference_pool3d( + h_in, h_out_ref, kernel_args, ReduceOpType{}); + + d_out_mem.FromDevice(h_out.data()); + + // Validate results + bool pass = ck_tile::check_err(h_out, h_out_ref); + std::cout << (pass ? "PASS" : "FAIL") << std::endl; + + return pass; + } +}; + +using Shape1_BlockWarps = ck_tile::sequence<4, 1>; +using Shape1_BlockTile = ck_tile::sequence<128, 128>; +using Shape1_WarpTile = ck_tile::sequence<32, 128>; +using Shape1_ThreadTile = ck_tile::sequence<8, 8>; + +// Cross-warp configuration +using Shape2_BlockWarps = ck_tile::sequence<2, 2>; +using Shape2_BlockTile = ck_tile::sequence<2, 1024>; +using Shape2_WarpTile = ck_tile::sequence<1, 512>; +using Shape2_ThreadTile = ck_tile::sequence<1, 8>; + +// Test configurations for different data types and operations +using TestConfig_F32_Max = std::tuple; + +using TestConfig_F16_Max = std::tuple; + +using TestConfig_F32_CrossWarp = std::tuple; + +using TestTypes = + ::testing::Types; + +TYPED_TEST_SUITE(TestCkTilePooling, TestTypes); + +TYPED_TEST(TestCkTilePooling, Pool3D_2x2x2) +{ + typename TestFixture::Config3D config = {1, // N - batch size + 4, // D - depth dimension + 4, // H - height dimension + 4, // W - width dimension + 32, // C - channel dimension + 2, // Z - pooling window depth + 2, // Y - pooling window height + 2, // X - pooling window width + 2, // Sz - window stride depth + 2, // Sy - window stride height + 2, // Sx - window stride width + 1, // Dz - window dilation depth + 1, // Dy - window dilation height + 1, // Dx - window dilation width + 0, // LeftPz - left padding depth + 0, // LeftPy - left padding height + 0, // LeftPx - left padding width + 0, // RightPz - right padding depth + 0, // RightPy - right padding height + 0, // RightPx - right padding width + "2x2x2 pooling"}; + bool pass = this->RunPool3D(config); + EXPECT_TRUE(pass); +} + +TYPED_TEST(TestCkTilePooling, Pool3D_3x3x3) +{ + typename TestFixture::Config3D config = {2, // N - batch size + 16, // D - depth dimension + 16, // H - height dimension + 16, // W - width dimension + 128, // C - channel dimension + 3, // Z - pooling window depth + 3, // Y - pooling window height + 3, // X - pooling window width + 2, // Sz - window stride depth + 2, // Sy - window stride height + 2, // Sx - window stride width + 1, // Dz - window dilation depth + 1, // Dy - window dilation height + 1, // Dx - window dilation width + 1, // LeftPz - left padding depth + 1, // LeftPy - left padding height + 1, // LeftPx - left padding width + 1, // RightPz - right padding depth + 1, // RightPy - right padding height + 1, // RightPx - right padding width + "3x3x3 pooling"}; + bool pass = this->RunPool3D(config); + EXPECT_TRUE(pass); +} From fb66b4f5e4b5b178e3eee04189224e139e939c0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=84=8D=F0=9D=95=A0=F0=9D=95=9D=F0=9D=95=9D=F0=9D=95=A0?= =?UTF-8?q?=F0=9D=95=A8=20=F0=9D=95=84=F0=9D=95=92=F0=9D=95=9F?= Date: Thu, 9 Oct 2025 17:43:41 +0300 Subject: [PATCH 002/262] [CK_TILE] fix pk_fp4 compilation for non-gfx950 GPUs (#2983) See build error log from https://github.com/ROCm/composable_kernel/issues/2271#issuecomment-3150218542 This PR make vector element access constexpr-safe by avoiding operator[] on ext_vector_type(2) and replace those sites in the pk_fp4 conversions so they can be used in constant expressions, as The operator[] on ext_vector_type(2) isn't allowed in constant expressions, which caused "constexpr function never produces a constant expression" with a note at x[0]. Using `bit_cast` to a trivial array representation keeps it constexpr-compatible. Signed-off-by: Hollow Man --- include/ck_tile/core/numeric/pk_fp4.hpp | 72 ++++++++++++++++++++++--- 1 file changed, 65 insertions(+), 7 deletions(-) diff --git a/include/ck_tile/core/numeric/pk_fp4.hpp b/include/ck_tile/core/numeric/pk_fp4.hpp index 8b78990d08..4f662095db 100644 --- a/include/ck_tile/core/numeric/pk_fp4.hpp +++ b/include/ck_tile/core/numeric/pk_fp4.hpp @@ -23,6 +23,51 @@ using fp32x2_t = float __attribute__((ext_vector_type(2))); using fp16x2_t = _Float16 __attribute__((ext_vector_type(2))); using bf16x2_t = bfloat16_t __attribute__((ext_vector_type(2))); +// Helpers: constexpr-safe access to elements of ext_vector_type(2) +// Some compilers don't allow operator[] in constant expressions for vector types. +// We use bit_cast to a trivially copyable representation to extract lanes. +namespace detail { +struct fp16x2_repr +{ + _Float16 e[2]; +}; +struct bf16x2_repr +{ + bfloat16_t e[2]; +}; +struct fp32x2_repr +{ + float e[2]; +}; + +CK_TILE_HOST_DEVICE constexpr _Float16 lane0(const fp16x2_t& v) +{ + return ck_tile::bit_cast(v).e[0]; +} +CK_TILE_HOST_DEVICE constexpr _Float16 lane1(const fp16x2_t& v) +{ + return ck_tile::bit_cast(v).e[1]; +} + +CK_TILE_HOST_DEVICE constexpr bfloat16_t lane0(const bf16x2_t& v) +{ + return ck_tile::bit_cast(v).e[0]; +} +CK_TILE_HOST_DEVICE constexpr bfloat16_t lane1(const bf16x2_t& v) +{ + return ck_tile::bit_cast(v).e[1]; +} + +CK_TILE_HOST_DEVICE constexpr float lane0(const fp32x2_t& v) +{ + return ck_tile::bit_cast(v).e[0]; +} +CK_TILE_HOST_DEVICE constexpr float lane1(const fp32x2_t& v) +{ + return ck_tile::bit_cast(v).e[1]; +} +} // namespace detail + struct pk_float4_e2m1_t; CK_TILE_HOST_DEVICE constexpr pk_float4_e2m1_t float_to_pk_fp4(const float& x, float scale = 1.f); @@ -166,15 +211,24 @@ template CK_TILE_DEVICE T _from_f4(pk_fp4_raw_t src, float scale = 1.0f) { if constexpr(std::is_same_v) - return fp32x2_t(__builtin_amdgcn_cvt_scalef32_pk_f32_fp4(src, scale, 0))[0]; + { + fp32x2_t tmp = __builtin_amdgcn_cvt_scalef32_pk_f32_fp4(src, scale, 0); + return detail::lane0(tmp); + } else if constexpr(std::is_same_v) return __builtin_amdgcn_cvt_scalef32_pk_f32_fp4(src, scale, 0); else if constexpr(std::is_same_v) - return fp16x2_t(__builtin_amdgcn_cvt_scalef32_pk_f16_fp4(src, scale, 0))[0]; + { + fp16x2_t tmp = __builtin_amdgcn_cvt_scalef32_pk_f16_fp4(src, scale, 0); + return detail::lane0(tmp); + } else if constexpr(std::is_same_v) return __builtin_amdgcn_cvt_scalef32_pk_f16_fp4(src, scale, 0); else if constexpr(std::is_same_v) - return bf16x2_t(__builtin_amdgcn_cvt_scalef32_pk_bf16_fp4(src, scale, 0))[0]; + { + bf16x2_t tmp = __builtin_amdgcn_cvt_scalef32_pk_bf16_fp4(src, scale, 0); + return detail::lane0(tmp); + } else if constexpr(std::is_same_v) return __builtin_amdgcn_cvt_scalef32_pk_bf16_fp4(src, scale, 0); else @@ -192,7 +246,8 @@ CK_TILE_DEVICE pk_fp4_raw_t _to_f4(T src, float scale = 1.0f) if constexpr(std::is_same_v) cvt.u32 = __builtin_amdgcn_cvt_scalef32_pk_fp4_f32(cvt.u32, src, src, scale, 0); else if constexpr(std::is_same_v) - cvt.u32 = __builtin_amdgcn_cvt_scalef32_pk_fp4_f32(cvt.u32, src[0], src[1], scale, 0); + cvt.u32 = __builtin_amdgcn_cvt_scalef32_pk_fp4_f32( + cvt.u32, detail::lane0(src), detail::lane1(src), scale, 0); else if constexpr(std::is_same_v) cvt.u32 = __builtin_amdgcn_cvt_scalef32_pk_fp4_f16(cvt.u32, fp16x2_t{src, src}, scale, 0); else if constexpr(std::is_same_v) @@ -269,7 +324,8 @@ CK_TILE_HOST_DEVICE constexpr pk_fp4_t fp16x2_to_pk_fp4(const fp16x2_t& x, float #if CK_TILE_FP4_CVT_DEVICE return impl::_to_f4(x, scale); #else - return pk_fp4_t::_pack(float_to_mxfp4(x[0], scale), float_to_mxfp4(x[1], scale)); + return pk_fp4_t::_pack(float_to_mxfp4(detail::lane0(x), scale), + float_to_mxfp4(detail::lane1(x), scale)); #endif } CK_TILE_HOST_DEVICE constexpr pk_fp4_t bf16x2_to_pk_fp4(const bf16x2_t& x, float scale) @@ -277,7 +333,8 @@ CK_TILE_HOST_DEVICE constexpr pk_fp4_t bf16x2_to_pk_fp4(const bf16x2_t& x, float #if CK_TILE_FP4_CVT_DEVICE return impl::_to_f4(x, scale); #else - return pk_fp4_t::_pack(float_to_mxfp4(x[0], scale), float_to_mxfp4(x[1], scale)); + return pk_fp4_t::_pack(float_to_mxfp4(detail::lane0(x), scale), + float_to_mxfp4(detail::lane1(x), scale)); #endif } CK_TILE_HOST_DEVICE constexpr pk_fp4_t fp32x2_to_pk_fp4(const fp32x2_t& x, float scale) @@ -285,7 +342,8 @@ CK_TILE_HOST_DEVICE constexpr pk_fp4_t fp32x2_to_pk_fp4(const fp32x2_t& x, float #if CK_TILE_FP4_CVT_DEVICE return impl::_to_f4(x, scale); #else - return pk_fp4_t::_pack(float_to_mxfp4(x[0], scale), float_to_mxfp4(x[1], scale)); + return pk_fp4_t::_pack(float_to_mxfp4(detail::lane0(x), scale), + float_to_mxfp4(detail::lane1(x), scale)); #endif } From b6036bc76a5ce55ef85b7f8578ae81c990f5932d Mon Sep 17 00:00:00 2001 From: Yi DING Date: Fri, 10 Oct 2025 11:34:47 +0800 Subject: [PATCH 003/262] [CK_TILE] FMHA Tests Enhancement (#2945) * fmha-gtest-wip * Thanks Copilot! --- test/ck_tile/fmha/CMakeLists.txt | 67 ++-- test/ck_tile/fmha/test_fmha_bwd.cpp | 248 +++++++++++++ test/ck_tile/fmha/test_fmha_bwd.inc | 347 ------------------ test/ck_tile/fmha/test_fmha_bwd_bf16.cpp | 21 -- test/ck_tile/fmha/test_fmha_bwd_fp16.cpp | 21 -- test/ck_tile/fmha/test_fmha_bwd_fp32.cpp | 20 - .../{test_fmha_fwd.inc => test_fmha_fwd.cpp} | 112 +++++- test/ck_tile/fmha/test_fmha_fwd_bf16.cpp | 44 --- test/ck_tile/fmha/test_fmha_fwd_fp16.cpp | 44 --- test/ck_tile/fmha/test_fmha_fwd_fp32.cpp | 39 -- test/ck_tile/fmha/test_fmha_fwd_fp8.cpp | 42 --- 11 files changed, 391 insertions(+), 614 deletions(-) create mode 100644 test/ck_tile/fmha/test_fmha_bwd.cpp delete mode 100644 test/ck_tile/fmha/test_fmha_bwd.inc delete mode 100644 test/ck_tile/fmha/test_fmha_bwd_bf16.cpp delete mode 100644 test/ck_tile/fmha/test_fmha_bwd_fp16.cpp delete mode 100644 test/ck_tile/fmha/test_fmha_bwd_fp32.cpp rename test/ck_tile/fmha/{test_fmha_fwd.inc => test_fmha_fwd.cpp} (92%) delete mode 100644 test/ck_tile/fmha/test_fmha_fwd_bf16.cpp delete mode 100644 test/ck_tile/fmha/test_fmha_fwd_fp16.cpp delete mode 100644 test/ck_tile/fmha/test_fmha_fwd_fp32.cpp delete mode 100644 test/ck_tile/fmha/test_fmha_fwd_fp8.cpp diff --git a/test/ck_tile/fmha/CMakeLists.txt b/test/ck_tile/fmha/CMakeLists.txt index 8e5cce4c0b..ca7b7b6324 100644 --- a/test/ck_tile/fmha/CMakeLists.txt +++ b/test/ck_tile/fmha/CMakeLists.txt @@ -5,35 +5,50 @@ endif() set(FMHA_BWD_INSTANCES "tile_fmha_bwd_instances") set(FMHA_FWD_INSTANCES "tile_fmha_fwd_instances") +set(TEST_NAME "test_ck_tile_fmha") -add_gtest_executable(test_ck_tile_fmha_bwd_fp32 test_fmha_bwd_fp32.cpp) -target_link_libraries(test_ck_tile_fmha_bwd_fp32 PRIVATE ${FMHA_BWD_INSTANCES}) +function(add_gtest_fwd test_group) + set(V_TYPES "fp16" "bf16" "fp8" "fp32") + set(CPP_TYPE_fp16 "FmhaFwdFp16") + set(CPP_TYPE_bf16 "FmhaFwdBf16") + set(CPP_TYPE_fp8 "FmhaFwdFp8") + set(CPP_TYPE_fp32 "FmhaFwdFp32") -add_gtest_executable(test_ck_tile_fmha_bwd_bf16 test_fmha_bwd_bf16.cpp) -target_link_libraries(test_ck_tile_fmha_bwd_bf16 PRIVATE ${FMHA_BWD_INSTANCES}) + set(all_tests) + foreach(type ${V_TYPES}) + set(name "${test_group}_${type}") + add_gtest_executable(${name} test_fmha_fwd.cpp) + get_test_property(${name} LABELS COMMON_LABELS) + set_tests_properties(${name} PROPERTIES LABELS "${COMMON_LABELS};${TEST_NAME};${test_group}") + target_compile_definitions(${name} PRIVATE DataTypeConfig=${CPP_TYPE_${type}}) + target_link_libraries(${name} PRIVATE ${FMHA_FWD_INSTANCES}) + list(APPEND all_tests ${name}) + endforeach() + message(STATUS "FMHA FWD tests: ${all_tests}") + add_custom_target(${test_group} DEPENDS ${all_tests}) +endfunction() -add_gtest_executable(test_ck_tile_fmha_bwd_fp16 test_fmha_bwd_fp16.cpp) -target_link_libraries(test_ck_tile_fmha_bwd_fp16 PRIVATE ${FMHA_BWD_INSTANCES}) +function(add_gtest_bwd test_group) + set(V_TYPES "fp16" "bf16" "fp32") + set(CPP_TYPE_fp16 "FmhaBwdFp16") + set(CPP_TYPE_bf16 "FmhaBwdBf16") + set(CPP_TYPE_fp32 "FmhaBwdFp32") -add_gtest_executable(test_ck_tile_fmha_fwd_fp32 test_fmha_fwd_fp32.cpp) -target_link_libraries(test_ck_tile_fmha_fwd_fp32 PRIVATE ${FMHA_FWD_INSTANCES}) + set(all_tests) + foreach(type ${V_TYPES}) + set(name "${test_group}_${type}") + add_gtest_executable(${name} test_fmha_bwd.cpp) + get_test_property(${name} LABELS COMMON_LABELS) + set_tests_properties(${name} PROPERTIES LABELS "${COMMON_LABELS};${TEST_NAME};${test_group}") + target_compile_definitions(${name} PRIVATE DataTypeConfig=${CPP_TYPE_${type}}) + target_link_libraries(${name} PRIVATE ${FMHA_BWD_INSTANCES}) + list(APPEND all_tests ${name}) + endforeach() + message(STATUS "FMHA BWD tests: ${all_tests}") + add_custom_target(${test_group} DEPENDS ${all_tests}) +endfunction() -add_gtest_executable(test_ck_tile_fmha_fwd_bf16 test_fmha_fwd_bf16.cpp) -target_link_libraries(test_ck_tile_fmha_fwd_bf16 PRIVATE ${FMHA_FWD_INSTANCES}) -add_gtest_executable(test_ck_tile_fmha_fwd_fp16 test_fmha_fwd_fp16.cpp) -target_link_libraries(test_ck_tile_fmha_fwd_fp16 PRIVATE ${FMHA_FWD_INSTANCES}) - -add_gtest_executable(test_ck_tile_fmha_fwd_fp8 test_fmha_fwd_fp8.cpp) -target_link_libraries(test_ck_tile_fmha_fwd_fp8 PRIVATE ${FMHA_FWD_INSTANCES}) - -add_custom_target(test_ck_tile_fmha - DEPENDS - test_ck_tile_fmha_bwd_fp32 - test_ck_tile_fmha_bwd_bf16 - test_ck_tile_fmha_bwd_fp16 - test_ck_tile_fmha_fwd_fp32 - test_ck_tile_fmha_fwd_bf16 - test_ck_tile_fmha_fwd_fp16 - test_ck_tile_fmha_fwd_fp8 -) +add_gtest_fwd(${TEST_NAME}_fwd) +add_gtest_bwd(${TEST_NAME}_bwd) +add_custom_target(${TEST_NAME} DEPENDS ${TEST_NAME}_fwd ${TEST_NAME}_bwd) diff --git a/test/ck_tile/fmha/test_fmha_bwd.cpp b/test/ck_tile/fmha/test_fmha_bwd.cpp new file mode 100644 index 0000000000..190cdd6452 --- /dev/null +++ b/test/ck_tile/fmha/test_fmha_bwd.cpp @@ -0,0 +1,248 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2025, Advanced Micro Devices, Inc. All rights reserved. + +#include "example/ck_tile/01_fmha/fmha_bwd.hpp" +#include "example/ck_tile/01_fmha/fmha_bwd_runner.hpp" + +#include "gtest/gtest.h" + +#ifndef DataTypeConfig +#define DataTypeConfig FmhaBwdFp16 // or FmhaBwdBf16 / FmhaBwdFp32 +#endif + +using ::testing::Bool; +using ::testing::Combine; +using ::testing::TestWithParam; +using ::testing::Values; +using ::testing::ValuesIn; + +template +struct TestConfigs +{ + static constexpr auto HDimValues = std::array{ + std::tuple{32, -1}, std::tuple{64, -1}, std::tuple{128, -1}, std::tuple{256, -1}}; +}; +template <> +struct TestConfigs +{ + static constexpr auto HDimValues = + std::array{std::tuple{32, -1}, std::tuple{64, -1}, std::tuple{128, -1}}; +}; +static auto HDimValues = ValuesIn(TestConfigs::HDimValues); +const auto ModeValues = ValuesIn(std::vector{mode_enum::batch, mode_enum::group}); +constexpr auto init_method = "uf"; + +// Random seed used for initializing input tensors. 0 for non-deterministic seed +CK_TILE_DECLARE_ENV_VAR(CK_TILE_TEST_SEED, uint64_t, 123456) + +// Whether to run long tests (from smoke_test_fwd.sh) +CK_TILE_DECLARE_ENV_VAR_BOOL(CK_TILE_FMHA_LONG_TESTS) + +const ck_tile::stream_config stream_config{ + nullptr, // stream_id_ + false, // time_kernel_ + 1, // log_level_ + 0, // cold_niters_ + 1, // nrepeat_ + true, // is_gpu_timer_ + false, // flush_cache_ + 1, // rotating_count_ +}; + +// batch, nhead, nhead_k, seqlen_q, seqlen_k, mask_str +using FmhaBwdDimsMaskParam = std::tuple; +using FmhaBwdTestParam = std::tuple< // + mode_enum, // mode + std::tuple, // hdim_q, hdim_v + std::tuple, // io_perm + std::string, // bias_str + bool, // use_dbias + float, // p_drop + std::tuple, // drop_seed, drop_offset, drop_prefs + FmhaBwdDimsMaskParam, + bool // deterministic + >; +void fmha_bwd_test(const FmhaBwdTestParam& param) +{ + auto [mode, hdims, perm, bias_str, use_dbias, p_drop, drop_misc, dims_mask, det] = param; + auto [hdim_q, hdim_v] = hdims; + auto [i_perm, o_perm] = perm; + auto [drop_seed, drop_offset, drop_prefs] = drop_misc; + auto [batch, nhead, nhead_k, seqlen_q, seqlen_k, mask_str] = dims_mask; + + auto result = fmha_bwd_run( + mode, + batch, + nhead, + nhead_k, + {seqlen_q}, + {seqlen_k}, + hdim_q, + hdim_v, + i_perm, + o_perm, + 0, // scale + bias_str, + use_dbias, + p_drop, + drop_seed, + drop_offset, + drop_prefs, + mask_str, + det, // deterministic + init_method, + static_cast(ck_tile::EnvValue(CK_TILE_ENV(CK_TILE_TEST_SEED))), + 1, + stream_config); + + if(result == bwd_result::no_instance) + GTEST_SKIP() << "No instance for current parameters"; + ASSERT_EQ(result, bwd_result::success); +} + +// Test cases from example/ck_tile/01_fmha/script/smoke_test_bwd.sh +class AllLong : public TestWithParam +{ +}; +GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(AllLong); +INSTANTIATE_TEST_SUITE_P(TestCkTileFmhaBwd, + AllLong, + Combine(ck_tile::EnvIsEnabled(CK_TILE_ENV(CK_TILE_FMHA_LONG_TESTS)) + ? ModeValues + : ValuesIn(std::vector{}), + HDimValues, + Values(std::tuple{true, true}, std::tuple{false, false}), // perm + Values("n", "a"), + Values(false), // use_dbias + Values(0.0f, 0.2f), // p_drop + Values(std::tuple{123, 1024, true}), // seed/offset/prefs + Values(std::tuple{1, 4, 2, 259, -1, "0"}, + std::tuple{2, 2, -1, 516, 253, "0"}, + std::tuple{1, 4, 1, 500, 251, "1"}, + std::tuple{1, 2, -1, 900, 258, "2"}, + std::tuple{2, 1, -1, 987, 219, "t:128,30"}, + std::tuple{2, 3, 1, 244, 499, "b:4,35"}), + Values(false) // deterministic + )); +TEST_P(AllLong, DataTypeConfig) { fmha_bwd_test(GetParam()); } + +class HDimPadding : public TestWithParam +{ +}; +INSTANTIATE_TEST_SUITE_P(TestCkTileFmhaBwd, + HDimPadding, + Combine(ModeValues, + Values(std::tuple{24, 48}, + std::tuple{48, 48}, + std::tuple{72, 72}, + std::tuple{96, 96}, + std::tuple{120, 160}, + std::tuple{256, 108}, + std::tuple{40, 64}), + Values(std::tuple{true, true}, std::tuple{false, false}), // perm + Values("n"), // bias_str + Values(false), // use_dbias + Values(0.0f), // p_drop + Values(std::tuple{0, 0, false}), // seed/offset/prefs + Values(std::tuple{1, 4, 2, 480, -1, "0"}, + std::tuple{2, 2, -1, 300, 400, "t:64,64"}, + std::tuple{1, 4, 1, 512, 201, "1"}, + std::tuple{1, 2, -1, 900, 256, "0"}, + std::tuple{2, 1, -1, 256, 256, "1"}), + Values(false) // deterministic + )); +TEST_P(HDimPadding, DataTypeConfig) { fmha_bwd_test(GetParam()); } + +class ElementwiseBias : public TestWithParam +{ +}; +INSTANTIATE_TEST_SUITE_P(TestCkTileFmhaBwd, + ElementwiseBias, + Combine(ModeValues, + HDimValues, + // layouts of bias and dbias are controlled by i_perm + Values(std::tuple{true, false}, std::tuple{false, false}), + Values("e:0", "e:1", "e:2"), + Bool(), // use_dbias + Values(0.0f), // p_drop + Values(std::tuple{0, 0, false}), // seed/offset/prefs + Values(std::tuple{1, 4, 2, 1024, 100, "0"}, + std::tuple{3, 2, -1, 128, 256, "2"}, + std::tuple{2, 2, -1, 130, 499, "t:50,64"}), + Values(false) // deterministic + )); +TEST_P(ElementwiseBias, DataTypeConfig) { fmha_bwd_test(GetParam()); } +class Alibi : public TestWithParam +{ +}; + +INSTANTIATE_TEST_SUITE_P( + TestCkTileFmhaBwd, + Alibi, + Combine(ModeValues, + HDimValues, + Values(std::tuple{true, true}), // perm + Values("a:0", "a:1"), + Values(false), // use_dbias + Values(0.0f), // p_drop + Values(std::tuple{0, 0, false}), // seed/offset/prefs + ValuesIn([]() { + const std::array dims{ + std::tuple{1, 3, 3, 1024, 1000}, + std::tuple{3, 5, 5, 128, 256}, + std::tuple{2, 8, 4, 130, 320}, + }; + const std::array mask_strs{"0", "t", "b", "t:50,64", "b:32,40"}; + std::vector dims_masks; + std::for_each(dims.begin(), dims.end(), [&](const auto& d) { + const auto& [b, h, hk, sq, sk] = d; + std::for_each(mask_strs.begin(), mask_strs.end(), [&](const auto& m) { + dims_masks.push_back(std::tuple{b, h, hk, sq, sk, m}); + }); + }); + return dims_masks; + }()), + Values(false) // deterministic + )); +TEST_P(Alibi, DataTypeConfig) { fmha_bwd_test(GetParam()); } + +class Dropout : public TestWithParam +{ +}; +INSTANTIATE_TEST_SUITE_P(TestCkTileFmhaBwd, + Dropout, + Combine(ModeValues, + HDimValues, + Values(std::tuple{true, true}), // perm + Values("n"), // bias_str + Values(false), // use_dbias + Values(0.123f, 0.5f), // p_drop + Values(std::tuple{10, 123, false}, // seed/offset/prefs + std::tuple{34534564645, 7876878876864, true}), + Values(std::tuple{2, 6, 2, 180, 512, "0"}, + std::tuple{3, 2, 2, 256, 128, "1"}, + std::tuple{4, 2, 1, 100, 768, "2"}), + Values(false) // deterministic + )); + +TEST_P(Dropout, DataTypeConfig) { fmha_bwd_test(GetParam()); } + +class Deterministic : public TestWithParam +{ +}; + +INSTANTIATE_TEST_SUITE_P(TestCkTileFmhaBwd, + Deterministic, + Combine(ModeValues, + HDimValues, + Values(std::tuple{false, true}, std::tuple{true, true}), // perm + Values("n"), // bias_str + Values(false), // use_dbias + Values(0.0f), // p_drop + Values(std::tuple{0, 0, false}), // seed/offset/prefs + Values(std::tuple{2, 6, 2, 180, 512, "0"}, + std::tuple{3, 3, 1, 256, 128, "1"}, + std::tuple{4, 2, 2, 768, 100, "2"}), + Values(true) // deterministic + )); +TEST_P(Deterministic, DataTypeConfig) { fmha_bwd_test(GetParam()); } diff --git a/test/ck_tile/fmha/test_fmha_bwd.inc b/test/ck_tile/fmha/test_fmha_bwd.inc deleted file mode 100644 index 704b5c7bf7..0000000000 --- a/test/ck_tile/fmha/test_fmha_bwd.inc +++ /dev/null @@ -1,347 +0,0 @@ -// SPDX-License-Identifier: MIT -// Copyright (c) 2025, Advanced Micro Devices, Inc. All rights reserved. - -using ::testing::Bool; -using ::testing::Combine; -using ::testing::TestWithParam; -using ::testing::Values; -using ::testing::ValuesIn; - -// Random seed used for initializing input tensors. 0 for non-deterministic seed -CK_TILE_DECLARE_ENV_VAR(CK_TILE_TEST_SEED, uint64_t, 123456) - -// Whether to run long tests (from smoke_test_fwd.sh) -CK_TILE_DECLARE_ENV_VAR_BOOL(CK_TILE_FMHA_LONG_TESTS) - -#define CHECK_RESULT(result) \ - do \ - { \ - if(result == bwd_result::no_instance) \ - GTEST_SKIP() << "No instance for current parameters"; \ - ASSERT_EQ(result, bwd_result::success); \ - } while(0) - -const ck_tile::stream_config stream_config{ - nullptr, // stream_id_ - false, // time_kernel_ - 1, // log_level_ - 0, // cold_niters_ - 1, // nrepeat_ - true, // is_gpu_timer_ - false, // flush_cache_ - 1, // rotating_count_ -}; - -#define COMMON_ARGS \ - init_method, static_cast(ck_tile::EnvValue(CK_TILE_ENV(CK_TILE_TEST_SEED))), 1, \ - stream_config - -auto EnableTestIf(bool condition) -{ - return ValuesIn(condition ? std::vector{true} : std::vector{}); -} - -class AllLong : public TestWithParam, - bool, - mode_enum, - std::string, - float, - std::tuple>> -{ -}; - -GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(AllLong); - -// Test cases from example/ck_tile/01_fmha/script/smoke_test_bwd.sh - -INSTANTIATE_TEST_SUITE_P( - TestCkTileFmhaBwd, - AllLong, - Combine(EnableTestIf(ck_tile::EnvIsEnabled(CK_TILE_ENV(CK_TILE_FMHA_LONG_TESTS))), - HDimValues, - Bool(), - ModeValues, - Values("n", "a"), - Values(0.0f, 0.2f), - Values(std::tuple{1, 4, 2, 259, -1, "0"}, - std::tuple{2, 2, -1, 516, 253, "0"}, - std::tuple{1, 4, 1, 500, 251, "1"}, - std::tuple{1, 2, -1, 900, 258, "2"}, - std::tuple{2, 1, -1, 987, 219, "t:128,30"}, - std::tuple{2, 3, 1, 244, 499, "b:4,35"}))); - -TEST_P(AllLong, Test) -{ - auto [_, hdims, perm, mode, bias_str, p_drop, dims_mask] = GetParam(); - auto [hdim_q, hdim_v] = hdims; - auto [batch, nhead, nhead_k, seqlen_q, seqlen_k, mask_str] = dims_mask; - - auto result = fmha_bwd_run(mode, - batch, - nhead, - nhead_k, - {seqlen_q}, - {seqlen_k}, - hdim_q, - hdim_v, - perm, // i_perm - perm, // o_perm - 0, // scale - bias_str, // bias_str - false, // use_dbias - p_drop, // p_drop - 123, // drop_seed - 1024, // drop_offset - true, // drop_prefs - mask_str, // mask_str - false, // deterministic - COMMON_ARGS); - CHECK_RESULT(result); -} - -class HDimPadding - : public TestWithParam, - bool, - mode_enum, - std::tuple>> -{ -}; - -INSTANTIATE_TEST_SUITE_P(TestCkTileFmhaBwd, - HDimPadding, - Combine(Values(std::tuple{24, 48}, - std::tuple{48, 48}, - std::tuple{72, 72}, - std::tuple{96, 96}, - std::tuple{120, 160}, - std::tuple{256, 108}, - std::tuple{40, 64}), - Bool(), - ModeValues, - Values(std::tuple{1, 4, 2, 480, -1, "0"}, - std::tuple{2, 2, -1, 300, 400, "t:64,64"}, - std::tuple{1, 4, 1, 512, 201, "1"}, - std::tuple{1, 2, -1, 900, 256, "0"}, - std::tuple{2, 1, -1, 256, 256, "1"}))); - -TEST_P(HDimPadding, Test) -{ - auto [hdims, perm, mode, dims_mask] = GetParam(); - auto [hdim_q, hdim_v] = hdims; - auto [batch, nhead, nhead_k, seqlen_q, seqlen_k, mask_str] = dims_mask; - - auto result = fmha_bwd_run(mode, - batch, - nhead, - nhead_k, - {seqlen_q}, - {seqlen_k}, - hdim_q, - hdim_v, - perm, // i_perm - perm, // o_perm - 0, // scale - "n", // bias_str - false, // use_dbias - 0.0f, // p_drop - 0, // drop_seed - 0, // drop_offset - false, // drop_prefs - mask_str, // mask_str - false, // deterministic - COMMON_ARGS); - CHECK_RESULT(result); -} - -class ElementwiseBias - : public TestWithParam, - bool, - mode_enum, - std::string, - bool, - std::tuple>> -{ -}; - -INSTANTIATE_TEST_SUITE_P(TestCkTileFmhaBwd, - ElementwiseBias, - Combine(HDimValues, - Bool(), // layouts of bias and dbias are controlled by i_perm - ModeValues, - Values("e:0", "e:1", "e:2"), - Bool(), - Values(std::tuple{1, 4, 2, 1024, 100, "0"}, - std::tuple{3, 2, -1, 128, 256, "2"}, - std::tuple{2, 2, -1, 130, 499, "t:50,64"}))); - -TEST_P(ElementwiseBias, Test) -{ - auto [hdims, i_perm, mode, bias_str, use_dbias, dims_mask] = GetParam(); - auto [hdim_q, hdim_v] = hdims; - auto [batch, nhead, nhead_k, seqlen_q, seqlen_k, mask_str] = dims_mask; - - auto result = fmha_bwd_run(mode, - batch, - nhead, - nhead_k, - {seqlen_q}, - {seqlen_k}, - hdim_q, - hdim_v, - i_perm, // i_perm - false, // o_perm - 0, // scale - bias_str, // bias_str - use_dbias, // use_dbias - 0.0f, // p_drop - 123, // drop_seed - 1024, // drop_offset - true, // drop_prefs - mask_str, // mask_str - false, // deterministic - COMMON_ARGS); - CHECK_RESULT(result); -} - -class Alibi : public TestWithParam, - mode_enum, - std::string, - std::tuple, - std::string>> -{ -}; - -INSTANTIATE_TEST_SUITE_P(TestCkTileFmhaBwd, - Alibi, - Combine(HDimValues, - ModeValues, - Values("a:0", "a:1"), - Values(std::tuple{1, 3, 3, 1024, 1000}, - std::tuple{3, 5, 5, 128, 256}, - std::tuple{2, 8, 4, 130, 320}), - Values("0", "t", "b", "t:50,64", "b:32,40"))); - -TEST_P(Alibi, Test) -{ - auto [hdims, mode, bias_str, dims, mask_str] = GetParam(); - auto [hdim_q, hdim_v] = hdims; - auto [batch, nhead, nhead_k, seqlen_q, seqlen_k] = dims; - - auto result = fmha_bwd_run(mode, - batch, - nhead, - nhead_k, - {seqlen_q}, - {seqlen_k}, - hdim_q, - hdim_v, - true, // i_perm - true, // o_perm - 0, // scale - bias_str, // bias_str - false, // use_dbias - 0.0f, // p_drop - 0, // drop_seed - 0, // drop_offset - false, // drop_prefs - mask_str, // mask_str - false, // deterministic - COMMON_ARGS); - CHECK_RESULT(result); -} - -class Dropout : public TestWithParam, - mode_enum, - float, - std::tuple, - std::tuple>> -{ -}; - -INSTANTIATE_TEST_SUITE_P(TestCkTileFmhaBwd, - Dropout, - Combine(HDimValues, - ModeValues, - Values(0.123f, 0.5f), - Values(std::tuple{10, 123, false}, - std::tuple{34534564645, 7876878876864, true}), - Values(std::tuple{2, 6, 2, 180, 512, "0"}, - std::tuple{3, 2, 2, 256, 128, "1"}, - std::tuple{4, 2, 1, 100, 768, "2"}))); - -TEST_P(Dropout, Test) -{ - auto [hdims, mode, p_drop, drop_seed_offset_prefs, dims_mask] = GetParam(); - auto [hdim_q, hdim_v] = hdims; - auto [drop_seed, drop_offset, drop_prefs] = drop_seed_offset_prefs; - auto [batch, nhead, nhead_k, seqlen_q, seqlen_k, mask_str] = dims_mask; - - auto result = fmha_bwd_run(mode, - batch, - nhead, - nhead_k, - {seqlen_q}, - {seqlen_k}, - hdim_q, - hdim_v, - true, // i_perm - true, // o_perm - 0.1f, // scale - "n", // bias_str - false, // use_dbias - p_drop, // p_drop - drop_seed, // drop_seed - drop_offset, // drop_offset - drop_prefs, // drop_prefs - mask_str, // mask_str - false, // deterministic - COMMON_ARGS); - CHECK_RESULT(result); -} - -class Deterministic - : public TestWithParam, - bool, - mode_enum, - std::tuple>> -{ -}; - -INSTANTIATE_TEST_SUITE_P(TestCkTileFmhaBwd, - Deterministic, - Combine(HDimValues, - Bool(), - ModeValues, - Values(std::tuple{2, 6, 2, 180, 512, "0"}, - std::tuple{3, 3, 1, 256, 128, "1"}, - std::tuple{4, 2, 2, 768, 100, "2"}))); - -TEST_P(Deterministic, Test) -{ - auto [hdims, i_perm, mode, dims_mask] = GetParam(); - auto [hdim_q, hdim_v] = hdims; - auto [batch, nhead, nhead_k, seqlen_q, seqlen_k, mask_str] = dims_mask; - - auto result = fmha_bwd_run(mode, - batch, - nhead, - nhead_k, - {seqlen_q}, - {seqlen_k}, - hdim_q, - hdim_v, - i_perm, // i_perm - true, // o_perm - 0, // scale - "n", // bias_str - false, // use_dbias - 0.0f, // p_drop - 0, // drop_seed - 0, // drop_offset - false, // drop_prefs - mask_str, // mask_str - true, // deterministic - COMMON_ARGS); - CHECK_RESULT(result); -} diff --git a/test/ck_tile/fmha/test_fmha_bwd_bf16.cpp b/test/ck_tile/fmha/test_fmha_bwd_bf16.cpp deleted file mode 100644 index 077e45a10d..0000000000 --- a/test/ck_tile/fmha/test_fmha_bwd_bf16.cpp +++ /dev/null @@ -1,21 +0,0 @@ -// SPDX-License-Identifier: MIT -// Copyright (c) 2025, Advanced Micro Devices, Inc. All rights reserved. - -#include "example/ck_tile/01_fmha/fmha_bwd.hpp" -#include "example/ck_tile/01_fmha/fmha_bwd_runner.hpp" - -#include "gtest/gtest.h" - -using DataTypeConfig = FmhaBwdBf16; - -using ::testing::Values; -using ::testing::ValuesIn; - -const auto HDimValues = - Values(std::tuple{32, -1}, std::tuple{64, -1}, std::tuple{128, -1}, std::tuple{256, -1}); - -const auto ModeValues = Values(mode_enum::batch, mode_enum::group); - -constexpr auto init_method = "uf"; - -#include "test_fmha_bwd.inc" diff --git a/test/ck_tile/fmha/test_fmha_bwd_fp16.cpp b/test/ck_tile/fmha/test_fmha_bwd_fp16.cpp deleted file mode 100644 index 86621b0494..0000000000 --- a/test/ck_tile/fmha/test_fmha_bwd_fp16.cpp +++ /dev/null @@ -1,21 +0,0 @@ -// SPDX-License-Identifier: MIT -// Copyright (c) 2025, Advanced Micro Devices, Inc. All rights reserved. - -#include "example/ck_tile/01_fmha/fmha_bwd.hpp" -#include "example/ck_tile/01_fmha/fmha_bwd_runner.hpp" - -#include "gtest/gtest.h" - -using DataTypeConfig = FmhaBwdFp16; - -using ::testing::Values; -using ::testing::ValuesIn; - -const auto HDimValues = - Values(std::tuple{32, -1}, std::tuple{64, -1}, std::tuple{128, -1}, std::tuple{256, -1}); - -const auto ModeValues = Values(mode_enum::batch, mode_enum::group); - -constexpr auto init_method = "uf"; - -#include "test_fmha_bwd.inc" diff --git a/test/ck_tile/fmha/test_fmha_bwd_fp32.cpp b/test/ck_tile/fmha/test_fmha_bwd_fp32.cpp deleted file mode 100644 index 09010d4b22..0000000000 --- a/test/ck_tile/fmha/test_fmha_bwd_fp32.cpp +++ /dev/null @@ -1,20 +0,0 @@ -// SPDX-License-Identifier: MIT -// Copyright (c) 2025, Advanced Micro Devices, Inc. All rights reserved. - -#include "example/ck_tile/01_fmha/fmha_bwd.hpp" -#include "example/ck_tile/01_fmha/fmha_bwd_runner.hpp" - -#include "gtest/gtest.h" - -using DataTypeConfig = FmhaBwdFp32; - -using ::testing::Values; -using ::testing::ValuesIn; - -const auto HDimValues = Values(std::tuple{32, -1}, std::tuple{64, -1}, std::tuple{128, -1}); - -const auto ModeValues = Values(mode_enum::batch, mode_enum::group); - -const std::string init_method = "uf"; - -#include "test_fmha_bwd.inc" diff --git a/test/ck_tile/fmha/test_fmha_fwd.inc b/test/ck_tile/fmha/test_fmha_fwd.cpp similarity index 92% rename from test/ck_tile/fmha/test_fmha_fwd.inc rename to test/ck_tile/fmha/test_fmha_fwd.cpp index ccca5cf969..6e4b547465 100644 --- a/test/ck_tile/fmha/test_fmha_fwd.inc +++ b/test/ck_tile/fmha/test_fmha_fwd.cpp @@ -1,12 +1,104 @@ // SPDX-License-Identifier: MIT // Copyright (c) 2025, Advanced Micro Devices, Inc. All rights reserved. +#include "example/ck_tile/01_fmha/fmha_fwd.hpp" +#include "example/ck_tile/01_fmha/fmha_fwd_runner.hpp" + +#include "gtest/gtest.h" + +#ifndef DataTypeConfig +#define DataTypeConfig FmhaFwdFp16 // or FmhaFwdBf16 / FmhaFwdFp8 / FmhaFwdFp32 +#endif + using ::testing::Bool; using ::testing::Combine; using ::testing::TestWithParam; using ::testing::Values; using ::testing::ValuesIn; +template +struct TestConfigs +{ + static constexpr auto HDimValues = std::array{ + std::tuple{32, -1}, + std::tuple{64, -1}, + std::tuple{96, 128}, + std::tuple{128, -1}, + std::tuple{192, 128}, + std::tuple{192, -1}, + std::tuple{256, -1}, + }; + static constexpr auto SplitKVHDimValues = std::array{ + std::tuple{32, -1}, + std::tuple{64, -1}, + std::tuple{96, -1}, + std::tuple{128, -1}, + std::tuple{256, -1}, + }; + static constexpr auto AppendKVHDimValues = std::array{ + std::tuple{32, -1}, std::tuple{64, -1}, std::tuple{128, -1}, std::tuple{256, -1}}; + static constexpr auto ModeValues = std::array{mode_enum::batch, mode_enum::group}; + static constexpr auto IsVRowmajorValues = std::array{false, true}; + static constexpr bool squant = false; + static constexpr bool def_lse = true; + static constexpr bool def_is_v_rowmajor = true; + static int adjust_seqlen(int seqlen) { return seqlen; } +}; +template <> +struct TestConfigs +{ + // Currently there are no fp8 instances for splitkv, pagedkv by default (the tests pass if such + // instances are added), however the corresponding tests are not disabled (they will be skipped) + // in case such instances will be added in the future. + + static constexpr auto HDimValues = std::array{std::tuple{64, -1}, std::tuple{128, -1}}; + static constexpr auto SplitKVHDimValues = std::array{std::tuple{64, -1}, std::tuple{128, -1}}; + static constexpr auto AppendKVHDimValues = std::array{std::tuple{64, -1}, std::tuple{128, -1}}; + // There are no fp8 instances with seqlen padding (mode_enum::group requires it) + static constexpr auto ModeValues = std::array{mode_enum::batch}; + static constexpr auto IsVRowmajorValues = std::array{false}; + static constexpr bool squant = true; + static constexpr bool def_lse = false; + static constexpr bool def_is_v_rowmajor = true; + static int adjust_seqlen(int seqlen) + { + // There are no fp8 instances with padding, pad seqlen to avoid skipping most of the tests + return ck_tile::integer_least_multiple(seqlen, 128); + } +}; +template <> +struct TestConfigs +{ + static constexpr auto HDimValues = std::array{ + std::tuple{32, -1}, + std::tuple{48, -1}, + std::tuple{64, -1}, + std::tuple{96, 128}, + std::tuple{128, -1}, + std::tuple{192, -1}, + std::tuple{256, -1}, + }; + static constexpr auto SplitKVHDimValues = std::array, 0>{}; + static constexpr auto AppendKVHDimValues = std::array, 0>{}; + static constexpr auto ModeValues = std::array{mode_enum::batch, mode_enum::group}; + static constexpr auto IsVRowmajorValues = std::array{true}; + static constexpr bool squant = false; + static constexpr bool def_lse = true; + static constexpr bool def_is_v_rowmajor = true; + static int adjust_seqlen(int seqlen) { return seqlen; } +}; + +static auto HDimValues = ValuesIn(TestConfigs::HDimValues); +static auto SplitKVHDimValues = ValuesIn(TestConfigs::SplitKVHDimValues); +static auto AppendKVHDimValues = ValuesIn(TestConfigs::AppendKVHDimValues); +static auto ModeValues = ValuesIn(TestConfigs::ModeValues); +static auto IsVRowmajorValues = ValuesIn(TestConfigs::IsVRowmajorValues); +constexpr bool squant = TestConfigs::squant; +constexpr bool def_lse = TestConfigs::def_lse; +constexpr bool def_is_v_rowmajor = TestConfigs::def_is_v_rowmajor; +int adjust_seqlen(int seqlen) { return TestConfigs::adjust_seqlen(seqlen); } +constexpr auto init_method = "uf"; + // Random seed used for initializing input tensors. 0 for non-deterministic seed CK_TILE_DECLARE_ENV_VAR(CK_TILE_TEST_SEED, uint64_t, 123456) @@ -79,7 +171,7 @@ INSTANTIATE_TEST_SUITE_P( std::tuple{1, 2, 1, -1, -1, 33, 0, -1, "2"}, std::tuple{1, 2, 1, -1, -1, 1, 10, 32, "2"}))); -TEST_P(AllLong, Test) +TEST_P(AllLong, DataTypeConfig) { auto [_, hdims, perm, is_v_rowmajor, mode, lse, bias_str, p_drop, dims_mask] = GetParam(); auto [hdim_q, hdim_v] = hdims; @@ -283,7 +375,7 @@ INSTANTIATE_TEST_SUITE_P(TestCkTileFmhaFwd, std::tuple{1, 2, -1, 900, 256, -1, "0"}, std::tuple{2, 1, -1, 256, 256, -1, "1"}))); -TEST_P(HDimPadding, Test) +TEST_P(HDimPadding, DataTypeConfig) { auto [hdims, perm, is_v_rowmajor, mode, dims_mask] = GetParam(); auto [hdim_q, hdim_v] = hdims; @@ -343,7 +435,7 @@ INSTANTIATE_TEST_SUITE_P(TestCkTileFmhaFwd, std::tuple{3, 2, -1, 128, 256, "2"}, std::tuple{2, 2, -1, 130, 499, "t:50,64"}))); -TEST_P(ElementwiseBias, Test) +TEST_P(ElementwiseBias, DataTypeConfig) { auto [hdims, i_perm, mode, bias_str, dims_mask] = GetParam(); auto [hdim_q, hdim_v] = hdims; @@ -402,7 +494,7 @@ INSTANTIATE_TEST_SUITE_P(TestCkTileFmhaFwd, std::tuple{2, 8, 2, 300, 355}), Values("0", "t", "b", "t:50,64", "b:32,40"))); -TEST_P(Alibi, Test) +TEST_P(Alibi, DataTypeConfig) { auto [hdims, mode, bias_str, dims, mask_str] = GetParam(); auto [hdim_q, hdim_v] = hdims; @@ -462,7 +554,7 @@ INSTANTIATE_TEST_SUITE_P(TestCkTileFmhaFwd, std::tuple{3, 2, 2, 256, 128, "1"}, std::tuple{4, 3, 1, 100, 768, "2"}))); -TEST_P(Dropout, Test) +TEST_P(Dropout, DataTypeConfig) { auto [hdims, mode, p_drop, drop_seed_offset_prefs, dims_mask] = GetParam(); auto [hdim_q, hdim_v] = hdims; @@ -528,7 +620,7 @@ INSTANTIATE_TEST_SUITE_P(TestCkTileFmhaFwd, std::tuple{3, 2, -1, 128, 768, "2"}, std::tuple{2, 2, -1, 230, 899, "t:50,64"}))); -TEST_P(PagedKV, Test) +TEST_P(PagedKV, DataTypeConfig) { auto [hdims, i_perm, is_v_rowmajor, mode, page_block_size, dims_mask] = GetParam(); auto [hdim_q, hdim_v] = hdims; @@ -597,7 +689,7 @@ INSTANTIATE_TEST_SUITE_P(TestCkTileFmhaFwd, std::tuple{2, 2, -1, 512, 2000, "0"}, std::tuple{3, 2, -1, 230, 899, "t:128,128"}))); -TEST_P(SplitKV, Test) +TEST_P(SplitKV, DataTypeConfig) { auto [hdims, i_perm, is_v_rowmajor, mode_use_cache_batch_idx, num_splits, dims_mask] = GetParam(); @@ -668,7 +760,7 @@ INSTANTIATE_TEST_SUITE_P( GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(AppendKV); -TEST_P(AppendKV, Test) +TEST_P(AppendKV, DataTypeConfig) { auto [hdims, i_perm, @@ -745,7 +837,7 @@ INSTANTIATE_TEST_SUITE_P(TestCkTileFmhaFwd, std::tuple{1, 2, 1, 128, 55, "0"}, std::tuple{3, 4, 2, 72, 128, "1"}))); -TEST_P(AppendKVRoPE, Test) +TEST_P(AppendKVRoPE, DataTypeConfig) { auto [_, hdims, i_perm, is_v_rowmajor, rotary, seqlen_knew, dims_mask] = GetParam(); auto [hdim_q, hdim_v] = hdims; @@ -1017,7 +1109,7 @@ static const std::vector kPaddingParams = BuildPaddingParams(); INSTANTIATE_TEST_SUITE_P(TestCkTileFmhaFwd_Padding, PaddingCases, ValuesIn(kPaddingParams)); -TEST_P(PaddingCases, Test) +TEST_P(PaddingCases, DataTypeConfig) { if constexpr(std::is_same_v) { diff --git a/test/ck_tile/fmha/test_fmha_fwd_bf16.cpp b/test/ck_tile/fmha/test_fmha_fwd_bf16.cpp deleted file mode 100644 index fbc6449a6a..0000000000 --- a/test/ck_tile/fmha/test_fmha_fwd_bf16.cpp +++ /dev/null @@ -1,44 +0,0 @@ -// SPDX-License-Identifier: MIT -// Copyright (c) 2025, Advanced Micro Devices, Inc. All rights reserved. - -#include "example/ck_tile/01_fmha/fmha_fwd.hpp" -#include "example/ck_tile/01_fmha/fmha_fwd_runner.hpp" - -#include "gtest/gtest.h" - -#include -#include - -using ::testing::Values; - -using DataTypeConfig = FmhaFwdBf16; - -const auto HDimValues = Values(std::tuple{32, -1}, - std::tuple{64, -1}, - std::tuple{96, 128}, - std::tuple{128, -1}, - std::tuple{192, 128}, - std::tuple{192, -1}, - std::tuple{256, -1}); - -const auto SplitKVHDimValues = Values(std::tuple{32, -1}, - std::tuple{64, -1}, - std::tuple{96, -1}, - std::tuple{128, -1}, - std::tuple{256, -1}); - -const auto AppendKVHDimValues = - Values(std::tuple{32, -1}, std::tuple{64, -1}, std::tuple{128, -1}, std::tuple{256, -1}); - -const auto ModeValues = Values(mode_enum::batch, mode_enum::group); - -const auto IsVRowmajorValues = Values(false, true); - -const bool squant = false; -const std::string init_method = "uf"; -const bool def_lse = true; -const bool def_is_v_rowmajor = true; - -int adjust_seqlen(int seqlen) { return seqlen; } - -#include "test_fmha_fwd.inc" diff --git a/test/ck_tile/fmha/test_fmha_fwd_fp16.cpp b/test/ck_tile/fmha/test_fmha_fwd_fp16.cpp deleted file mode 100644 index abc2c44726..0000000000 --- a/test/ck_tile/fmha/test_fmha_fwd_fp16.cpp +++ /dev/null @@ -1,44 +0,0 @@ -// SPDX-License-Identifier: MIT -// Copyright (c) 2025, Advanced Micro Devices, Inc. All rights reserved. - -#include "example/ck_tile/01_fmha/fmha_fwd.hpp" -#include "example/ck_tile/01_fmha/fmha_fwd_runner.hpp" - -#include "gtest/gtest.h" - -#include -#include - -using ::testing::Values; - -using DataTypeConfig = FmhaFwdFp16; - -const auto HDimValues = Values(std::tuple{32, -1}, - std::tuple{64, -1}, - std::tuple{96, 128}, - std::tuple{128, -1}, - std::tuple{192, 128}, - std::tuple{192, -1}, - std::tuple{256, -1}); - -const auto SplitKVHDimValues = Values(std::tuple{32, -1}, - std::tuple{64, -1}, - std::tuple{96, -1}, - std::tuple{128, -1}, - std::tuple{256, -1}); - -const auto AppendKVHDimValues = - Values(std::tuple{32, -1}, std::tuple{64, -1}, std::tuple{128, -1}, std::tuple{256, -1}); - -const auto ModeValues = Values(mode_enum::batch, mode_enum::group); - -const auto IsVRowmajorValues = Values(false, true); - -const bool squant = false; -const std::string init_method = "uf"; -const bool def_lse = true; -const bool def_is_v_rowmajor = true; - -int adjust_seqlen(int seqlen) { return seqlen; } - -#include "test_fmha_fwd.inc" diff --git a/test/ck_tile/fmha/test_fmha_fwd_fp32.cpp b/test/ck_tile/fmha/test_fmha_fwd_fp32.cpp deleted file mode 100644 index 00f1eb0629..0000000000 --- a/test/ck_tile/fmha/test_fmha_fwd_fp32.cpp +++ /dev/null @@ -1,39 +0,0 @@ -// SPDX-License-Identifier: MIT -// Copyright (c) 2025, Advanced Micro Devices, Inc. All rights reserved. - -#include "example/ck_tile/01_fmha/fmha_fwd.hpp" -#include "example/ck_tile/01_fmha/fmha_fwd_runner.hpp" - -#include "gtest/gtest.h" - -#include -#include - -using ::testing::Values; - -using DataTypeConfig = FmhaFwdFp32; - -const auto HDimValues = Values(std::tuple{32, -1}, - std::tuple{48, -1}, - std::tuple{64, -1}, - std::tuple{96, 128}, - std::tuple{128, -1}, - std::tuple{192, -1}, - std::tuple{256, -1}); - -const auto SplitKVHDimValues = Values(); - -const auto AppendKVHDimValues = Values(); - -const auto ModeValues = Values(mode_enum::batch, mode_enum::group); - -const auto IsVRowmajorValues = Values(true); - -const bool squant = false; -const std::string init_method = "uf"; -const bool def_lse = true; -const bool def_is_v_rowmajor = true; - -int adjust_seqlen(int seqlen) { return seqlen; } - -#include "test_fmha_fwd.inc" diff --git a/test/ck_tile/fmha/test_fmha_fwd_fp8.cpp b/test/ck_tile/fmha/test_fmha_fwd_fp8.cpp deleted file mode 100644 index b99c304d1f..0000000000 --- a/test/ck_tile/fmha/test_fmha_fwd_fp8.cpp +++ /dev/null @@ -1,42 +0,0 @@ -// SPDX-License-Identifier: MIT -// Copyright (c) 2025, Advanced Micro Devices, Inc. All rights reserved. - -#include "example/ck_tile/01_fmha/fmha_fwd.hpp" -#include "example/ck_tile/01_fmha/fmha_fwd_runner.hpp" - -#include "gtest/gtest.h" - -#include -#include - -using ::testing::Values; - -using DataTypeConfig = FmhaFwdFp8; - -// Currently there are no fp8 instances for splitkv, pagedkv by default (the tests pass if such -// instances are added), however the corresponding tests are not disabled (they will be skipped) -// in case such instances will be added in the future. - -const auto HDimValues = Values(std::tuple{64, -1}, std::tuple{128, -1}); - -const auto SplitKVHDimValues = Values(std::tuple{64, -1}, std::tuple{128, -1}); - -const auto AppendKVHDimValues = Values(std::tuple{64, -1}, std::tuple{128, -1}); - -// There are no fp8 instances with seqlen padding (mode_enum::group requires it) -const auto ModeValues = Values(mode_enum::batch); - -const auto IsVRowmajorValues = Values(false); - -const auto squant = true; -const std::string init_method = "uf"; -const bool def_lse = false; -const bool def_is_v_rowmajor = true; - -int adjust_seqlen(int seqlen) -{ - // There are no fp8 instances with padding, pad seqlen to avoid skipping most of the tests - return ck_tile::integer_least_multiple(seqlen, 128); -} - -#include "test_fmha_fwd.inc" From ad7a215aba80308e2ec49849a017f73da7bfa437 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20Kocot?= Date: Fri, 10 Oct 2025 09:24:21 +0200 Subject: [PATCH 004/262] Fix splitK for grouped conv bwd data (#2991) --- .../operator_transform/transform_conv_bwd_data_to_gemm_v1.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/ck/tensor_operation/operator_transform/transform_conv_bwd_data_to_gemm_v1.hpp b/include/ck/tensor_operation/operator_transform/transform_conv_bwd_data_to_gemm_v1.hpp index 03c1945d95..b989d63e0e 100644 --- a/include/ck/tensor_operation/operator_transform/transform_conv_bwd_data_to_gemm_v1.hpp +++ b/include/ck/tensor_operation/operator_transform/transform_conv_bwd_data_to_gemm_v1.hpp @@ -814,7 +814,7 @@ struct TransformConvBwdDataToGemm_v1 IWTildeSliceBegin, GcdStrideDilationH_, GcdStrideDilationW_, - AK0, + AK0 * batch_k_, AK1, GemmMPerBlock, GemmKPerBlock)), From fada1a3cae190aa6c1568b44eac7d6b2d4e33740 Mon Sep 17 00:00:00 2001 From: yinglu Date: Fri, 10 Oct 2025 15:28:17 +0800 Subject: [PATCH 005/262] Conv:TF32: add more instances - 2 (#2879) * add instances of device_grouped_conv_fwd_xdl_f32_comp_instances * add instances of device_grouped_conv_fwd_xdl_f32_tf32_mem_instances * add instances of device_grouped_conv_fwd_xdl_large_tensor_f32_tf32_instances * tf32:conv:add instances for base class DeviceConvFwd * tf32:conv:add instances for base class DeviceGroupedConvBwdDataMultipleD * tf32:conv:add instances for base class DeviceGroupedConvBwdWeight * add tf32 in profiler * remove gnhwc/ngchw/ngcdhw instances * remove non-ndhwgc/nhwgc/nhwc instances * add check in IsSupportedArgument() --- include/ck/library/utility/check_err.hpp | 116 +++++++++++--- ...nv_bwd_data_multiple_d_xdl_cshuffle_v1.hpp | 16 ++ ...onv_bwd_weight_multiple_d_xdl_cshuffle.hpp | 16 ++ ...conv_bwd_weight_two_stage_xdl_cshuffle.hpp | 17 +++ ...e_grouped_conv_bwd_weight_xdl_cshuffle.hpp | 16 ++ ...rouped_conv_bwd_weight_xdl_cshuffle_v3.hpp | 17 +++ ..._conv_fwd_multiple_abd_xdl_cshuffle_v3.hpp | 19 +++ ...d_multiple_d_xdl_large_tensor_cshuffle.hpp | 17 +++ .../grid/gridwise_gemm_xdlops_bwd_weight.hpp | 18 ++- include/ck/utility/numeric_utils.hpp | 18 +++ .../cpu/reference_conv_bwd_data.hpp | 22 ++- ...d_conv_bwd_data_transpose_xdl_instance.hpp | 7 +- ...ed_conv_bwd_data_xdl_bilinear_instance.hpp | 38 +++++ ...ice_grouped_conv_bwd_data_xdl_instance.hpp | 113 ++++++++++++++ ...ouped_conv_bwd_data_xdl_scale_instance.hpp | 38 +++++ ...rouped_conv_bwd_weight_v3_xdl_instance.hpp | 19 +++ ..._conv_bwd_weight_xdl_bilinear_instance.hpp | 34 +++++ ...e_grouped_conv_bwd_weight_xdl_instance.hpp | 57 +++++++ ...ped_conv_bwd_weight_xdl_scale_instance.hpp | 34 +++++ .../gpu/grouped_convolution_backward_data.hpp | 54 +++++-- .../grouped_convolution_backward_data_xdl.inc | 96 ++++++++++++ .../grouped_convolution_backward_weight.hpp | 83 +++++++--- ...d_convolution_backward_weight_bilinear.hpp | 32 +++- ...uped_convolution_backward_weight_scale.hpp | 33 +++- ...rouped_convolution_backward_weight_xdl.inc | 142 +++++++++++++++++- ...ped_convolution_forward_bias_clamp_xdl.inc | 16 ++ .../grouped_conv2d_bwd_data/CMakeLists.txt | 3 + ...gc_gkyxc_nhwgk_f32_tf32_16_16_instance.cpp | 51 +++++++ ...dl_nhwgc_gkyxc_nhwgk_f32_tf32_instance.cpp | 51 +++++++ ...hwgk_f32_tf32_optimized_loads_instance.cpp | 52 +++++++ .../grouped_conv2d_bwd_weight/CMakeLists.txt | 5 + ...nhwgk_f32_tf32_default_pipev2_instance.cpp | 42 ++++++ ...nhwgk_f32_tf32_default_pipev5_instance.cpp | 42 ++++++ ...dl_nhwgc_gkyxc_nhwgk_f32_tf32_instance.cpp | 48 ++++++ ...xc_nhwgk_f32_tf32_pad0_pipev2_instance.cpp | 42 ++++++ ...xc_nhwgk_f32_tf32_pad0_pipev5_instance.cpp | 42 ++++++ .../grouped_conv3d_bwd_data/CMakeLists.txt | 3 + ..._gkzyxc_ndhwgk_f32_tf32_16_16_instance.cpp | 51 +++++++ ...ndhwgc_gkzyxc_ndhwgk_f32_tf32_instance.cpp | 51 +++++++ ...hwgk_f32_tf32_optimized_loads_instance.cpp | 52 +++++++ .../grouped_conv3d_bwd_weight/CMakeLists.txt | 5 + ...dhwgk_f32_tf32_default_pipev2_instance.cpp | 42 ++++++ ...dhwgk_f32_tf32_default_pipev5_instance.cpp | 42 ++++++ ...ndhwgc_gkzyxc_ndhwgk_f32_tf32_instance.cpp | 48 ++++++ ...c_ndhwgk_f32_tf32_pad0_pipev2_instance.cpp | 42 ++++++ ...c_ndhwgk_f32_tf32_pad0_pipev5_instance.cpp | 42 ++++++ .../CMakeLists.txt | 1 + ...ndhwgc_gkzyxc_ndhwgk_f32_tf32_instance.cpp | 52 +++++++ .../CMakeLists.txt | 1 + ...ndhwgc_gkzyxc_ndhwgk_f32_tf32_instance.cpp | 52 +++++++ .../profile_grouped_conv_bwd_data_impl.hpp | 46 +++--- .../src/profile_grouped_conv_bwd_data.cpp | 120 +++++++++++---- .../src/profile_grouped_conv_bwd_weight.cpp | 49 +++++- profiler/src/profile_grouped_conv_fwd.cpp | 36 +++++ .../profile_grouped_conv_fwd_bias_clamp.cpp | 35 +++-- .../src/profile_grouped_conv_fwd_clamp.cpp | 35 +++-- 56 files changed, 2119 insertions(+), 152 deletions(-) create mode 100644 library/src/tensor_operation_instance/gpu/grouped_conv2d_bwd_data/xdl/device_grouped_conv2d_bwd_data_xdl_nhwgc_gkyxc_nhwgk_f32_tf32_16_16_instance.cpp create mode 100644 library/src/tensor_operation_instance/gpu/grouped_conv2d_bwd_data/xdl/device_grouped_conv2d_bwd_data_xdl_nhwgc_gkyxc_nhwgk_f32_tf32_instance.cpp create mode 100644 library/src/tensor_operation_instance/gpu/grouped_conv2d_bwd_data/xdl/device_grouped_conv2d_bwd_data_xdl_nhwgc_gkyxc_nhwgk_f32_tf32_optimized_loads_instance.cpp create mode 100644 library/src/tensor_operation_instance/gpu/grouped_conv2d_bwd_weight/xdl/nhwgc_gkyxc_nhwgk/device_grouped_conv2d_bwd_weight_xdl_nhwgc_gkyxc_nhwgk_f32_tf32_default_pipev2_instance.cpp create mode 100644 library/src/tensor_operation_instance/gpu/grouped_conv2d_bwd_weight/xdl/nhwgc_gkyxc_nhwgk/device_grouped_conv2d_bwd_weight_xdl_nhwgc_gkyxc_nhwgk_f32_tf32_default_pipev5_instance.cpp create mode 100644 library/src/tensor_operation_instance/gpu/grouped_conv2d_bwd_weight/xdl/nhwgc_gkyxc_nhwgk/device_grouped_conv2d_bwd_weight_xdl_nhwgc_gkyxc_nhwgk_f32_tf32_instance.cpp create mode 100644 library/src/tensor_operation_instance/gpu/grouped_conv2d_bwd_weight/xdl/nhwgc_gkyxc_nhwgk/device_grouped_conv2d_bwd_weight_xdl_nhwgc_gkyxc_nhwgk_f32_tf32_pad0_pipev2_instance.cpp create mode 100644 library/src/tensor_operation_instance/gpu/grouped_conv2d_bwd_weight/xdl/nhwgc_gkyxc_nhwgk/device_grouped_conv2d_bwd_weight_xdl_nhwgc_gkyxc_nhwgk_f32_tf32_pad0_pipev5_instance.cpp create mode 100644 library/src/tensor_operation_instance/gpu/grouped_conv3d_bwd_data/xdl/device_grouped_conv3d_bwd_data_xdl_ndhwgc_gkzyxc_ndhwgk_f32_tf32_16_16_instance.cpp create mode 100644 library/src/tensor_operation_instance/gpu/grouped_conv3d_bwd_data/xdl/device_grouped_conv3d_bwd_data_xdl_ndhwgc_gkzyxc_ndhwgk_f32_tf32_instance.cpp create mode 100644 library/src/tensor_operation_instance/gpu/grouped_conv3d_bwd_data/xdl/device_grouped_conv3d_bwd_data_xdl_ndhwgc_gkzyxc_ndhwgk_f32_tf32_optimized_loads_instance.cpp create mode 100644 library/src/tensor_operation_instance/gpu/grouped_conv3d_bwd_weight/xdl/ndhwgc_gkzyxc_ndhwgk/device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_f32_tf32_default_pipev2_instance.cpp create mode 100644 library/src/tensor_operation_instance/gpu/grouped_conv3d_bwd_weight/xdl/ndhwgc_gkzyxc_ndhwgk/device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_f32_tf32_default_pipev5_instance.cpp create mode 100644 library/src/tensor_operation_instance/gpu/grouped_conv3d_bwd_weight/xdl/ndhwgc_gkzyxc_ndhwgk/device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_f32_tf32_instance.cpp create mode 100644 library/src/tensor_operation_instance/gpu/grouped_conv3d_bwd_weight/xdl/ndhwgc_gkzyxc_ndhwgk/device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_f32_tf32_pad0_pipev2_instance.cpp create mode 100644 library/src/tensor_operation_instance/gpu/grouped_conv3d_bwd_weight/xdl/ndhwgc_gkzyxc_ndhwgk/device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_f32_tf32_pad0_pipev5_instance.cpp create mode 100644 library/src/tensor_operation_instance/gpu/grouped_conv3d_bwd_weight_bilinear/xdl/device_grouped_conv3d_bwd_weight_xdl_bilinear_ndhwgc_gkzyxc_ndhwgk_f32_tf32_instance.cpp create mode 100644 library/src/tensor_operation_instance/gpu/grouped_conv3d_bwd_weight_scale/xdl/device_grouped_conv3d_bwd_weight_xdl_scale_ndhwgc_gkzyxc_ndhwgk_f32_tf32_instance.cpp diff --git a/include/ck/library/utility/check_err.hpp b/include/ck/library/utility/check_err.hpp index 185166f7ec..3637053e14 100644 --- a/include/ck/library/utility/check_err.hpp +++ b/include/ck/library/utility/check_err.hpp @@ -31,13 +31,15 @@ double get_relative_threshold(const int number_of_accumulations = 1) using F16 = ck::half_t; using BF16 = ck::bhalf_t; using F32 = float; + using TF32 = ck::tf32_t; using I8 = int8_t; using I32 = int32_t; static_assert(is_same_v || is_same_v || is_same_v || is_same_v || - is_same_v || is_same_v || - is_same_v || is_same_v, + is_same_v || is_same_v || + is_same_v || is_same_v || + is_same_v, "Warning: Unhandled ComputeDataType for setting up the relative threshold!"); double compute_error = 0; if constexpr(is_same_v || is_same_v || @@ -52,8 +54,9 @@ double get_relative_threshold(const int number_of_accumulations = 1) static_assert(is_same_v || is_same_v || is_same_v || is_same_v || - is_same_v || is_same_v || - is_same_v || is_same_v, + is_same_v || is_same_v || + is_same_v || is_same_v || + is_same_v, "Warning: Unhandled OutDataType for setting up the relative threshold!"); double output_error = 0; if constexpr(is_same_v || is_same_v || @@ -69,8 +72,9 @@ double get_relative_threshold(const int number_of_accumulations = 1) static_assert(is_same_v || is_same_v || is_same_v || is_same_v || - is_same_v || is_same_v || - is_same_v || is_same_v, + is_same_v || is_same_v || + is_same_v || is_same_v || + is_same_v, "Warning: Unhandled AccDataType for setting up the relative threshold!"); double acc_error = 0; if constexpr(is_same_v || is_same_v || @@ -93,13 +97,15 @@ double get_absolute_threshold(const double max_possible_num, const int number_of using F16 = ck::half_t; using BF16 = ck::bhalf_t; using F32 = float; + using TF32 = ck::tf32_t; using I8 = int8_t; using I32 = int32_t; static_assert(is_same_v || is_same_v || is_same_v || is_same_v || - is_same_v || is_same_v || - is_same_v || is_same_v, + is_same_v || is_same_v || + is_same_v || is_same_v || + is_same_v, "Warning: Unhandled ComputeDataType for setting up the absolute threshold!"); auto expo = std::log2(std::abs(max_possible_num)); double compute_error = 0; @@ -115,8 +121,9 @@ double get_absolute_threshold(const double max_possible_num, const int number_of static_assert(is_same_v || is_same_v || is_same_v || is_same_v || - is_same_v || is_same_v || - is_same_v || is_same_v, + is_same_v || is_same_v || + is_same_v || is_same_v || + is_same_v, "Warning: Unhandled OutDataType for setting up the absolute threshold!"); double output_error = 0; if constexpr(is_same_v || is_same_v || @@ -132,8 +139,9 @@ double get_absolute_threshold(const double max_possible_num, const int number_of static_assert(is_same_v || is_same_v || is_same_v || is_same_v || - is_same_v || is_same_v || - is_same_v || is_same_v, + is_same_v || is_same_v || + is_same_v || is_same_v || + is_same_v, "Warning: Unhandled AccDataType for setting up the absolute threshold!"); double acc_error = 0; if constexpr(is_same_v || is_same_v || @@ -149,11 +157,67 @@ double get_absolute_threshold(const double max_possible_num, const int number_of return std::max(acc_error, midway_error); } -template +template > +typename std::enable_if< + std::is_same_v, ranges::range_value_t> && + std::is_same_v, float> && + std::is_same_v, + bool>::type +check_err(const Range& out, + const RefRange& ref, + const std::string& msg = "Error: Incorrect results!", + double rtol = 1e-5, + double atol = 3e-5) +{ + if(out.size() != ref.size()) + { + std::cerr << msg << " out.size() != ref.size(), :" << out.size() << " != " << ref.size() + << std::endl; + return false; + } + + bool res{true}; + int err_count = 0; + double err = 0; + double max_err = std::numeric_limits::min(); + for(std::size_t i = 0; i < ref.size(); ++i) + { + const double o = *std::next(std::begin(out), i); + const double r = *std::next(std::begin(ref), i); + err = std::abs(o - r); + if(err > atol + rtol * std::abs(r) || !std::isfinite(o) || !std::isfinite(r)) + { + max_err = err > max_err ? err : max_err; + if(err_count < 5) + { + std::cerr << msg << std::setw(12) << std::setprecision(7) << " out[" << i + << "] != ref[" << i << "]: " << o << " != " << r << std::endl; + } + res = false; + err_count++; + } + } + if(!res) + { + const float error_percent = + static_cast(err_count) / static_cast(out.size()) * 100.f; + std::cerr << "max err: " << max_err; + std::cerr << ", number of errors: " << err_count; + std::cerr << ", " << error_percent << "% wrong values" << std::endl; + } + return res; +} + +template > typename std::enable_if< std::is_same_v, ranges::range_value_t> && std::is_floating_point_v> && - !std::is_same_v, half_t>, + !std::is_same_v, half_t> && + !std::is_same_v, bool>::type check_err(const Range& out, const RefRange& ref, @@ -200,7 +264,9 @@ check_err(const Range& out, return res; } -template +template > typename std::enable_if< std::is_same_v, ranges::range_value_t> && std::is_same_v, bhalf_t>, @@ -251,7 +317,9 @@ check_err(const Range& out, return res; } -template +template > typename std::enable_if< std::is_same_v, ranges::range_value_t> && std::is_same_v, half_t>, @@ -301,7 +369,9 @@ check_err(const Range& out, return res; } -template +template > std::enable_if_t<(std::is_same_v, ranges::range_value_t> && std::is_integral_v> && !std::is_same_v, bhalf_t> && @@ -358,7 +428,9 @@ check_err(const Range& out, return res; } -template +template > std::enable_if_t<(std::is_same_v, ranges::range_value_t> && std::is_same_v, f8_t>), bool> @@ -407,7 +479,9 @@ check_err(const Range& out, return res; } -template +template > std::enable_if_t<(std::is_same_v, ranges::range_value_t> && std::is_same_v, bf8_t>), bool> @@ -452,7 +526,9 @@ check_err(const Range& out, return res; } -template +template > std::enable_if_t<(std::is_same_v, ranges::range_value_t> && std::is_same_v, f4_t>), bool> diff --git a/include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_bwd_data_multiple_d_xdl_cshuffle_v1.hpp b/include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_bwd_data_multiple_d_xdl_cshuffle_v1.hpp index 3d6f34f121..47832e2153 100644 --- a/include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_bwd_data_multiple_d_xdl_cshuffle_v1.hpp +++ b/include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_bwd_data_multiple_d_xdl_cshuffle_v1.hpp @@ -1499,6 +1499,22 @@ struct DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1 { return false; } + if constexpr(is_same_v || is_same_v) + { + if(!is_tf32_supported()) + { + return false; + } + if constexpr(!is_same_v) + { + if(ck::EnvIsEnabled(CK_ENV(CK_LOGGING))) + { + std::cout << "ComputeDataType for A and B should be same while using TF32" + << std::endl; + } + return false; + } + } if constexpr(!IsSplitKSupported) { diff --git a/include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_bwd_weight_multiple_d_xdl_cshuffle.hpp b/include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_bwd_weight_multiple_d_xdl_cshuffle.hpp index 987a1e273a..ab185700b6 100644 --- a/include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_bwd_weight_multiple_d_xdl_cshuffle.hpp +++ b/include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_bwd_weight_multiple_d_xdl_cshuffle.hpp @@ -951,6 +951,22 @@ struct DeviceGroupedConvBwdWeightMultipleD_Xdl_CShuffle { return false; } + if constexpr(is_same_v || is_same_v) + { + if(!is_tf32_supported()) + { + return false; + } + if constexpr(!is_same_v) + { + if(ck::EnvIsEnabled(CK_ENV(CK_LOGGING))) + { + std::cout << "ComputeDataType for A and B should be same while using TF32" + << std::endl; + } + return false; + } + } if constexpr(NDimSpatial == 1) { if constexpr(!is_GNWC_GKXC_GNWK()) diff --git a/include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_bwd_weight_two_stage_xdl_cshuffle.hpp b/include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_bwd_weight_two_stage_xdl_cshuffle.hpp index e38768b2fa..50796f78b4 100644 --- a/include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_bwd_weight_two_stage_xdl_cshuffle.hpp +++ b/include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_bwd_weight_two_stage_xdl_cshuffle.hpp @@ -1687,6 +1687,23 @@ struct DeviceGroupedConvBwdWeightTwoStage_Xdl_CShuffle const index_t GemmK = arg.a_grid_desc_k0_m_k1_.GetLength(I0) * arg.a_grid_desc_k0_m_k1_.GetLength(I2); + if constexpr(is_same_v || is_same_v) + { + if(!is_tf32_supported()) + { + return false; + } + if constexpr(!is_same_v) + { + if(ck::EnvIsEnabled(CK_ENV(CK_LOGGING))) + { + std::cout << "ComputeDataType for A and B should be same while using TF32" + << std::endl; + } + return false; + } + } + if(get_warp_size() == 64) { if constexpr(NXdlPerWave64 > 0) diff --git a/include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_bwd_weight_xdl_cshuffle.hpp b/include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_bwd_weight_xdl_cshuffle.hpp index 22fc13bae4..c7ee3e9ecf 100644 --- a/include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_bwd_weight_xdl_cshuffle.hpp +++ b/include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_bwd_weight_xdl_cshuffle.hpp @@ -950,6 +950,22 @@ struct DeviceGroupedConvBwdWeight_Xdl_CShuffle { return false; } + if constexpr(is_same_v || is_same_v) + { + if(!is_tf32_supported()) + { + return false; + } + if constexpr(!is_same_v) + { + if(ck::EnvIsEnabled(CK_ENV(CK_LOGGING))) + { + std::cout << "ComputeDataType for A and B should be same while using TF32" + << std::endl; + } + return false; + } + } if constexpr(NDimSpatial == 1) { if constexpr(!is_GNWC_GKXC_GNWK()) diff --git a/include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_bwd_weight_xdl_cshuffle_v3.hpp b/include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_bwd_weight_xdl_cshuffle_v3.hpp index 735eebbdf6..07722155fd 100644 --- a/include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_bwd_weight_xdl_cshuffle_v3.hpp +++ b/include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_bwd_weight_xdl_cshuffle_v3.hpp @@ -1289,6 +1289,23 @@ struct DeviceGroupedConvBwdWeight_Xdl_CShuffleV3 const index_t GemmK = arg.a_grid_desc_kbatch_k0_m_k1_.GetLength(I0) * arg.a_grid_desc_kbatch_k0_m_k1_.GetLength(I2); + if constexpr(is_same_v || is_same_v) + { + if(!is_tf32_supported()) + { + return false; + } + if constexpr(!is_same_v) + { + if(ck::EnvIsEnabled(CK_ENV(CK_LOGGING))) + { + std::cout << "ComputeDataType for A and B should be same while using TF32" + << std::endl; + } + return false; + } + } + if(get_warp_size() == 64) { if constexpr(NXdlPerWave64 > 0) diff --git a/include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_fwd_multiple_abd_xdl_cshuffle_v3.hpp b/include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_fwd_multiple_abd_xdl_cshuffle_v3.hpp index dd2e429a01..dbc60e3fdc 100644 --- a/include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_fwd_multiple_abd_xdl_cshuffle_v3.hpp +++ b/include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_fwd_multiple_abd_xdl_cshuffle_v3.hpp @@ -1399,6 +1399,25 @@ struct DeviceGroupedConvFwdMultipleABD_Xdl_CShuffle_V3 } return false; } + + if constexpr(is_same_v || + is_same_v) + { + if(!is_tf32_supported()) + { + return false; + } + if constexpr(!is_same_v) + { + if(ck::EnvIsEnabled(CK_ENV(CK_LOGGING))) + { + std::cout << "ComputeDataType for A and B should be same while using TF32" + << std::endl; + } + return false; + } + } + // check ConvolutionForwardSpecialization if constexpr(ConvForwardSpecialization == ConvolutionForwardSpecialization::Filter1x1Stride1Pad0) diff --git a/include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_fwd_multiple_d_xdl_large_tensor_cshuffle.hpp b/include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_fwd_multiple_d_xdl_large_tensor_cshuffle.hpp index 25afe46690..020b3dc5a6 100644 --- a/include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_fwd_multiple_d_xdl_large_tensor_cshuffle.hpp +++ b/include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_fwd_multiple_d_xdl_large_tensor_cshuffle.hpp @@ -820,6 +820,23 @@ struct DeviceGroupedConvFwdMultipleD_Xdl_CShuffle_Large_Tensor { return false; } + if constexpr(is_same_v || + is_same_v) + { + if(!is_tf32_supported()) + { + return false; + } + if constexpr(!is_same_v) + { + if(ck::EnvIsEnabled(CK_ENV(CK_LOGGING))) + { + std::cout << "ComputeDataType for A and B should be same while using TF32" + << std::endl; + } + return false; + } + } // check ConvolutionForwardSpecialization if constexpr(ConvForwardSpecialization == ConvolutionForwardSpecialization::Filter1x1Stride1Pad0) diff --git a/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdlops_bwd_weight.hpp b/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdlops_bwd_weight.hpp index abb8c52e0f..cb841c36ea 100644 --- a/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdlops_bwd_weight.hpp +++ b/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdlops_bwd_weight.hpp @@ -280,8 +280,8 @@ struct GridwiseGemm_bk0mk1_bk0nk1_mn_xdlops_bwd_weight using FloatBAdjusted = conditional_t, ck::bhalf_t, ComputeTypeB>; #else - using FloatAAdjusted = ComputeTypeA; - using FloatBAdjusted = ComputeTypeB; + using FloatAAdjusted = conditional_t, float, ComputeTypeA>; + using FloatBAdjusted = conditional_t, float, ComputeTypeB>; #endif // M0/M1/M1Padding @@ -760,19 +760,19 @@ struct GridwiseGemm_bk0mk1_bk0nk1_mn_xdlops_bwd_weight // register // sanity check constexpr bool is_single_rate_mfma = - (((is_same::value || is_same::value) && + (((is_same::value || is_same::value) && K1 <= 4) || - (is_same::value && K1 <= 8) || - ((is_same::value || is_same::value) && + (is_same::value && K1 <= 8) || + ((is_same::value || is_same::value) && K1 < 32)) ? true : false; constexpr auto is_scale_mfma = false; constexpr index_t KPack = math::max(K1, - MfmaSelector::selected_mfma.k_per_blk); @@ -787,7 +787,9 @@ struct GridwiseGemm_bk0mk1_bk0nk1_mn_xdlops_bwd_weight NPerXdl, MRepeat, NRepeat, - KPack>{}; + KPack, + ComputeTypeA, + ComputeTypeB>{}; auto c_thread_buf = blockwise_gemm.GetCThreadBuffer(); diff --git a/include/ck/utility/numeric_utils.hpp b/include/ck/utility/numeric_utils.hpp index 399bc0c3e8..ab84bd765f 100644 --- a/include/ck/utility/numeric_utils.hpp +++ b/include/ck/utility/numeric_utils.hpp @@ -45,6 +45,24 @@ struct NumericUtils using bitwise_type = uint32_t; }; +template <> +struct NumericUtils +{ + static constexpr int exp = 8; + static constexpr int mant = 10; + static constexpr int bias = 127; + static constexpr uint32_t nan_mask = 0x7F800000; + static constexpr uint32_t head_mask = 0xFF800000; + static constexpr uint32_t mant_mask = 0x7FFFFF; + static constexpr uint32_t exp_mask = 0xFF; + static constexpr uint32_t Inf = 0x7F800000; + static constexpr uint32_t NegInf = 0xFF800000; + static constexpr uint32_t NaN = 0x7F800001; + static constexpr uint32_t Neg0 = 0x80000000; + static constexpr bool has_inf = true; + using bitwise_type = uint32_t; +}; + template <> struct NumericUtils { diff --git a/library/include/ck/library/reference_tensor_operation/cpu/reference_conv_bwd_data.hpp b/library/include/ck/library/reference_tensor_operation/cpu/reference_conv_bwd_data.hpp index 10b169c21e..54f190b3ec 100644 --- a/library/include/ck/library/reference_tensor_operation/cpu/reference_conv_bwd_data.hpp +++ b/library/include/ck/library/reference_tensor_operation/cpu/reference_conv_bwd_data.hpp @@ -28,6 +28,7 @@ template = 1 && NDimSpatial <= 3, bool>::type = false> struct ReferenceConvBwdData : public device::BaseOperator { @@ -142,8 +143,10 @@ struct ReferenceConvBwdData : public device::BaseOperator c, x); - v_acc += ck::type_convert(v_out) * - ck::type_convert(v_wei); + v_acc += ck::type_convert( + ck::type_convert(v_out)) * + ck::type_convert( + ck::type_convert(v_wei)); } } } @@ -235,8 +238,11 @@ struct ReferenceConvBwdData : public device::BaseOperator y, x); - v_acc += ck::type_convert(v_out) * - ck::type_convert(v_wei); + v_acc += + ck::type_convert( + ck::type_convert(v_out)) * + ck::type_convert( + ck::type_convert(v_wei)); } } } @@ -354,8 +360,12 @@ struct ReferenceConvBwdData : public device::BaseOperator x); v_acc += - ck::type_convert(v_out) * - ck::type_convert(v_wei); + ck::type_convert( + ck::type_convert< + ComputeDataType>(v_out)) * + ck::type_convert( + ck::type_convert< + ComputeDataType>(v_wei)); } } } diff --git a/library/include/ck/library/tensor_operation_instance/gpu/grouped_conv_bwd_data/device_grouped_conv_bwd_data_transpose_xdl_instance.hpp b/library/include/ck/library/tensor_operation_instance/gpu/grouped_conv_bwd_data/device_grouped_conv_bwd_data_transpose_xdl_instance.hpp index e535ba0170..04165382f4 100644 --- a/library/include/ck/library/tensor_operation_instance/gpu/grouped_conv_bwd_data/device_grouped_conv_bwd_data_transpose_xdl_instance.hpp +++ b/library/include/ck/library/tensor_operation_instance/gpu/grouped_conv_bwd_data/device_grouped_conv_bwd_data_transpose_xdl_instance.hpp @@ -18,6 +18,7 @@ namespace instance { using BF16 = ck::bhalf_t; using F16 = ck::half_t; using F32 = float; +using TF32 = ck::tf32_t; using BF8 = ck::bf8_t; using F8 = ck::f8_t; @@ -84,17 +85,17 @@ using device_grouped_conv_bwd_data_transpose_xdl_bf16_instances = DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, 1, S<4, 8, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 8, 8, 1, 1, 1, S<1, 32, 1, 4>, 8, make_default_loop_scheduler(), BF16, BF16, 2, 2>, DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, 1, S<4, 8, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 8, 8, 1, 1, 1, S<1, 16, 1, 4>, 8, make_default_loop_scheduler(), BF16, BF16, 2, 2>, DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, 1, S<4, 4, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 8, 8, 1, 1, 1, S<1, 16, 1, 4>, 8, make_default_loop_scheduler(), BF16, BF16, 2, 2>, - + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, 1, S<4, 16, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 8, 8, 1, 1, 1, S<1, 32, 1, 8>, 8, make_default_loop_scheduler(), BF16, BF16, 4, 4>, DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, 1, S<4, 8, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 8, 8, 1, 1, 1, S<1, 32, 1, 4>, 8, make_default_loop_scheduler(), BF16, BF16, 4, 4>, DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, 1, S<4, 8, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 8, 8, 1, 1, 1, S<1, 16, 1, 4>, 8, make_default_loop_scheduler(), BF16, BF16, 4, 4>, DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, 1, S<4, 4, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 8, 8, 1, 1, 1, S<1, 16, 1, 4>, 8, make_default_loop_scheduler(), BF16, BF16, 4, 4>, - + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, 1, S<4, 16, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 8, 8, 1, 1, 1, S<1, 32, 1, 8>, 8, make_default_loop_scheduler(), BF16, BF16, 1, 2>, DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, 1, S<4, 8, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 8, 8, 1, 1, 1, S<1, 32, 1, 4>, 8, make_default_loop_scheduler(), BF16, BF16, 1, 2>, DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, 1, S<4, 8, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 8, 8, 1, 1, 1, S<1, 16, 1, 4>, 8, make_default_loop_scheduler(), BF16, BF16, 1, 2>, DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, 1, S<4, 4, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 8, 8, 1, 1, 1, S<1, 16, 1, 4>, 8, make_default_loop_scheduler(), BF16, BF16, 1, 2>, - + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, 1, S<4, 16, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 8, 8, 1, 1, 1, S<1, 32, 1, 8>, 8, make_default_loop_scheduler(), BF16, BF16, 2, 1>, DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, 1, S<4, 8, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 8, 8, 1, 1, 1, S<1, 32, 1, 4>, 8, make_default_loop_scheduler(), BF16, BF16, 2, 1>, DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, 1, S<4, 8, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 8, 8, 1, 1, 1, S<1, 16, 1, 4>, 8, make_default_loop_scheduler(), BF16, BF16, 2, 1>, diff --git a/library/include/ck/library/tensor_operation_instance/gpu/grouped_conv_bwd_data/device_grouped_conv_bwd_data_xdl_bilinear_instance.hpp b/library/include/ck/library/tensor_operation_instance/gpu/grouped_conv_bwd_data/device_grouped_conv_bwd_data_xdl_bilinear_instance.hpp index 216b4e2fe7..e5f1bdc3e7 100644 --- a/library/include/ck/library/tensor_operation_instance/gpu/grouped_conv_bwd_data/device_grouped_conv_bwd_data_xdl_bilinear_instance.hpp +++ b/library/include/ck/library/tensor_operation_instance/gpu/grouped_conv_bwd_data/device_grouped_conv_bwd_data_xdl_bilinear_instance.hpp @@ -18,6 +18,7 @@ namespace instance { using BF16 = ck::bhalf_t; using F16 = ck::half_t; using F32 = float; +using TF32 = ck::tf32_t; template using S = ck::Sequence; @@ -143,6 +144,43 @@ using device_grouped_conv_bwd_data_xdl_bilinear_f32_instances = // clang-format on >; +// f32_f32_f32_f32 tf32 +template +using device_grouped_conv_bwd_data_xdl_bilinear_f32_tf32_instances = + std::tuple< + // clang-format off + // ##############################################| NDim| ALayout| BLayout| DsLayout| ELayout| AData| BData| AccData| CShuffle| DsData| EData| AElementwise| BElementwise| CDEElementwise| ConvolutionBackward| DoPad| DoPad| NumGemmK| Block| MPer| NPer| KPer| AK1| BK1| MPer| NPer| MXdl| NXdl| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockLds| BBlockTransfer| BBlockTransfer| BBlockTransfer| BBlockTransfer| BBlockTransfer| BBlockTransfer| BBlockLds| CShuffleMXdl| CShuffleNXdl| CDEBlockTransfer| CDEBlockTransfer| + // ##############################################| Spatial| | | | | Type| Type| Type| DataType| Type| Type| Operation| Operation| Operation| DataSpecialization| GemmM| GemmN| PrefetchStage| Size| Block| Block| Block| | | XDL| XDL| PerWave| PerWave| ThreadCluster| ThreadCluster| SrcAccessOrder| SrcVectorDim| SrcScalar| DstScalar| ExtraM| ThreadCluster| ThreadCluster| SrcAccessOrder| SrcVectorDim| SrcScalar| DstScalar| ExtraN| PerWave| PerWave| _MBlock_MPerBlock| ScalarPerVector| + // ##############################################| | | | | | | | | | | | | | | | | | | | | | | | | | | | | Lengths_AK0_M_AK1| ArrangeOrder| | | PerVector| PerVector_AK1| | Lengths_BK0_N_BK1| ArrangeOrder| | | PerVector| PerVector_BK1| | PerShuffle| PerShuffle| _NBlock_NPerBlock| _NPerBlock| + // ##############################################| | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | + // generic instance + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1< NDimSpatial, ALayout, BLayout, DsLayout, ELayout, F32, F32, F32, F32, Tuple, F32, PassThrough, PassThrough, Bilinear, ConvSpec, true, true, 1, 64, 64, 64, 32, 8, 8, 32, 32, 2, 2, S<4, 16, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 1, 4, 1, S<4, 16, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 1, 4, 1, 1, 1, S<1, 16, 1, 4>, 1, make_default_loop_scheduler(), TF32, TF32>, + // instances for small conv.K and conv.C + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1< NDimSpatial, ALayout, BLayout, DsLayout, ELayout, F32, F32, F32, F32, Tuple, F32, PassThrough, PassThrough, Bilinear, ConvSpec, true, true, 1, 256, 64, 128, 32, 8, 8, 32, 32, 1, 2, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 1, 4, 1, S<4, 32, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 4, 4, 1, 1, 1, S<1, 32, 1, 8> , 4, make_default_loop_scheduler(), TF32, TF32>, + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1< NDimSpatial, ALayout, BLayout, DsLayout, ELayout, F32, F32, F32, F32, Tuple, F32, PassThrough, PassThrough, Bilinear, ConvSpec, true, true, 1, 128, 128, 32, 32, 8, 8, 32, 32, 2, 1, S<4, 32, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 4, 4, 1, S<4, 8, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 1, 4, 1, 1, 1, S<1, 32, 1, 4>, 1, make_default_loop_scheduler(), TF32, TF32>, + + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1< NDimSpatial, ALayout, BLayout, DsLayout, ELayout, F32, F32, F32, F32, Tuple, F32, PassThrough, PassThrough, Bilinear, ConvSpec, true, true, 1, 256, 128, 256, 32, 8, 2, 32, 32, 2, 4, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 4, 4, 1, S<4, 64, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 4, 2, 0, 1, 1, S<1, 32, 1, 8> , 4, make_default_loop_scheduler(), TF32, TF32>, + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1< NDimSpatial, ALayout, BLayout, DsLayout, ELayout, F32, F32, F32, F32, Tuple, F32, PassThrough, PassThrough, Bilinear, ConvSpec, true, true, 1, 256, 256, 128, 32, 8, 8, 32, 32, 4, 2, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 4, 4, 1, S<4, 32, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 4, 4, 1, 1, 1, S<1, 32, 1, 8> , 4, make_default_loop_scheduler(), TF32, TF32>, + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1< NDimSpatial, ALayout, BLayout, DsLayout, ELayout, F32, F32, F32, F32, Tuple, F32, PassThrough, PassThrough, Bilinear, ConvSpec, true, true, 1, 256, 128, 256, 32, 8, 8, 32, 32, 2, 4, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 4, 4, 1, S<4, 64, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 4, 4, 1, 1, 1, S<1, 32, 1, 8> , 4, make_default_loop_scheduler(), TF32, TF32>, + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1< NDimSpatial, ALayout, BLayout, DsLayout, ELayout, F32, F32, F32, F32, Tuple, F32, PassThrough, PassThrough, Bilinear, ConvSpec, true, true, 1, 128, 128, 128, 32, 8, 8, 32, 32, 4, 2, S<4, 32, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 4, 4, 1, S<4, 32, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 4, 4, 1, 1, 1, S<1, 16, 1, 8> , 4, make_default_loop_scheduler(), TF32, TF32>, + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1< NDimSpatial, ALayout, BLayout, DsLayout, ELayout, F32, F32, F32, F32, Tuple, F32, PassThrough, PassThrough, Bilinear, ConvSpec, true, true, 1, 256, 128, 128, 32, 8, 8, 32, 32, 2, 2, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 4, 4, 1, S<4, 32, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 4, 4, 1, 1, 1, S<1, 32, 1, 8> , 4, make_default_loop_scheduler(), TF32, TF32>, + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1< NDimSpatial, ALayout, BLayout, DsLayout, ELayout, F32, F32, F32, F32, Tuple, F32, PassThrough, PassThrough, Bilinear, ConvSpec, true, true, 1, 128, 128, 64, 32, 8, 8, 32, 32, 2, 2, S<4, 32, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 4, 4, 1, S<4, 16, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 4, 4, 1, 1, 1, S<1, 32, 1, 4> , 4, make_default_loop_scheduler(), TF32, TF32>, + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1< NDimSpatial, ALayout, BLayout, DsLayout, ELayout, F32, F32, F32, F32, Tuple, F32, PassThrough, PassThrough, Bilinear, ConvSpec, true, true, 1, 128, 64, 128, 32, 8, 8, 32, 32, 2, 2, S<4, 32, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 4, 4, 1, S<4, 32, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 4, 4, 1, 1, 1, S<1, 16, 1, 8> , 4, make_default_loop_scheduler(), TF32, TF32>, + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1< NDimSpatial, ALayout, BLayout, DsLayout, ELayout, F32, F32, F32, F32, Tuple, F32, PassThrough, PassThrough, Bilinear, ConvSpec, true, true, 1, 64, 64, 64, 32, 8, 8, 32, 32, 2, 2, S<4, 16, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 4, 4, 1, S<4, 16, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 4, 4, 1, 1, 1, S<1, 16, 1, 4> , 4, make_default_loop_scheduler(), TF32, TF32>, + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1< NDimSpatial, ALayout, BLayout, DsLayout, ELayout, F32, F32, F32, F32, Tuple, F32, PassThrough, PassThrough, Bilinear, ConvSpec, true, true, 1, 256, 128, 64, 32, 8, 8, 32, 32, 2, 1, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 4, 4, 1, S<4, 16, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 4, 4, 1, 1, 1, S<1, 32, 1, 8> , 4, make_default_loop_scheduler(), TF32, TF32>, + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1< NDimSpatial, ALayout, BLayout, DsLayout, ELayout, F32, F32, F32, F32, Tuple, F32, PassThrough, PassThrough, Bilinear, ConvSpec, true, true, 1, 256, 64, 128, 32, 8, 8, 32, 32, 1, 2, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 4, 4, 1, S<4, 32, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 4, 4, 1, 1, 1, S<1, 32, 1, 8> , 4, make_default_loop_scheduler(), TF32, TF32>, + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1< NDimSpatial, ALayout, BLayout, DsLayout, ELayout, F32, F32, F32, F32, Tuple, F32, PassThrough, PassThrough, Bilinear, ConvSpec, true, true, 1, 128, 128, 32, 32, 8, 8, 32, 32, 2, 1, S<4, 32, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 4, 4, 1, S<4, 8, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 4, 4, 1, 1, 1, S<1, 32, 1, 4> , 4, make_default_loop_scheduler(), TF32, TF32>, + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1< NDimSpatial, ALayout, BLayout, DsLayout, ELayout, F32, F32, F32, F32, Tuple, F32, PassThrough, PassThrough, Bilinear, ConvSpec, true, true, 1, 128, 32, 128, 32, 8, 8, 32, 32, 1, 2, S<4, 32, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 4, 4, 1, S<4, 32, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 4, 4, 1, 1, 1, S<1, 16, 1, 8> , 4, make_default_loop_scheduler(), TF32, TF32>, + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1< NDimSpatial, ALayout, BLayout, DsLayout, ELayout, F32, F32, F32, F32, Tuple, F32, PassThrough, PassThrough, Bilinear, ConvSpec, true, true, 1, 64, 64, 32, 32, 8, 8, 32, 32, 2, 1, S<4, 16, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 4, 4, 1, S<4, 8, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 4, 4, 1, 1, 1, S<1, 16, 1, 4> , 4, make_default_loop_scheduler(), TF32, TF32>, + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1< NDimSpatial, ALayout, BLayout, DsLayout, ELayout, F32, F32, F32, F32, Tuple, F32, PassThrough, PassThrough, Bilinear, ConvSpec, true, true, 1, 64, 32, 64, 32, 8, 8, 32, 32, 1, 2, S<4, 16, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, 1, S<4, 8, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 8, 8, 1, 1, 1, S<1, 16, 1, 4> , 8, make_default_loop_scheduler(), TF32, TF32> + // clang-format on + >; + } // namespace instance } // namespace device } // namespace tensor_operation diff --git a/library/include/ck/library/tensor_operation_instance/gpu/grouped_conv_bwd_data/device_grouped_conv_bwd_data_xdl_instance.hpp b/library/include/ck/library/tensor_operation_instance/gpu/grouped_conv_bwd_data/device_grouped_conv_bwd_data_xdl_instance.hpp index f16a345e14..9c4b9cb512 100644 --- a/library/include/ck/library/tensor_operation_instance/gpu/grouped_conv_bwd_data/device_grouped_conv_bwd_data_xdl_instance.hpp +++ b/library/include/ck/library/tensor_operation_instance/gpu/grouped_conv_bwd_data/device_grouped_conv_bwd_data_xdl_instance.hpp @@ -18,6 +18,7 @@ namespace instance { using BF16 = ck::bhalf_t; using F16 = ck::half_t; using F32 = float; +using TF32 = ck::tf32_t; using BF8 = ck::bf8_t; using F8 = ck::f8_t; @@ -375,6 +376,41 @@ using device_grouped_conv_bwd_data_xdl_f32_optimized_loads_instances = // clang-format on >; +template +using device_grouped_conv_bwd_data_xdl_f32_tf32_optimized_loads_instances = + std::tuple< + // clang-format off + // ##############################################| NDim| ALayout| BLayout| DsLayout| ELayout| AData| BData| AccData| CShuffle| DsData| EData| AElementwise| BElementwise| CDEElementwise| ConvolutionBackward| DoPad| DoPad| NumGemmK| Block| MPer| NPer| KPer| AK1| BK1| MPer| NPer| MXdl| NXdl| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockLds| BBlockTransfer| BBlockTransfer| BBlockTransfer| BBlockTransfer| BBlockTransfer| BBlockTransfer| BBlockLds| CShuffleMXdl| CShuffleNXdl| CDEBlockTransfer| CDEBlockTransfer| + // ##############################################| Spatial| | | | | Type| Type| Type| DataType| Type| Type| Operation| Operation| Operation| DataSpecialization| GemmM| GemmN| PrefetchStage| Size| Block| Block| Block| | | XDL| XDL| PerWave| PerWave| ThreadCluster| ThreadCluster| SrcAccessOrder| SrcVectorDim| SrcScalar| DstScalar| ExtraM| ThreadCluster| ThreadCluster| SrcAccessOrder| SrcVectorDim| SrcScalar| DstScalar| ExtraN| PerWave| PerWave| _MBlock_MPerBlock| ScalarPerVector| + // ##############################################| | | | | | | | | | | | | | | | | | | | | | | | | | | | | Lengths_AK0_M_AK1| ArrangeOrder| | | PerVector| PerVector_AK1| | Lengths_BK0_N_BK1| ArrangeOrder| | | PerVector| PerVector_BK1| | PerShuffle| PerShuffle| _NBlock_NPerBlock| _NPerBlock| + // ##############################################| | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | + // A K1 one access for each thread per load + // 32x32 + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1< NDimSpatial, ALayout, BLayout, DsLayout, ELayout, F32, F32, F32, F32, Empty_Tuple, F32, PassThrough, PassThrough, PassThrough, ConvSpec, true, true, 1, 256, 128, 32, 32, 8, 8, 32, 32, 1, 1, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 4, 1, S<4, 4, 8>, S<0, 2, 1>, S<0, 2, 1>, 1, 8, 1, 1, 1, 1, S<1, 32, 1, 8>, 4, make_default_loop_scheduler(), TF32, TF32>, + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1< NDimSpatial, ALayout, BLayout, DsLayout, ELayout, F32, F32, F32, F32, Empty_Tuple, F32, PassThrough, PassThrough, PassThrough, ConvSpec, true, true, 1, 256, 128, 32, 16, 4, 4, 32, 32, 1, 1, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 4, 4, 1, S<4, 4, 4>, S<0, 2, 1>, S<0, 2, 1>, 1, 8, 1, 1, 1, 1, S<1, 32, 1, 8>, 4, make_default_loop_scheduler(), TF32, TF32>, + + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1< NDimSpatial, ALayout, BLayout, DsLayout, ELayout, F32, F32, F32, F32, Empty_Tuple, F32, PassThrough, PassThrough, PassThrough, ConvSpec, true, true, 1, 256, 128, 32, 32, 8, 8, 32, 32, 1, 1, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 4, 1, S<4, 8, 8>, S<0, 2, 1>, S<0, 2, 1>, 1, 2, 1, 1, 1, 1, S<1, 16, 1, 16>, 2, make_default_loop_scheduler(), TF32, TF32>, + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1< NDimSpatial, ALayout, BLayout, DsLayout, ELayout, F32, F32, F32, F32, Empty_Tuple, F32, PassThrough, PassThrough, PassThrough, ConvSpec, true, true, 1, 256, 128, 32, 16, 4, 4, 32, 32, 1, 1, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 4, 4, 1, S<4, 8, 4>, S<0, 2, 1>, S<0, 2, 1>, 1, 2, 1, 1, 1, 1, S<1, 16, 1, 16>, 2, make_default_loop_scheduler(), TF32, TF32>, + + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1< NDimSpatial, ALayout, BLayout, DsLayout, ELayout, F32, F32, F32, F32, Empty_Tuple, F32, PassThrough, PassThrough, PassThrough, ConvSpec, true, true, 1, 256, 128, 32, 32, 8, 8, 32, 32, 1, 1, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 4, 1, S<4, 8, 8>, S<0, 2, 1>, S<0, 2, 1>, 1, 1, 1, 1, 1, 1, S<1, 8, 1, 32>, 1, make_default_loop_scheduler(), TF32, TF32>, + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1< NDimSpatial, ALayout, BLayout, DsLayout, ELayout, F32, F32, F32, F32, Empty_Tuple, F32, PassThrough, PassThrough, PassThrough, ConvSpec, true, true, 1, 256, 128, 32, 16, 4, 4, 32, 32, 1, 1, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 4, 4, 1, S<4, 16, 4>, S<0, 2, 1>, S<0, 2, 1>, 1, 1, 1, 1, 1, 1, S<1, 8, 1, 32>, 1, make_default_loop_scheduler(), TF32, TF32>, + // 16x16 + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1< NDimSpatial, ALayout, BLayout, DsLayout, ELayout, F32, F32, F32, F32, Empty_Tuple, F32, PassThrough, PassThrough, PassThrough, ConvSpec, true, true, 1, 256, 64, 16, 32, 8, 8, 16, 16, 1, 1, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 4, 1, S<4, 2, 8>, S<0, 2, 1>, S<0, 2, 1>, 1, 8, 1, 1, 1, 1, S<1, 64, 1, 4>, 4, make_default_loop_scheduler(), TF32, TF32>, + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1< NDimSpatial, ALayout, BLayout, DsLayout, ELayout, F32, F32, F32, F32, Empty_Tuple, F32, PassThrough, PassThrough, PassThrough, ConvSpec, true, true, 1, 256, 64, 16, 16, 4, 4, 16, 16, 1, 1, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 4, 4, 1, S<4, 2, 4>, S<0, 2, 1>, S<0, 2, 1>, 1, 8, 1, 1, 1, 1, S<1, 64, 1, 4>, 4, make_default_loop_scheduler(), TF32, TF32>, + + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1< NDimSpatial, ALayout, BLayout, DsLayout, ELayout, F32, F32, F32, F32, Empty_Tuple, F32, PassThrough, PassThrough, PassThrough, ConvSpec, true, true, 1, 256, 64, 16, 32, 8, 8, 16, 16, 1, 1, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 4, 1, S<4, 8, 8>, S<0, 2, 1>, S<0, 2, 1>, 1, 2, 1, 1, 1, 1, S<1, 32, 1, 8>, 2, make_default_loop_scheduler(), TF32, TF32>, + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1< NDimSpatial, ALayout, BLayout, DsLayout, ELayout, F32, F32, F32, F32, Empty_Tuple, F32, PassThrough, PassThrough, PassThrough, ConvSpec, true, true, 1, 256, 64, 16, 16, 4, 4, 16, 16, 1, 1, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 4, 4, 1, S<4, 8, 4>, S<0, 2, 1>, S<0, 2, 1>, 1, 2, 1, 1, 1, 1, S<1, 32, 1, 8>, 2, make_default_loop_scheduler(), TF32, TF32>, + + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1< NDimSpatial, ALayout, BLayout, DsLayout, ELayout, F32, F32, F32, F32, Empty_Tuple, F32, PassThrough, PassThrough, PassThrough, ConvSpec, true, true, 1, 256, 64, 16, 32, 8, 8, 16, 16, 1, 1, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 4, 1, S<4, 8, 8>, S<0, 2, 1>, S<0, 2, 1>, 1, 1, 1, 1, 1, 1, S<1, 16, 1, 16>, 1, make_default_loop_scheduler(), TF32, TF32>, + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1< NDimSpatial, ALayout, BLayout, DsLayout, ELayout, F32, F32, F32, F32, Empty_Tuple, F32, PassThrough, PassThrough, PassThrough, ConvSpec, true, true, 1, 256, 64, 16, 16, 4, 4, 16, 16, 1, 1, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 4, 4, 1, S<4, 16, 4>, S<0, 2, 1>, S<0, 2, 1>, 1, 1, 1, 1, 1, 1, S<1, 16, 1, 16>, 1, make_default_loop_scheduler(), TF32, TF32> + // clang-format on + >; + template ; +// f32_f32_f32_f32 tf32 +template +using device_grouped_conv_bwd_data_xdl_f32_tf32_generic_instances = + std::tuple< + // clang-format off + // ##############################################| NDim| ALayout| BLayout| DsLayout| ELayout| AData| BData| AccData| CShuffle| DsData| EData| AElementwise| BElementwise| CDEElementwise| ConvolutionBackward| DoPad| DoPad| NumGemmK| Block| MPer| NPer| KPer| AK1| BK1| MPer| NPer| MXdl| NXdl| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockLds| BBlockTransfer| BBlockTransfer| BBlockTransfer| BBlockTransfer| BBlockTransfer| BBlockTransfer| BBlockLds| CShuffleMXdl| CShuffleNXdl| CDEBlockTransfer| CDEBlockTransfer| + // ##############################################| Spatial| | | | | Type| Type| Type| DataType| Type| Type| Operation| Operation| Operation| DataSpecialization| GemmM| GemmN| PrefetchStage| Size| Block| Block| Block| | | XDL| XDL| PerWave| PerWave| ThreadCluster| ThreadCluster| SrcAccessOrder| SrcVectorDim| SrcScalar| DstScalar| ExtraM| ThreadCluster| ThreadCluster| SrcAccessOrder| SrcVectorDim| SrcScalar| DstScalar| ExtraN| PerWave| PerWave| _MBlock_MPerBlock| ScalarPerVector| + // ##############################################| | | | | | | | | | | | | | | | | | | | | | | | | | | | | Lengths_AK0_M_AK1| ArrangeOrder| | | PerVector| PerVector_AK1| | Lengths_BK0_N_BK1| ArrangeOrder| | | PerVector| PerVector_BK1| | PerShuffle| PerShuffle| _NBlock_NPerBlock| _NPerBlock| + // ##############################################| | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | + // generic instance + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1< NDimSpatial, ALayout, BLayout, DsLayout, ELayout, F32, F32, F32, F32, Empty_Tuple, F32, PassThrough, PassThrough, PassThrough, ConvSpec, true, true, 1, 64, 64, 64, 32, 8, 8, 32, 32, 2, 2, S<4, 16, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 1, 4, 1, S<4, 16, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 1, 4, 1, 1, 1, S<1, 16, 1, 4>, 1, make_default_loop_scheduler(), TF32, TF32> + // clang-format on + >; + +template +using device_grouped_conv_bwd_data_xdl_f32_tf32_16_16_instances = + std::tuple< + // clang-format off + // ##############################################| NDim| ALayout| BLayout| DsLayout| ELayout| AData| BData| AccData| CShuffle| DsData| EData| AElementwise| BElementwise| CDEElementwise| ConvolutionBackward| DoPad| DoPad| NumGemmK| Block| MPer| NPer| KPer| AK1| BK1| MPer| NPer| MXdl| NXdl| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockLds| BBlockTransfer| BBlockTransfer| BBlockTransfer| BBlockTransfer| BBlockTransfer| BBlockTransfer| BBlockLds| CShuffleMXdl| CShuffleNXdl| CDEBlockTransfer| CDEBlockTransfer| + // ##############################################| Spatial| | | | | Type| Type| Type| DataType| Type| Type| Operation| Operation| Operation| DataSpecialization| GemmM| GemmN| PrefetchStage| Size| Block| Block| Block| | | XDL| XDL| PerWave| PerWave| ThreadCluster| ThreadCluster| SrcAccessOrder| SrcVectorDim| SrcScalar| DstScalar| ExtraM| ThreadCluster| ThreadCluster| SrcAccessOrder| SrcVectorDim| SrcScalar| DstScalar| ExtraN| PerWave| PerWave| _MBlock_MPerBlock| ScalarPerVector| + // ##############################################| | | | | | | | | | | | | | | | | | | | | | | | | | | | | Lengths_AK0_M_AK1| ArrangeOrder| | | PerVector| PerVector_AK1| | Lengths_BK0_N_BK1| ArrangeOrder| | | PerVector| PerVector_BK1| | PerShuffle| PerShuffle| _NBlock_NPerBlock| _NPerBlock| + // ##############################################| | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1< NDimSpatial, ALayout, BLayout, DsLayout, ELayout, F32, F32, F32, F32, Empty_Tuple, F32, PassThrough, PassThrough, PassThrough, ConvSpec, true, true, 1, 64, 16, 64, 32, 8, 8, 16, 16, 1, 4, S<4, 16, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 4, 8, 1, S<4, 16, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 4, 8, 1, 1, 1, S<1, 16, 1, 4>, 4, make_default_loop_scheduler(), TF32, TF32>, + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1< NDimSpatial, ALayout, BLayout, DsLayout, ELayout, F32, F32, F32, F32, Empty_Tuple, F32, PassThrough, PassThrough, PassThrough, ConvSpec, true, true, 1, 64, 16, 64, 32, 8, 8, 16, 16, 1, 4, S<4, 16, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 4, 8, 1, S<4, 16, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 1, 8, 1, 1, 1, S<1, 16, 1, 4>, 1, make_default_loop_scheduler(), TF32, TF32>, + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1< NDimSpatial, ALayout, BLayout, DsLayout, ELayout, F32, F32, F32, F32, Empty_Tuple, F32, PassThrough, PassThrough, PassThrough, ConvSpec, true, true, 1, 64, 16, 64, 32, 8, 8, 16, 16, 1, 4, S<4, 16, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 1, 8, 1, S<4, 16, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 4, 8, 1, 1, 1, S<1, 16, 1, 4>, 4, make_default_loop_scheduler(), TF32, TF32>, + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1< NDimSpatial, ALayout, BLayout, DsLayout, ELayout, F32, F32, F32, F32, Empty_Tuple, F32, PassThrough, PassThrough, PassThrough, ConvSpec, true, true, 1, 64, 64, 16, 32, 8, 8, 16, 16, 4, 1, S<4, 16, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 4, 8, 1, S<4, 4, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 4, 8, 1, 1, 1, S<1, 16, 1, 4>, 4, make_default_loop_scheduler(), TF32, TF32>, + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1< NDimSpatial, ALayout, BLayout, DsLayout, ELayout, F32, F32, F32, F32, Empty_Tuple, F32, PassThrough, PassThrough, PassThrough, ConvSpec, true, true, 1, 64, 64, 16, 32, 8, 8, 16, 16, 4, 1, S<4, 16, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 4, 8, 1, S<4, 16, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 1, 8, 1, 1, 1, S<1, 16, 1, 4>, 1, make_default_loop_scheduler(), TF32, TF32>, + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1< NDimSpatial, ALayout, BLayout, DsLayout, ELayout, F32, F32, F32, F32, Empty_Tuple, F32, PassThrough, PassThrough, PassThrough, ConvSpec, true, true, 1, 64, 64, 16, 32, 8, 8, 16, 16, 4, 1, S<4, 16, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 1, 8, 1, S<4, 4, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 4, 8, 1, 1, 1, S<1, 16, 1, 4>, 4, make_default_loop_scheduler(), TF32, TF32> + // clang-format on + >; + +template +using device_grouped_conv_bwd_data_xdl_f32_tf32_instances = + std::tuple< + // clang-format off + // ##############################################| NDim| ALayout| BLayout| DsLayout| ELayout| AData| BData| AccData| CShuffle| DsData| EData| AElementwise| BElementwise| CDEElementwise| ConvolutionBackward| DoPad| DoPad| NumGemmK| Block| MPer| NPer| KPer| AK1| BK1| MPer| NPer| MXdl| NXdl| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockLds| BBlockTransfer| BBlockTransfer| BBlockTransfer| BBlockTransfer| BBlockTransfer| BBlockTransfer| BBlockLds| CShuffleMXdl| CShuffleNXdl| CDEBlockTransfer| CDEBlockTransfer| + // ##############################################| Spatial| | | | | Type| Type| Type| DataType| Type| Type| Operation| Operation| Operation| DataSpecialization| GemmM| GemmN| PrefetchStage| Size| Block| Block| Block| | | XDL| XDL| PerWave| PerWave| ThreadCluster| ThreadCluster| SrcAccessOrder| SrcVectorDim| SrcScalar| DstScalar| ExtraM| ThreadCluster| ThreadCluster| SrcAccessOrder| SrcVectorDim| SrcScalar| DstScalar| ExtraN| PerWave| PerWave| _MBlock_MPerBlock| ScalarPerVector| + // ##############################################| | | | | | | | | | | | | | | | | | | | | | | | | | | | | Lengths_AK0_M_AK1| ArrangeOrder| | | PerVector| PerVector_AK1| | Lengths_BK0_N_BK1| ArrangeOrder| | | PerVector| PerVector_BK1| | PerShuffle| PerShuffle| _NBlock_NPerBlock| _NPerBlock| + // ##############################################| | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | + // generic instance + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1< NDimSpatial, ALayout, BLayout, DsLayout, ELayout, F32, F32, F32, F32, Empty_Tuple, F32, PassThrough, PassThrough, PassThrough, ConvSpec, true, true, 1, 64, 64, 64, 32, 8, 8, 32, 32, 2, 2, S<4, 16, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 1, 4, 1, S<4, 16, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 1, 4, 1, 1, 1, S<1, 16, 1, 4>, 1, make_default_loop_scheduler(), TF32, TF32>, + // instances for small conv.K and conv.C + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1< NDimSpatial, ALayout, BLayout, DsLayout, ELayout, F32, F32, F32, F32, Empty_Tuple, F32, PassThrough, PassThrough, PassThrough, ConvSpec, true, true, 1, 256, 64, 128, 32, 8, 8, 32, 32, 1, 2, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 1, 4, 1, S<4, 32, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 4, 4, 1, 1, 1, S<1, 32, 1, 8>, 4, make_default_loop_scheduler(), TF32, TF32>, + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1< NDimSpatial, ALayout, BLayout, DsLayout, ELayout, F32, F32, F32, F32, Empty_Tuple, F32, PassThrough, PassThrough, PassThrough, ConvSpec, true, true, 1, 128, 128, 32, 32, 8, 8, 32, 32, 2, 1, S<4, 32, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 4, 4, 1, S<4, 8, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 1, 4, 1, 1, 1, S<1, 32, 1, 4>, 1, make_default_loop_scheduler(), TF32, TF32>, + + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1< NDimSpatial, ALayout, BLayout, DsLayout, ELayout, F32, F32, F32, F32, Empty_Tuple, F32, PassThrough, PassThrough, PassThrough, ConvSpec, true, true, 1, 256, 128, 256, 32, 8, 8, 32, 32, 2, 4, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 4, 4, 1, S<4, 64, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 4, 4, 0, 1, 1, S<1, 32, 1, 8>, 4, make_default_loop_scheduler(), TF32, TF32>, + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1< NDimSpatial, ALayout, BLayout, DsLayout, ELayout, F32, F32, F32, F32, Empty_Tuple, F32, PassThrough, PassThrough, PassThrough, ConvSpec, true, true, 1, 256, 256, 128, 32, 8, 8, 32, 32, 4, 2, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 4, 4, 1, S<4, 32, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 4, 4, 1, 1, 1, S<1, 32, 1, 8>, 4, make_default_loop_scheduler(), TF32, TF32>, + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1< NDimSpatial, ALayout, BLayout, DsLayout, ELayout, F32, F32, F32, F32, Empty_Tuple, F32, PassThrough, PassThrough, PassThrough, ConvSpec, true, true, 1, 256, 128, 256, 32, 8, 8, 32, 32, 2, 4, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 4, 4, 1, S<4, 64, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 4, 4, 1, 1, 1, S<1, 32, 1, 8>, 4, make_default_loop_scheduler(), TF32, TF32>, + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1< NDimSpatial, ALayout, BLayout, DsLayout, ELayout, F32, F32, F32, F32, Empty_Tuple, F32, PassThrough, PassThrough, PassThrough, ConvSpec, true, true, 1, 128, 128, 128, 32, 8, 8, 32, 32, 4, 2, S<4, 32, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 4, 4, 1, S<4, 32, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 4, 4, 1, 1, 1, S<1, 16, 1, 8>, 4, make_default_loop_scheduler(), TF32, TF32>, + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1< NDimSpatial, ALayout, BLayout, DsLayout, ELayout, F32, F32, F32, F32, Empty_Tuple, F32, PassThrough, PassThrough, PassThrough, ConvSpec, true, true, 1, 256, 128, 128, 32, 8, 8, 32, 32, 2, 2, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 4, 4, 1, S<4, 32, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 4, 4, 1, 1, 1, S<1, 32, 1, 8>, 4, make_default_loop_scheduler(), TF32, TF32>, + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1< NDimSpatial, ALayout, BLayout, DsLayout, ELayout, F32, F32, F32, F32, Empty_Tuple, F32, PassThrough, PassThrough, PassThrough, ConvSpec, true, true, 1, 128, 128, 64, 32, 8, 8, 32, 32, 2, 2, S<4, 32, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 4, 4, 1, S<4, 16, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 4, 4, 1, 1, 1, S<1, 32, 1, 4>, 4, make_default_loop_scheduler(), TF32, TF32>, + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1< NDimSpatial, ALayout, BLayout, DsLayout, ELayout, F32, F32, F32, F32, Empty_Tuple, F32, PassThrough, PassThrough, PassThrough, ConvSpec, true, true, 1, 128, 64, 128, 32, 8, 8, 32, 32, 2, 2, S<4, 32, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 4, 4, 1, S<4, 32, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 4, 4, 1, 1, 1, S<1, 16, 1, 8>, 4, make_default_loop_scheduler(), TF32, TF32>, + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1< NDimSpatial, ALayout, BLayout, DsLayout, ELayout, F32, F32, F32, F32, Empty_Tuple, F32, PassThrough, PassThrough, PassThrough, ConvSpec, true, true, 1, 64, 64, 64, 32, 8, 8, 32, 32, 2, 2, S<4, 16, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 4, 4, 1, S<4, 16, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 4, 4, 1, 1, 1, S<1, 16, 1, 4>, 4, make_default_loop_scheduler(), TF32, TF32>, + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1< NDimSpatial, ALayout, BLayout, DsLayout, ELayout, F32, F32, F32, F32, Empty_Tuple, F32, PassThrough, PassThrough, PassThrough, ConvSpec, true, true, 1, 256, 128, 64, 32, 8, 8, 32, 32, 2, 1, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 4, 4, 1, S<4, 16, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 4, 4, 1, 1, 1, S<1, 32, 1, 8>, 4, make_default_loop_scheduler(), TF32, TF32>, + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1< NDimSpatial, ALayout, BLayout, DsLayout, ELayout, F32, F32, F32, F32, Empty_Tuple, F32, PassThrough, PassThrough, PassThrough, ConvSpec, true, true, 1, 256, 64, 128, 32, 8, 8, 32, 32, 1, 2, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 4, 4, 1, S<4, 32, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 4, 4, 1, 1, 1, S<1, 32, 1, 8>, 4, make_default_loop_scheduler(), TF32, TF32>, + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1< NDimSpatial, ALayout, BLayout, DsLayout, ELayout, F32, F32, F32, F32, Empty_Tuple, F32, PassThrough, PassThrough, PassThrough, ConvSpec, true, true, 1, 128, 128, 32, 32, 8, 8, 32, 32, 2, 1, S<4, 32, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 4, 4, 1, S<4, 8, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 4, 4, 1, 1, 1, S<1, 32, 1, 4>, 4, make_default_loop_scheduler(), TF32, TF32>, + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1< NDimSpatial, ALayout, BLayout, DsLayout, ELayout, F32, F32, F32, F32, Empty_Tuple, F32, PassThrough, PassThrough, PassThrough, ConvSpec, true, true, 1, 128, 32, 128, 32, 8, 8, 32, 32, 1, 2, S<4, 32, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 4, 4, 1, S<4, 32, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 4, 4, 1, 1, 1, S<1, 16, 1, 8>, 4, make_default_loop_scheduler(), TF32, TF32>, + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1< NDimSpatial, ALayout, BLayout, DsLayout, ELayout, F32, F32, F32, F32, Empty_Tuple, F32, PassThrough, PassThrough, PassThrough, ConvSpec, true, true, 1, 64, 64, 32, 32, 8, 8, 32, 32, 2, 1, S<4, 16, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 4, 4, 1, S<4, 8, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 4, 4, 1, 1, 1, S<1, 16, 1, 4>, 4, make_default_loop_scheduler(), TF32, TF32>, + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1< NDimSpatial, ALayout, BLayout, DsLayout, ELayout, F32, F32, F32, F32, Empty_Tuple, F32, PassThrough, PassThrough, PassThrough, ConvSpec, true, true, 1, 64, 32, 64, 32, 8, 8, 32, 32, 1, 2, S<4, 16, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 4, 8, 1, S<4, 8, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 4, 8, 1, 1, 1, S<1, 16, 1, 4>, 4, make_default_loop_scheduler(), TF32, TF32> + // clang-format on + >; + // f16_f16_f16_comp_f8 template using S = ck::Sequence; @@ -143,6 +144,43 @@ using device_grouped_conv_bwd_data_xdl_scale_f32_instances = // clang-format on >; +// f32_f32_f32_f32 tf32 +template +using device_grouped_conv_bwd_data_xdl_scale_f32_tf32_instances = + std::tuple< + // clang-format off + // ##############################################| NDim| ALayout| BLayout| DsLayout| ELayout| AData| BData| AccData| CShuffle| DsData| EData| AElementwise| BElementwise| CDEElementwise| ConvolutionBackward| DoPad| DoPad| NumGemmK| Block| MPer| NPer| KPer| AK1| BK1| MPer| NPer| MXdl| NXdl| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockLds| BBlockTransfer| BBlockTransfer| BBlockTransfer| BBlockTransfer| BBlockTransfer| BBlockTransfer| BBlockLds| CShuffleMXdl| CShuffleNXdl| CDEBlockTransfer| CDEBlockTransfer| + // ##############################################| Spatial| | | | | Type| Type| Type| DataType| Type| Type| Operation| Operation| Operation| DataSpecialization| GemmM| GemmN| PrefetchStage| Size| Block| Block| Block| | | XDL| XDL| PerWave| PerWave| ThreadCluster| ThreadCluster| SrcAccessOrder| SrcVectorDim| SrcScalar| DstScalar| ExtraM| ThreadCluster| ThreadCluster| SrcAccessOrder| SrcVectorDim| SrcScalar| DstScalar| ExtraN| PerWave| PerWave| _MBlock_MPerBlock| ScalarPerVector| + // ##############################################| | | | | | | | | | | | | | | | | | | | | | | | | | | | | Lengths_AK0_M_AK1| ArrangeOrder| | | PerVector| PerVector_AK1| | Lengths_BK0_N_BK1| ArrangeOrder| | | PerVector| PerVector_BK1| | PerShuffle| PerShuffle| _NBlock_NPerBlock| _NPerBlock| + // ##############################################| | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | + // generic instance + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1< NDimSpatial, ALayout, BLayout, DsLayout, ELayout, F32, F32, F32, F32, Tuple<>, F32, PassThrough, PassThrough, Scale, ConvSpec, true, true, 1, 64, 64, 64, 32, 8, 8, 32, 32, 2, 2, S<4, 16, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 1, 4, 1, S<4, 16, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 1, 4, 1, 1, 1, S<1, 16, 1, 4>, 1>, + // instances for small conv.K and conv.C + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1< NDimSpatial, ALayout, BLayout, DsLayout, ELayout, F32, F32, F32, F32, Tuple<>, F32, PassThrough, PassThrough, Scale, ConvSpec, true, true, 1, 256, 64, 128, 32, 8, 8, 32, 32, 1, 2, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 1, 4, 1, S<4, 32, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 4, 4, 1, 1, 1, S<1, 32, 1, 8>, 4>, + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1< NDimSpatial, ALayout, BLayout, DsLayout, ELayout, F32, F32, F32, F32, Tuple<>, F32, PassThrough, PassThrough, Scale, ConvSpec, true, true, 1, 128, 128, 32, 32, 8, 8, 32, 32, 2, 1, S<4, 32, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 4, 4, 1, S<4, 8, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 1, 4, 1, 1, 1, S<1, 32, 1, 4>, 1>, + + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1< NDimSpatial, ALayout, BLayout, DsLayout, ELayout, F32, F32, F32, F32, Tuple<>, F32, PassThrough, PassThrough, Scale, ConvSpec, true, true, 1, 256, 128, 256, 32, 8, 2, 32, 32, 2, 4, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 4, 4, 1, S<4, 64, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 4, 2, 0, 1, 1, S<1, 32, 1, 8>, 4>, + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1< NDimSpatial, ALayout, BLayout, DsLayout, ELayout, F32, F32, F32, F32, Tuple<>, F32, PassThrough, PassThrough, Scale, ConvSpec, true, true, 1, 256, 256, 128, 32, 8, 8, 32, 32, 4, 2, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 4, 4, 1, S<4, 32, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 4, 4, 1, 1, 1, S<1, 32, 1, 8>, 4>, + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1< NDimSpatial, ALayout, BLayout, DsLayout, ELayout, F32, F32, F32, F32, Tuple<>, F32, PassThrough, PassThrough, Scale, ConvSpec, true, true, 1, 256, 128, 256, 32, 8, 8, 32, 32, 2, 4, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 4, 4, 1, S<4, 64, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 4, 4, 1, 1, 1, S<1, 32, 1, 8>, 4>, + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1< NDimSpatial, ALayout, BLayout, DsLayout, ELayout, F32, F32, F32, F32, Tuple<>, F32, PassThrough, PassThrough, Scale, ConvSpec, true, true, 1, 128, 128, 128, 32, 8, 8, 32, 32, 4, 2, S<4, 32, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 4, 4, 1, S<4, 32, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 4, 4, 1, 1, 1, S<1, 16, 1, 8>, 4>, + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1< NDimSpatial, ALayout, BLayout, DsLayout, ELayout, F32, F32, F32, F32, Tuple<>, F32, PassThrough, PassThrough, Scale, ConvSpec, true, true, 1, 256, 128, 128, 32, 8, 8, 32, 32, 2, 2, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 4, 4, 1, S<4, 32, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 4, 4, 1, 1, 1, S<1, 32, 1, 8>, 4>, + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1< NDimSpatial, ALayout, BLayout, DsLayout, ELayout, F32, F32, F32, F32, Tuple<>, F32, PassThrough, PassThrough, Scale, ConvSpec, true, true, 1, 128, 128, 64, 32, 8, 8, 32, 32, 2, 2, S<4, 32, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 4, 4, 1, S<4, 16, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 4, 4, 1, 1, 1, S<1, 32, 1, 4>, 4>, + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1< NDimSpatial, ALayout, BLayout, DsLayout, ELayout, F32, F32, F32, F32, Tuple<>, F32, PassThrough, PassThrough, Scale, ConvSpec, true, true, 1, 128, 64, 128, 32, 8, 8, 32, 32, 2, 2, S<4, 32, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 4, 4, 1, S<4, 32, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 4, 4, 1, 1, 1, S<1, 16, 1, 8>, 4>, + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1< NDimSpatial, ALayout, BLayout, DsLayout, ELayout, F32, F32, F32, F32, Tuple<>, F32, PassThrough, PassThrough, Scale, ConvSpec, true, true, 1, 64, 64, 64, 32, 8, 8, 32, 32, 2, 2, S<4, 16, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 4, 4, 1, S<4, 16, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 4, 4, 1, 1, 1, S<1, 16, 1, 4>, 4>, + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1< NDimSpatial, ALayout, BLayout, DsLayout, ELayout, F32, F32, F32, F32, Tuple<>, F32, PassThrough, PassThrough, Scale, ConvSpec, true, true, 1, 256, 128, 64, 32, 8, 8, 32, 32, 2, 1, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 4, 4, 1, S<4, 16, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 4, 4, 1, 1, 1, S<1, 32, 1, 8>, 4>, + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1< NDimSpatial, ALayout, BLayout, DsLayout, ELayout, F32, F32, F32, F32, Tuple<>, F32, PassThrough, PassThrough, Scale, ConvSpec, true, true, 1, 256, 64, 128, 32, 8, 8, 32, 32, 1, 2, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 4, 4, 1, S<4, 32, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 4, 4, 1, 1, 1, S<1, 32, 1, 8>, 4>, + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1< NDimSpatial, ALayout, BLayout, DsLayout, ELayout, F32, F32, F32, F32, Tuple<>, F32, PassThrough, PassThrough, Scale, ConvSpec, true, true, 1, 128, 128, 32, 32, 8, 8, 32, 32, 2, 1, S<4, 32, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 4, 4, 1, S<4, 8, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 4, 4, 1, 1, 1, S<1, 32, 1, 4>, 4>, + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1< NDimSpatial, ALayout, BLayout, DsLayout, ELayout, F32, F32, F32, F32, Tuple<>, F32, PassThrough, PassThrough, Scale, ConvSpec, true, true, 1, 128, 32, 128, 32, 8, 8, 32, 32, 1, 2, S<4, 32, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 4, 4, 1, S<4, 32, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 4, 4, 1, 1, 1, S<1, 16, 1, 8>, 4>, + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1< NDimSpatial, ALayout, BLayout, DsLayout, ELayout, F32, F32, F32, F32, Tuple<>, F32, PassThrough, PassThrough, Scale, ConvSpec, true, true, 1, 64, 64, 32, 32, 8, 8, 32, 32, 2, 1, S<4, 16, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 4, 4, 1, S<4, 8, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 4, 4, 1, 1, 1, S<1, 16, 1, 4>, 4>, + DeviceGroupedConvBwdDataMultipleD_Xdl_CShuffle_v1< NDimSpatial, ALayout, BLayout, DsLayout, ELayout, F32, F32, F32, F32, Tuple<>, F32, PassThrough, PassThrough, Scale, ConvSpec, true, true, 1, 64, 32, 64, 32, 8, 8, 32, 32, 1, 2, S<4, 16, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, 1, S<4, 8, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 8, 8, 1, 1, 1, S<1, 16, 1, 4>, 8> + // clang-format on + >; + } // namespace instance } // namespace device } // namespace tensor_operation diff --git a/library/include/ck/library/tensor_operation_instance/gpu/grouped_conv_bwd_weight/device_grouped_conv_bwd_weight_v3_xdl_instance.hpp b/library/include/ck/library/tensor_operation_instance/gpu/grouped_conv_bwd_weight/device_grouped_conv_bwd_weight_v3_xdl_instance.hpp index b445e0001d..4e096e5b44 100644 --- a/library/include/ck/library/tensor_operation_instance/gpu/grouped_conv_bwd_weight/device_grouped_conv_bwd_weight_v3_xdl_instance.hpp +++ b/library/include/ck/library/tensor_operation_instance/gpu/grouped_conv_bwd_weight/device_grouped_conv_bwd_weight_v3_xdl_instance.hpp @@ -18,6 +18,7 @@ using namespace ck::tensor_layout::convolution; using BF16 = ck::bhalf_t; using F16 = ck::half_t; using F32 = float; +using TF32 = ck::tf32_t; #ifdef CK_ENABLE_FP8 using F8 = ck::f8_t; @@ -58,6 +59,24 @@ using device_grouped_conv_bwd_weight_v3_xdl_c_shuffle_f32_instances = std::tuple // clang-format on >; +template +using device_grouped_conv_bwd_weight_v3_xdl_c_shuffle_f32_tf32_instances = std::tuple< + // clang-format off + //#########################################| Num| InLayout| WeiLayout| OutLayout| InData| WeiData| OutData| AccData| In| Wei| Out| ConvBackward| Block| MPer| NPer| K0Per| K1| MPer| NPer| MXdl| NXdl| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockLds| BBlockTransfer| BBlockTransfer| BBlockTransfer| BlockTransfer| BBlockTransfer| BBlockTransfer| BBlockLds| CShuffle| CShuffle| CBlockTransfer| CBlockTransfer| BlockGemm| BlockGemm| + //#########################################| Dim| | | | Type| Type| Type| Type| Elementwise| Elementwise| Elementwise| Weight| Size| Block| Block| Block| | XDL| XDL| Per| Per| ThreadCluster| ThreadCluster| SrcAccessOrder| SrcVectorDim| SrcScalar| DstScalar| AddExtraM| ThreadCluster| ThreadCluster| SrcAccessOrder| SrcVectorDim| SrcScalar| DstScalar| AddExtraN| MXdlPerWave| NXdlPerWave| ClusterLengths| ScalarPerVector| Pipeline| Pipeline| + //#########################################| Spatial| | | | | | | | Operation| Operation| Operation| Specialization| | | | | | | | Wave| Wave| Lengths_K0_M_K1| ArrangeOrder| | | PerVector| PerVector_K1| | Lengths_K0_N_K1| ArrangeOrder| | | PerVector| PerVector_K1| | PerShuffle| PerShuffle| MBlock_MPerBlock| NWaveNPerXdl| Scheduler| Version| + //#########################################| | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | NBlock_NPerBlock| | | | + // generic instance + DeviceGroupedConvBwdWeight_Xdl_CShuffleV3< NDimSpatial, ALayout, BLayout, ELayout, F32, F32, F32, F32, PassThrough, PassThrough, PassThrough, ConvSpec, 64, 16, 16, 32, 8, 16, 16, 1, 1, S<4, 8, 1>, S<2, 0, 1>, S<1, 0, 2>, 1, 1, 4, false, S<4, 8, 1>, S<2, 0, 1>, S<1, 0, 2>, 1, 1, 4, false, 1, 1, S<1, 8, 1, 8>, 2, Scheduler, PipelineVersion, TF32, TF32> + // clang-format on + >; + template ; +template +using device_grouped_conv_bwd_weight_xdl_c_shuffle_f32_tf32_bilinear_instances = std::tuple< + // clang-format off + //#########################################| Num| InLayout| WeiLayout| OutLayout| DsData| InData| WeiData| OutData| AccData| DsData| In| Wei| Out| ConvBackward| Block| MPer| NPer| K0Per| K1| MPer| NPer| MXdl| NXdl| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockLds| BBlockTransfer| BBlockTransfer| BBlockTransfer| BlockTransfer| BBlockTransfer| BBlockTransfer| BBlockLds| CShuffle| CShuffle| CBlockTransfer| CBlockTransfer| + //#########################################| Dim| | | | Layout| Type| Type| Type| Type| Type| Elementwise| Elementwise| Elementwise| Weight| Size| Block| Block| Block| | XDL| XDL| Per| Per| ThreadCluster| ThreadCluster| SrcAccessOrder| SrcVectorDim| SrcScalar| DstScalar| AddExtraM| ThreadCluster| ThreadCluster| SrcAccessOrder| SrcVectorDim| SrcScalar| DstScalar| AddExtraN| MXdlPerWave| NXdlPerWave| ClusterLengths| ScalarPerVector| + //#########################################| Spatial| | | | | | | | | | Operation| Operation| Operation| Specialization| | | | | | | | Wave| Wave| Lengths_K0_M_K1| ArrangeOrder| | | PerVector| PerVector_K1| | Lengths_K0_N_K1| ArrangeOrder| | | PerVector| PerVector_K1| | PerShuffle| PerShuffle| MBlock_MPerBlock| NWaveNPerXdl| + //#########################################| | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | NBlock_NPerBlock| | + // generic instance + DeviceGroupedConvBwdWeightMultipleD_Xdl_CShuffle< NDimSpatial, ALayout, BLayout, ELayout, Tuple, F32, F32, F32, F32, Tuple, PassThrough, Bilinear, PassThrough, ConvSpec, 64, 64, 64, 4, 4, 32, 32, 2, 2, S<1, 4, 16, 1>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 1, 4, true, S<1, 4, 16, 1>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 1, 4, true, 1, 1, S<1, 16, 1, 4>, 1, TF32, TF32>, + // instances for small conv.K and conv.C + DeviceGroupedConvBwdWeightMultipleD_Xdl_CShuffle< NDimSpatial, ALayout, BLayout, ELayout, Tuple, F32, F32, F32, F32, Tuple, PassThrough, Bilinear, PassThrough, ConvSpec, 128, 128, 32, 4, 4, 32, 32, 2, 1, S<1, 4, 32, 1>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 4, true, S<1, 4, 8, 4>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 1, 1, true, 1, 1, S<1, 32, 1, 4>, 1, TF32, TF32>, + DeviceGroupedConvBwdWeightMultipleD_Xdl_CShuffle< NDimSpatial, ALayout, BLayout, ELayout, Tuple, F32, F32, F32, F32, Tuple, PassThrough, Bilinear, PassThrough, ConvSpec, 64, 32, 64, 4, 4, 32, 32, 1, 2, S<1, 4, 8, 2>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 1, 2, true, S<1, 4, 16, 1>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 4, true, 1, 1, S<1, 16, 1, 4>, 4, TF32, TF32>, + + DeviceGroupedConvBwdWeightMultipleD_Xdl_CShuffle< NDimSpatial, ALayout, BLayout, ELayout, Tuple, F32, F32, F32, F32, Tuple, PassThrough, Bilinear, PassThrough, ConvSpec, 256, 256, 128, 4, 4, 32, 32, 4, 2, S<1, 4, 64, 1>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 4, true, S<1, 4, 32, 2>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 2, true, 1, 1, S<1, 32, 1, 8>, 4, TF32, TF32>, + DeviceGroupedConvBwdWeightMultipleD_Xdl_CShuffle< NDimSpatial, ALayout, BLayout, ELayout, Tuple, F32, F32, F32, F32, Tuple, PassThrough, Bilinear, PassThrough, ConvSpec, 256, 128, 256, 4, 4, 32, 32, 2, 4, S<1, 4, 32, 2>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 2, true, S<1, 4, 64, 1>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 4, true, 1, 1, S<1, 32, 1, 8>, 4, TF32, TF32>, + DeviceGroupedConvBwdWeightMultipleD_Xdl_CShuffle< NDimSpatial, ALayout, BLayout, ELayout, Tuple, F32, F32, F32, F32, Tuple, PassThrough, Bilinear, PassThrough, ConvSpec, 128, 128, 128, 4, 4, 32, 32, 4, 2, S<1, 4, 32, 1>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 4, true, S<1, 4, 32, 1>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 4, true, 1, 1, S<1, 32, 1, 4>, 4, TF32, TF32>, + DeviceGroupedConvBwdWeightMultipleD_Xdl_CShuffle< NDimSpatial, ALayout, BLayout, ELayout, Tuple, F32, F32, F32, F32, Tuple, PassThrough, Bilinear, PassThrough, ConvSpec, 256, 128, 128, 4, 4, 32, 32, 2, 2, S<1, 4, 32, 2>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 2, true, S<1, 4, 32, 2>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 2, true, 1, 1, S<1, 32, 1, 4>, 4, TF32, TF32>, + DeviceGroupedConvBwdWeightMultipleD_Xdl_CShuffle< NDimSpatial, ALayout, BLayout, ELayout, Tuple, F32, F32, F32, F32, Tuple, PassThrough, Bilinear, PassThrough, ConvSpec, 128, 128, 64, 4, 4, 32, 32, 2, 2, S<1, 4, 32, 1>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 4, true, S<1, 4, 16, 2>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 2, true, 1, 1, S<1, 32, 1, 4>, 4, TF32, TF32>, + DeviceGroupedConvBwdWeightMultipleD_Xdl_CShuffle< NDimSpatial, ALayout, BLayout, ELayout, Tuple, F32, F32, F32, F32, Tuple, PassThrough, Bilinear, PassThrough, ConvSpec, 128, 64, 128, 4, 4, 32, 32, 2, 2, S<1, 4, 16, 2>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 2, true, S<1, 4, 32, 1>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 4, true, 1, 1, S<1, 32, 1, 4>, 4, TF32, TF32>, + DeviceGroupedConvBwdWeightMultipleD_Xdl_CShuffle< NDimSpatial, ALayout, BLayout, ELayout, Tuple, F32, F32, F32, F32, Tuple, PassThrough, Bilinear, PassThrough, ConvSpec, 64, 64, 64, 4, 4, 32, 32, 2, 2, S<1, 4, 16, 1>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 4, true, S<1, 4, 16, 1>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 4, true, 1, 1, S<1, 16, 1, 4>, 4, TF32, TF32>, + DeviceGroupedConvBwdWeightMultipleD_Xdl_CShuffle< NDimSpatial, ALayout, BLayout, ELayout, Tuple, F32, F32, F32, F32, Tuple, PassThrough, Bilinear, PassThrough, ConvSpec, 256, 128, 64, 4, 4, 32, 32, 2, 1, S<1, 4, 32, 2>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 2, true, S<1, 4, 16, 4>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 1, true, 1, 1, S<1, 32, 1, 4>, 4, TF32, TF32>, + DeviceGroupedConvBwdWeightMultipleD_Xdl_CShuffle< NDimSpatial, ALayout, BLayout, ELayout, Tuple, F32, F32, F32, F32, Tuple, PassThrough, Bilinear, PassThrough, ConvSpec, 256, 64, 128, 4, 4, 32, 32, 1, 2, S<1, 4, 16, 4>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 1, true, S<1, 4, 32, 2>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 2, true, 1, 1, S<1, 32, 1, 4>, 4, TF32, TF32>, + DeviceGroupedConvBwdWeightMultipleD_Xdl_CShuffle< NDimSpatial, ALayout, BLayout, ELayout, Tuple, F32, F32, F32, F32, Tuple, PassThrough, Bilinear, PassThrough, ConvSpec, 128, 128, 32, 4, 4, 32, 32, 2, 1, S<1, 4, 32, 1>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 4, true, S<1, 4, 8, 4>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 1, true, 1, 1, S<1, 32, 1, 4>, 4, TF32, TF32>, + DeviceGroupedConvBwdWeightMultipleD_Xdl_CShuffle< NDimSpatial, ALayout, BLayout, ELayout, Tuple, F32, F32, F32, F32, Tuple, PassThrough, Bilinear, PassThrough, ConvSpec, 128, 32, 128, 4, 4, 32, 32, 1, 2, S<1, 4, 8, 4>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 1, true, S<1, 4, 32, 1>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 4, true, 1, 1, S<1, 32, 1, 4>, 4, TF32, TF32>, + DeviceGroupedConvBwdWeightMultipleD_Xdl_CShuffle< NDimSpatial, ALayout, BLayout, ELayout, Tuple, F32, F32, F32, F32, Tuple, PassThrough, Bilinear, PassThrough, ConvSpec, 64, 64, 32, 4, 4, 32, 32, 2, 1, S<1, 4, 16, 1>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 4, true, S<1, 4, 8, 2>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 2, true, 1, 1, S<1, 16, 1, 4>, 4, TF32, TF32>, + DeviceGroupedConvBwdWeightMultipleD_Xdl_CShuffle< NDimSpatial, ALayout, BLayout, ELayout, Tuple, F32, F32, F32, F32, Tuple, PassThrough, Bilinear, PassThrough, ConvSpec, 64, 32, 64, 4, 4, 32, 32, 1, 2, S<1, 4, 8, 2>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 2, true, S<1, 4, 16, 1>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 4, true, 1, 1, S<1, 16, 1, 4>, 4, TF32, TF32> + // clang-format on + >; + template ; +template +using device_grouped_conv_bwd_weight_xdl_c_shuffle_f32_tf32_generic_instances = std::tuple< + // clang-format off + //#########################################| Num| InLayout| WeiLayout| OutLayout| InData| WeiData| OutData| AccData| In| Wei| Out| ConvBackward| Block| MPer| NPer| K0Per| K1| MPer| NPer| MXdl| NXdl| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockLds| BBlockTransfer| BBlockTransfer| BBlockTransfer| BlockTransfer| BBlockTransfer| BBlockTransfer| BBlockLds| CShuffle| CShuffle| CBlockTransfer| CBlockTransfer| + //#########################################| Dim| | | | Type| Type| Type| Type| Elementwise| Elementwise| Elementwise| Weight| Size| Block| Block| Block| | XDL| XDL| Per| Per| ThreadCluster| ThreadCluster| SrcAccessOrder| SrcVectorDim| SrcScalar| DstScalar| AddExtraM| ThreadCluster| ThreadCluster| SrcAccessOrder| SrcVectorDim| SrcScalar| DstScalar| AddExtraN| MXdlPerWave| NXdlPerWave| ClusterLengths| ScalarPerVector| + //#########################################| Spatial| | | | | | | | Operation| Operation| Operation| Specialization| | | | | | | | Wave| Wave| Lengths_K0_M_K1| ArrangeOrder| | | PerVector| PerVector_K1| | Lengths_K0_N_K1| ArrangeOrder| | | PerVector| PerVector_K1| | PerShuffle| PerShuffle| MBlock_MPerBlock| NWaveNPerXdl| + //#########################################| | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | NBlock_NPerBlock| | + // generic instance + DeviceGroupedConvBwdWeight_Xdl_CShuffle< NDimSpatial, ALayout, BLayout, ELayout, F32, F32, F32, F32, PassThrough, PassThrough, PassThrough, ConvSpec, 64, 64, 64, 4, 4, 32, 32, 2, 2, S<1, 4, 16, 1>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 1, 4, true, S<1, 4, 16, 1>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 1, 4, true, 1, 1, S<1, 16, 1, 4>, 1, TF32, TF32> + // clang-format on + >; + +template +using device_grouped_conv_bwd_weight_xdl_c_shuffle_f32_tf32_instances = std::tuple< + // clang-format off + //#########################################| Num| InLayout| WeiLayout| OutLayout| InData| WeiData| OutData| AccData| In| Wei| Out| ConvBackward| Block| MPer| NPer| K0Per| K1| MPer| NPer| MXdl| NXdl| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockLds| BBlockTransfer| BBlockTransfer| BBlockTransfer| BlockTransfer| BBlockTransfer| BBlockTransfer| BBlockLds| CShuffle| CShuffle| CBlockTransfer| CBlockTransfer| Compute| Compute| MaxTranspose| MaxTranspose| + //#########################################| Dim| | | | Type| Type| Type| Type| Elementwise| Elementwise| Elementwise| Weight| Size| Block| Block| Block| | XDL| XDL| Per| Per| ThreadCluster| ThreadCluster| SrcAccessOrder| SrcVectorDim| SrcScalar| DstScalar| AddExtraM| ThreadCluster| ThreadCluster| SrcAccessOrder| SrcVectorDim| SrcScalar| DstScalar| AddExtraN| MXdlPerWave| NXdlPerWave| ClusterLengths| ScalarPerVector| TypeA| TypeB| TransferSrc| TransferDst| + //#########################################| Spatial| | | | | | | | Operation| Operation| Operation| Specialization| | | | | | | | Wave| Wave| Lengths_K0_M_K1| ArrangeOrder| | | PerVector| PerVector_K1| | Lengths_K0_N_K1| ArrangeOrder| | | PerVector| PerVector_K1| | PerShuffle| PerShuffle| MBlock_MPerBlock| NWaveNPerXdl| | | ScalarPerVector| ScalarPerVector| + //#########################################| | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | NBlock_NPerBlock| | | | | | + // generic instance + DeviceGroupedConvBwdWeight_Xdl_CShuffle< NDimSpatial, ALayout, BLayout, ELayout, F32, F32, F32, F32, PassThrough, PassThrough, PassThrough, ConvSpec, 64, 64, 64, 4, 4, 32, 32, 2, 2, S<1, 4, 16, 1>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 1, 4, true, S<1, 4, 16, 1>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 1, 4, true, 1, 1, S<1, 16, 1, 4>, 1, TF32, TF32, MaxTransposeTransferSrcScalarPerVector, MaxTransposeTransferDstScalarPerVector>, + // instances for small conv.K and conv.C + DeviceGroupedConvBwdWeight_Xdl_CShuffle< NDimSpatial, ALayout, BLayout, ELayout, F32, F32, F32, F32, PassThrough, PassThrough, PassThrough, ConvSpec, 128, 128, 32, 4, 4, 32, 32, 2, 1, S<1, 4, 32, 1>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 4, true, S<1, 4, 8, 4>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 1, 1, true, 1, 1, S<1, 32, 1, 4>, 1, TF32, TF32, MaxTransposeTransferSrcScalarPerVector, MaxTransposeTransferDstScalarPerVector>, + DeviceGroupedConvBwdWeight_Xdl_CShuffle< NDimSpatial, ALayout, BLayout, ELayout, F32, F32, F32, F32, PassThrough, PassThrough, PassThrough, ConvSpec, 64, 32, 64, 4, 4, 32, 32, 1, 2, S<1, 4, 8, 2>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 1, 2, true, S<1, 4, 16, 1>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 4, true, 1, 1, S<1, 16, 1, 4>, 4, TF32, TF32, MaxTransposeTransferSrcScalarPerVector, MaxTransposeTransferDstScalarPerVector>, + + DeviceGroupedConvBwdWeight_Xdl_CShuffle< NDimSpatial, ALayout, BLayout, ELayout, F32, F32, F32, F32, PassThrough, PassThrough, PassThrough, ConvSpec, 256, 256, 128, 4, 4, 32, 32, 4, 2, S<1, 4, 64, 1>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 4, true, S<1, 4, 32, 2>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 2, true, 1, 1, S<1, 32, 1, 8>, 4, TF32, TF32, MaxTransposeTransferSrcScalarPerVector, MaxTransposeTransferDstScalarPerVector>, + DeviceGroupedConvBwdWeight_Xdl_CShuffle< NDimSpatial, ALayout, BLayout, ELayout, F32, F32, F32, F32, PassThrough, PassThrough, PassThrough, ConvSpec, 256, 128, 256, 4, 4, 32, 32, 2, 4, S<1, 4, 32, 2>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 2, true, S<1, 4, 64, 1>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 4, true, 1, 1, S<1, 32, 1, 8>, 4, TF32, TF32, MaxTransposeTransferSrcScalarPerVector, MaxTransposeTransferDstScalarPerVector>, + DeviceGroupedConvBwdWeight_Xdl_CShuffle< NDimSpatial, ALayout, BLayout, ELayout, F32, F32, F32, F32, PassThrough, PassThrough, PassThrough, ConvSpec, 128, 128, 128, 4, 4, 32, 32, 4, 2, S<1, 4, 32, 1>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 4, true, S<1, 4, 32, 1>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 4, true, 1, 1, S<1, 32, 1, 4>, 4, TF32, TF32, MaxTransposeTransferSrcScalarPerVector, MaxTransposeTransferDstScalarPerVector>, + DeviceGroupedConvBwdWeight_Xdl_CShuffle< NDimSpatial, ALayout, BLayout, ELayout, F32, F32, F32, F32, PassThrough, PassThrough, PassThrough, ConvSpec, 256, 128, 128, 4, 4, 32, 32, 2, 2, S<1, 4, 32, 2>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 2, true, S<1, 4, 32, 2>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 2, true, 1, 1, S<1, 32, 1, 4>, 4, TF32, TF32, MaxTransposeTransferSrcScalarPerVector, MaxTransposeTransferDstScalarPerVector>, + DeviceGroupedConvBwdWeight_Xdl_CShuffle< NDimSpatial, ALayout, BLayout, ELayout, F32, F32, F32, F32, PassThrough, PassThrough, PassThrough, ConvSpec, 128, 128, 64, 4, 4, 32, 32, 2, 2, S<1, 4, 32, 1>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 4, true, S<1, 4, 16, 2>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 2, true, 1, 1, S<1, 32, 1, 4>, 4, TF32, TF32, MaxTransposeTransferSrcScalarPerVector, MaxTransposeTransferDstScalarPerVector>, + DeviceGroupedConvBwdWeight_Xdl_CShuffle< NDimSpatial, ALayout, BLayout, ELayout, F32, F32, F32, F32, PassThrough, PassThrough, PassThrough, ConvSpec, 128, 64, 128, 4, 4, 32, 32, 2, 2, S<1, 4, 16, 2>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 2, true, S<1, 4, 32, 1>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 4, true, 1, 1, S<1, 32, 1, 4>, 4, TF32, TF32, MaxTransposeTransferSrcScalarPerVector, MaxTransposeTransferDstScalarPerVector>, + DeviceGroupedConvBwdWeight_Xdl_CShuffle< NDimSpatial, ALayout, BLayout, ELayout, F32, F32, F32, F32, PassThrough, PassThrough, PassThrough, ConvSpec, 64, 64, 64, 4, 4, 32, 32, 2, 2, S<1, 4, 16, 1>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 4, true, S<1, 4, 16, 1>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 4, true, 1, 1, S<1, 16, 1, 4>, 4, TF32, TF32, MaxTransposeTransferSrcScalarPerVector, MaxTransposeTransferDstScalarPerVector>, + DeviceGroupedConvBwdWeight_Xdl_CShuffle< NDimSpatial, ALayout, BLayout, ELayout, F32, F32, F32, F32, PassThrough, PassThrough, PassThrough, ConvSpec, 256, 128, 64, 4, 4, 32, 32, 2, 1, S<1, 4, 32, 2>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 2, true, S<1, 4, 16, 4>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 1, true, 1, 1, S<1, 32, 1, 4>, 4, TF32, TF32, MaxTransposeTransferSrcScalarPerVector, MaxTransposeTransferDstScalarPerVector>, + DeviceGroupedConvBwdWeight_Xdl_CShuffle< NDimSpatial, ALayout, BLayout, ELayout, F32, F32, F32, F32, PassThrough, PassThrough, PassThrough, ConvSpec, 256, 64, 128, 4, 4, 32, 32, 1, 2, S<1, 4, 16, 4>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 1, true, S<1, 4, 32, 2>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 2, true, 1, 1, S<1, 32, 1, 4>, 4, TF32, TF32, MaxTransposeTransferSrcScalarPerVector, MaxTransposeTransferDstScalarPerVector>, + DeviceGroupedConvBwdWeight_Xdl_CShuffle< NDimSpatial, ALayout, BLayout, ELayout, F32, F32, F32, F32, PassThrough, PassThrough, PassThrough, ConvSpec, 128, 128, 32, 4, 4, 32, 32, 2, 1, S<1, 4, 32, 1>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 4, true, S<1, 4, 8, 4>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 1, true, 1, 1, S<1, 32, 1, 4>, 4, TF32, TF32, MaxTransposeTransferSrcScalarPerVector, MaxTransposeTransferDstScalarPerVector>, + DeviceGroupedConvBwdWeight_Xdl_CShuffle< NDimSpatial, ALayout, BLayout, ELayout, F32, F32, F32, F32, PassThrough, PassThrough, PassThrough, ConvSpec, 128, 32, 128, 4, 4, 32, 32, 1, 2, S<1, 4, 8, 4>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 1, true, S<1, 4, 32, 1>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 4, true, 1, 1, S<1, 32, 1, 4>, 4, TF32, TF32, MaxTransposeTransferSrcScalarPerVector, MaxTransposeTransferDstScalarPerVector>, + DeviceGroupedConvBwdWeight_Xdl_CShuffle< NDimSpatial, ALayout, BLayout, ELayout, F32, F32, F32, F32, PassThrough, PassThrough, PassThrough, ConvSpec, 64, 64, 32, 4, 4, 32, 32, 2, 1, S<1, 4, 16, 1>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 4, true, S<1, 4, 8, 2>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 2, true, 1, 1, S<1, 16, 1, 4>, 4, TF32, TF32, MaxTransposeTransferSrcScalarPerVector, MaxTransposeTransferDstScalarPerVector>, + DeviceGroupedConvBwdWeight_Xdl_CShuffle< NDimSpatial, ALayout, BLayout, ELayout, F32, F32, F32, F32, PassThrough, PassThrough, PassThrough, ConvSpec, 64, 32, 64, 4, 4, 32, 32, 1, 2, S<1, 4, 8, 2>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 2, true, S<1, 4, 16, 1>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 4, true, 1, 1, S<1, 16, 1, 4>, 4, TF32, TF32, MaxTransposeTransferSrcScalarPerVector, MaxTransposeTransferDstScalarPerVector>, + + DeviceGroupedConvBwdWeight_Xdl_CShuffle< NDimSpatial, ALayout, BLayout, ELayout, F32, F32, F32, F32, PassThrough, PassThrough, PassThrough, ConvSpec, 256, 64, 64, 8, 8, 32, 32, 1, 1, S<1, 8, 16, 1>, S<0, 3, 1, 2>, S<0, 3, 1, 2>, 2, 4, 4, true, S<1, 8, 16, 1>, S<0, 3, 1, 2>, S<0, 3, 1, 2>, 2, 4, 4, true, 1, 1, S<1, 16, 1, 16>, 4, TF32, TF32, MaxTransposeTransferSrcScalarPerVector, MaxTransposeTransferDstScalarPerVector>, + DeviceGroupedConvBwdWeight_Xdl_CShuffle< NDimSpatial, ALayout, BLayout, ELayout, F32, F32, F32, F32, PassThrough, PassThrough, PassThrough, ConvSpec, 256, 64, 64, 8, 8, 32, 32, 1, 1, S<1, 8, 16, 1>, S<0, 3, 1, 2>, S<0, 3, 1, 2>, 2, 4, 4, true, S<1, 8, 32, 1>, S<0, 3, 1, 2>, S<0, 3, 1, 2>, 2, 1, 4, true, 1, 1, S<1, 4, 1, 64>, 1, TF32, TF32, MaxTransposeTransferSrcScalarPerVector, MaxTransposeTransferDstScalarPerVector>, + DeviceGroupedConvBwdWeight_Xdl_CShuffle< NDimSpatial, ALayout, BLayout, ELayout, F32, F32, F32, F32, PassThrough, PassThrough, PassThrough, ConvSpec, 256, 64, 64, 8, 8, 32, 32, 1, 1, S<1, 8, 32, 1>, S<0, 3, 1, 2>, S<0, 3, 1, 2>, 2, 1, 4, true, S<1, 8, 16, 1>, S<0, 3, 1, 2>, S<0, 3, 1, 2>, 2, 4, 4, true, 1, 1, S<1, 16, 1, 16>, 4, TF32, TF32, MaxTransposeTransferSrcScalarPerVector, MaxTransposeTransferDstScalarPerVector>, + DeviceGroupedConvBwdWeight_Xdl_CShuffle< NDimSpatial, ALayout, BLayout, ELayout, F32, F32, F32, F32, PassThrough, PassThrough, PassThrough, ConvSpec, 256, 64, 64, 8, 8, 32, 32, 1, 1, S<1, 8, 32, 1>, S<0, 3, 1, 2>, S<0, 3, 1, 2>, 2, 1, 4, true, S<1, 8, 32, 1>, S<0, 3, 1, 2>, S<0, 3, 1, 2>, 2, 1, 4, true, 1, 1, S<1, 4, 1, 64>, 1, TF32, TF32, MaxTransposeTransferSrcScalarPerVector, MaxTransposeTransferDstScalarPerVector> + // clang-format on + >; + template ; +template +using device_grouped_conv_bwd_weight_xdl_c_shuffle_f32_tf32_scale_instances = std::tuple< + // clang-format off + //#########################################| Num| InLayout| WeiLayout| OutLayout| DsData| InData| WeiData| OutData| AccData| DsData| In| Wei| Out| ConvBackward| Block| MPer| NPer| K0Per| K1| MPer| NPer| MXdl| NXdl| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockLds| BBlockTransfer| BBlockTransfer| BBlockTransfer| BlockTransfer| BBlockTransfer| BBlockTransfer| BBlockLds| CShuffle| CShuffle| CBlockTransfer| CBlockTransfer| + //#########################################| Dim| | | | Layout| Type| Type| Type| Type| Type| Elementwise| Elementwise| Elementwise| Weight| Size| Block| Block| Block| | XDL| XDL| Per| Per| ThreadCluster| ThreadCluster| SrcAccessOrder| SrcVectorDim| SrcScalar| DstScalar| AddExtraM| ThreadCluster| ThreadCluster| SrcAccessOrder| SrcVectorDim| SrcScalar| DstScalar| AddExtraN| MXdlPerWave| NXdlPerWave| ClusterLengths| ScalarPerVector| + //#########################################| Spatial| | | | | | | | | | Operation| Operation| Operation| Specialization| | | | | | | | Wave| Wave| Lengths_K0_M_K1| ArrangeOrder| | | PerVector| PerVector_K1| | Lengths_K0_N_K1| ArrangeOrder| | | PerVector| PerVector_K1| | PerShuffle| PerShuffle| MBlock_MPerBlock| NWaveNPerXdl| + //#########################################| | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | NBlock_NPerBlock| | + // generic instance + DeviceGroupedConvBwdWeightMultipleD_Xdl_CShuffle< NDimSpatial, ALayout, BLayout, ELayout, Empty_Tuple, F32, F32, F32, F32, Empty_Tuple, PassThrough, Scale, PassThrough, ConvSpec, 64, 64, 64, 4, 4, 32, 32, 2, 2, S<1, 4, 16, 1>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 1, 4, true, S<1, 4, 16, 1>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 1, 4, true, 1, 1, S<1, 16, 1, 4>, 1, TF32, TF32>, + // instances for small conv.K and conv.C + DeviceGroupedConvBwdWeightMultipleD_Xdl_CShuffle< NDimSpatial, ALayout, BLayout, ELayout, Empty_Tuple, F32, F32, F32, F32, Empty_Tuple, PassThrough, Scale, PassThrough, ConvSpec, 128, 128, 32, 4, 4, 32, 32, 2, 1, S<1, 4, 32, 1>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 4, true, S<1, 4, 8, 4>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 1, 1, true, 1, 1, S<1, 32, 1, 4>, 1, TF32, TF32>, + DeviceGroupedConvBwdWeightMultipleD_Xdl_CShuffle< NDimSpatial, ALayout, BLayout, ELayout, Empty_Tuple, F32, F32, F32, F32, Empty_Tuple, PassThrough, Scale, PassThrough, ConvSpec, 64, 32, 64, 4, 4, 32, 32, 1, 2, S<1, 4, 8, 2>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 1, 2, true, S<1, 4, 16, 1>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 4, true, 1, 1, S<1, 16, 1, 4>, 4, TF32, TF32>, + + DeviceGroupedConvBwdWeightMultipleD_Xdl_CShuffle< NDimSpatial, ALayout, BLayout, ELayout, Empty_Tuple, F32, F32, F32, F32, Empty_Tuple, PassThrough, Scale, PassThrough, ConvSpec, 256, 256, 128, 4, 4, 32, 32, 4, 2, S<1, 4, 64, 1>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 4, true, S<1, 4, 32, 2>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 2, true, 1, 1, S<1, 32, 1, 8>, 4, TF32, TF32>, + DeviceGroupedConvBwdWeightMultipleD_Xdl_CShuffle< NDimSpatial, ALayout, BLayout, ELayout, Empty_Tuple, F32, F32, F32, F32, Empty_Tuple, PassThrough, Scale, PassThrough, ConvSpec, 256, 128, 256, 4, 4, 32, 32, 2, 4, S<1, 4, 32, 2>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 2, true, S<1, 4, 64, 1>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 4, true, 1, 1, S<1, 32, 1, 8>, 4, TF32, TF32>, + DeviceGroupedConvBwdWeightMultipleD_Xdl_CShuffle< NDimSpatial, ALayout, BLayout, ELayout, Empty_Tuple, F32, F32, F32, F32, Empty_Tuple, PassThrough, Scale, PassThrough, ConvSpec, 128, 128, 128, 4, 4, 32, 32, 4, 2, S<1, 4, 32, 1>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 4, true, S<1, 4, 32, 1>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 4, true, 1, 1, S<1, 32, 1, 4>, 4, TF32, TF32>, + DeviceGroupedConvBwdWeightMultipleD_Xdl_CShuffle< NDimSpatial, ALayout, BLayout, ELayout, Empty_Tuple, F32, F32, F32, F32, Empty_Tuple, PassThrough, Scale, PassThrough, ConvSpec, 256, 128, 128, 4, 4, 32, 32, 2, 2, S<1, 4, 32, 2>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 2, true, S<1, 4, 32, 2>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 2, true, 1, 1, S<1, 32, 1, 4>, 4, TF32, TF32>, + DeviceGroupedConvBwdWeightMultipleD_Xdl_CShuffle< NDimSpatial, ALayout, BLayout, ELayout, Empty_Tuple, F32, F32, F32, F32, Empty_Tuple, PassThrough, Scale, PassThrough, ConvSpec, 128, 128, 64, 4, 4, 32, 32, 2, 2, S<1, 4, 32, 1>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 4, true, S<1, 4, 16, 2>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 2, true, 1, 1, S<1, 32, 1, 4>, 4, TF32, TF32>, + DeviceGroupedConvBwdWeightMultipleD_Xdl_CShuffle< NDimSpatial, ALayout, BLayout, ELayout, Empty_Tuple, F32, F32, F32, F32, Empty_Tuple, PassThrough, Scale, PassThrough, ConvSpec, 128, 64, 128, 4, 4, 32, 32, 2, 2, S<1, 4, 16, 2>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 2, true, S<1, 4, 32, 1>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 4, true, 1, 1, S<1, 32, 1, 4>, 4, TF32, TF32>, + DeviceGroupedConvBwdWeightMultipleD_Xdl_CShuffle< NDimSpatial, ALayout, BLayout, ELayout, Empty_Tuple, F32, F32, F32, F32, Empty_Tuple, PassThrough, Scale, PassThrough, ConvSpec, 64, 64, 64, 4, 4, 32, 32, 2, 2, S<1, 4, 16, 1>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 4, true, S<1, 4, 16, 1>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 4, true, 1, 1, S<1, 16, 1, 4>, 4, TF32, TF32>, + DeviceGroupedConvBwdWeightMultipleD_Xdl_CShuffle< NDimSpatial, ALayout, BLayout, ELayout, Empty_Tuple, F32, F32, F32, F32, Empty_Tuple, PassThrough, Scale, PassThrough, ConvSpec, 256, 128, 64, 4, 4, 32, 32, 2, 1, S<1, 4, 32, 2>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 2, true, S<1, 4, 16, 4>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 1, true, 1, 1, S<1, 32, 1, 4>, 4, TF32, TF32>, + DeviceGroupedConvBwdWeightMultipleD_Xdl_CShuffle< NDimSpatial, ALayout, BLayout, ELayout, Empty_Tuple, F32, F32, F32, F32, Empty_Tuple, PassThrough, Scale, PassThrough, ConvSpec, 256, 64, 128, 4, 4, 32, 32, 1, 2, S<1, 4, 16, 4>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 1, true, S<1, 4, 32, 2>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 2, true, 1, 1, S<1, 32, 1, 4>, 4, TF32, TF32>, + DeviceGroupedConvBwdWeightMultipleD_Xdl_CShuffle< NDimSpatial, ALayout, BLayout, ELayout, Empty_Tuple, F32, F32, F32, F32, Empty_Tuple, PassThrough, Scale, PassThrough, ConvSpec, 128, 128, 32, 4, 4, 32, 32, 2, 1, S<1, 4, 32, 1>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 4, true, S<1, 4, 8, 4>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 1, true, 1, 1, S<1, 32, 1, 4>, 4, TF32, TF32>, + DeviceGroupedConvBwdWeightMultipleD_Xdl_CShuffle< NDimSpatial, ALayout, BLayout, ELayout, Empty_Tuple, F32, F32, F32, F32, Empty_Tuple, PassThrough, Scale, PassThrough, ConvSpec, 128, 32, 128, 4, 4, 32, 32, 1, 2, S<1, 4, 8, 4>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 1, true, S<1, 4, 32, 1>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 4, true, 1, 1, S<1, 32, 1, 4>, 4, TF32, TF32>, + DeviceGroupedConvBwdWeightMultipleD_Xdl_CShuffle< NDimSpatial, ALayout, BLayout, ELayout, Empty_Tuple, F32, F32, F32, F32, Empty_Tuple, PassThrough, Scale, PassThrough, ConvSpec, 64, 64, 32, 4, 4, 32, 32, 2, 1, S<1, 4, 16, 1>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 4, true, S<1, 4, 8, 2>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 2, true, 1, 1, S<1, 16, 1, 4>, 4, TF32, TF32>, + DeviceGroupedConvBwdWeightMultipleD_Xdl_CShuffle< NDimSpatial, ALayout, BLayout, ELayout, Empty_Tuple, F32, F32, F32, F32, Empty_Tuple, PassThrough, Scale, PassThrough, ConvSpec, 64, 32, 64, 4, 4, 32, 32, 1, 2, S<1, 4, 8, 2>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 2, true, S<1, 4, 16, 1>, S<0, 3, 1, 2>, S<0, 2, 1, 3>, 2, 4, 4, true, 1, 1, S<1, 16, 1, 4>, 4, TF32, TF32> + // clang-format on + >; + template && is_same_v && - is_same_v && is_same_v && - is_same_v) + is_same_v) { - add_device_grouped_conv2d_bwd_data_xdl_nhwgk_gkyxc_nhwgc_f32_instances(op_ptrs); - add_device_grouped_conv2d_bwd_data_xdl_nhwgk_gkyxc_nhwgc_f32_16_16_instances( - op_ptrs); - add_device_grouped_conv2d_bwd_data_xdl_nhwgk_gkyxc_nhwgc_f32_optimized_loads_instances( - op_ptrs); + static_assert(is_same_v, + "Error: this operator requires the same compute type"); + if constexpr(is_same_v) + { + add_device_grouped_conv2d_bwd_data_xdl_nhwgk_gkyxc_nhwgc_f32_tf32_instances( + op_ptrs); + add_device_grouped_conv2d_bwd_data_xdl_nhwgk_gkyxc_nhwgc_f32_tf32_16_16_instances( + op_ptrs); + add_device_grouped_conv2d_bwd_data_xdl_nhwgk_gkyxc_nhwgc_f32_tf32_optimized_loads_instances( + op_ptrs); + } + else + { + add_device_grouped_conv2d_bwd_data_xdl_nhwgk_gkyxc_nhwgc_f32_instances( + op_ptrs); + add_device_grouped_conv2d_bwd_data_xdl_nhwgk_gkyxc_nhwgc_f32_16_16_instances( + op_ptrs); + add_device_grouped_conv2d_bwd_data_xdl_nhwgk_gkyxc_nhwgc_f32_optimized_loads_instances( + op_ptrs); + } } #endif #ifdef CK_ENABLE_BF16 @@ -275,12 +289,26 @@ struct DeviceOperationInstanceFactory< is_same_v && is_same_v && is_same_v) { - add_device_grouped_conv3d_bwd_data_xdl_ndhwgk_gkzyxc_ndhwgc_f32_instances( - op_ptrs); - add_device_grouped_conv3d_bwd_data_xdl_ndhwgk_gkzyxc_ndhwgc_f32_16_16_instances( - op_ptrs); - add_device_grouped_conv3d_bwd_data_xdl_ndhwgk_gkzyxc_ndhwgc_f32_optimized_loads_instances( - op_ptrs); + static_assert(is_same_v, + "Error: this operator requires the same compute type"); + if constexpr(is_same_v) + { + add_device_grouped_conv3d_bwd_data_xdl_ndhwgk_gkzyxc_ndhwgc_f32_instances( + op_ptrs); + add_device_grouped_conv3d_bwd_data_xdl_ndhwgk_gkzyxc_ndhwgc_f32_16_16_instances( + op_ptrs); + add_device_grouped_conv3d_bwd_data_xdl_ndhwgk_gkzyxc_ndhwgc_f32_optimized_loads_instances( + op_ptrs); + } + else if constexpr(is_same_v) + { + add_device_grouped_conv3d_bwd_data_xdl_ndhwgk_gkzyxc_ndhwgc_f32_tf32_instances( + op_ptrs); + add_device_grouped_conv3d_bwd_data_xdl_ndhwgk_gkzyxc_ndhwgc_f32_tf32_16_16_instances( + op_ptrs); + add_device_grouped_conv3d_bwd_data_xdl_ndhwgk_gkzyxc_ndhwgc_f32_tf32_optimized_loads_instances( + op_ptrs); + } } #endif #ifdef CK_ENABLE_BF16 diff --git a/library/include/ck/library/tensor_operation_instance/gpu/grouped_convolution_backward_data_xdl.inc b/library/include/ck/library/tensor_operation_instance/gpu/grouped_convolution_backward_data_xdl.inc index 10362a31c4..505f67d1c3 100644 --- a/library/include/ck/library/tensor_operation_instance/gpu/grouped_convolution_backward_data_xdl.inc +++ b/library/include/ck/library/tensor_operation_instance/gpu/grouped_convolution_backward_data_xdl.inc @@ -127,6 +127,38 @@ void add_device_grouped_conv2d_bwd_data_xdl_nhwgk_gkyxc_nhwgc_f32_16_16_instance PassThrough, PassThrough>>>& instances); +void add_device_grouped_conv2d_bwd_data_xdl_nhwgk_gkyxc_nhwgc_f32_tf32_instances( + std::vector>>& instances); + +void add_device_grouped_conv2d_bwd_data_xdl_nhwgk_gkyxc_nhwgc_f32_tf32_16_16_instances( + std::vector>>& instances); + void add_device_grouped_conv2d_bwd_data_xdl_nhwgk_gkyxc_nhwgc_f32_optimized_loads_instances( std::vector>>& instances); + +void add_device_grouped_conv2d_bwd_data_xdl_nhwgk_gkyxc_nhwgc_f32_tf32_optimized_loads_instances( + std::vector>>& instances); #endif #ifdef CK_ENABLE_BF16 void add_device_grouped_conv2d_bwd_data_xdl_nhwgk_gkyxc_nhwgc_bf16_instances( @@ -479,6 +527,38 @@ void add_device_grouped_conv3d_bwd_data_xdl_ndhwgk_gkzyxc_ndhwgc_f32_16_16_insta PassThrough, PassThrough>>>& instances); +void add_device_grouped_conv3d_bwd_data_xdl_ndhwgk_gkzyxc_ndhwgc_f32_tf32_instances( + std::vector>>& instances); + +void add_device_grouped_conv3d_bwd_data_xdl_ndhwgk_gkzyxc_ndhwgc_f32_tf32_16_16_instances( + std::vector>>& instances); + void add_device_grouped_conv3d_bwd_data_xdl_ndhwgk_gkzyxc_ndhwgc_f32_optimized_loads_instances( std::vector>>& instances); + +void add_device_grouped_conv3d_bwd_data_xdl_ndhwgk_gkzyxc_ndhwgc_f32_tf32_optimized_loads_instances( + std::vector>>& instances); #endif #ifdef CK_ENABLE_BF16 void add_device_grouped_conv3d_bwd_data_xdl_ndhwgk_gkzyxc_ndhwgc_bf16_instances( diff --git a/library/include/ck/library/tensor_operation_instance/gpu/grouped_convolution_backward_weight.hpp b/library/include/ck/library/tensor_operation_instance/gpu/grouped_convolution_backward_weight.hpp index 3c0784eef3..7f91d0bee9 100644 --- a/library/include/ck/library/tensor_operation_instance/gpu/grouped_convolution_backward_weight.hpp +++ b/library/include/ck/library/tensor_operation_instance/gpu/grouped_convolution_backward_weight.hpp @@ -349,20 +349,37 @@ struct DeviceOperationInstanceFactory && is_same_v && - is_same_v && is_same_v && - is_same_v) + is_same_v) { - add_device_grouped_conv2d_bwd_weight_xdl_nhwgc_gkyxc_nhwgk_f32_instances( - op_ptrs); + static_assert(is_same_v, + "Error: ComputeTypeA and ComputeTypeB should be the same"); + if constexpr(is_same_v) + { + add_device_grouped_conv2d_bwd_weight_xdl_nhwgc_gkyxc_nhwgk_f32_instances( + op_ptrs); - add_device_grouped_conv2d_bwd_weight_xdl_nhwgc_gkyxc_nhwgk_f32_default_pipev2_instances( - op_ptrs); - add_device_grouped_conv2d_bwd_weight_xdl_nhwgc_gkyxc_nhwgk_f32_default_pipev5_instances( - op_ptrs); - add_device_grouped_conv2d_bwd_weight_xdl_nhwgc_gkyxc_nhwgk_f32_pad0_pipev2_instances( - op_ptrs); - add_device_grouped_conv2d_bwd_weight_xdl_nhwgc_gkyxc_nhwgk_f32_pad0_pipev5_instances( - op_ptrs); + add_device_grouped_conv2d_bwd_weight_xdl_nhwgc_gkyxc_nhwgk_f32_default_pipev2_instances( + op_ptrs); + add_device_grouped_conv2d_bwd_weight_xdl_nhwgc_gkyxc_nhwgk_f32_default_pipev5_instances( + op_ptrs); + add_device_grouped_conv2d_bwd_weight_xdl_nhwgc_gkyxc_nhwgk_f32_pad0_pipev2_instances( + op_ptrs); + add_device_grouped_conv2d_bwd_weight_xdl_nhwgc_gkyxc_nhwgk_f32_pad0_pipev5_instances( + op_ptrs); + } + else if constexpr(is_same_v) + { + add_device_grouped_conv2d_bwd_weight_xdl_nhwgc_gkyxc_nhwgk_f32_tf32_instances( + op_ptrs); + add_device_grouped_conv2d_bwd_weight_xdl_nhwgc_gkyxc_nhwgk_f32_tf32_default_pipev2_instances( + op_ptrs); + add_device_grouped_conv2d_bwd_weight_xdl_nhwgc_gkyxc_nhwgk_f32_tf32_default_pipev5_instances( + op_ptrs); + add_device_grouped_conv2d_bwd_weight_xdl_nhwgc_gkyxc_nhwgk_f32_tf32_pad0_pipev2_instances( + op_ptrs); + add_device_grouped_conv2d_bwd_weight_xdl_nhwgc_gkyxc_nhwgk_f32_tf32_pad0_pipev5_instances( + op_ptrs); + } } #endif #ifdef CK_ENABLE_FP16 @@ -595,20 +612,36 @@ struct DeviceOperationInstanceFactory && is_same_v && - is_same_v && is_same_v && - is_same_v) + is_same_v) { - add_device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_f32_instances( - op_ptrs); - - add_device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_f32_default_pipev2_instances( - op_ptrs); - add_device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_f32_default_pipev5_instances( - op_ptrs); - add_device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_f32_pad0_pipev2_instances( - op_ptrs); - add_device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_f32_pad0_pipev2_instances( - op_ptrs); + static_assert(is_same_v, + "Error: ComputeTypeA and ComputeTypeB should be the same"); + if constexpr(is_same_v) + { + add_device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_f32_instances( + op_ptrs); + add_device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_f32_default_pipev2_instances( + op_ptrs); + add_device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_f32_default_pipev5_instances( + op_ptrs); + add_device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_f32_pad0_pipev2_instances( + op_ptrs); + add_device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_f32_pad0_pipev5_instances( + op_ptrs); + } + else if constexpr(is_same_v) + { + add_device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_f32_tf32_instances( + op_ptrs); + add_device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_f32_tf32_default_pipev2_instances( + op_ptrs); + add_device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_f32_tf32_default_pipev5_instances( + op_ptrs); + add_device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_f32_tf32_pad0_pipev2_instances( + op_ptrs); + add_device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_f32_tf32_pad0_pipev5_instances( + op_ptrs); + } } #endif #ifdef CK_ENABLE_FP16 diff --git a/library/include/ck/library/tensor_operation_instance/gpu/grouped_convolution_backward_weight_bilinear.hpp b/library/include/ck/library/tensor_operation_instance/gpu/grouped_convolution_backward_weight_bilinear.hpp index 50b6f0b6d8..ffe98602a3 100644 --- a/library/include/ck/library/tensor_operation_instance/gpu/grouped_convolution_backward_weight_bilinear.hpp +++ b/library/include/ck/library/tensor_operation_instance/gpu/grouped_convolution_backward_weight_bilinear.hpp @@ -62,6 +62,21 @@ void add_device_grouped_conv3d_bwd_weight_xdl_bilinear_ndhwgc_gkzyxc_ndhwgk_f32_ PassThrough, Bilinear, PassThrough>>>& instances); +void add_device_grouped_conv3d_bwd_weight_xdl_bilinear_ndhwgc_gkzyxc_ndhwgk_f32_tf32_instances( + std::vector, + F32, + F32, + F32, + Tuple, + PassThrough, + Bilinear, + PassThrough, + TF32, + TF32>>>& instances); #endif #if defined CK_ENABLE_FP16 && defined CK_ENABLE_FP8 && defined CK_ENABLE_BF8 void add_device_grouped_conv3d_bwd_weight_xdl_bilinear_ndhwgc_gkzyxc_ndhwgk_f16_comp_bf8_f8_instances( @@ -138,11 +153,20 @@ struct DeviceOperationInstanceFactory< { #ifdef CK_ENABLE_FP32 if constexpr(is_same_v && is_same_v && - is_same_v && is_same_v && - is_same_v) + is_same_v) { - add_device_grouped_conv3d_bwd_weight_xdl_bilinear_ndhwgc_gkzyxc_ndhwgk_f32_instances( - op_ptrs); + static_assert(is_same_v, + "Error: this operator requires the same compute type"); + if constexpr(is_same_v) + { + add_device_grouped_conv3d_bwd_weight_xdl_bilinear_ndhwgc_gkzyxc_ndhwgk_f32_tf32_instances( + op_ptrs); + } + else + { + add_device_grouped_conv3d_bwd_weight_xdl_bilinear_ndhwgc_gkzyxc_ndhwgk_f32_instances( + op_ptrs); + } } #endif #ifdef CK_ENABLE_FP16 diff --git a/library/include/ck/library/tensor_operation_instance/gpu/grouped_convolution_backward_weight_scale.hpp b/library/include/ck/library/tensor_operation_instance/gpu/grouped_convolution_backward_weight_scale.hpp index 89a2848920..46ddba312a 100644 --- a/library/include/ck/library/tensor_operation_instance/gpu/grouped_convolution_backward_weight_scale.hpp +++ b/library/include/ck/library/tensor_operation_instance/gpu/grouped_convolution_backward_weight_scale.hpp @@ -62,6 +62,22 @@ void add_device_grouped_conv3d_bwd_weight_xdl_scale_ndhwgc_gkzyxc_ndhwgk_f32_ins PassThrough, Scale, PassThrough>>>& instances); + +void add_device_grouped_conv3d_bwd_weight_xdl_scale_ndhwgc_gkzyxc_ndhwgk_f32_tf32_instances( + std::vector, + F32, + F32, + F32, + Tuple<>, + PassThrough, + Scale, + PassThrough, + TF32, + TF32>>>& instances); #endif #if defined CK_ENABLE_FP16 && defined CK_ENABLE_FP8 && defined CK_ENABLE_BF8 void add_device_grouped_conv3d_bwd_weight_xdl_scale_ndhwgc_gkzyxc_ndhwgk_f16_comp_bf8_f8_instances( @@ -138,11 +154,20 @@ struct DeviceOperationInstanceFactory< { #ifdef CK_ENABLE_FP32 if constexpr(is_same_v && is_same_v && - is_same_v && is_same_v && - is_same_v) + is_same_v) { - add_device_grouped_conv3d_bwd_weight_xdl_scale_ndhwgc_gkzyxc_ndhwgk_f32_instances( - op_ptrs); + static_assert(is_same_v, + "Error: this operator requires the same compute type"); + if constexpr(is_same_v) + { + add_device_grouped_conv3d_bwd_weight_xdl_scale_ndhwgc_gkzyxc_ndhwgk_f32_tf32_instances( + op_ptrs); + } + else + { + add_device_grouped_conv3d_bwd_weight_xdl_scale_ndhwgc_gkzyxc_ndhwgk_f32_instances( + op_ptrs); + } } #endif #ifdef CK_ENABLE_FP16 diff --git a/library/include/ck/library/tensor_operation_instance/gpu/grouped_convolution_backward_weight_xdl.inc b/library/include/ck/library/tensor_operation_instance/gpu/grouped_convolution_backward_weight_xdl.inc index 31926ce908..0d3159210d 100644 --- a/library/include/ck/library/tensor_operation_instance/gpu/grouped_convolution_backward_weight_xdl.inc +++ b/library/include/ck/library/tensor_operation_instance/gpu/grouped_convolution_backward_weight_xdl.inc @@ -570,6 +570,20 @@ void add_device_grouped_conv2d_bwd_weight_xdl_nhwgc_gkyxc_nhwgk_f32_instances( PassThrough, PassThrough>>>& instances); +void add_device_grouped_conv2d_bwd_weight_xdl_nhwgc_gkyxc_nhwgk_f32_tf32_instances( + std::vector>>& instances); + void add_device_grouped_conv2d_bwd_weight_xdl_ngchw_gkyxc_ngkhw_f32_instances( std::vector>>& instances); +void add_device_grouped_conv2d_bwd_weight_xdl_nhwgc_gkyxc_nhwgk_f32_tf32_default_pipev2_instances( + std::vector>>& instances); + void add_device_grouped_conv2d_bwd_weight_xdl_nhwgc_gkyxc_nhwgk_f32_default_pipev5_instances( std::vector>>& instances); +void add_device_grouped_conv2d_bwd_weight_xdl_nhwgc_gkyxc_nhwgk_f32_tf32_default_pipev5_instances( + std::vector>>& instances); + void add_device_grouped_conv2d_bwd_weight_xdl_nhwgc_gkyxc_nhwgk_f32_pad0_pipev2_instances( std::vector>>& instances); +void add_device_grouped_conv2d_bwd_weight_xdl_nhwgc_gkyxc_nhwgk_f32_tf32_pad0_pipev2_instances( + std::vector>>& instances); + void add_device_grouped_conv2d_bwd_weight_xdl_nhwgc_gkyxc_nhwgk_f32_pad0_pipev5_instances( std::vector>>& instances); + +void add_device_grouped_conv2d_bwd_weight_xdl_nhwgc_gkyxc_nhwgk_f32_tf32_pad0_pipev5_instances( + std::vector>>& instances); #endif // conv3d backward weight #ifdef CK_ENABLE_BF16 @@ -1177,7 +1247,7 @@ void add_device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_f32_pad0_pipe PassThrough, PassThrough>>>& instances); -void add_device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_f32_pad0_pipev2_instances( +void add_device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_f32_pad0_pipev5_instances( std::vector>>& instances); + +void add_device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_f32_tf32_instances( + std::vector>>& instances); + +void add_device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_f32_tf32_default_pipev2_instances( + std::vector>>& instances); + +void add_device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_f32_tf32_default_pipev5_instances( + std::vector>>& instances); + +void add_device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_f32_tf32_pad0_pipev2_instances( + std::vector>>& instances); + +void add_device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_f32_tf32_pad0_pipev5_instances( + std::vector>>& instances); #endif #if defined CK_ENABLE_FP16 && defined CK_ENABLE_FP8 && defined CK_ENABLE_BF8 void add_device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_f16_comp_bf8_f8_instances( diff --git a/library/include/ck/library/tensor_operation_instance/gpu/grouped_convolution_forward_bias_clamp_xdl.inc b/library/include/ck/library/tensor_operation_instance/gpu/grouped_convolution_forward_bias_clamp_xdl.inc index 4678ab6c66..da80ca9add 100644 --- a/library/include/ck/library/tensor_operation_instance/gpu/grouped_convolution_forward_bias_clamp_xdl.inc +++ b/library/include/ck/library/tensor_operation_instance/gpu/grouped_convolution_forward_bias_clamp_xdl.inc @@ -570,6 +570,22 @@ void add_device_grouped_conv2d_fwd_bias_clamp_xdl_merged_groups_nhwgc_gkyxc_nhwg TF32, TF32>>>& instances); +void add_device_grouped_conv2d_fwd_bias_clamp_xdl_merged_groups_nhwgc_gkyxc_nhwgk_f32_tf32_instances( + std::vector, + NHWGK, + F32, + F32, + Tuple, + F32, + PassThrough, + PassThrough, + AddClamp, + TF32, + TF32>>>& instances); + void add_device_grouped_conv2d_fwd_bias_clamp_xdl_nhwgc_gkyxc_nhwgk_f32_comp_instances( std::vector>>& instances) +{ + // 1. Default + add_device_operation_instances( + instances, + device_grouped_conv_bwd_data_xdl_f32_tf32_16_16_instances<2, + NHWGK, + GKYXC, + Empty_Tuple, + NHWGC, + ConvBwdDataDefault>{}); + // 2. Filter1x1Stride1Pad0 + add_device_operation_instances(instances, + device_grouped_conv_bwd_data_xdl_f32_tf32_16_16_instances< + 2, + NHWGK, + GKYXC, + Empty_Tuple, + NHWGC, + ConvBwdDataFilter1x1Stride1Pad0>{}); +} + +} // namespace instance +} // namespace device +} // namespace tensor_operation +} // namespace ck diff --git a/library/src/tensor_operation_instance/gpu/grouped_conv2d_bwd_data/xdl/device_grouped_conv2d_bwd_data_xdl_nhwgc_gkyxc_nhwgk_f32_tf32_instance.cpp b/library/src/tensor_operation_instance/gpu/grouped_conv2d_bwd_data/xdl/device_grouped_conv2d_bwd_data_xdl_nhwgc_gkyxc_nhwgk_f32_tf32_instance.cpp new file mode 100644 index 0000000000..56dc9222d4 --- /dev/null +++ b/library/src/tensor_operation_instance/gpu/grouped_conv2d_bwd_data/xdl/device_grouped_conv2d_bwd_data_xdl_nhwgc_gkyxc_nhwgk_f32_tf32_instance.cpp @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2018-2025, Advanced Micro Devices, Inc. All rights reserved. + +#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" +#include "ck/library/tensor_operation_instance/gpu/grouped_conv_bwd_data/device_grouped_conv_bwd_data_xdl_instance.hpp" + +namespace ck { +namespace tensor_operation { +namespace device { +namespace instance { + +void add_device_grouped_conv2d_bwd_data_xdl_nhwgk_gkyxc_nhwgc_f32_tf32_instances( + std::vector>>& instances) +{ + // 1. Default + add_device_operation_instances( + instances, + device_grouped_conv_bwd_data_xdl_f32_tf32_instances<2, + NHWGK, + GKYXC, + Empty_Tuple, + NHWGC, + ConvBwdDataDefault>{}); + // 2. Filter1x1Stride1Pad0 + add_device_operation_instances( + instances, + device_grouped_conv_bwd_data_xdl_f32_tf32_instances<2, + NHWGK, + GKYXC, + Empty_Tuple, + NHWGC, + ConvBwdDataFilter1x1Stride1Pad0>{}); +} + +} // namespace instance +} // namespace device +} // namespace tensor_operation +} // namespace ck diff --git a/library/src/tensor_operation_instance/gpu/grouped_conv2d_bwd_data/xdl/device_grouped_conv2d_bwd_data_xdl_nhwgc_gkyxc_nhwgk_f32_tf32_optimized_loads_instance.cpp b/library/src/tensor_operation_instance/gpu/grouped_conv2d_bwd_data/xdl/device_grouped_conv2d_bwd_data_xdl_nhwgc_gkyxc_nhwgk_f32_tf32_optimized_loads_instance.cpp new file mode 100644 index 0000000000..e328b03db2 --- /dev/null +++ b/library/src/tensor_operation_instance/gpu/grouped_conv2d_bwd_data/xdl/device_grouped_conv2d_bwd_data_xdl_nhwgc_gkyxc_nhwgk_f32_tf32_optimized_loads_instance.cpp @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2025, Advanced Micro Devices, Inc. All rights reserved. + +#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" +#include "ck/library/tensor_operation_instance/gpu/grouped_conv_bwd_data/device_grouped_conv_bwd_data_xdl_instance.hpp" + +namespace ck { +namespace tensor_operation { +namespace device { +namespace instance { + +void add_device_grouped_conv2d_bwd_data_xdl_nhwgk_gkyxc_nhwgc_f32_tf32_optimized_loads_instances( + std::vector>>& instances) +{ + // 1. Default + add_device_operation_instances( + instances, + device_grouped_conv_bwd_data_xdl_f32_tf32_optimized_loads_instances<2, + NHWGK, + GKYXC, + Empty_Tuple, + NHWGC, + ConvBwdDataDefault>{}); + // 2. Filter1x1Stride1Pad0 + add_device_operation_instances( + instances, + device_grouped_conv_bwd_data_xdl_f32_tf32_optimized_loads_instances< + 2, + NHWGK, + GKYXC, + Empty_Tuple, + NHWGC, + ConvBwdDataFilter1x1Stride1Pad0>{}); +} + +} // namespace instance +} // namespace device +} // namespace tensor_operation +} // namespace ck diff --git a/library/src/tensor_operation_instance/gpu/grouped_conv2d_bwd_weight/CMakeLists.txt b/library/src/tensor_operation_instance/gpu/grouped_conv2d_bwd_weight/CMakeLists.txt index 7264c4688d..f042e09e69 100644 --- a/library/src/tensor_operation_instance/gpu/grouped_conv2d_bwd_weight/CMakeLists.txt +++ b/library/src/tensor_operation_instance/gpu/grouped_conv2d_bwd_weight/CMakeLists.txt @@ -10,6 +10,7 @@ set(GROUPED_CONV2D_BWD_WEIGHT xdl/nhwgc_gkyxc_nhwgk/device_grouped_conv2d_bwd_weight_xdl_nhwgc_gkyxc_nhwgk_f16_instance.cpp xdl/nhwgc_gkyxc_nhwgk/device_grouped_conv2d_bwd_weight_xdl_nhwgc_gkyxc_nhwgk_f32_instance.cpp + xdl/nhwgc_gkyxc_nhwgk/device_grouped_conv2d_bwd_weight_xdl_nhwgc_gkyxc_nhwgk_f32_tf32_instance.cpp xdl/nhwgc_gkyxc_nhwgk/device_grouped_conv2d_bwd_weight_xdl_nhwgc_gkyxc_nhwgk_bf16_f32_bf16_instance.cpp xdl/nhwgc_gkyxc_nhwgk/device_grouped_conv2d_bwd_weight_xdl_nhwgc_gkyxc_nhwgk_bf16_instance.cpp xdl/nhwgc_gkyxc_nhwgk/device_grouped_conv2d_bwd_weight_xdl_nhwgc_gkyxc_nhwgk_bf16_default_pipev2_instance.cpp @@ -21,9 +22,13 @@ set(GROUPED_CONV2D_BWD_WEIGHT xdl/nhwgc_gkyxc_nhwgk/device_grouped_conv2d_bwd_weight_xdl_nhwgc_gkyxc_nhwgk_f16_pad0_pipev2_instance.cpp xdl/nhwgc_gkyxc_nhwgk/device_grouped_conv2d_bwd_weight_xdl_nhwgc_gkyxc_nhwgk_f16_pad0_pipev5_instance.cpp xdl/nhwgc_gkyxc_nhwgk/device_grouped_conv2d_bwd_weight_xdl_nhwgc_gkyxc_nhwgk_f32_default_pipev2_instance.cpp + xdl/nhwgc_gkyxc_nhwgk/device_grouped_conv2d_bwd_weight_xdl_nhwgc_gkyxc_nhwgk_f32_tf32_default_pipev2_instance.cpp xdl/nhwgc_gkyxc_nhwgk/device_grouped_conv2d_bwd_weight_xdl_nhwgc_gkyxc_nhwgk_f32_default_pipev5_instance.cpp + xdl/nhwgc_gkyxc_nhwgk/device_grouped_conv2d_bwd_weight_xdl_nhwgc_gkyxc_nhwgk_f32_tf32_default_pipev5_instance.cpp xdl/nhwgc_gkyxc_nhwgk/device_grouped_conv2d_bwd_weight_xdl_nhwgc_gkyxc_nhwgk_f32_pad0_pipev2_instance.cpp + xdl/nhwgc_gkyxc_nhwgk/device_grouped_conv2d_bwd_weight_xdl_nhwgc_gkyxc_nhwgk_f32_tf32_pad0_pipev2_instance.cpp xdl/nhwgc_gkyxc_nhwgk/device_grouped_conv2d_bwd_weight_xdl_nhwgc_gkyxc_nhwgk_f32_pad0_pipev5_instance.cpp + xdl/nhwgc_gkyxc_nhwgk/device_grouped_conv2d_bwd_weight_xdl_nhwgc_gkyxc_nhwgk_f32_tf32_pad0_pipev5_instance.cpp xdl/nhwgc_gkyxc_nhwgk/device_grouped_conv2d_bwd_weight_two_stage_xdl_nhwgc_gkyxc_nhwgk_f16_pipev2_instance.cpp xdl/nhwgc_gkyxc_nhwgk/device_grouped_conv2d_bwd_weight_two_stage_xdl_nhwgc_gkyxc_nhwgk_f16_pipev5_instance.cpp xdl/nhwgc_gkyxc_nhwgk/device_grouped_conv2d_bwd_weight_two_stage_xdl_nhwgc_gkyxc_nhwgk_bf16_pipev2_instance.cpp diff --git a/library/src/tensor_operation_instance/gpu/grouped_conv2d_bwd_weight/xdl/nhwgc_gkyxc_nhwgk/device_grouped_conv2d_bwd_weight_xdl_nhwgc_gkyxc_nhwgk_f32_tf32_default_pipev2_instance.cpp b/library/src/tensor_operation_instance/gpu/grouped_conv2d_bwd_weight/xdl/nhwgc_gkyxc_nhwgk/device_grouped_conv2d_bwd_weight_xdl_nhwgc_gkyxc_nhwgk_f32_tf32_default_pipev2_instance.cpp new file mode 100644 index 0000000000..e3161c5ff4 --- /dev/null +++ b/library/src/tensor_operation_instance/gpu/grouped_conv2d_bwd_weight/xdl/nhwgc_gkyxc_nhwgk/device_grouped_conv2d_bwd_weight_xdl_nhwgc_gkyxc_nhwgk_f32_tf32_default_pipev2_instance.cpp @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2025, Advanced Micro Devices, Inc. All rights reserved. + +#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" +#include "ck/library/tensor_operation_instance/gpu/grouped_conv_bwd_weight/device_grouped_conv_bwd_weight_v3_xdl_instance.hpp" + +namespace ck { +namespace tensor_operation { +namespace device { +namespace instance { + +// Compilation parameters for in[n, hi, wi, g, c] * wei[g, k, y, x, c] = out[n, ho, wo, g, k] +void add_device_grouped_conv2d_bwd_weight_xdl_nhwgc_gkyxc_nhwgk_f32_tf32_default_pipev2_instances( + std::vector>>& instances) +{ + add_device_operation_instances( + instances, + device_grouped_conv_bwd_weight_v3_xdl_c_shuffle_f32_tf32_instances< + 2, + NHWGC, + GKYXC, + NHWGK, + ConvBwdWeightDefault, + BlockGemmPipelineScheduler::Intrawave, + BlockGemmPipelineVersion::v2>{}); +} + +} // namespace instance +} // namespace device +} // namespace tensor_operation +} // namespace ck diff --git a/library/src/tensor_operation_instance/gpu/grouped_conv2d_bwd_weight/xdl/nhwgc_gkyxc_nhwgk/device_grouped_conv2d_bwd_weight_xdl_nhwgc_gkyxc_nhwgk_f32_tf32_default_pipev5_instance.cpp b/library/src/tensor_operation_instance/gpu/grouped_conv2d_bwd_weight/xdl/nhwgc_gkyxc_nhwgk/device_grouped_conv2d_bwd_weight_xdl_nhwgc_gkyxc_nhwgk_f32_tf32_default_pipev5_instance.cpp new file mode 100644 index 0000000000..65b811e068 --- /dev/null +++ b/library/src/tensor_operation_instance/gpu/grouped_conv2d_bwd_weight/xdl/nhwgc_gkyxc_nhwgk/device_grouped_conv2d_bwd_weight_xdl_nhwgc_gkyxc_nhwgk_f32_tf32_default_pipev5_instance.cpp @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2025, Advanced Micro Devices, Inc. All rights reserved. + +#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" +#include "ck/library/tensor_operation_instance/gpu/grouped_conv_bwd_weight/device_grouped_conv_bwd_weight_v3_xdl_instance.hpp" + +namespace ck { +namespace tensor_operation { +namespace device { +namespace instance { + +// Compilation parameters for in[n, hi, wi, g, c] * wei[g, k, y, x, c] = out[n, ho, wo, g, k] +void add_device_grouped_conv2d_bwd_weight_xdl_nhwgc_gkyxc_nhwgk_f32_tf32_default_pipev5_instances( + std::vector>>& instances) +{ + add_device_operation_instances( + instances, + device_grouped_conv_bwd_weight_v3_xdl_c_shuffle_f32_tf32_instances< + 2, + NHWGC, + GKYXC, + NHWGK, + ConvBwdWeightDefault, + BlockGemmPipelineScheduler::Intrawave, + BlockGemmPipelineVersion::v5>{}); +} + +} // namespace instance +} // namespace device +} // namespace tensor_operation +} // namespace ck diff --git a/library/src/tensor_operation_instance/gpu/grouped_conv2d_bwd_weight/xdl/nhwgc_gkyxc_nhwgk/device_grouped_conv2d_bwd_weight_xdl_nhwgc_gkyxc_nhwgk_f32_tf32_instance.cpp b/library/src/tensor_operation_instance/gpu/grouped_conv2d_bwd_weight/xdl/nhwgc_gkyxc_nhwgk/device_grouped_conv2d_bwd_weight_xdl_nhwgc_gkyxc_nhwgk_f32_tf32_instance.cpp new file mode 100644 index 0000000000..6bfbf4ee73 --- /dev/null +++ b/library/src/tensor_operation_instance/gpu/grouped_conv2d_bwd_weight/xdl/nhwgc_gkyxc_nhwgk/device_grouped_conv2d_bwd_weight_xdl_nhwgc_gkyxc_nhwgk_f32_tf32_instance.cpp @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2018-2025, Advanced Micro Devices, Inc. All rights reserved. + +#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" +#include "ck/library/tensor_operation_instance/gpu/grouped_conv_bwd_weight/device_grouped_conv_bwd_weight_xdl_instance.hpp" + +namespace ck { +namespace tensor_operation { +namespace device { +namespace instance { + +// Compilation parameters for in[n, hi, wi, g, c] * wei[g, k, y, x, c] = out[n, ho, wo, g, k] +void add_device_grouped_conv2d_bwd_weight_xdl_nhwgc_gkyxc_nhwgk_f32_tf32_instances( + std::vector>>& instances) +{ + // 1. Default + add_device_operation_instances( + instances, + device_grouped_conv_bwd_weight_xdl_c_shuffle_f32_tf32_instances<2, + NHWGC, + GKYXC, + NHWGK, + ConvBwdWeightDefault>{}); + // 2. Filter1x1Stride1Pad0 + add_device_operation_instances(instances, + device_grouped_conv_bwd_weight_xdl_c_shuffle_f32_tf32_instances< + 2, + NHWGC, + GKYXC, + NHWGK, + ConvBwdWeightFilter1x1Stride1Pad0>{}); +} + +} // namespace instance +} // namespace device +} // namespace tensor_operation +} // namespace ck diff --git a/library/src/tensor_operation_instance/gpu/grouped_conv2d_bwd_weight/xdl/nhwgc_gkyxc_nhwgk/device_grouped_conv2d_bwd_weight_xdl_nhwgc_gkyxc_nhwgk_f32_tf32_pad0_pipev2_instance.cpp b/library/src/tensor_operation_instance/gpu/grouped_conv2d_bwd_weight/xdl/nhwgc_gkyxc_nhwgk/device_grouped_conv2d_bwd_weight_xdl_nhwgc_gkyxc_nhwgk_f32_tf32_pad0_pipev2_instance.cpp new file mode 100644 index 0000000000..f1d425fab4 --- /dev/null +++ b/library/src/tensor_operation_instance/gpu/grouped_conv2d_bwd_weight/xdl/nhwgc_gkyxc_nhwgk/device_grouped_conv2d_bwd_weight_xdl_nhwgc_gkyxc_nhwgk_f32_tf32_pad0_pipev2_instance.cpp @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2025, Advanced Micro Devices, Inc. All rights reserved. + +#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" +#include "ck/library/tensor_operation_instance/gpu/grouped_conv_bwd_weight/device_grouped_conv_bwd_weight_v3_xdl_instance.hpp" + +namespace ck { +namespace tensor_operation { +namespace device { +namespace instance { + +// Compilation parameters for in[n, hi, wi, g, c] * wei[g, k, y, x, c] = out[n, ho, wo, g, k] +void add_device_grouped_conv2d_bwd_weight_xdl_nhwgc_gkyxc_nhwgk_f32_tf32_pad0_pipev2_instances( + std::vector>>& instances) +{ + add_device_operation_instances( + instances, + device_grouped_conv_bwd_weight_v3_xdl_c_shuffle_f32_tf32_instances< + 2, + NHWGC, + GKYXC, + NHWGK, + ConvBwdWeightFilter1x1Stride1Pad0, + BlockGemmPipelineScheduler::Intrawave, + BlockGemmPipelineVersion::v2>{}); +} + +} // namespace instance +} // namespace device +} // namespace tensor_operation +} // namespace ck diff --git a/library/src/tensor_operation_instance/gpu/grouped_conv2d_bwd_weight/xdl/nhwgc_gkyxc_nhwgk/device_grouped_conv2d_bwd_weight_xdl_nhwgc_gkyxc_nhwgk_f32_tf32_pad0_pipev5_instance.cpp b/library/src/tensor_operation_instance/gpu/grouped_conv2d_bwd_weight/xdl/nhwgc_gkyxc_nhwgk/device_grouped_conv2d_bwd_weight_xdl_nhwgc_gkyxc_nhwgk_f32_tf32_pad0_pipev5_instance.cpp new file mode 100644 index 0000000000..70b1739ee1 --- /dev/null +++ b/library/src/tensor_operation_instance/gpu/grouped_conv2d_bwd_weight/xdl/nhwgc_gkyxc_nhwgk/device_grouped_conv2d_bwd_weight_xdl_nhwgc_gkyxc_nhwgk_f32_tf32_pad0_pipev5_instance.cpp @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2025, Advanced Micro Devices, Inc. All rights reserved. + +#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" +#include "ck/library/tensor_operation_instance/gpu/grouped_conv_bwd_weight/device_grouped_conv_bwd_weight_v3_xdl_instance.hpp" + +namespace ck { +namespace tensor_operation { +namespace device { +namespace instance { + +// Compilation parameters for in[n, hi, wi, g, c] * wei[g, k, y, x, c] = out[n, ho, wo, g, k] +void add_device_grouped_conv2d_bwd_weight_xdl_nhwgc_gkyxc_nhwgk_f32_tf32_pad0_pipev5_instances( + std::vector>>& instances) +{ + add_device_operation_instances( + instances, + device_grouped_conv_bwd_weight_v3_xdl_c_shuffle_f32_tf32_instances< + 2, + NHWGC, + GKYXC, + NHWGK, + ConvBwdWeightFilter1x1Stride1Pad0, + BlockGemmPipelineScheduler::Intrawave, + BlockGemmPipelineVersion::v5>{}); +} + +} // namespace instance +} // namespace device +} // namespace tensor_operation +} // namespace ck diff --git a/library/src/tensor_operation_instance/gpu/grouped_conv3d_bwd_data/CMakeLists.txt b/library/src/tensor_operation_instance/gpu/grouped_conv3d_bwd_data/CMakeLists.txt index 8652d9fd9d..f5b2f0d021 100644 --- a/library/src/tensor_operation_instance/gpu/grouped_conv3d_bwd_data/CMakeLists.txt +++ b/library/src/tensor_operation_instance/gpu/grouped_conv3d_bwd_data/CMakeLists.txt @@ -6,12 +6,15 @@ set(GROUPED_CONV3D_BWD_DATA xdl/device_grouped_conv3d_bwd_data_xdl_ndhwgc_gkzyxc_ndhwgk_f16_instance.cpp xdl/device_grouped_conv3d_bwd_data_xdl_ndhwgc_gkzyxc_ndhwgk_bf16_instance.cpp xdl/device_grouped_conv3d_bwd_data_xdl_ndhwgc_gkzyxc_ndhwgk_f32_instance.cpp + xdl/device_grouped_conv3d_bwd_data_xdl_ndhwgc_gkzyxc_ndhwgk_f32_tf32_instance.cpp xdl/device_grouped_conv3d_bwd_data_xdl_ndhwgc_gkzyxc_ndhwgk_f16_16_16_instance.cpp xdl/device_grouped_conv3d_bwd_data_xdl_ndhwgc_gkzyxc_ndhwgk_bf16_16_16_instance.cpp xdl/device_grouped_conv3d_bwd_data_xdl_ndhwgc_gkzyxc_ndhwgk_f32_16_16_instance.cpp + xdl/device_grouped_conv3d_bwd_data_xdl_ndhwgc_gkzyxc_ndhwgk_f32_tf32_16_16_instance.cpp xdl/device_grouped_conv3d_bwd_data_xdl_ndhwgc_gkzyxc_ndhwgk_f16_optimized_loads_instance.cpp xdl/device_grouped_conv3d_bwd_data_xdl_ndhwgc_gkzyxc_ndhwgk_bf16_optimized_loads_instance.cpp xdl/device_grouped_conv3d_bwd_data_xdl_ndhwgc_gkzyxc_ndhwgk_f32_optimized_loads_instance.cpp + xdl/device_grouped_conv3d_bwd_data_xdl_ndhwgc_gkzyxc_ndhwgk_f32_tf32_optimized_loads_instance.cpp xdl/device_grouped_conv3d_bwd_data_xdl_ngcdhw_gkzyxc_ngkdhw_f16_instance.cpp xdl/device_grouped_conv3d_bwd_data_xdl_ngcdhw_gkzyxc_ngkdhw_bf16_instance.cpp xdl/device_grouped_conv3d_bwd_data_xdl_ngcdhw_gkzyxc_ngkdhw_f32_instance.cpp diff --git a/library/src/tensor_operation_instance/gpu/grouped_conv3d_bwd_data/xdl/device_grouped_conv3d_bwd_data_xdl_ndhwgc_gkzyxc_ndhwgk_f32_tf32_16_16_instance.cpp b/library/src/tensor_operation_instance/gpu/grouped_conv3d_bwd_data/xdl/device_grouped_conv3d_bwd_data_xdl_ndhwgc_gkzyxc_ndhwgk_f32_tf32_16_16_instance.cpp new file mode 100644 index 0000000000..63e90333a9 --- /dev/null +++ b/library/src/tensor_operation_instance/gpu/grouped_conv3d_bwd_data/xdl/device_grouped_conv3d_bwd_data_xdl_ndhwgc_gkzyxc_ndhwgk_f32_tf32_16_16_instance.cpp @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2025, Advanced Micro Devices, Inc. All rights reserved. + +#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" +#include "ck/library/tensor_operation_instance/gpu/grouped_conv_bwd_data/device_grouped_conv_bwd_data_xdl_instance.hpp" + +namespace ck { +namespace tensor_operation { +namespace device { +namespace instance { + +void add_device_grouped_conv3d_bwd_data_xdl_ndhwgk_gkzyxc_ndhwgc_f32_tf32_16_16_instances( + std::vector>>& instances) +{ + // 1. Default + add_device_operation_instances( + instances, + device_grouped_conv_bwd_data_xdl_f32_tf32_16_16_instances<3, + NDHWGK, + GKZYXC, + Empty_Tuple, + NDHWGC, + ConvBwdDataDefault>{}); + // 2. Filter1x1Stride1Pad0 + add_device_operation_instances(instances, + device_grouped_conv_bwd_data_xdl_f32_tf32_16_16_instances< + 3, + NDHWGK, + GKZYXC, + Empty_Tuple, + NDHWGC, + ConvBwdDataFilter1x1Stride1Pad0>{}); +} + +} // namespace instance +} // namespace device +} // namespace tensor_operation +} // namespace ck diff --git a/library/src/tensor_operation_instance/gpu/grouped_conv3d_bwd_data/xdl/device_grouped_conv3d_bwd_data_xdl_ndhwgc_gkzyxc_ndhwgk_f32_tf32_instance.cpp b/library/src/tensor_operation_instance/gpu/grouped_conv3d_bwd_data/xdl/device_grouped_conv3d_bwd_data_xdl_ndhwgc_gkzyxc_ndhwgk_f32_tf32_instance.cpp new file mode 100644 index 0000000000..1db6494479 --- /dev/null +++ b/library/src/tensor_operation_instance/gpu/grouped_conv3d_bwd_data/xdl/device_grouped_conv3d_bwd_data_xdl_ndhwgc_gkzyxc_ndhwgk_f32_tf32_instance.cpp @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2018-2025, Advanced Micro Devices, Inc. All rights reserved. + +#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" +#include "ck/library/tensor_operation_instance/gpu/grouped_conv_bwd_data/device_grouped_conv_bwd_data_xdl_instance.hpp" + +namespace ck { +namespace tensor_operation { +namespace device { +namespace instance { + +void add_device_grouped_conv3d_bwd_data_xdl_ndhwgk_gkzyxc_ndhwgc_f32_tf32_instances( + std::vector>>& instances) +{ + // 1. Default + add_device_operation_instances( + instances, + device_grouped_conv_bwd_data_xdl_f32_tf32_instances<3, + NDHWGK, + GKZYXC, + Empty_Tuple, + NDHWGC, + ConvBwdDataDefault>{}); + // 2. Filter1x1Stride1Pad0 + add_device_operation_instances( + instances, + device_grouped_conv_bwd_data_xdl_f32_tf32_instances<3, + NDHWGK, + GKZYXC, + Empty_Tuple, + NDHWGC, + ConvBwdDataFilter1x1Stride1Pad0>{}); +} + +} // namespace instance +} // namespace device +} // namespace tensor_operation +} // namespace ck diff --git a/library/src/tensor_operation_instance/gpu/grouped_conv3d_bwd_data/xdl/device_grouped_conv3d_bwd_data_xdl_ndhwgc_gkzyxc_ndhwgk_f32_tf32_optimized_loads_instance.cpp b/library/src/tensor_operation_instance/gpu/grouped_conv3d_bwd_data/xdl/device_grouped_conv3d_bwd_data_xdl_ndhwgc_gkzyxc_ndhwgk_f32_tf32_optimized_loads_instance.cpp new file mode 100644 index 0000000000..7fe082b8e0 --- /dev/null +++ b/library/src/tensor_operation_instance/gpu/grouped_conv3d_bwd_data/xdl/device_grouped_conv3d_bwd_data_xdl_ndhwgc_gkzyxc_ndhwgk_f32_tf32_optimized_loads_instance.cpp @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2025, Advanced Micro Devices, Inc. All rights reserved. + +#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" +#include "ck/library/tensor_operation_instance/gpu/grouped_conv_bwd_data/device_grouped_conv_bwd_data_xdl_instance.hpp" + +namespace ck { +namespace tensor_operation { +namespace device { +namespace instance { + +void add_device_grouped_conv3d_bwd_data_xdl_ndhwgk_gkzyxc_ndhwgc_f32_tf32_optimized_loads_instances( + std::vector>>& instances) +{ + // 1. Default + add_device_operation_instances( + instances, + device_grouped_conv_bwd_data_xdl_f32_tf32_optimized_loads_instances<3, + NDHWGK, + GKZYXC, + Empty_Tuple, + NDHWGC, + ConvBwdDataDefault>{}); + // 2. Filter1x1Stride1Pad0 + add_device_operation_instances( + instances, + device_grouped_conv_bwd_data_xdl_f32_tf32_optimized_loads_instances< + 3, + NDHWGK, + GKZYXC, + Empty_Tuple, + NDHWGC, + ConvBwdDataFilter1x1Stride1Pad0>{}); +} + +} // namespace instance +} // namespace device +} // namespace tensor_operation +} // namespace ck diff --git a/library/src/tensor_operation_instance/gpu/grouped_conv3d_bwd_weight/CMakeLists.txt b/library/src/tensor_operation_instance/gpu/grouped_conv3d_bwd_weight/CMakeLists.txt index 5574cf82f9..f9922b1f37 100644 --- a/library/src/tensor_operation_instance/gpu/grouped_conv3d_bwd_weight/CMakeLists.txt +++ b/library/src/tensor_operation_instance/gpu/grouped_conv3d_bwd_weight/CMakeLists.txt @@ -6,6 +6,7 @@ set(GROUPED_CONV3D_BWD_WEIGHT xdl/ndhwgc_gkzyxc_ndhwgk/device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_f16_instance.cpp xdl/ndhwgc_gkzyxc_ndhwgk/device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_f32_instance.cpp + xdl/ndhwgc_gkzyxc_ndhwgk/device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_f32_tf32_instance.cpp xdl/ndhwgc_gkzyxc_ndhwgk/device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_bf16_f32_bf16_instance.cpp xdl/ndhwgc_gkzyxc_ndhwgk/device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_bf16_instance.cpp xdl/ndhwgc_gkzyxc_ndhwgk/device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_bf16_default_pipev2_instance.cpp @@ -17,9 +18,13 @@ set(GROUPED_CONV3D_BWD_WEIGHT xdl/ndhwgc_gkzyxc_ndhwgk/device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_f16_pad0_pipev2_instance.cpp xdl/ndhwgc_gkzyxc_ndhwgk/device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_f16_pad0_pipev5_instance.cpp xdl/ndhwgc_gkzyxc_ndhwgk/device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_f32_default_pipev2_instance.cpp + xdl/ndhwgc_gkzyxc_ndhwgk/device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_f32_tf32_default_pipev2_instance.cpp xdl/ndhwgc_gkzyxc_ndhwgk/device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_f32_default_pipev5_instance.cpp + xdl/ndhwgc_gkzyxc_ndhwgk/device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_f32_tf32_default_pipev5_instance.cpp xdl/ndhwgc_gkzyxc_ndhwgk/device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_f32_pad0_pipev2_instance.cpp + xdl/ndhwgc_gkzyxc_ndhwgk/device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_f32_tf32_pad0_pipev2_instance.cpp xdl/ndhwgc_gkzyxc_ndhwgk/device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_f32_pad0_pipev5_instance.cpp + xdl/ndhwgc_gkzyxc_ndhwgk/device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_f32_tf32_pad0_pipev5_instance.cpp xdl/ndhwgc_gkzyxc_ndhwgk/device_grouped_conv3d_bwd_weight_two_stage_xdl_ndhwgc_gkzyxc_ndhwgk_f16_pipev2_instance.cpp xdl/ndhwgc_gkzyxc_ndhwgk/device_grouped_conv3d_bwd_weight_two_stage_xdl_ndhwgc_gkzyxc_ndhwgk_f16_pipev5_instance.cpp xdl/ndhwgc_gkzyxc_ndhwgk/device_grouped_conv3d_bwd_weight_two_stage_xdl_ndhwgc_gkzyxc_ndhwgk_bf16_pipev2_instance.cpp diff --git a/library/src/tensor_operation_instance/gpu/grouped_conv3d_bwd_weight/xdl/ndhwgc_gkzyxc_ndhwgk/device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_f32_tf32_default_pipev2_instance.cpp b/library/src/tensor_operation_instance/gpu/grouped_conv3d_bwd_weight/xdl/ndhwgc_gkzyxc_ndhwgk/device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_f32_tf32_default_pipev2_instance.cpp new file mode 100644 index 0000000000..dab91ec747 --- /dev/null +++ b/library/src/tensor_operation_instance/gpu/grouped_conv3d_bwd_weight/xdl/ndhwgc_gkzyxc_ndhwgk/device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_f32_tf32_default_pipev2_instance.cpp @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2025, Advanced Micro Devices, Inc. All rights reserved. + +#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" +#include "ck/library/tensor_operation_instance/gpu/grouped_conv_bwd_weight/device_grouped_conv_bwd_weight_v3_xdl_instance.hpp" + +namespace ck { +namespace tensor_operation { +namespace device { +namespace instance { + +// Compilation parameters for in[n, hi, wi, g, c] * wei[g, k, y, x, c] = out[n, ho, wo, g, k] +void add_device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_f32_tf32_default_pipev2_instances( + std::vector>>& instances) +{ + add_device_operation_instances( + instances, + device_grouped_conv_bwd_weight_v3_xdl_c_shuffle_f32_tf32_instances< + 3, + NDHWGC, + GKZYXC, + NDHWGK, + ConvBwdWeightDefault, + BlockGemmPipelineScheduler::Intrawave, + BlockGemmPipelineVersion::v2>{}); +} + +} // namespace instance +} // namespace device +} // namespace tensor_operation +} // namespace ck diff --git a/library/src/tensor_operation_instance/gpu/grouped_conv3d_bwd_weight/xdl/ndhwgc_gkzyxc_ndhwgk/device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_f32_tf32_default_pipev5_instance.cpp b/library/src/tensor_operation_instance/gpu/grouped_conv3d_bwd_weight/xdl/ndhwgc_gkzyxc_ndhwgk/device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_f32_tf32_default_pipev5_instance.cpp new file mode 100644 index 0000000000..01229234ff --- /dev/null +++ b/library/src/tensor_operation_instance/gpu/grouped_conv3d_bwd_weight/xdl/ndhwgc_gkzyxc_ndhwgk/device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_f32_tf32_default_pipev5_instance.cpp @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2025, Advanced Micro Devices, Inc. All rights reserved. + +#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" +#include "ck/library/tensor_operation_instance/gpu/grouped_conv_bwd_weight/device_grouped_conv_bwd_weight_v3_xdl_instance.hpp" + +namespace ck { +namespace tensor_operation { +namespace device { +namespace instance { + +// Compilation parameters for in[n, hi, wi, g, c] * wei[g, k, y, x, c] = out[n, ho, wo, g, k] +void add_device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_f32_tf32_default_pipev5_instances( + std::vector>>& instances) +{ + add_device_operation_instances( + instances, + device_grouped_conv_bwd_weight_v3_xdl_c_shuffle_f32_tf32_instances< + 3, + NDHWGC, + GKZYXC, + NDHWGK, + ConvBwdWeightDefault, + BlockGemmPipelineScheduler::Intrawave, + BlockGemmPipelineVersion::v5>{}); +} + +} // namespace instance +} // namespace device +} // namespace tensor_operation +} // namespace ck diff --git a/library/src/tensor_operation_instance/gpu/grouped_conv3d_bwd_weight/xdl/ndhwgc_gkzyxc_ndhwgk/device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_f32_tf32_instance.cpp b/library/src/tensor_operation_instance/gpu/grouped_conv3d_bwd_weight/xdl/ndhwgc_gkzyxc_ndhwgk/device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_f32_tf32_instance.cpp new file mode 100644 index 0000000000..ac6c3b60e4 --- /dev/null +++ b/library/src/tensor_operation_instance/gpu/grouped_conv3d_bwd_weight/xdl/ndhwgc_gkzyxc_ndhwgk/device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_f32_tf32_instance.cpp @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2018-2025, Advanced Micro Devices, Inc. All rights reserved. + +#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" +#include "ck/library/tensor_operation_instance/gpu/grouped_conv_bwd_weight/device_grouped_conv_bwd_weight_xdl_instance.hpp" + +namespace ck { +namespace tensor_operation { +namespace device { +namespace instance { + +// Compilation parameters for in[n, hi, wi, g, c] * wei[g, k, y, x, c] = out[n, ho, wo, g, k] +void add_device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_f32_tf32_instances( + std::vector>>& instances) +{ + // 1. Default + add_device_operation_instances( + instances, + device_grouped_conv_bwd_weight_xdl_c_shuffle_f32_tf32_instances<3, + NDHWGC, + GKZYXC, + NDHWGK, + ConvBwdWeightDefault>{}); + // 2. Filter1x1Stride1Pad0 + add_device_operation_instances(instances, + device_grouped_conv_bwd_weight_xdl_c_shuffle_f32_tf32_instances< + 3, + NDHWGC, + GKZYXC, + NDHWGK, + ConvBwdWeightFilter1x1Stride1Pad0>{}); +} + +} // namespace instance +} // namespace device +} // namespace tensor_operation +} // namespace ck diff --git a/library/src/tensor_operation_instance/gpu/grouped_conv3d_bwd_weight/xdl/ndhwgc_gkzyxc_ndhwgk/device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_f32_tf32_pad0_pipev2_instance.cpp b/library/src/tensor_operation_instance/gpu/grouped_conv3d_bwd_weight/xdl/ndhwgc_gkzyxc_ndhwgk/device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_f32_tf32_pad0_pipev2_instance.cpp new file mode 100644 index 0000000000..c479cc2048 --- /dev/null +++ b/library/src/tensor_operation_instance/gpu/grouped_conv3d_bwd_weight/xdl/ndhwgc_gkzyxc_ndhwgk/device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_f32_tf32_pad0_pipev2_instance.cpp @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2025, Advanced Micro Devices, Inc. All rights reserved. + +#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" +#include "ck/library/tensor_operation_instance/gpu/grouped_conv_bwd_weight/device_grouped_conv_bwd_weight_v3_xdl_instance.hpp" + +namespace ck { +namespace tensor_operation { +namespace device { +namespace instance { + +// Compilation parameters for in[n, hi, wi, g, c] * wei[g, k, y, x, c] = out[n, ho, wo, g, k] +void add_device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_f32_tf32_pad0_pipev2_instances( + std::vector>>& instances) +{ + add_device_operation_instances( + instances, + device_grouped_conv_bwd_weight_v3_xdl_c_shuffle_f32_tf32_instances< + 3, + NDHWGC, + GKZYXC, + NDHWGK, + ConvBwdWeightFilter1x1Stride1Pad0, + BlockGemmPipelineScheduler::Intrawave, + BlockGemmPipelineVersion::v2>{}); +} + +} // namespace instance +} // namespace device +} // namespace tensor_operation +} // namespace ck diff --git a/library/src/tensor_operation_instance/gpu/grouped_conv3d_bwd_weight/xdl/ndhwgc_gkzyxc_ndhwgk/device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_f32_tf32_pad0_pipev5_instance.cpp b/library/src/tensor_operation_instance/gpu/grouped_conv3d_bwd_weight/xdl/ndhwgc_gkzyxc_ndhwgk/device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_f32_tf32_pad0_pipev5_instance.cpp new file mode 100644 index 0000000000..cfb0e8a65e --- /dev/null +++ b/library/src/tensor_operation_instance/gpu/grouped_conv3d_bwd_weight/xdl/ndhwgc_gkzyxc_ndhwgk/device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_f32_tf32_pad0_pipev5_instance.cpp @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2025, Advanced Micro Devices, Inc. All rights reserved. + +#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" +#include "ck/library/tensor_operation_instance/gpu/grouped_conv_bwd_weight/device_grouped_conv_bwd_weight_v3_xdl_instance.hpp" + +namespace ck { +namespace tensor_operation { +namespace device { +namespace instance { + +// Compilation parameters for in[n, hi, wi, g, c] * wei[g, k, y, x, c] = out[n, ho, wo, g, k] +void add_device_grouped_conv3d_bwd_weight_xdl_ndhwgc_gkzyxc_ndhwgk_f32_tf32_pad0_pipev5_instances( + std::vector>>& instances) +{ + add_device_operation_instances( + instances, + device_grouped_conv_bwd_weight_v3_xdl_c_shuffle_f32_tf32_instances< + 3, + NDHWGC, + GKZYXC, + NDHWGK, + ConvBwdWeightFilter1x1Stride1Pad0, + BlockGemmPipelineScheduler::Intrawave, + BlockGemmPipelineVersion::v5>{}); +} + +} // namespace instance +} // namespace device +} // namespace tensor_operation +} // namespace ck diff --git a/library/src/tensor_operation_instance/gpu/grouped_conv3d_bwd_weight_bilinear/CMakeLists.txt b/library/src/tensor_operation_instance/gpu/grouped_conv3d_bwd_weight_bilinear/CMakeLists.txt index 329e8e4c7f..b8621e73aa 100644 --- a/library/src/tensor_operation_instance/gpu/grouped_conv3d_bwd_weight_bilinear/CMakeLists.txt +++ b/library/src/tensor_operation_instance/gpu/grouped_conv3d_bwd_weight_bilinear/CMakeLists.txt @@ -2,6 +2,7 @@ set(GROUPED_CONV3D_BWD_WEIGHT_BILINEAR xdl/device_grouped_conv3d_bwd_weight_xdl_bilinear_ndhwgc_gkzyxc_ndhwgk_f16_instance.cpp xdl/device_grouped_conv3d_bwd_weight_xdl_bilinear_ndhwgc_gkzyxc_ndhwgk_f32_instance.cpp + xdl/device_grouped_conv3d_bwd_weight_xdl_bilinear_ndhwgc_gkzyxc_ndhwgk_f32_tf32_instance.cpp xdl/device_grouped_conv3d_bwd_weight_xdl_bilinear_ndhwgc_gkzyxc_ndhwgk_bf16_instance.cpp) if((DTYPES MATCHES "fp8" AND DTYPES MATCHES "bf8" AND DTYPES MATCHES "fp16") OR NOT DEFINED DTYPES) diff --git a/library/src/tensor_operation_instance/gpu/grouped_conv3d_bwd_weight_bilinear/xdl/device_grouped_conv3d_bwd_weight_xdl_bilinear_ndhwgc_gkzyxc_ndhwgk_f32_tf32_instance.cpp b/library/src/tensor_operation_instance/gpu/grouped_conv3d_bwd_weight_bilinear/xdl/device_grouped_conv3d_bwd_weight_xdl_bilinear_ndhwgc_gkzyxc_ndhwgk_f32_tf32_instance.cpp new file mode 100644 index 0000000000..a71c02aec1 --- /dev/null +++ b/library/src/tensor_operation_instance/gpu/grouped_conv3d_bwd_weight_bilinear/xdl/device_grouped_conv3d_bwd_weight_xdl_bilinear_ndhwgc_gkzyxc_ndhwgk_f32_tf32_instance.cpp @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2025, Advanced Micro Devices, Inc. All rights reserved. + +#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" +#include "ck/library/tensor_operation_instance/gpu/grouped_conv_bwd_weight/device_grouped_conv_bwd_weight_xdl_bilinear_instance.hpp" + +namespace ck { +namespace tensor_operation { +namespace device { +namespace instance { + +// Compilation parameters for in[n, hi, wi, g, c] * wei[g, k, y, x, c] = out[n, ho, wo, g, k] +void add_device_grouped_conv3d_bwd_weight_xdl_bilinear_ndhwgc_gkzyxc_ndhwgk_f32_tf32_instances( + std::vector, + F32, + F32, + F32, + Tuple, + PassThrough, + Bilinear, + PassThrough, + TF32, + TF32>>>& instances) +{ + // 1. Default + add_device_operation_instances( + instances, + device_grouped_conv_bwd_weight_xdl_c_shuffle_f32_tf32_bilinear_instances< + 3, + NDHWGC, + GKZYXC, + NDHWGK, + ConvBwdWeightDefault>{}); + // 2. Filter1x1Stride1Pad0 + add_device_operation_instances( + instances, + device_grouped_conv_bwd_weight_xdl_c_shuffle_f32_tf32_bilinear_instances< + 3, + NDHWGC, + GKZYXC, + NDHWGK, + ConvBwdWeightFilter1x1Stride1Pad0>{}); +} + +} // namespace instance +} // namespace device +} // namespace tensor_operation +} // namespace ck diff --git a/library/src/tensor_operation_instance/gpu/grouped_conv3d_bwd_weight_scale/CMakeLists.txt b/library/src/tensor_operation_instance/gpu/grouped_conv3d_bwd_weight_scale/CMakeLists.txt index 9a42d1ec3a..5277b04ed4 100644 --- a/library/src/tensor_operation_instance/gpu/grouped_conv3d_bwd_weight_scale/CMakeLists.txt +++ b/library/src/tensor_operation_instance/gpu/grouped_conv3d_bwd_weight_scale/CMakeLists.txt @@ -2,6 +2,7 @@ set(GROUPED_CONV3D_BWD_WEIGHT_SCALE xdl/device_grouped_conv3d_bwd_weight_xdl_scale_ndhwgc_gkzyxc_ndhwgk_f16_instance.cpp xdl/device_grouped_conv3d_bwd_weight_xdl_scale_ndhwgc_gkzyxc_ndhwgk_f32_instance.cpp + xdl/device_grouped_conv3d_bwd_weight_xdl_scale_ndhwgc_gkzyxc_ndhwgk_f32_tf32_instance.cpp xdl/device_grouped_conv3d_bwd_weight_xdl_scale_ndhwgc_gkzyxc_ndhwgk_bf16_instance.cpp) if((DTYPES MATCHES "fp8" AND DTYPES MATCHES "bf8" AND DTYPES MATCHES "fp16") OR NOT DEFINED DTYPES) diff --git a/library/src/tensor_operation_instance/gpu/grouped_conv3d_bwd_weight_scale/xdl/device_grouped_conv3d_bwd_weight_xdl_scale_ndhwgc_gkzyxc_ndhwgk_f32_tf32_instance.cpp b/library/src/tensor_operation_instance/gpu/grouped_conv3d_bwd_weight_scale/xdl/device_grouped_conv3d_bwd_weight_xdl_scale_ndhwgc_gkzyxc_ndhwgk_f32_tf32_instance.cpp new file mode 100644 index 0000000000..65f79141b3 --- /dev/null +++ b/library/src/tensor_operation_instance/gpu/grouped_conv3d_bwd_weight_scale/xdl/device_grouped_conv3d_bwd_weight_xdl_scale_ndhwgc_gkzyxc_ndhwgk_f32_tf32_instance.cpp @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2025, Advanced Micro Devices, Inc. All rights reserved. + +#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" +#include "ck/library/tensor_operation_instance/gpu/grouped_conv_bwd_weight/device_grouped_conv_bwd_weight_xdl_scale_instance.hpp" + +namespace ck { +namespace tensor_operation { +namespace device { +namespace instance { + +// Compilation parameters for in[n, hi, wi, g, c] * wei[g, k, y, x, c] = out[n, ho, wo, g, k] +void add_device_grouped_conv3d_bwd_weight_xdl_scale_ndhwgc_gkzyxc_ndhwgk_f32_tf32_instances( + std::vector, + F32, + F32, + F32, + Tuple<>, + PassThrough, + Scale, + PassThrough, + TF32, + TF32>>>& instances) +{ + // 1. Default + add_device_operation_instances( + instances, + device_grouped_conv_bwd_weight_xdl_c_shuffle_f32_tf32_scale_instances< + 3, + NDHWGC, + GKZYXC, + NDHWGK, + ConvBwdWeightDefault>{}); + // 2. Filter1x1Stride1Pad0 + add_device_operation_instances( + instances, + device_grouped_conv_bwd_weight_xdl_c_shuffle_f32_tf32_scale_instances< + 3, + NDHWGC, + GKZYXC, + NDHWGK, + ConvBwdWeightFilter1x1Stride1Pad0>{}); +} + +} // namespace instance +} // namespace device +} // namespace tensor_operation +} // namespace ck diff --git a/profiler/include/profiler/profile_grouped_conv_bwd_data_impl.hpp b/profiler/include/profiler/profile_grouped_conv_bwd_data_impl.hpp index a7c6717f58..0b73fe7adf 100644 --- a/profiler/include/profiler/profile_grouped_conv_bwd_data_impl.hpp +++ b/profiler/include/profiler/profile_grouped_conv_bwd_data_impl.hpp @@ -29,7 +29,8 @@ template + typename InDataType, + typename ComputeDataType = InDataType> bool profile_grouped_conv_bwd_data_impl(int do_verification, int init_method, bool do_log, @@ -96,7 +97,11 @@ bool profile_grouped_conv_bwd_data_impl(int do_verification, OutDataType, InElementOp, WeiElementOp, - OutElementOp>(); + OutElementOp, + 0, + 0, + 0, + ComputeDataType>(); auto ref_invoker = ref_conv.MakeInvoker(); @@ -171,9 +176,13 @@ bool profile_grouped_conv_bwd_data_impl(int do_verification, { in_device_buf.FromDevice(in_device.mData.data()); - using ComputeType = std::conditional_t; + using ComputeType_ = std::conditional_t; + using ComputeType = + std::conditional_t; using AccDataType = std::conditional_t, int32_t, float>; const index_t num_accums = conv_param.K_; @@ -222,18 +231,21 @@ bool profile_grouped_conv_bwd_data_impl(int do_verification, }; // do GEMM - using DeviceOp = ck::tensor_operation::device::DeviceGroupedConvBwdDataMultipleD, - InLayout, - OutDataType, - WeiDataType, - ck::Tuple<>, - InDataType, - OutElementOp, - WeiElementOp, - InElementOp>; + using DeviceOp = + ck::tensor_operation::device::DeviceGroupedConvBwdDataMultipleD, + InLayout, + OutDataType, + WeiDataType, + ck::Tuple<>, + InDataType, + OutElementOp, + WeiElementOp, + InElementOp, + ComputeDataType, + ComputeDataType>; // get device op instances const auto op_ptrs = ck::tensor_operation::device::instance::DeviceOperationInstanceFactory< diff --git a/profiler/src/profile_grouped_conv_bwd_data.cpp b/profiler/src/profile_grouped_conv_bwd_data.cpp index 5cdece499e..95098e2301 100644 --- a/profiler/src/profile_grouped_conv_bwd_data.cpp +++ b/profiler/src/profile_grouped_conv_bwd_data.cpp @@ -21,9 +21,10 @@ enum struct ConvLayout enum struct ConvDataType { - F32_F32_F32, // 0 - F16_F16_F16, // 1 - BF16_BF16_BF16, // 2 + F32_F32_F32, // 0 + F16_F16_F16, // 1 + BF16_BF16_BF16, // 2 + F32_F32_F32_TF32, // 3 }; #define OP_NAME "grouped_conv_bwd_data" @@ -37,6 +38,7 @@ static void print_helper_msg() << "arg2: data type (0: Output fp32, Weight fp32, Input fp32\n" << " 1: Output fp16, Weight fp16, Input fp16\n" << " 2: Output bf16, Weight bf16, Input bf16\n" + << " 3: Output fp32, Weight fp32, Input fp32, Compute tf32)\n" << "arg3: tensor layout (0: Output[G, N, Ho, Wo, C], Weight[G, K, Y, X, C], Input[G, N, Hi, Wi, K]\n" << " 1: Output[N, Ho, Wo, G, C], Weight[G, K, Y, X, C], Input[N, Hi, Wi, G, K])\n" << " 2: Output[N, G, C, Ho, Wo], Weight[G, K, Y, X, C], Input[N, G, K, Hi, Wi])\n" @@ -82,6 +84,9 @@ int profile_grouped_conv_bwd_data(int argc, char* argv[]) using F32 = float; using F16 = ck::half_t; using BF16 = ck::bhalf_t; +#if defined(__gfx942__) + using TF32 = ck::tf32_t; +#endif using namespace ck::tensor_layout::convolution; @@ -94,16 +99,18 @@ int profile_grouped_conv_bwd_data(int argc, char* argv[]) auto in_layout, auto wei_type, auto out_type, - auto in_type) { + auto in_type, + auto compute_type) { constexpr ck::index_t NDimSpatial = num_dim_spatial_tmp.value; using OutLayout = decltype(out_layout); using WeiLayout = decltype(wei_layout); using InLayout = decltype(in_layout); - using OutDataType = decltype(out_type); - using WeiDataType = decltype(wei_type); - using InDataType = decltype(in_type); + using OutDataType = decltype(out_type); + using WeiDataType = decltype(wei_type); + using InDataType = decltype(in_type); + using ComputeDataType = decltype(compute_type); bool pass = ck::profiler::profile_grouped_conv_bwd_data_impl( + InDataType, + ComputeDataType>( do_verification, init_method, do_log, time_kernel, params, split_k); return pass ? 0 : 1; @@ -123,60 +131,84 @@ int profile_grouped_conv_bwd_data(int argc, char* argv[]) { if(data_type == ConvDataType::F32_F32_F32) { - return profile(I2, GNHWK{}, GKYXC{}, GNHWC{}, F32{}, F32{}, F32{}); + return profile(I2, GNHWK{}, GKYXC{}, GNHWC{}, F32{}, F32{}, F32{}, F32{}); } else if(data_type == ConvDataType::F16_F16_F16) { - return profile(I2, GNHWK{}, GKYXC{}, GNHWC{}, F16{}, F16{}, F16{}); + return profile(I2, GNHWK{}, GKYXC{}, GNHWC{}, F16{}, F16{}, F16{}, F16{}); } else if(data_type == ConvDataType::BF16_BF16_BF16) { - return profile(I2, GNHWK{}, GKYXC{}, GNHWC{}, BF16{}, BF16{}, BF16{}); + return profile(I2, GNHWK{}, GKYXC{}, GNHWC{}, BF16{}, BF16{}, BF16{}, BF16{}); + } + else if(data_type == ConvDataType::F32_F32_F32_TF32) + { +#if defined(__gfx942__) + return profile(I2, GNHWK{}, GKYXC{}, GNHWC{}, F32{}, F32{}, F32{}, TF32{}); +#endif } } else if(layout == ConvLayout::NHWGC_GKYXC_NHWGK) { if(data_type == ConvDataType::F32_F32_F32) { - return profile(I2, NHWGK{}, GKYXC{}, NHWGC{}, F32{}, F32{}, F32{}); + return profile(I2, NHWGK{}, GKYXC{}, NHWGC{}, F32{}, F32{}, F32{}, F32{}); } else if(data_type == ConvDataType::F16_F16_F16) { - return profile(I2, NHWGK{}, GKYXC{}, NHWGC{}, F16{}, F16{}, F16{}); + return profile(I2, NHWGK{}, GKYXC{}, NHWGC{}, F16{}, F16{}, F16{}, F16{}); } else if(data_type == ConvDataType::BF16_BF16_BF16) { - return profile(I2, NHWGK{}, GKYXC{}, NHWGC{}, BF16{}, BF16{}, BF16{}); + return profile(I2, NHWGK{}, GKYXC{}, NHWGC{}, BF16{}, BF16{}, BF16{}, BF16{}); + } + else if(data_type == ConvDataType::F32_F32_F32_TF32) + { +#if defined(__gfx942__) + return profile(I2, NHWGK{}, GKYXC{}, NHWGC{}, F32{}, F32{}, F32{}, TF32{}); +#endif } } else if(layout == ConvLayout::NGCHW_GKYXC_NGKHW) { if(data_type == ConvDataType::F32_F32_F32) { - return profile(I2, NGKHW{}, GKYXC{}, NGCHW{}, F32{}, F32{}, F32{}); + return profile(I2, NGKHW{}, GKYXC{}, NGCHW{}, F32{}, F32{}, F32{}, F32{}); } else if(data_type == ConvDataType::F16_F16_F16) { - return profile(I2, NGKHW{}, GKYXC{}, NGCHW{}, F16{}, F16{}, F16{}); + return profile(I2, NGKHW{}, GKYXC{}, NGCHW{}, F16{}, F16{}, F16{}, F16{}); } else if(data_type == ConvDataType::BF16_BF16_BF16) { - return profile(I2, NGKHW{}, GKYXC{}, NGCHW{}, BF16{}, BF16{}, BF16{}); + return profile(I2, NGKHW{}, GKYXC{}, NGCHW{}, BF16{}, BF16{}, BF16{}, BF16{}); + } + else if(data_type == ConvDataType::F32_F32_F32_TF32) + { +#if defined(__gfx942__) + return profile(I2, NGKHW{}, GKYXC{}, NGCHW{}, F32{}, F32{}, F32{}, TF32{}); +#endif } } else if(layout == ConvLayout::NGCHW_GKCYX_NGKHW) { if(data_type == ConvDataType::F32_F32_F32) { - return profile(I2, NGKHW{}, GKCYX{}, NGCHW{}, F32{}, F32{}, F32{}); + return profile(I2, NGKHW{}, GKCYX{}, NGCHW{}, F32{}, F32{}, F32{}, F32{}); } else if(data_type == ConvDataType::F16_F16_F16) { - return profile(I2, NGKHW{}, GKCYX{}, NGCHW{}, F16{}, F16{}, F16{}); + return profile(I2, NGKHW{}, GKCYX{}, NGCHW{}, F16{}, F16{}, F16{}, F16{}); } else if(data_type == ConvDataType::BF16_BF16_BF16) { - return profile(I2, NGKHW{}, GKCYX{}, NGCHW{}, BF16{}, BF16{}, BF16{}); + return profile(I2, NGKHW{}, GKCYX{}, NGCHW{}, BF16{}, BF16{}, BF16{}, BF16{}); + } + else if(data_type == ConvDataType::F32_F32_F32_TF32) + { +#if defined(__gfx942__) + return profile(I2, NGKHW{}, GKCYX{}, NGCHW{}, F32{}, F32{}, F32{}, TF32{}); +#endif } } } @@ -186,60 +218,84 @@ int profile_grouped_conv_bwd_data(int argc, char* argv[]) { if(data_type == ConvDataType::F32_F32_F32) { - return profile(I3, GNDHWK{}, GKZYXC{}, GNDHWC{}, F32{}, F32{}, F32{}); + return profile(I3, GNDHWK{}, GKZYXC{}, GNDHWC{}, F32{}, F32{}, F32{}, F32{}); } else if(data_type == ConvDataType::F16_F16_F16) { - return profile(I3, GNDHWK{}, GKZYXC{}, GNDHWC{}, F16{}, F16{}, F16{}); + return profile(I3, GNDHWK{}, GKZYXC{}, GNDHWC{}, F16{}, F16{}, F16{}, F16{}); } else if(data_type == ConvDataType::BF16_BF16_BF16) { - return profile(I3, GNDHWK{}, GKZYXC{}, GNDHWC{}, BF16{}, BF16{}, BF16{}); + return profile(I3, GNDHWK{}, GKZYXC{}, GNDHWC{}, BF16{}, BF16{}, BF16{}, BF16{}); + } + else if(data_type == ConvDataType::F32_F32_F32_TF32) + { +#if defined(__gfx942__) + return profile(I3, GNDHWK{}, GKZYXC{}, GNDHWC{}, F32{}, F32{}, F32{}, TF32{}); +#endif } } else if(layout == ConvLayout::NHWGC_GKYXC_NHWGK) { if(data_type == ConvDataType::F32_F32_F32) { - return profile(I3, NDHWGK{}, GKZYXC{}, NDHWGC{}, F32{}, F32{}, F32{}); + return profile(I3, NDHWGK{}, GKZYXC{}, NDHWGC{}, F32{}, F32{}, F32{}, F32{}); } else if(data_type == ConvDataType::F16_F16_F16) { - return profile(I3, NDHWGK{}, GKZYXC{}, NDHWGC{}, F16{}, F16{}, F16{}); + return profile(I3, NDHWGK{}, GKZYXC{}, NDHWGC{}, F16{}, F16{}, F16{}, F16{}); } else if(data_type == ConvDataType::BF16_BF16_BF16) { - return profile(I3, NDHWGK{}, GKZYXC{}, NDHWGC{}, BF16{}, BF16{}, BF16{}); + return profile(I3, NDHWGK{}, GKZYXC{}, NDHWGC{}, BF16{}, BF16{}, BF16{}, BF16{}); + } + else if(data_type == ConvDataType::F32_F32_F32_TF32) + { +#if defined(__gfx942__) + return profile(I3, NDHWGK{}, GKZYXC{}, NDHWGC{}, F32{}, F32{}, F32{}, TF32{}); +#endif } } else if(layout == ConvLayout::NGCHW_GKYXC_NGKHW) { if(data_type == ConvDataType::F32_F32_F32) { - return profile(I3, NGKDHW{}, GKZYXC{}, NGCDHW{}, F32{}, F32{}, F32{}); + return profile(I3, NGKDHW{}, GKZYXC{}, NGCDHW{}, F32{}, F32{}, F32{}, F32{}); } else if(data_type == ConvDataType::F16_F16_F16) { - return profile(I3, NGKDHW{}, GKZYXC{}, NGCDHW{}, F16{}, F16{}, F16{}); + return profile(I3, NGKDHW{}, GKZYXC{}, NGCDHW{}, F16{}, F16{}, F16{}, F16{}); } else if(data_type == ConvDataType::BF16_BF16_BF16) { - return profile(I3, NGKDHW{}, GKZYXC{}, NGCDHW{}, BF16{}, BF16{}, BF16{}); + return profile(I3, NGKDHW{}, GKZYXC{}, NGCDHW{}, BF16{}, BF16{}, BF16{}, BF16{}); + } + else if(data_type == ConvDataType::F32_F32_F32_TF32) + { +#if defined(__gfx942__) + return profile(I3, NGKDHW{}, GKZYXC{}, NGCDHW{}, F32{}, F32{}, F32{}, TF32{}); +#endif } } else if(layout == ConvLayout::NGCHW_GKYXC_NGKHW) { if(data_type == ConvDataType::F32_F32_F32) { - return profile(I3, NGKDHW{}, GKCZYX{}, NGCDHW{}, F32{}, F32{}, F32{}); + return profile(I3, NGKDHW{}, GKCZYX{}, NGCDHW{}, F32{}, F32{}, F32{}, F32{}); } else if(data_type == ConvDataType::F16_F16_F16) { - return profile(I3, NGKDHW{}, GKCZYX{}, NGCDHW{}, F16{}, F16{}, F16{}); + return profile(I3, NGKDHW{}, GKCZYX{}, NGCDHW{}, F16{}, F16{}, F16{}, F16{}); } else if(data_type == ConvDataType::BF16_BF16_BF16) { - return profile(I3, NGKDHW{}, GKCZYX{}, NGCDHW{}, BF16{}, BF16{}, BF16{}); + return profile(I3, NGKDHW{}, GKCZYX{}, NGCDHW{}, BF16{}, BF16{}, BF16{}, BF16{}); + } + else if(data_type == ConvDataType::F32_F32_F32_TF32) + { +#if defined(__gfx942__) + return profile(I3, NGKDHW{}, GKCZYX{}, NGCDHW{}, F32{}, F32{}, F32{}, TF32{}); +#endif } } } diff --git a/profiler/src/profile_grouped_conv_bwd_weight.cpp b/profiler/src/profile_grouped_conv_bwd_weight.cpp index 8347ce0e42..7d3f1ad6c0 100644 --- a/profiler/src/profile_grouped_conv_bwd_weight.cpp +++ b/profiler/src/profile_grouped_conv_bwd_weight.cpp @@ -28,6 +28,7 @@ enum struct ConvDataType F16_F16_F16_BF8_F8, // 3 I8_I8_I8, // 4 BF16_BF16_BF16, // 5 + F32_F32_F32_TF32, // 6 }; #define OP_NAME "grouped_conv_bwd_weight" @@ -41,7 +42,8 @@ static void print_helper_msg() << " 2: Input bf16, Weight fp32, Output bf16\n" << " 3: Input fp16, Weight fp16, Output fp16, Gemm bf8@fp8\n" << " 4: Input int8, Weight int8, Output int8\n" - << " 5: Input bf16, Weight bf16, Output bf16)\n" + << " 5: Input bf16, Weight bf16, Output bf16\n" + << " 6: Input fp32, Weight fp32, Output fp32, Compute tf32)\n" << "arg3: tensor layout (0: Input[G, N, C, Hi, Wi], Weight[G, K, C, Y, X], Output[G, " "N, K, Ho, Wo]\n" << " 1: Input[G, N, Hi, Wi, C], Weight[G, K, Y, X, C], Output[G, " @@ -97,6 +99,9 @@ int profile_grouped_conv_bwd_weight(int argc, char* argv[]) using BF16 = ck::bhalf_t; using F8 = ck::f8_t; using BF8 = ck::bf8_t; +#if defined(__gfx942__) + using TF32 = ck::tf32_t; +#endif using namespace ck::tensor_layout::convolution; @@ -155,6 +160,12 @@ int profile_grouped_conv_bwd_weight(int argc, char* argv[]) // fp32 atomic add is used for weight tensor in bf16 kernel return profile(I1, GNWC{}, GKXC{}, GNWK{}, BF16{}, F32{}, BF16{}, BF16{}, BF16{}); } + else if(data_type == ConvDataType::F32_F32_F32_TF32) + { +#if defined(__gfx942__) + return profile(I1, GNWC{}, GKXC{}, GNWK{}, F32{}, F32{}, F32{}, TF32{}, TF32{}); +#endif + } } if(num_dim_spatial == 2 && layout == ConvLayout::GNHWC_GKYXC_GNHWK) { @@ -171,6 +182,12 @@ int profile_grouped_conv_bwd_weight(int argc, char* argv[]) // fp32 atomic add is used for weight tensor in bf16 kernel return profile(I2, GNHWC{}, GKYXC{}, GNHWK{}, BF16{}, F32{}, BF16{}, BF16{}, BF16{}); } + else if(data_type == ConvDataType::F32_F32_F32_TF32) + { +#if defined(__gfx942__) + return profile(I2, GNHWC{}, GKYXC{}, GNHWK{}, F32{}, F32{}, F32{}, TF32{}, TF32{}); +#endif + } } if(num_dim_spatial == 2 && layout == ConvLayout::NHWGC_GKYXC_NHWGK) { @@ -191,6 +208,12 @@ int profile_grouped_conv_bwd_weight(int argc, char* argv[]) { return profile(I2, NHWGC{}, GKYXC{}, NHWGK{}, BF16{}, BF16{}, BF16{}, BF16{}, BF16{}); } + else if(data_type == ConvDataType::F32_F32_F32_TF32) + { +#if defined(__gfx942__) + return profile(I2, NHWGC{}, GKYXC{}, NHWGK{}, F32{}, F32{}, F32{}, TF32{}, TF32{}); +#endif + } } else if(num_dim_spatial == 2 && layout == ConvLayout::NGCHW_GKYXC_NGKHW) { @@ -218,6 +241,12 @@ int profile_grouped_conv_bwd_weight(int argc, char* argv[]) { return profile(I2, NGCHW{}, GKCYX{}, NGKHW{}, F32{}, F32{}, F32{}, F32{}, F32{}); } + else if(data_type == ConvDataType::F32_F32_F32_TF32) + { +#if defined(__gfx942__) + return profile(I2, NGCHW{}, GKCYX{}, NGKHW{}, F32{}, F32{}, F32{}, TF32{}, TF32{}); +#endif + } } if(num_dim_spatial == 3 && layout == ConvLayout::GNHWC_GKYXC_GNHWK) { @@ -239,6 +268,12 @@ int profile_grouped_conv_bwd_weight(int argc, char* argv[]) return profile( I3, GNDHWC{}, GKZYXC{}, GNDHWK{}, int8_t{}, int8_t{}, int8_t{}, int8_t{}, int8_t{}); } + else if(data_type == ConvDataType::F32_F32_F32_TF32) + { +#if defined(__gfx942__) + return profile(I3, GNDHWC{}, GKZYXC{}, GNDHWK{}, F32{}, F32{}, F32{}, TF32{}, TF32{}); +#endif + } } if(num_dim_spatial == 3 && layout == ConvLayout::NHWGC_GKYXC_NHWGK) { @@ -269,6 +304,12 @@ int profile_grouped_conv_bwd_weight(int argc, char* argv[]) return profile( I3, NDHWGC{}, GKZYXC{}, NDHWGK{}, int8_t{}, int8_t{}, int8_t{}, int8_t{}, int8_t{}); } + else if(data_type == ConvDataType::F32_F32_F32_TF32) + { +#if defined(__gfx942__) + return profile(I3, NDHWGC{}, GKZYXC{}, NDHWGK{}, F32{}, F32{}, F32{}, TF32{}, TF32{}); +#endif + } } else if(num_dim_spatial == 3 && layout == ConvLayout::NGCHW_GKYXC_NGKHW) { @@ -297,6 +338,12 @@ int profile_grouped_conv_bwd_weight(int argc, char* argv[]) { return profile(I3, NGCDHW{}, GKCZYX{}, NGKDHW{}, F32{}, F32{}, F32{}, F32{}, F32{}); } + else if(data_type == ConvDataType::F32_F32_F32_TF32) + { +#if defined(__gfx942__) + return profile(I3, NGCDHW{}, GKCZYX{}, NGKDHW{}, F32{}, F32{}, F32{}, TF32{}, TF32{}); +#endif + } } std::cout << "this data_type & layout is not implemented" << std::endl; diff --git a/profiler/src/profile_grouped_conv_fwd.cpp b/profiler/src/profile_grouped_conv_fwd.cpp index a8d343405d..13f5cd1cda 100644 --- a/profiler/src/profile_grouped_conv_fwd.cpp +++ b/profiler/src/profile_grouped_conv_fwd.cpp @@ -226,6 +226,12 @@ int profile_grouped_conv_fwd(int argc, char* argv[]) { return profile(I1, GNWC{}, GKXC{}, GNWK{}, INT8{}, INT8{}, INT8{}, INT8{}, INT8{}); } + else if(data_type == ConvDataType::F32_F32_F32_TF32) + { +#if defined(__gfx942__) + return profile(I1, GNWC{}, GKXC{}, GNWK{}, F32{}, F32{}, F32{}, TF32{}, TF32{}); +#endif + } } else if(num_dim_spatial == 2 && layout == ConvLayout::GNHWC_GKYXC_GNHWK) { @@ -245,6 +251,12 @@ int profile_grouped_conv_fwd(int argc, char* argv[]) { return profile(I2, GNHWC{}, GKYXC{}, GNHWK{}, INT8{}, INT8{}, INT8{}, INT8{}, INT8{}); } + else if(data_type == ConvDataType::F32_F32_F32_TF32) + { +#if defined(__gfx942__) + return profile(I2, GNHWC{}, GKYXC{}, GNHWK{}, F32{}, F32{}, F32{}, TF32{}, TF32{}); +#endif + } } else if(num_dim_spatial == 3 && layout == ConvLayout::GNHWC_GKYXC_GNHWK) { @@ -292,6 +304,12 @@ int profile_grouped_conv_fwd(int argc, char* argv[]) { return profile(I1, NWGC{}, GKXC{}, NWGK{}, INT8{}, INT8{}, INT8{}, INT8{}, INT8{}); } + else if(data_type == ConvDataType::F32_F32_F32_TF32) + { +#if defined(__gfx942__) + return profile(I1, NWGC{}, GKXC{}, NWGK{}, F32{}, F32{}, F32{}, TF32{}, TF32{}); +#endif + } } else if(num_dim_spatial == 2 && layout == ConvLayout::NHWGC_GKYXC_NHWGK) { @@ -311,6 +329,12 @@ int profile_grouped_conv_fwd(int argc, char* argv[]) { return profile(I2, NHWGC{}, GKYXC{}, NHWGK{}, INT8{}, INT8{}, INT8{}, INT8{}, INT8{}); } + else if(data_type == ConvDataType::F32_F32_F32_TF32) + { +#if defined(__gfx942__) + return profile(I2, NHWGC{}, GKYXC{}, NHWGK{}, F32{}, F32{}, F32{}, TF32{}, TF32{}); +#endif + } } else if(num_dim_spatial == 2 && layout == ConvLayout::NGCHW_GKYXC_NGKHW) { @@ -326,6 +350,12 @@ int profile_grouped_conv_fwd(int argc, char* argv[]) { return profile(I2, NGCHW{}, GKYXC{}, NGKHW{}, BF16{}, BF16{}, BF16{}, BF16{}, BF16{}); } + else if(data_type == ConvDataType::F32_F32_F32_TF32) + { +#if defined(__gfx942__) + return profile(I2, NGCHW{}, GKYXC{}, NGKHW{}, F32{}, F32{}, F32{}, TF32{}, TF32{}); +#endif + } } else if(num_dim_spatial == 2 && layout == ConvLayout::NGCHW_GKCYX_NGKHW) { @@ -341,6 +371,12 @@ int profile_grouped_conv_fwd(int argc, char* argv[]) { return profile(I2, NGCHW{}, GKCYX{}, NGKHW{}, BF16{}, BF16{}, BF16{}, BF16{}, BF16{}); } + else if(data_type == ConvDataType::F32_F32_F32_TF32) + { +#if defined(__gfx942__) + return profile(I2, NGCHW{}, GKCYX{}, NGKHW{}, F32{}, F32{}, F32{}, TF32{}, TF32{}); +#endif + } } else if(num_dim_spatial == 3 && layout == ConvLayout::NHWGC_GKYXC_NHWGK) { diff --git a/profiler/src/profile_grouped_conv_fwd_bias_clamp.cpp b/profiler/src/profile_grouped_conv_fwd_bias_clamp.cpp index 34b3df1c65..fb1eedf2a7 100644 --- a/profiler/src/profile_grouped_conv_fwd_bias_clamp.cpp +++ b/profiler/src/profile_grouped_conv_fwd_bias_clamp.cpp @@ -20,14 +20,15 @@ enum struct ConvLayout enum struct ConvDataType { - F32_F32_F32, // 0 - F16_F16_F16, // 1 - BF16_BF16_BF16, // 2 - INT8_INT8_INT8, // 3 - F8_F8_F8, // 4 - BF8_BF8_F8, // 5 - F8_BF8_F8, // 6 - BF8_F8_F8, // 7 + F32_F32_F32, // 0 + F16_F16_F16, // 1 + BF16_BF16_BF16, // 2 + INT8_INT8_INT8, // 3 + F8_F8_F8, // 4 + BF8_BF8_F8, // 5 + F8_BF8_F8, // 6 + BF8_F8_F8, // 7 + F32_F32_F32_TF32, // 8 }; enum struct IndexType @@ -51,7 +52,8 @@ static void print_helper_msg() << " 4: Input fp8, Weight fp8, Output fp8\n" << " 5: Input bf8, Weight bf8, Output fp8\n" << " 6: Input fp8, Weight bf8, Output fp8\n" - << " 7: Input bf8, Weight fp8, Output fp8)\n" + << " 7: Input bf8, Weight fp8, Output fp8\n" + << " 8: Input fp32, Weight fp32, Output fp32, Compute tf32)\n" << "arg3: tensor layout (0: Input[G, N, Hi, Wi, C], Weight[G, K, Y, X, C], Output[G, N, Ho, Wo, K]\n" << " 1: Input[N, Hi, Wi, G, C], Weight[G, K, Y, X, C], Output[N, Ho, Wo, G, K]\n" << " 2: Input[N, G, C, Hi, Wi], Weight[G, K, Y, X, C], Output[N, " @@ -103,6 +105,9 @@ int grouped_conv_fwd_bias_clamp(int argc, char* argv[]) using F32 = float; using BF16 = ck::bhalf_t; using F16 = ck::half_t; +#if defined(__gfx942__) + using TF32 = ck::tf32_t; +#endif using GKZYXC = ck::tensor_layout::convolution::GKZYXC; using NDHWGC = ck::tensor_layout::convolution::NDHWGC; @@ -165,6 +170,12 @@ int grouped_conv_fwd_bias_clamp(int argc, char* argv[]) { return profile(I2, NHWGC{}, GKYXC{}, NHWGK{}, BF16{}, BF16{}, BF16{}, BF16{}, BF16{}); } + else if(data_type == ConvDataType::F32_F32_F32_TF32) + { +#if defined(__gfx942__) + return profile(I2, NHWGC{}, GKYXC{}, NHWGK{}, F32{}, F32{}, F32{}, TF32{}, TF32{}); +#endif + } } else if(num_dim_spatial == 3 && layout == ConvLayout::NHWGC_GKYXC_NHWGK) { @@ -181,6 +192,12 @@ int grouped_conv_fwd_bias_clamp(int argc, char* argv[]) return profile( I3, NDHWGC{}, GKZYXC{}, NDHWGK{}, BF16{}, BF16{}, BF16{}, BF16{}, BF16{}); } + else if(data_type == ConvDataType::F32_F32_F32_TF32) + { +#if defined(__gfx942__) + return profile(I3, NDHWGC{}, GKZYXC{}, NDHWGK{}, F32{}, F32{}, F32{}, TF32{}, TF32{}); +#endif + } } std::cout << "this data_type & layout is not implemented" << std::endl; diff --git a/profiler/src/profile_grouped_conv_fwd_clamp.cpp b/profiler/src/profile_grouped_conv_fwd_clamp.cpp index 600f91744a..1b100ff867 100644 --- a/profiler/src/profile_grouped_conv_fwd_clamp.cpp +++ b/profiler/src/profile_grouped_conv_fwd_clamp.cpp @@ -20,14 +20,15 @@ enum struct ConvLayout enum struct ConvDataType { - F32_F32_F32, // 0 - F16_F16_F16, // 1 - BF16_BF16_BF16, // 2 - INT8_INT8_INT8, // 3 - F8_F8_F8, // 4 - BF8_BF8_F8, // 5 - F8_BF8_F8, // 6 - BF8_F8_F8, // 7 + F32_F32_F32, // 0 + F16_F16_F16, // 1 + BF16_BF16_BF16, // 2 + INT8_INT8_INT8, // 3 + F8_F8_F8, // 4 + BF8_BF8_F8, // 5 + F8_BF8_F8, // 6 + BF8_F8_F8, // 7 + F32_F32_F32_TF32, // 8 }; enum struct IndexType @@ -51,7 +52,8 @@ static void print_helper_msg() << " 4: Input fp8, Weight fp8, Output fp8\n" << " 5: Input bf8, Weight bf8, Output fp8\n" << " 6: Input fp8, Weight bf8, Output fp8\n" - << " 7: Input bf8, Weight fp8, Output fp8)\n" + << " 7: Input bf8, Weight fp8, Output fp8\n" + << " 8: Input fp32, Weight fp32, Output fp32, Compute tf32)\n" << "arg3: tensor layout (0: Input[G, N, Hi, Wi, C], Weight[G, K, Y, X, C], Output[G, N, Ho, Wo, K]\n" << " 1: Input[N, Hi, Wi, G, C], Weight[G, K, Y, X, C], Output[N, Ho, Wo, G, K]\n" << " 2: Input[N, G, C, Hi, Wi], Weight[G, K, Y, X, C], Output[N, " @@ -103,6 +105,9 @@ int grouped_conv_fwd_clamp(int argc, char* argv[]) using F32 = float; using BF16 = ck::bhalf_t; using F16 = ck::half_t; +#if defined(__gfx942__) + using TF32 = ck::tf32_t; +#endif using GKZYXC = ck::tensor_layout::convolution::GKZYXC; using NDHWGC = ck::tensor_layout::convolution::NDHWGC; @@ -168,6 +173,12 @@ int grouped_conv_fwd_clamp(int argc, char* argv[]) { return profile(I2, NHWGC{}, GKYXC{}, NHWGK{}, BF16{}, BF16{}, BF16{}, BF16{}, BF16{}); } + else if(data_type == ConvDataType::F32_F32_F32_TF32) + { +#if defined(__gfx942__) + return profile(I2, NHWGC{}, GKYXC{}, NHWGK{}, F32{}, F32{}, F32{}, TF32{}, TF32{}); +#endif + } } else if(num_dim_spatial == 3 && layout == ConvLayout::NHWGC_GKYXC_NHWGK) { @@ -184,6 +195,12 @@ int grouped_conv_fwd_clamp(int argc, char* argv[]) return profile( I3, NDHWGC{}, GKZYXC{}, NDHWGK{}, BF16{}, BF16{}, BF16{}, BF16{}, BF16{}); } + else if(data_type == ConvDataType::F32_F32_F32_TF32) + { +#if defined(__gfx942__) + return profile(I3, NDHWGC{}, GKZYXC{}, NDHWGK{}, F32{}, F32{}, F32{}, TF32{}, TF32{}); +#endif + } } std::cout << "this data_type & layout is not implemented" << std::endl; From 9d060d3e3c7c943a6609a95e11ff48c35b30edef Mon Sep 17 00:00:00 2001 From: Max Podkorytov <4273004+tenpercent@users.noreply.github.com> Date: Fri, 10 Oct 2025 12:57:50 -0700 Subject: [PATCH 006/262] [CK-Tile] functional support for transposed inputs in compute-bound double-lds-buffer pipeline with async loads from global memory to LDS (#2984) * reuse local prefetch logic from compute v4 pipeline add single-tile test explicit lambda capture reuse lds block descriptors from base policy for the transposed case match the test case kernel configuration with compute v4 * add comments --- .../gemm_pipeline_ag_bg_cr_comp_async.hpp | 112 ++++++++++-------- ...ine_ag_bg_cr_comp_async_default_policy.hpp | 82 +++++++++---- .../gemm/test_gemm_pipeline_kernel_types.hpp | 5 +- .../gemm/test_gemm_pipeline_ut_cases.inc | 5 + 4 files changed, 128 insertions(+), 76 deletions(-) diff --git a/include/ck_tile/ops/gemm/pipeline/gemm_pipeline_ag_bg_cr_comp_async.hpp b/include/ck_tile/ops/gemm/pipeline/gemm_pipeline_ag_bg_cr_comp_async.hpp index 2c8d008127..fa7f9fc788 100644 --- a/include/ck_tile/ops/gemm/pipeline/gemm_pipeline_ag_bg_cr_comp_async.hpp +++ b/include/ck_tile/ops/gemm/pipeline/gemm_pipeline_ag_bg_cr_comp_async.hpp @@ -152,6 +152,9 @@ struct GemmPipelineAgBgCrCompAsync : public BaseGemmPipelineAgBgCrCompAsync{}; + static constexpr auto is_b_load_tr_v = bool_constant{}; + CK_TILE_HOST_DEVICE static constexpr index_t GetSmemSize() { return Policy::template GetSmemSize(); @@ -249,11 +252,6 @@ struct GemmPipelineAgBgCrCompAsync : public BaseGemmPipelineAgBgCrCompAsync; constexpr bool is_b_row_major = std::is_same_v; - // TODO currently only support A matrix row major, B matrix col major; if A matrix is - // col major or B is row major, need to combine with transpose load api - static_assert(!(is_a_col_major || is_b_row_major), - "only support A matrix is row major, B matrix is col major!"); - static_assert(is_a_col_major ? (KPerBlock == ADramBlockWindowTmp{}.get_window_lengths()[I0{}] && MPerBlock == ADramBlockWindowTmp{}.get_window_lengths()[I1{}]) @@ -293,18 +291,29 @@ struct GemmPipelineAgBgCrCompAsync : public BaseGemmPipelineAgBgCrCompAsync{}, number{}); + else + return make_tuple(number{}, number{}); + }(); + + constexpr auto b_lds_shape = []() { + if constexpr(is_b_load_tr_v) + return make_tuple(number{}, number{}); + else + return make_tuple(number{}, number{}); + }(); + // LDS tile windows for storing, one per LDS buffer - auto a_copy_lds_window0 = make_tile_window( - a_lds_block0, make_tuple(number{}, number{}), {0, 0}); + auto a_copy_lds_window0 = make_tile_window(a_lds_block0, a_lds_shape, {0, 0}); - auto a_copy_lds_window1 = make_tile_window( - a_lds_block1, make_tuple(number{}, number{}), {0, 0}); + auto a_copy_lds_window1 = make_tile_window(a_lds_block1, a_lds_shape, {0, 0}); - auto b_copy_lds_window0 = make_tile_window( - b_lds_block0, make_tuple(number{}, number{}), {0, 0}); + auto b_copy_lds_window0 = make_tile_window(b_lds_block0, b_lds_shape, {0, 0}); - auto b_copy_lds_window1 = make_tile_window( - b_lds_block1, make_tuple(number{}, number{}), {0, 0}); + auto b_copy_lds_window1 = make_tile_window(b_lds_block1, b_lds_shape, {0, 0}); // initialize DRAM window steps, used to advance the DRAM windows using ADramTileWindowStep = typename ADramBlockWindowTmp::BottomTensorIndex; @@ -336,44 +345,49 @@ struct GemmPipelineAgBgCrCompAsync : public BaseGemmPipelineAgBgCrCompAsync{}], b_dram_tile_window_step); - constexpr auto ALdsTileDistr = decltype(make_static_tile_distribution( - BlockGemm::MakeABlockDistributionEncode())){}; - constexpr auto BLdsTileDistr = decltype(make_static_tile_distribution( - BlockGemm::MakeBBlockDistributionEncode())){}; + // tile distribution for the register tiles + constexpr auto ALdsTileDistr = + make_static_tile_distribution(BlockGemm::MakeABlockDistributionEncode()); + constexpr auto BLdsTileDistr = + make_static_tile_distribution(BlockGemm::MakeBBlockDistributionEncode()); using ALdsTile = decltype(make_static_distributed_tensor(ALdsTileDistr)); using BLdsTile = decltype(make_static_distributed_tensor(BLdsTileDistr)); // register tiles; double buffering -> a register tile corresponds to a LDS tile window - ALdsTile a_block_tile0; - ALdsTile a_block_tile1; + ALdsTile a_block_tile0, a_block_tile1; + BLdsTile b_block_tile0, b_block_tile1; - BLdsTile b_block_tile0; - BLdsTile b_block_tile1; + constexpr auto a_lds_input_tile_distr = [ALdsTileDistr]() { + if constexpr(is_a_load_tr_v) + return make_static_tile_distribution( + typename InputTileDistributionTraits< + typename decltype(ALdsTileDistr)::DstrEncode, + typename Problem::ADataType>::TransposedDstrEncode{}); + else + return ALdsTileDistr; + }(); + constexpr auto b_lds_input_tile_distr = [BLdsTileDistr]() { + if constexpr(is_b_load_tr_v) + return make_static_tile_distribution( + typename InputTileDistributionTraits< + typename decltype(BLdsTileDistr)::DstrEncode, + typename Problem::BDataType>::TransposedDstrEncode{}); + else + return BLdsTileDistr; + }(); // LDS tile windows for reading; // they share the data pointer with the LDS windows for storing // but also associate with a distribution to produce a register tile when reading auto a_lds_ld_window0 = - make_tile_window(a_lds_block0, - make_tuple(number{}, number{}), - {0, 0}, - ALdsTileDistr); + make_tile_window(a_lds_block0, a_lds_shape, {0, 0}, a_lds_input_tile_distr); auto a_lds_ld_window1 = - make_tile_window(a_lds_block1, - make_tuple(number{}, number{}), - {0, 0}, - ALdsTileDistr); + make_tile_window(a_lds_block1, a_lds_shape, {0, 0}, a_lds_input_tile_distr); auto b_lds_ld_window0 = - make_tile_window(b_lds_block0, - make_tuple(number{}, number{}), - {0, 0}, - BLdsTileDistr); + make_tile_window(b_lds_block0, b_lds_shape, {0, 0}, b_lds_input_tile_distr); auto b_lds_ld_window1 = - make_tile_window(b_lds_block1, - make_tuple(number{}, number{}), - {0, 0}, - BLdsTileDistr); + make_tile_window(b_lds_block1, b_lds_shape, {0, 0}, b_lds_input_tile_distr); static_assert(!(is_tile_window_linear_v) && !(is_tile_window_linear_v) && @@ -384,8 +398,8 @@ struct GemmPipelineAgBgCrCompAsync : public BaseGemmPipelineAgBgCrCompAsync(); + if constexpr(is_a_load_tr) + { + // TODO: better LDS descriptor for performance + // This branch is reusing the logic from + // UniversalGemmBasePolicy::MakeALdsBlockDescriptor + constexpr auto a_lds_block_desc_0 = make_naive_tensor_descriptor( // + make_tuple(number{}, number{}), + make_tuple(number{}, number<1>{}), + number{}, + number<1>{}); + return a_lds_block_desc_0; + } + else + { + constexpr index_t KPack = GetSmemPackA(); - constexpr auto a_lds_block_desc_0 = make_naive_tensor_descriptor( - make_tuple(number{}, number{}, number{}), - make_tuple(number{}, number{}, number<1>{}), - number{}, - number<1>{}); + constexpr auto a_lds_block_desc_0 = make_naive_tensor_descriptor( + make_tuple(number{}, number{}, number{}), + make_tuple(number{}, number{}, number<1>{}), + number{}, + number<1>{}); - return transform_tensor_descriptor( - a_lds_block_desc_0, - make_tuple( - make_pass_through_transform(number{}), - make_merge_transform(make_tuple(number{}, number{}))), - make_tuple(sequence<1>{}, sequence<0, 2>{}), - make_tuple(sequence<0>{}, sequence<1>{})); + return transform_tensor_descriptor( + a_lds_block_desc_0, + make_tuple( + make_pass_through_transform(number{}), + make_merge_transform(make_tuple(number{}, number{}))), + make_tuple(sequence<1>{}, sequence<0, 2>{}), + make_tuple(sequence<0>{}, sequence<1>{})); + } } template @@ -45,21 +60,36 @@ struct GemmPipelineAgBgCrCompAsyncDefaultPolicy { constexpr index_t NPerBlock = Problem::BlockGemmShape::kN; constexpr index_t KPerBlock = Problem::BlockGemmShape::kK; - constexpr index_t KPack = GetSmemPackB(); + if constexpr(is_b_load_tr) + { + // TODO: better LDS descriptor for performance + // This branch is reusing the logic from + // UniversalGemmBasePolicy::MakeBLdsBlockDescriptor + constexpr auto b_lds_block_desc_0 = + make_naive_tensor_descriptor(make_tuple(number{}, number{}), + make_tuple(number{}, number<1>{}), + number{}, + number<1>{}); + return b_lds_block_desc_0; + } + else + { + constexpr index_t KPack = GetSmemPackB(); - constexpr auto b_lds_block_desc_0 = make_naive_tensor_descriptor( - make_tuple(number{}, number{}, number{}), - make_tuple(number{}, number{}, number<1>{}), - number{}, - number<1>{}); + constexpr auto b_lds_block_desc_0 = make_naive_tensor_descriptor( + make_tuple(number{}, number{}, number{}), + make_tuple(number{}, number{}, number<1>{}), + number{}, + number<1>{}); - return transform_tensor_descriptor( - b_lds_block_desc_0, - make_tuple( - make_pass_through_transform(number{}), - make_merge_transform(make_tuple(number{}, number{}))), - make_tuple(sequence<1>{}, sequence<0, 2>{}), - make_tuple(sequence<0>{}, sequence<1>{})); + return transform_tensor_descriptor( + b_lds_block_desc_0, + make_tuple( + make_pass_through_transform(number{}), + make_merge_transform(make_tuple(number{}, number{}))), + make_tuple(sequence<1>{}, sequence<0, 2>{}), + make_tuple(sequence<0>{}, sequence<1>{})); + } } template diff --git a/test/ck_tile/gemm/test_gemm_pipeline_kernel_types.hpp b/test/ck_tile/gemm/test_gemm_pipeline_kernel_types.hpp index 243a823653..bba106174c 100644 --- a/test/ck_tile/gemm/test_gemm_pipeline_kernel_types.hpp +++ b/test/ck_tile/gemm/test_gemm_pipeline_kernel_types.hpp @@ -131,7 +131,10 @@ using KernelTypesCompV4 = ::testing::Types< >; using KernelTypesCompAsync = ::testing::Types< - std::tuple< Row, Col, Row, F16, F16, F32, F16, I256, I256, I32, I32, I32, I16, Intrawave, CompAsync> + std::tuple< Row, Row, Row, F16, F16, F32, F16, I256, I256, I32, I32, I32, I16, Intrawave, CompAsync>, + std::tuple< Row, Col, Row, F16, F16, F32, F16, I256, I256, I32, I32, I32, I16, Intrawave, CompAsync>, + std::tuple< Col, Row, Row, F16, F16, F32, F16, I256, I256, I32, I32, I32, I16, Intrawave, CompAsync>, + std::tuple< Col, Col, Row, F16, F16, F32, F16, I256, I256, I32, I32, I32, I16, Intrawave, CompAsync> >; using KernelTypesCompV4Wmma = ::testing::Types< diff --git a/test/ck_tile/gemm/test_gemm_pipeline_ut_cases.inc b/test/ck_tile/gemm/test_gemm_pipeline_ut_cases.inc index 66ef05b0ba..ae91631a00 100644 --- a/test/ck_tile/gemm/test_gemm_pipeline_ut_cases.inc +++ b/test/ck_tile/gemm/test_gemm_pipeline_ut_cases.inc @@ -33,6 +33,11 @@ TYPED_TEST(TEST_SUITE_NAME, SmallM) } } +TYPED_TEST(TEST_SUITE_NAME, SingleTile) +{ + this->Run(TestFixture::M_Tile, TestFixture::N_Tile, TestFixture::K_Tile); +} + TYPED_TEST(TEST_SUITE_NAME, MidLargeM) { std::vector Ms{127, 255, 312, 799, 1573}; From 3c39d279ab4569d1b33399e7746465744ed662c0 Mon Sep 17 00:00:00 2001 From: Khushbu Agarwal Date: Fri, 10 Oct 2025 15:36:24 -0700 Subject: [PATCH 007/262] supporting prefill shapes for preshuffle block scale gemm (#2975) * debugging * debugging for prefill shapes * comment unused code * fix for prefill shapes * clearing up the code * add int4 to universal gemm example * clang formatted * adding test for prefill shapes in block scale gemm * lil improv on the block pipeline * Address Review Comment --------- Co-authored-by: ThomasNing --- .../03_gemm/gemm_weight_preshuffle.cpp | 7 ++ .../gemm_weight_preshuffle_invoker.hpp | 5 +- example/ck_tile/03_gemm/run_gemm_example.inc | 16 +-- example/ck_tile/38_block_scale_gemm/README.md | 14 ++- .../38_block_scale_gemm/gemm_quant_basic.cpp | 3 +- .../38_block_scale_gemm/gemm_utils.hpp | 22 +++- ...ock_universal_gemm_ar_flatbr_bquant_cr.hpp | 104 ++++++++++-------- .../gemm_quant/kernel/gemm_quant_kernel.hpp | 1 - .../test_gemm_quant_fixtures.hpp | 41 +++---- .../test_gemm_quant_typed.cpp | 13 ++- 10 files changed, 137 insertions(+), 89 deletions(-) mode change 100755 => 100644 include/ck_tile/ops/gemm_quant/block/block_universal_gemm_ar_flatbr_bquant_cr.hpp diff --git a/example/ck_tile/03_gemm/gemm_weight_preshuffle.cpp b/example/ck_tile/03_gemm/gemm_weight_preshuffle.cpp index 0f323cb0e3..89f177b781 100644 --- a/example/ck_tile/03_gemm/gemm_weight_preshuffle.cpp +++ b/example/ck_tile/03_gemm/gemm_weight_preshuffle.cpp @@ -75,6 +75,13 @@ int run_gemm_example(ck_tile::ArgParser& arg_parser) ck_tile::bf8_t, ck_tile::half_t>(a_layout, b_layout, arg_parser); } + else if(data_type == "int4") + { + return run_gemm_example_prec_type, + ck_tile::fp8_t, + ck_tile::pk_int4_t, + ck_tile::half_t>(a_layout, b_layout, arg_parser); + } else { throw std::runtime_error("Unsupported data type for this operation !!!"); diff --git a/example/ck_tile/03_gemm/gemm_weight_preshuffle_invoker.hpp b/example/ck_tile/03_gemm/gemm_weight_preshuffle_invoker.hpp index d737a0f864..023b0336fe 100644 --- a/example/ck_tile/03_gemm/gemm_weight_preshuffle_invoker.hpp +++ b/example/ck_tile/03_gemm/gemm_weight_preshuffle_invoker.hpp @@ -194,10 +194,7 @@ struct WeightPreshuffleInvoker } else { - Run(has_hot_loop_, - tail_number_, - ck_tile::integral_constant{}); + throw std::runtime_error("split-k is not supported yet!"); } }; diff --git a/example/ck_tile/03_gemm/run_gemm_example.inc b/example/ck_tile/03_gemm/run_gemm_example.inc index e6875f97d5..42a2d70692 100644 --- a/example/ck_tile/03_gemm/run_gemm_example.inc +++ b/example/ck_tile/03_gemm/run_gemm_example.inc @@ -300,16 +300,8 @@ int run_gemm_example_with_layouts(ck_tile::ArgParser& arg_parser, if(init_method == 0) { - if constexpr(preshuffle) - { - ck_tile::FillUniformDistribution{-.5f, .5f}(a_m_k); - ck_tile::FillUniformDistribution{-.5f, .5f}(b_k_n); - } - else - { - ck_tile::FillUniformDistribution{-5.f, 5.f}(a_m_k); - ck_tile::FillUniformDistribution{-5.f, 5.f}(b_k_n); - } + ck_tile::FillUniformDistribution{-5.f, 5.f}(a_m_k); + ck_tile::FillUniformDistribution{-5.f, 5.f}(b_k_n); } else if(init_method == 1) { @@ -353,6 +345,10 @@ int run_gemm_example_with_layouts(ck_tile::ArgParser& arg_parser, } }(); // shuffled buffer B for device implementation + if constexpr(std::is_same_v) + { + ck_tile::permute_vectors_i4x4_b(b_shuffle_host); + } b_k_n_dev_buf.ToDevice(b_shuffle_host.data()); } else diff --git a/example/ck_tile/38_block_scale_gemm/README.md b/example/ck_tile/38_block_scale_gemm/README.md index 7f8aba7b3d..b7b14f9d13 100644 --- a/example/ck_tile/38_block_scale_gemm/README.md +++ b/example/ck_tile/38_block_scale_gemm/README.md @@ -4,8 +4,18 @@ This folder contains examples of quant GEMMs using the ck_tile tile-programming - AQuant kernel with blocks of A matrix sharing scales: custom GEMM pipeline - BQuant kernel with blocks of B matrix sharing scales: custom GEMM pipeline -- Row and Column-wise scaled: scaling implemented in Epilogue -- Tensor-wise scaled: scaling implemented in Epilogue +- Row and Column-wise scaled: All of the rowwise elements in A Matrix and columwise elements in B Matrix will share the same quantization element and the elementwisde operation will complete in epilogue. +- Tensor-wise scaled: Share the same scalar scale across the whole tensor of A or B + +--- + +## Features + +- **Preshuffled GEMM**: Shuffle the GEMM of B (weight) matrix in the warp layout and bypass the shared memory to do the GEMM calculation. Best performance solution for GEMM. +- **TransposeC**: Transpose the C Matrix Output layout to have the best coalesced scale reading +- **Preshuffled Quant**: Preshuffle the input matrix to load multiple Quant warp blocks along the selected dimension. +- **Precision**: Supports fp16, bf16, fp8, bf8, int4 (for B Matrix). +- **Validation**: CPU/GPU validation and error tolerance options. ## build ``` diff --git a/example/ck_tile/38_block_scale_gemm/gemm_quant_basic.cpp b/example/ck_tile/38_block_scale_gemm/gemm_quant_basic.cpp index 00d1af5aaa..c9cc56d033 100644 --- a/example/ck_tile/38_block_scale_gemm/gemm_quant_basic.cpp +++ b/example/ck_tile/38_block_scale_gemm/gemm_quant_basic.cpp @@ -47,6 +47,7 @@ float gemm_calc_quant(const ck_tile::QuantGemmHostArgs& args, const ck_tile::str QuantMode, ALayout, // for AQLayout BLayout, // for BQLayout + false, GemmConfig::DoubleSmemBuffer>; using GemmPipelineProblem = ck_tile::GemmPipelineProblemBase(argc, argv); } +int main(int argc, char* argv[]) { return !run_gemm_example(argc, argv); } diff --git a/example/ck_tile/38_block_scale_gemm/gemm_utils.hpp b/example/ck_tile/38_block_scale_gemm/gemm_utils.hpp index cfe7b72af9..0206aa88a8 100644 --- a/example/ck_tile/38_block_scale_gemm/gemm_utils.hpp +++ b/example/ck_tile/38_block_scale_gemm/gemm_utils.hpp @@ -166,6 +166,26 @@ struct GemmConfigPreshuffleB_Bquant_decode : public GemmConfigBase static constexpr bool DoubleSmemBuffer = true; }; +template +struct GemmConfigPreshuffleB_Bquant_prefill : public GemmConfigBase +{ + static constexpr ck_tile::index_t M_Tile = 128; + static constexpr ck_tile::index_t N_Tile = 128; + static constexpr ck_tile::index_t K_Tile = 128 / sizeof(PrecType); + + static constexpr ck_tile::index_t M_Warp = 1; + static constexpr ck_tile::index_t N_Warp = 4; + static constexpr ck_tile::index_t K_Warp = 1; + + static constexpr ck_tile::index_t M_Warp_Tile = 16; + static constexpr ck_tile::index_t N_Warp_Tile = 16; + static constexpr ck_tile::index_t K_Warp_Tile = + get_k_from_preshuffled_warp_tile(); + + static constexpr bool PreshuffleB = true; + static constexpr bool DoubleSmemBuffer = true; +}; + template {}; - static_for<0, QScalesPerBlockRow, 1>{}([&](auto kQScale) { - CWarpTensor c_warp_tensor; - static_for<0, KIterPerQScale, 1>{}([&](auto kIterInQScale) { - static_for<0, MIterPerWarp, 1>{}([&](auto mIter) { - static_for<0, NIterPerWarp, 1>{}([&](auto nIter) { - constexpr auto kIter = kQScale * KIterPerQScale + kIterInQScale; + statically_indexed_array, MIterPerWarp> + c_acc; - constexpr auto AwarpIter = (kIter * MIterPerWarp + mIter) % m_preload; - - // warp GEMM - if constexpr(kIterInQScale == 0) - c_warp_tensor = WG{}(a_warp_tensor(number{}), - b_warp_tensor(nIter)(number{})); - else - WG{}(c_warp_tensor, - a_warp_tensor(number{}), - b_warp_tensor(nIter)(number{})); - - __builtin_amdgcn_sched_barrier(0x7F6); - // preload next A from lds - if constexpr((kIter * MIterPerWarp + mIter) < - (KIterPerWarp * MIterPerWarp - m_preload)) - { - constexpr auto AmIter = (mIter + m_preload) % MIterPerWarp; - constexpr auto AkIter = (kIter + (mIter + m_preload) / MIterPerWarp); - a_warp_tensor(number{}) = - load_tile(a_warp_windows(number{})(number{})); - } - // barrier - if constexpr((kIter == KIterPerWarp - 1) && (mIter == MIter_2nd_last)) - { - block_sync_lds(); - } - }); + auto zero_accumulators = [&] { + static_for<0, MIterPerWarp, 1>{}([&](auto mIter) { + static_for<0, NIterPerWarp, 1>{}([&](auto nIter) { + static_for<0, (WG::kM * WG::kN) / warp_size, 1>{}([&](auto i) { + c_acc(mIter)(nIter).get_thread_buffer()[i] = 0.0f; + }); // make sure WG::CWarpTensor exposes a clear/zero }); }); + }; + static_for<0, QScalesPerBlockRow, 1>{}([&](auto kQScale) { + zero_accumulators(); + static_for<0, KIterPerQScale, 1>{}([&](auto kIterInQScale) { + constexpr auto kIter = kQScale * KIterPerQScale + kIterInQScale; + static_for<0, MIterPerWarp, 1>{}([&](auto mIter) { + constexpr auto AwarpIter = (kIter * MIterPerWarp + mIter) % m_preload; + static_for<0, NIterPerWarp, 1>{}([&](auto nIter) { + // warp GEMM + WG{}(c_acc(mIter)(nIter), + a_warp_tensor(number{}), + b_warp_tensor(nIter)(number{})); + }); + __builtin_amdgcn_sched_barrier(0x7F6); + // preload next A from lds + if constexpr((kIter * MIterPerWarp + mIter) < + (KIterPerWarp * MIterPerWarp - m_preload)) + { + constexpr auto AmIter = (mIter + m_preload) % MIterPerWarp; + constexpr auto AkIter = (kIter + (mIter + m_preload) / MIterPerWarp); + a_warp_tensor(number{}) = + load_tile(a_warp_windows(number{})(number{})); + } + // barrier + // Could be deleted + if constexpr((mIter == MIter_2nd_last)) + { + block_sync_lds(); + } + }); + }); + static_for<0, MIterPerWarp, 1>{}([&](auto mIter) { + static_for<0, NIterPerWarp, 1>{}([&](auto nIter) { + constexpr auto tbuf_offset = + number{}, + c_warp_y_index_zeros)) / + CBlockTensor::PackedSize>{}; - constexpr auto tbuf_offset = - number{}, number<0>{}>{}, c_warp_y_index_zeros)) / - CBlockTensor::PackedSize>{}; + constexpr index_t reg_offset = nIter * KPerBlockBQ + kQScale; - constexpr index_t reg_offset = kQScale; - // nIter * KPerBlockBQ + kQScale; //((kIter * WG::kK) / kQuantGroupSize); + auto& scale_reg = bq_block_tensor.get_thread_buffer()[reg_offset]; + float scale_reg_f = cvt_scale_to_fp32(scale_reg); - auto& scale_reg = bq_block_tensor.get_thread_buffer()[reg_offset]; - float scale_reg_f = cvt_scale_to_fp32(scale_reg); - - static_for<0, WG::kM * WG::kN / warp_size, 1>{}([&](auto c_row) { - c_block_tensor.get_thread_buffer()[tbuf_offset + c_row] += - (c_warp_tensor.get_thread_buffer()[c_row] * scale_reg_f); + static_for<0, WG::kM * WG::kN / warp_size, 1>{}([&](auto c_row) { + auto& c_ref = c_block_tensor.get_thread_buffer()[tbuf_offset + c_row]; + const auto acc_val = c_acc(mIter)(nIter).get_thread_buffer()[c_row]; + c_ref = c_ref + acc_val * scale_reg_f; + }); + }); }); }); } diff --git a/include/ck_tile/ops/gemm_quant/kernel/gemm_quant_kernel.hpp b/include/ck_tile/ops/gemm_quant/kernel/gemm_quant_kernel.hpp index bba2bc8400..bc2c9c603a 100644 --- a/include/ck_tile/ops/gemm_quant/kernel/gemm_quant_kernel.hpp +++ b/include/ck_tile/ops/gemm_quant/kernel/gemm_quant_kernel.hpp @@ -1111,7 +1111,6 @@ struct QuantGemmKernel // allocate LDS __shared__ char smem_ptr_0[GetSmemSize()]; - assert(kargs.k_batch == 1); if constexpr(GemmPipeline::DoubleSmemBuffer == true) { diff --git a/test/ck_tile/gemm_block_scale/test_gemm_quant_fixtures.hpp b/test/ck_tile/gemm_block_scale/test_gemm_quant_fixtures.hpp index 21eabd6041..21f586499e 100644 --- a/test/ck_tile/gemm_block_scale/test_gemm_quant_fixtures.hpp +++ b/test/ck_tile/gemm_block_scale/test_gemm_quant_fixtures.hpp @@ -57,26 +57,10 @@ struct GemmConfigPreshuffleQuantTransposeC : public GemmConfigBase static constexpr bool TransposeC = true; }; -struct GemmConfigPreshuffleB +struct GemmConfigPreshuffleBDecode : public GemmConfigBase { - static constexpr bool kPadM = false; - static constexpr bool kPadN = false; - static constexpr bool kPadK = false; - - static constexpr bool PermuteA = false; - static constexpr bool PermuteB = false; - - static constexpr bool TransposeC = false; - static constexpr bool UseStructuredSparsity = false; - - static constexpr int kBlockPerCu = 1; - static constexpr ck_tile::index_t TileParitionerGroupNum = 8; - static constexpr ck_tile::index_t TileParitionerM01 = 4; - static constexpr auto Scheduler = ck_tile::GemmPipelineScheduler::Intrawave; - static constexpr ck_tile::index_t NumWaveGroups = 1; - static constexpr bool PreshuffleQuant = false; - static constexpr bool PreshuffleB = true; - static constexpr bool DoubleSmemBuffer = true; + static constexpr bool PreshuffleB = true; + static constexpr bool DoubleSmemBuffer = true; // Default GEMM tile sizes for tests static constexpr ck_tile::index_t M_Tile = 16; @@ -92,6 +76,25 @@ struct GemmConfigPreshuffleB static constexpr ck_tile::index_t K_Warp_Tile = 64; }; +struct GemmConfigPreshuffleBPrefill : public GemmConfigBase +{ + static constexpr bool PreshuffleB = true; + static constexpr bool DoubleSmemBuffer = true; + + // Default GEMM tile sizes for tests + static constexpr ck_tile::index_t M_Tile = 128; + static constexpr ck_tile::index_t N_Tile = 128; + static constexpr ck_tile::index_t K_Tile = 128; + + static constexpr ck_tile::index_t M_Warp = 1; + static constexpr ck_tile::index_t N_Warp = 4; + static constexpr ck_tile::index_t K_Warp = 1; + + static constexpr ck_tile::index_t M_Warp_Tile = 16; + static constexpr ck_tile::index_t N_Warp_Tile = 16; + static constexpr ck_tile::index_t K_Warp_Tile = 64; +}; + template class TestCkTileGemmAQuant : public TestCkTileGemmQuantBase> { diff --git a/test/ck_tile/gemm_block_scale/test_gemm_quant_typed.cpp b/test/ck_tile/gemm_block_scale/test_gemm_quant_typed.cpp index ea7a88febb..b4c11d5c5a 100644 --- a/test/ck_tile/gemm_block_scale/test_gemm_quant_typed.cpp +++ b/test/ck_tile/gemm_block_scale/test_gemm_quant_typed.cpp @@ -62,10 +62,15 @@ using BQuantTypes = ::testing::Types< // clang-format off using BPreshuffleBQuantTypes = ::testing::Types< - std::tuple, - std::tuple, - std::tuple, - std::tuple + std::tuple, + std::tuple, + std::tuple, + std::tuple, + + std::tuple, + std::tuple, + std::tuple, + std::tuple >; // clang-format off From 0843815db7763cf5650f7803185a3ab9d24194d7 Mon Sep 17 00:00:00 2001 From: John Shumway Date: Fri, 10 Oct 2025 19:13:34 -0700 Subject: [PATCH 008/262] Fix GCC 7 CTAD compilation error in test_fmha_bwd.cpp (#3001) Fixes compilation error on SLES15 with GCC 7 for gfx942 builds: error: 'vector' may not intend to support class template argument deduction [-Werror,-Wctad-maybe-unsupported] Changes: - Explicitly specify template argument for `std::vector` instead of relying on C++17 CTAD - Maintains compatibility with both older (GCC 7) and newer compilers --- test/ck_tile/fmha/test_fmha_bwd.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/ck_tile/fmha/test_fmha_bwd.cpp b/test/ck_tile/fmha/test_fmha_bwd.cpp index 190cdd6452..1279b98383 100644 --- a/test/ck_tile/fmha/test_fmha_bwd.cpp +++ b/test/ck_tile/fmha/test_fmha_bwd.cpp @@ -29,7 +29,7 @@ struct TestConfigs std::array{std::tuple{32, -1}, std::tuple{64, -1}, std::tuple{128, -1}}; }; static auto HDimValues = ValuesIn(TestConfigs::HDimValues); -const auto ModeValues = ValuesIn(std::vector{mode_enum::batch, mode_enum::group}); +const auto ModeValues = ValuesIn(std::vector{mode_enum::batch, mode_enum::group}); constexpr auto init_method = "uf"; // Random seed used for initializing input tensors. 0 for non-deterministic seed From f5708882a3c0f391b7d02f5af926964170bd8f4e Mon Sep 17 00:00:00 2001 From: Christopher Millette <63608002+cgmillette@users.noreply.github.com> Date: Sat, 11 Oct 2025 07:53:40 -0500 Subject: [PATCH 009/262] Streamk functional tests (#2974) * Add initial fp16_mem_128x128x32_2x2x1_32x32x16_NonPersistent test suite * Account for stride when computing K offsets for A and B tensor This change ensures that the correct stride is used when computing the K offsets into the A and B tensors in the Stream-K Kernel's operator() function. This ensures that the kernel executes correct regardless of whether A and B are row or column major. * Move helper code to test_gemm_streamk_util.hpp * Separate tests into smoke/regression/extended. Add bf16 datatype * Run clang-format * Refactor combinatorial macro expansion and naming * Adjust the initialization values to account for better tolerance on bf16 * Correct BF16 datatypes in comments * Move the extended tests under the REGRESSION_TESTS label * Apply suggestions from code review Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --------- Co-authored-by: Emily Martins Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../ops/gemm/kernel/streamk_gemm_kernel.hpp | 46 ++- test/CMakeLists.txt | 1 + test/ck_tile/gemm_streamk/CMakeLists.txt | 116 ++++++- ...28x128x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...56x256x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...28x128x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...56x256x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...28x128x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...56x256x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...28x128x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...56x256x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...28x128x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...56x256x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...28x128x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...56x256x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...28x128x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...56x256x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...28x128x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...56x256x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...28x128x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...56x256x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...28x128x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...56x256x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...28x128x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...56x256x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...28x128x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...56x256x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...28x128x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...56x256x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...28x128x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...56x256x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...28x128x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...56x256x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...28x128x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...56x256x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...28x128x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...56x256x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...28x128x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...56x256x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...28x128x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...56x256x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...28x128x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...56x256x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...28x128x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...56x256x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...28x128x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...56x256x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...28x128x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...56x256x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...28x128x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...56x256x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...28x128x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...56x256x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...28x128x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...56x256x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...28x128x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...56x256x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...28x128x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...56x256x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...28x128x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...56x256x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...28x128x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...56x256x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...28x128x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...56x256x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...28x128x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...56x256x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...28x128x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...28x128x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...28x128x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...28x128x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...28x128x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...28x128x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...28x128x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...28x128x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...28x128x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...28x128x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...28x128x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...28x128x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...28x128x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...28x128x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...28x128x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...28x128x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...56x256x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...56x256x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...56x256x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...56x256x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...56x256x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...56x256x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...56x256x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...56x256x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...56x256x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...56x256x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...56x256x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...56x256x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...56x256x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...56x256x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...56x256x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + ...56x256x32_2x2x1_32x32x16_NonPersistent.cpp | 11 + .../gemm_streamk/test_gemm_streamk.hpp | 269 +++++++++++++++ .../gemm_streamk/test_gemm_streamk_cases.inc | 220 +++++++----- ... => test_gemm_streamk_common_includes.hpp} | 10 +- .../gemm_streamk/test_gemm_streamk_types.hpp | 112 ++++++- .../test_gemm_streamk_types_bf16.hpp | 76 +++++ .../test_gemm_streamk_types_fp16.hpp | 77 +++++ .../gemm_streamk/test_gemm_streamk_util.hpp | 313 ++++-------------- 106 files changed, 1952 insertions(+), 344 deletions(-) create mode 100644 test/ck_tile/gemm_streamk/extended_tests/compv3/bf16_ccc_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/compv3/bf16_ccc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/compv3/bf16_ccr_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/compv3/bf16_ccr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/compv3/bf16_crc_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/compv3/bf16_crc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/compv3/bf16_crr_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/compv3/bf16_crr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/compv3/bf16_rcc_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/compv3/bf16_rcc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/compv3/bf16_rcr_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/compv3/bf16_rcr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/compv3/bf16_rrc_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/compv3/bf16_rrc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/compv3/bf16_rrr_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/compv3/bf16_rrr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/compv3/f16_ccc_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/compv3/f16_ccc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/compv3/f16_ccr_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/compv3/f16_ccr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/compv3/f16_crc_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/compv3/f16_crc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/compv3/f16_crr_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/compv3/f16_crr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/compv3/f16_rcc_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/compv3/f16_rcc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/compv3/f16_rcr_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/compv3/f16_rcr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/compv3/f16_rrc_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/compv3/f16_rrc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/compv3/f16_rrr_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/compv3/f16_rrr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/compv4/bf16_ccc_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/compv4/bf16_ccc_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/compv4/bf16_ccr_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/compv4/bf16_ccr_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/compv4/bf16_crc_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/compv4/bf16_crc_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/compv4/bf16_crr_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/compv4/bf16_crr_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/compv4/bf16_rcc_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/compv4/bf16_rcc_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/compv4/bf16_rcr_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/compv4/bf16_rcr_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/compv4/bf16_rrc_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/compv4/bf16_rrc_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/compv4/bf16_rrr_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/compv4/bf16_rrr_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/compv4/f16_ccc_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/compv4/f16_ccc_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/compv4/f16_ccr_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/compv4/f16_ccr_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/compv4/f16_crc_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/compv4/f16_crc_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/compv4/f16_crr_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/compv4/f16_crr_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/compv4/f16_rcc_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/compv4/f16_rcc_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/compv4/f16_rcr_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/compv4/f16_rcr_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/compv4/f16_rrc_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/compv4/f16_rrc_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/compv4/f16_rrr_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/compv4/f16_rrr_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/mem/bf16_ccc_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/mem/bf16_ccr_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/mem/bf16_crc_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/mem/bf16_crr_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/mem/bf16_rcc_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/mem/bf16_rcr_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/mem/bf16_rrc_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/mem/bf16_rrr_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/mem/f16_ccc_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/mem/f16_ccr_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/mem/f16_crc_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/mem/f16_crr_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/mem/f16_rcc_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/mem/f16_rcr_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/mem/f16_rrc_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/extended_tests/mem/f16_rrr_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/smoke_tests/bf16_ccc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/smoke_tests/bf16_ccr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/smoke_tests/bf16_crc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/smoke_tests/bf16_crr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/smoke_tests/bf16_rcc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/smoke_tests/bf16_rcr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/smoke_tests/bf16_rrc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/smoke_tests/bf16_rrr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/smoke_tests/f16_ccc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/smoke_tests/f16_ccr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/smoke_tests/f16_crc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/smoke_tests/f16_crr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/smoke_tests/f16_rcc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/smoke_tests/f16_rcr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/smoke_tests/f16_rrc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/smoke_tests/f16_rrr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp create mode 100644 test/ck_tile/gemm_streamk/test_gemm_streamk.hpp rename test/ck_tile/gemm_streamk/{test_gemm_streamk.cpp => test_gemm_streamk_common_includes.hpp} (54%) create mode 100644 test/ck_tile/gemm_streamk/test_gemm_streamk_types_bf16.hpp create mode 100644 test/ck_tile/gemm_streamk/test_gemm_streamk_types_fp16.hpp diff --git a/include/ck_tile/ops/gemm/kernel/streamk_gemm_kernel.hpp b/include/ck_tile/ops/gemm/kernel/streamk_gemm_kernel.hpp index ad85b5392d..58bce4795f 100644 --- a/include/ck_tile/ops/gemm/kernel/streamk_gemm_kernel.hpp +++ b/include/ck_tile/ops/gemm/kernel/streamk_gemm_kernel.hpp @@ -303,19 +303,20 @@ struct StreamKKernel auto spatial_idx = kargs.tile_partitioner.GetOutputTileIndex(tile_idx); // Get the offsets in A, B, C tensors. - index_t i_m = static_cast(spatial_idx[UniversalGemmKernel::I0] * + index_t i_m = static_cast(spatial_idx[UniversalGemmKernel::I0] * TilePartitioner::MPerBlock); - index_t i_n = static_cast(spatial_idx[UniversalGemmKernel::I1] * + index_t i_n = static_cast(spatial_idx[UniversalGemmKernel::I1] * TilePartitioner::NPerBlock); - index_t i_k = static_cast(iter_offset) * TilePartitioner::KPerBlock; + auto [i_k_a, i_k_b] = GetKOffsets( + static_cast(iter_offset), kargs.stride_As[0], kargs.stride_Bs[0]); // Determine the total size along the K dimension the WG is using in this iteration // (used to construct tensor views). index_t k_size = static_cast(current_iter_length * TilePartitioner::KPerBlock); // Update pointer offsets for A, B, and C. - const ADataType* a_ptr = static_cast(kargs.as_ptr[0]) + i_k; - const BDataType* b_ptr = static_cast(kargs.bs_ptr[0]) + i_k; + const ADataType* a_ptr = static_cast(kargs.as_ptr[0]) + i_k_a; + const BDataType* b_ptr = static_cast(kargs.bs_ptr[0]) + i_k_b; CDataType* c_ptr = static_cast(kargs.e_ptr); // Run the GEMM pipeline and Epilogue. @@ -339,6 +340,41 @@ struct StreamKKernel } private: + /// @brief Computes the K offsets in the A and B tensors given iter_offset, where iter_offset is + /// the starting macro tile index in the K dimension for the workgroup. + /// @return A tuple containing the offsets into the A and B tensors accounting for the layouts + /// of A and B. + /// @note The default case is that A is assumed to be row major and B is assumed to be column + /// major. + template + CK_TILE_DEVICE static tuple + GetKOffsets(index_t iter_offset, index_t stride_a, index_t stride_b) + { + index_t stride_offset_a; + index_t stride_offset_b; + if constexpr(std::is_same_v) + { + stride_offset_a = stride_a; + } + else + { + stride_offset_a = 1; + } + + if constexpr(std::is_same_v) + { + stride_offset_b = stride_b; + } + else + { + stride_offset_b = 1; + } + + index_t base_offset = iter_offset * TilePartitioner::KPerBlock; + + return make_tuple(base_offset * stride_offset_a, base_offset * stride_offset_b); + } + CK_TILE_HOST static int NumCU() { hipDeviceProp_t dev_prop; diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 292bc41a0b..96df4e32a1 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -45,6 +45,7 @@ set(REGRESSION_TESTS test_ck_tile_fmha_fwd_bf16 test_ck_tile_fmha_fwd_fp16 test_ck_tile_fmha_fwd_fp8 + test_ck_tile_streamk_extended ) function(add_test_executable TEST_NAME) diff --git a/test/ck_tile/gemm_streamk/CMakeLists.txt b/test/ck_tile/gemm_streamk/CMakeLists.txt index e00874ba07..ae527a24f7 100644 --- a/test/ck_tile/gemm_streamk/CMakeLists.txt +++ b/test/ck_tile/gemm_streamk/CMakeLists.txt @@ -1,7 +1,121 @@ # Currently test_ck_tile_streamk is only built on gfx9 if(GPU_TARGETS MATCHES "gfx9") + + include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}) + #TODO: support all arches - add_gtest_executable(test_ck_tile_streamk test_gemm_streamk.cpp) + #TODO: current stream-k c-shuffle only supports C layout as R + add_gtest_executable(test_ck_tile_streamk_smoke + ${CMAKE_CURRENT_SOURCE_DIR}/smoke_tests/f16_rrr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp + #${CMAKE_CURRENT_SOURCE_DIR}/smoke_tests/f16_rrc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/smoke_tests/f16_rcr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp + #${CMAKE_CURRENT_SOURCE_DIR}/smoke_tests/f16_rcc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/smoke_tests/f16_crr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp + #${CMAKE_CURRENT_SOURCE_DIR}/smoke_tests/f16_crc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/smoke_tests/f16_ccr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp + #${CMAKE_CURRENT_SOURCE_DIR}/smoke_tests/f16_ccc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp + + ${CMAKE_CURRENT_SOURCE_DIR}/smoke_tests/bf16_rrr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp + #${CMAKE_CURRENT_SOURCE_DIR}/smoke_tests/bf16_rrc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/smoke_tests/bf16_rcr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp + #${CMAKE_CURRENT_SOURCE_DIR}/smoke_tests/bf16_rcc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/smoke_tests/bf16_crr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp + #${CMAKE_CURRENT_SOURCE_DIR}/smoke_tests/bf16_crc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/smoke_tests/bf16_ccr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp + #${CMAKE_CURRENT_SOURCE_DIR}/smoke_tests/bf16_ccc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp + ) + + add_gtest_executable(test_ck_tile_streamk_extended + # compv3 pipeline + ${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/compv3/f16_rrr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp + #${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/compv3/f16_rrc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/compv3/f16_rcr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp + #${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/compv3/f16_rcc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/compv3/f16_crr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp + #${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/compv3/f16_crc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/compv3/f16_ccr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp + #${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/compv3/f16_ccc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/compv3/f16_rrr_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp + #${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/compv3/f16_rrc_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/compv3/f16_rcr_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp + #${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/compv3/f16_rcc_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/compv3/f16_crr_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp + #${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/compv3/f16_crc_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/compv3/f16_ccr_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp + #${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/compv3/f16_ccc_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp + + ${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/compv3/bf16_rrr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp + #${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/compv3/bf16_rrc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/compv3/bf16_rcr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp + #${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/compv3/bf16_rcc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/compv3/bf16_crr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp + #${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/compv3/bf16_crc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/compv3/bf16_ccr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp + #${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/compv3/bf16_ccc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/compv3/bf16_rrr_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp + #${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/compv3/bf16_rrc_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/compv3/bf16_rcr_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp + #${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/compv3/bf16_rcc_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/compv3/bf16_crr_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp + #${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/compv3/bf16_crc_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/compv3/bf16_ccr_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp + #${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/compv3/bf16_ccc_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp + + # TODO: add compv4 pipeline + # ${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/compv4/f16_rrr_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp + # #${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/compv4/f16_rrc_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp + # ${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/compv4/f16_rcr_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp + # #${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/compv4/f16_rcc_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp + # ${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/compv4/f16_crr_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp + # #${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/compv4/f16_crc_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp + # ${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/compv4/f16_ccr_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp + # #${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/compv4/f16_ccc_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp + # ${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/compv4/f16_rrr_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp + # #${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/compv4/f16_rrc_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp + # ${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/compv4/f16_rcr_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp + # #${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/compv4/f16_rcc_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp + # ${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/compv4/f16_crr_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp + # #${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/compv4/f16_crc_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp + # ${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/compv4/f16_ccr_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp + # #${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/compv4/f16_ccc_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp + + # ${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/compv4/bf16_rrr_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp + # #${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/compv4/bf16_rrc_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp + # ${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/compv4/bf16_rcr_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp + # #${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/compv4/bf16_rcc_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp + # ${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/compv4/bf16_crr_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp + # #${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/compv4/bf16_crc_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp + # ${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/compv4/bf16_ccr_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp + # #${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/compv4/bf16_ccc_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp + # ${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/compv4/bf16_rrr_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp + # #${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/compv4/bf16_rrc_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp + # ${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/compv4/bf16_rcr_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp + # #${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/compv4/bf16_rcc_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp + # ${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/compv4/bf16_crr_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp + # #${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/compv4/bf16_crc_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp + # ${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/compv4/bf16_ccr_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp + # #${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/compv4/bf16_ccc_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp + + + # mem pipeline + ${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/mem/f16_rrr_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp + #${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/mem/f16_rrc_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/mem/f16_rcr_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp + #${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/mem/f16_rcc_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/mem/f16_crr_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp + #${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/mem/f16_crc_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/mem/f16_ccr_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp + #${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/mem/f16_ccc_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp + + ${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/mem/bf16_rrr_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp + #${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/mem/bf16_rrc_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/mem/bf16_rcr_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp + #${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/mem/bf16_rcc_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/mem/bf16_crr_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp + #${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/mem/bf16_crc_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/mem/bf16_ccr_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp + #${CMAKE_CURRENT_SOURCE_DIR}/extended_tests/mem/bf16_ccc_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp + ) else() message(DEBUG "Skipping test_ck_tile_streamk tests for current target") endif() diff --git a/test/ck_tile/gemm_streamk/extended_tests/compv3/bf16_ccc_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/compv3/bf16_ccc_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..9bd736feb3 --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/compv3/bf16_ccc_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS BF16_CCC_CompV3_128x128x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/compv3/bf16_ccc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/compv3/bf16_ccc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..9f43b7a0a7 --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/compv3/bf16_ccc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS BF16_CCC_CompV3_256x256x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/compv3/bf16_ccr_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/compv3/bf16_ccr_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..f71515503d --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/compv3/bf16_ccr_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS BF16_CCR_CompV3_128x128x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/compv3/bf16_ccr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/compv3/bf16_ccr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..cda7f2a72b --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/compv3/bf16_ccr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS BF16_CCR_CompV3_256x256x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/compv3/bf16_crc_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/compv3/bf16_crc_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..be38c2b7d0 --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/compv3/bf16_crc_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS BF16_CRC_CompV3_128x128x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/compv3/bf16_crc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/compv3/bf16_crc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..115c5449d4 --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/compv3/bf16_crc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS BF16_CRC_CompV3_256x256x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/compv3/bf16_crr_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/compv3/bf16_crr_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..78c35c557c --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/compv3/bf16_crr_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS BF16_CRR_CompV3_128x128x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/compv3/bf16_crr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/compv3/bf16_crr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..b1cd42d599 --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/compv3/bf16_crr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS BF16_CRR_CompV3_256x256x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/compv3/bf16_rcc_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/compv3/bf16_rcc_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..a0a20d5843 --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/compv3/bf16_rcc_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS BF16_RCC_CompV3_128x128x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/compv3/bf16_rcc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/compv3/bf16_rcc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..944b6b1960 --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/compv3/bf16_rcc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS BF16_RCC_CompV3_256x256x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/compv3/bf16_rcr_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/compv3/bf16_rcr_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..f434380f50 --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/compv3/bf16_rcr_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS BF16_RCR_CompV3_128x128x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/compv3/bf16_rcr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/compv3/bf16_rcr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..165f8349e9 --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/compv3/bf16_rcr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS BF16_RCR_CompV3_256x256x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/compv3/bf16_rrc_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/compv3/bf16_rrc_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..9003ece236 --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/compv3/bf16_rrc_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS BF16_RRC_CompV3_128x128x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/compv3/bf16_rrc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/compv3/bf16_rrc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..9705060b18 --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/compv3/bf16_rrc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS BF16_RRC_CompV3_256x256x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/compv3/bf16_rrr_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/compv3/bf16_rrr_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..dc30023521 --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/compv3/bf16_rrr_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS BF16_RRR_CompV3_128x128x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/compv3/bf16_rrr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/compv3/bf16_rrr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..5b3350534a --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/compv3/bf16_rrr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS BF16_RRR_CompV3_256x256x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/compv3/f16_ccc_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/compv3/f16_ccc_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..b5e1ac7478 --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/compv3/f16_ccc_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS F16_CCC_CompV3_128x128x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/compv3/f16_ccc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/compv3/f16_ccc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..e51fb3b959 --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/compv3/f16_ccc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS F16_CCC_CompV3_256x256x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/compv3/f16_ccr_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/compv3/f16_ccr_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..9991d59995 --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/compv3/f16_ccr_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS F16_CCR_CompV3_128x128x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/compv3/f16_ccr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/compv3/f16_ccr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..22d417bdde --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/compv3/f16_ccr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS F16_CCR_CompV3_256x256x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/compv3/f16_crc_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/compv3/f16_crc_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..dc79747889 --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/compv3/f16_crc_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS F16_CRC_CompV3_128x128x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/compv3/f16_crc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/compv3/f16_crc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..3d45ac02ab --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/compv3/f16_crc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS F16_CRC_CompV3_256x256x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/compv3/f16_crr_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/compv3/f16_crr_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..b681704dc4 --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/compv3/f16_crr_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS F16_CRR_CompV3_128x128x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/compv3/f16_crr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/compv3/f16_crr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..dac4308d66 --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/compv3/f16_crr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS F16_CRR_CompV3_256x256x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/compv3/f16_rcc_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/compv3/f16_rcc_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..a2294b5742 --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/compv3/f16_rcc_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS F16_RCC_CompV3_128x128x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/compv3/f16_rcc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/compv3/f16_rcc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..52b11dd8a2 --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/compv3/f16_rcc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS F16_RCC_CompV3_256x256x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/compv3/f16_rcr_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/compv3/f16_rcr_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..699a5dd6f1 --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/compv3/f16_rcr_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS F16_RCR_CompV3_128x128x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/compv3/f16_rcr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/compv3/f16_rcr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..0cb8d1d338 --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/compv3/f16_rcr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS F16_RCR_CompV3_256x256x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/compv3/f16_rrc_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/compv3/f16_rrc_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..75487a1c70 --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/compv3/f16_rrc_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS F16_RRC_CompV3_128x128x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/compv3/f16_rrc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/compv3/f16_rrc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..ce9ec9244a --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/compv3/f16_rrc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS F16_RRC_CompV3_256x256x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/compv3/f16_rrr_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/compv3/f16_rrr_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..c1239bba1a --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/compv3/f16_rrr_compv3_128x128x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS F16_RRR_CompV3_128x128x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/compv3/f16_rrr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/compv3/f16_rrr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..93f2f90048 --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/compv3/f16_rrr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS F16_RRR_CompV3_256x256x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/compv4/bf16_ccc_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/compv4/bf16_ccc_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..346e865808 --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/compv4/bf16_ccc_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS BF16_CCC_CompV4_128x128x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/compv4/bf16_ccc_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/compv4/bf16_ccc_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..42779c1d0c --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/compv4/bf16_ccc_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS BF16_CCC_CompV4_256x256x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/compv4/bf16_ccr_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/compv4/bf16_ccr_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..f0d76a25b4 --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/compv4/bf16_ccr_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS BF16_CCR_CompV4_128x128x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/compv4/bf16_ccr_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/compv4/bf16_ccr_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..702dacf603 --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/compv4/bf16_ccr_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS BF16_CCR_CompV4_256x256x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/compv4/bf16_crc_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/compv4/bf16_crc_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..6792a252e4 --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/compv4/bf16_crc_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS BF16_CRC_CompV4_128x128x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/compv4/bf16_crc_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/compv4/bf16_crc_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..c8395d1702 --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/compv4/bf16_crc_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS BF16_CRC_CompV4_256x256x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/compv4/bf16_crr_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/compv4/bf16_crr_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..2eba9e5e74 --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/compv4/bf16_crr_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS BF16_CRR_CompV4_128x128x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/compv4/bf16_crr_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/compv4/bf16_crr_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..ea8ec795f7 --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/compv4/bf16_crr_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS BF16_CRR_CompV4_256x256x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/compv4/bf16_rcc_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/compv4/bf16_rcc_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..55b66d1313 --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/compv4/bf16_rcc_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS BF16_RCC_CompV4_128x128x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/compv4/bf16_rcc_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/compv4/bf16_rcc_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..9351154bc8 --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/compv4/bf16_rcc_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS BF16_RCC_CompV4_256x256x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/compv4/bf16_rcr_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/compv4/bf16_rcr_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..826525ed01 --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/compv4/bf16_rcr_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS BF16_RCR_CompV4_128x128x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/compv4/bf16_rcr_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/compv4/bf16_rcr_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..a584346e8c --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/compv4/bf16_rcr_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS BF16_RCR_CompV4_256x256x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/compv4/bf16_rrc_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/compv4/bf16_rrc_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..714a610ee9 --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/compv4/bf16_rrc_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS BF16_RRC_CompV4_128x128x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/compv4/bf16_rrc_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/compv4/bf16_rrc_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..c7ba8154eb --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/compv4/bf16_rrc_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS BF16_RRC_CompV4_256x256x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/compv4/bf16_rrr_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/compv4/bf16_rrr_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..4f52ade2b6 --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/compv4/bf16_rrr_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS BF16_RRR_CompV4_128x128x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/compv4/bf16_rrr_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/compv4/bf16_rrr_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..6ee0580a60 --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/compv4/bf16_rrr_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS BF16_RRR_CompV4_256x256x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/compv4/f16_ccc_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/compv4/f16_ccc_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..cdc1d4534e --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/compv4/f16_ccc_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS F16_CCC_CompV4_128x128x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/compv4/f16_ccc_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/compv4/f16_ccc_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..f934a06df5 --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/compv4/f16_ccc_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS F16_CCC_CompV4_256x256x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/compv4/f16_ccr_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/compv4/f16_ccr_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..3b95352c13 --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/compv4/f16_ccr_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS F16_CCR_CompV4_128x128x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/compv4/f16_ccr_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/compv4/f16_ccr_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..d8683a20c5 --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/compv4/f16_ccr_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS F16_CCR_CompV4_256x256x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/compv4/f16_crc_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/compv4/f16_crc_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..5e675f0069 --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/compv4/f16_crc_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS F16_CRC_CompV4_128x128x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/compv4/f16_crc_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/compv4/f16_crc_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..253d539241 --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/compv4/f16_crc_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS F16_CRC_CompV4_256x256x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/compv4/f16_crr_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/compv4/f16_crr_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..a4420a8fc3 --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/compv4/f16_crr_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS F16_CRR_CompV4_128x128x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/compv4/f16_crr_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/compv4/f16_crr_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..0dcb4afe05 --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/compv4/f16_crr_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS F16_CRR_CompV4_256x256x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/compv4/f16_rcc_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/compv4/f16_rcc_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..eb46e78375 --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/compv4/f16_rcc_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS F16_RCC_CompV4_128x128x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/compv4/f16_rcc_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/compv4/f16_rcc_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..5653a05ccc --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/compv4/f16_rcc_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS F16_RCC_CompV4_256x256x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/compv4/f16_rcr_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/compv4/f16_rcr_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..ae94416336 --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/compv4/f16_rcr_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS F16_RCR_CompV4_128x128x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/compv4/f16_rcr_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/compv4/f16_rcr_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..28673bc2d8 --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/compv4/f16_rcr_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS F16_RCR_CompV4_256x256x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/compv4/f16_rrc_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/compv4/f16_rrc_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..d61389c941 --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/compv4/f16_rrc_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS F16_RRC_CompV4_128x128x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/compv4/f16_rrc_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/compv4/f16_rrc_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..d276153efc --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/compv4/f16_rrc_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS F16_RRC_CompV4_256x256x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/compv4/f16_rrr_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/compv4/f16_rrr_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..a31d8ed592 --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/compv4/f16_rrr_compv4_128x128x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS F16_RRR_CompV4_128x128x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/compv4/f16_rrr_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/compv4/f16_rrr_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..c68e67658a --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/compv4/f16_rrr_compv4_256x256x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS F16_RRR_CompV4_256x256x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/mem/bf16_ccc_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/mem/bf16_ccc_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..851765f0aa --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/mem/bf16_ccc_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS BF16_CCC_Mem_128x128x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/mem/bf16_ccr_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/mem/bf16_ccr_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..0f7a3f8ca8 --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/mem/bf16_ccr_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS BF16_CCR_Mem_128x128x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/mem/bf16_crc_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/mem/bf16_crc_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..0e7f1e1fad --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/mem/bf16_crc_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS BF16_CRC_Mem_128x128x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/mem/bf16_crr_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/mem/bf16_crr_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..8a85738652 --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/mem/bf16_crr_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS BF16_CRR_Mem_128x128x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/mem/bf16_rcc_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/mem/bf16_rcc_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..ab0220f4ef --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/mem/bf16_rcc_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS BF16_RCC_Mem_128x128x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/mem/bf16_rcr_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/mem/bf16_rcr_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..9e60ef1717 --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/mem/bf16_rcr_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS BF16_RCR_Mem_128x128x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/mem/bf16_rrc_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/mem/bf16_rrc_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..61cd772d51 --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/mem/bf16_rrc_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS BF16_RRC_Mem_128x128x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/mem/bf16_rrr_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/mem/bf16_rrr_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..66e0e80c46 --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/mem/bf16_rrr_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS BF16_RRR_Mem_128x128x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/mem/f16_ccc_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/mem/f16_ccc_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..a2f26b768e --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/mem/f16_ccc_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS F16_CCC_Mem_128x128x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/mem/f16_ccr_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/mem/f16_ccr_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..d94547daa7 --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/mem/f16_ccr_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS F16_CCR_Mem_128x128x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/mem/f16_crc_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/mem/f16_crc_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..090b472d45 --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/mem/f16_crc_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS F16_CRC_Mem_128x128x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/mem/f16_crr_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/mem/f16_crr_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..5535325436 --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/mem/f16_crr_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS F16_CRR_Mem_128x128x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/mem/f16_rcc_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/mem/f16_rcc_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..a2f999a69d --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/mem/f16_rcc_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS F16_RCC_Mem_128x128x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/mem/f16_rcr_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/mem/f16_rcr_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..f6d4b50c4a --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/mem/f16_rcr_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS F16_RCR_Mem_128x128x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/mem/f16_rrc_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/mem/f16_rrc_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..7dd85dbf0d --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/mem/f16_rrc_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS F16_RRC_Mem_128x128x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/extended_tests/mem/f16_rrr_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/extended_tests/mem/f16_rrr_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..58ba61ad38 --- /dev/null +++ b/test/ck_tile/gemm_streamk/extended_tests/mem/f16_rrr_mem_128x128x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS F16_RRR_Mem_128x128x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/smoke_tests/bf16_ccc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/smoke_tests/bf16_ccc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..9f43b7a0a7 --- /dev/null +++ b/test/ck_tile/gemm_streamk/smoke_tests/bf16_ccc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS BF16_CCC_CompV3_256x256x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/smoke_tests/bf16_ccr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/smoke_tests/bf16_ccr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..cda7f2a72b --- /dev/null +++ b/test/ck_tile/gemm_streamk/smoke_tests/bf16_ccr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS BF16_CCR_CompV3_256x256x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/smoke_tests/bf16_crc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/smoke_tests/bf16_crc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..115c5449d4 --- /dev/null +++ b/test/ck_tile/gemm_streamk/smoke_tests/bf16_crc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS BF16_CRC_CompV3_256x256x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/smoke_tests/bf16_crr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/smoke_tests/bf16_crr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..b1cd42d599 --- /dev/null +++ b/test/ck_tile/gemm_streamk/smoke_tests/bf16_crr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS BF16_CRR_CompV3_256x256x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/smoke_tests/bf16_rcc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/smoke_tests/bf16_rcc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..944b6b1960 --- /dev/null +++ b/test/ck_tile/gemm_streamk/smoke_tests/bf16_rcc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS BF16_RCC_CompV3_256x256x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/smoke_tests/bf16_rcr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/smoke_tests/bf16_rcr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..165f8349e9 --- /dev/null +++ b/test/ck_tile/gemm_streamk/smoke_tests/bf16_rcr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS BF16_RCR_CompV3_256x256x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/smoke_tests/bf16_rrc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/smoke_tests/bf16_rrc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..9705060b18 --- /dev/null +++ b/test/ck_tile/gemm_streamk/smoke_tests/bf16_rrc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS BF16_RRC_CompV3_256x256x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/smoke_tests/bf16_rrr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/smoke_tests/bf16_rrr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..5b3350534a --- /dev/null +++ b/test/ck_tile/gemm_streamk/smoke_tests/bf16_rrr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS BF16_RRR_CompV3_256x256x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/smoke_tests/f16_ccc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/smoke_tests/f16_ccc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..e51fb3b959 --- /dev/null +++ b/test/ck_tile/gemm_streamk/smoke_tests/f16_ccc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS F16_CCC_CompV3_256x256x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/smoke_tests/f16_ccr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/smoke_tests/f16_ccr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..22d417bdde --- /dev/null +++ b/test/ck_tile/gemm_streamk/smoke_tests/f16_ccr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS F16_CCR_CompV3_256x256x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/smoke_tests/f16_crc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/smoke_tests/f16_crc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..3d45ac02ab --- /dev/null +++ b/test/ck_tile/gemm_streamk/smoke_tests/f16_crc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS F16_CRC_CompV3_256x256x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/smoke_tests/f16_crr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/smoke_tests/f16_crr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..dac4308d66 --- /dev/null +++ b/test/ck_tile/gemm_streamk/smoke_tests/f16_crr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS F16_CRR_CompV3_256x256x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/smoke_tests/f16_rcc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/smoke_tests/f16_rcc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..52b11dd8a2 --- /dev/null +++ b/test/ck_tile/gemm_streamk/smoke_tests/f16_rcc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS F16_RCC_CompV3_256x256x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/smoke_tests/f16_rcr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/smoke_tests/f16_rcr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..0cb8d1d338 --- /dev/null +++ b/test/ck_tile/gemm_streamk/smoke_tests/f16_rcr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS F16_RCR_CompV3_256x256x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/smoke_tests/f16_rrc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/smoke_tests/f16_rrc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..ce9ec9244a --- /dev/null +++ b/test/ck_tile/gemm_streamk/smoke_tests/f16_rrc_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS F16_RRC_CompV3_256x256x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/smoke_tests/f16_rrr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp b/test/ck_tile/gemm_streamk/smoke_tests/f16_rrr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp new file mode 100644 index 0000000000..93f2f90048 --- /dev/null +++ b/test/ck_tile/gemm_streamk/smoke_tests/f16_rrr_compv3_256x256x32_2x2x1_32x32x16_NonPersistent.cpp @@ -0,0 +1,11 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_streamk_common_includes.hpp" + +#define TEST_SUITE_PARAMS F16_RRR_CompV3_256x256x32_2x2x1_32x32x16_NonPersistent +#define TEST_SUITE_NAME MAKE_TEST_SUITE_NAME(TEST_SUITE_PARAMS) + +DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS); + +#include "test_gemm_streamk_cases.inc" diff --git a/test/ck_tile/gemm_streamk/test_gemm_streamk.hpp b/test/ck_tile/gemm_streamk/test_gemm_streamk.hpp new file mode 100644 index 0000000000..da0b8d153d --- /dev/null +++ b/test/ck_tile/gemm_streamk/test_gemm_streamk.hpp @@ -0,0 +1,269 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#pragma once + +#include +#include +#include +#include +#include + +#include "ck_tile/host.hpp" +#include "ck_tile/ops/epilogue.hpp" +#include "ck_tile/ops/gemm.hpp" + +#include "test_gemm_streamk_util.hpp" + +template +class TestCkTileStreamK : public ::testing::Test +{ + protected: + using ALayout = std::tuple_element_t<0, Tuple>; + using BLayout = std::tuple_element_t<1, Tuple>; + using CLayout = std::tuple_element_t<2, Tuple>; + using ADataType = std::tuple_element_t<3, Tuple>; + using BDataType = std::tuple_element_t<4, Tuple>; + using AccDataType = std::tuple_element_t<5, Tuple>; + using CDataType = std::tuple_element_t<6, Tuple>; + using DsLayout = ck_tile::tuple<>; + using DsDataType = ck_tile::tuple<>; + + static constexpr ck_tile::index_t M_Tile = std::tuple_element_t<7, Tuple>::value; + static constexpr ck_tile::index_t N_Tile = std::tuple_element_t<8, Tuple>::value; + static constexpr ck_tile::index_t K_Tile = std::tuple_element_t<9, Tuple>::value; + + static constexpr ck_tile::index_t M_Warp = std::tuple_element_t<10, Tuple>::value; + static constexpr ck_tile::index_t N_Warp = std::tuple_element_t<11, Tuple>::value; + static constexpr ck_tile::index_t K_Warp = std::tuple_element_t<12, Tuple>::value; + + static constexpr ck_tile::index_t M_Warp_Tile = std::tuple_element_t<13, Tuple>::value; + static constexpr ck_tile::index_t N_Warp_Tile = std::tuple_element_t<14, Tuple>::value; + static constexpr ck_tile::index_t K_Warp_Tile = std::tuple_element_t<15, Tuple>::value; + + static constexpr GemmPipelineType PipelineType = std::tuple_element_t<16, Tuple>::value; + static constexpr bool Persistent = std::tuple_element_t<17, Tuple>::value; + + template + bool invoke_streamk(const ck_tile::StreamKHostArgs& args, + const ck_tile::stream_config& s, + int num_cu, + int occupancy) + { + constexpr bool kPadM = PadM; + constexpr bool kPadN = PadN; + constexpr bool kPadK = PadK; + constexpr bool preshuffle = Preshuffle; + + constexpr bool DoubleSmemBuffer = false; + constexpr int kBlockPerCu = 1; + constexpr bool StructuredSparsity = false; + constexpr bool NumWaveGroup = 1; + + using GemmShape = + ck_tile::TileGemmShape, + ck_tile::sequence, + ck_tile::sequence>; + + using TilePartitioner = ck_tile::StreamKTilePartitioner; + + using GemmUniversalTraits = ck_tile::TileGemmUniversalTraits; + + const auto Run = [&](const auto memory_operation_) { + constexpr auto memory_operation = memory_operation_.value; + constexpr auto scheduler = ck_tile::GemmPipelineScheduler::Intrawave; + + // We create the GEMM pipeline without specifying has_hot_loop or tail_num. + // This is because num_loop can vary (a) per WG and (b) per iteration of the Stream-K + // while loop. Instead, has_hot_loop and tail_num are determined in the Stream-K + // Kernel's RunGemm function. This is a similar pattern used by grouped GEMM. + using UniversalGemmProblem = ck_tile::UniversalGemmPipelineProblem; + // For initial testing, we will just test with one pipeline. + // More extensive testing is coming later and will test other pipelines. + using GemmPipeline = + typename GemmPipelineTypeSelector::pipeline; + + using GemmEpilogue = ck_tile::CShuffleEpilogue< + ck_tile::CShuffleEpilogueProblem, + AccDataType, + CDataType, + ck_tile::tuple<>, + CLayout, + ck_tile::element_wise::PassThrough, + TilePartitioner::MPerBlock, + TilePartitioner::NPerBlock, + M_Warp, + N_Warp, + M_Warp_Tile, + N_Warp_Tile, + K_Warp_Tile, + UniversalGemmProblem::TransposeC, + memory_operation>>; + + using Kernel = ck_tile::StreamKKernel; + + auto kargs = Kernel::MakeKernelArgs(args, num_cu, occupancy); + + if(!Kernel::IsSupportedArgument(kargs)) + { + return false; + } + + dim3 grid_dims = Kernel::GridSize(kargs.tile_partitioner); + dim3 block_dims = Kernel::BlockSize(); + + ck_tile::launch_kernel( + s, ck_tile::make_kernel(Kernel{}, grid_dims, block_dims, 0, kargs)); + + return true; + }; + + return Run(ck_tile::integral_constant{}); + } + + public: + // Since Stream-K is build on gfx9, the lower bound for CUs is 104. Thus, we default num_cu to + // 104 and occupancy to 1 to ensure tests are reproducible on different architectures. + void Run(ck_tile::index_t M, + ck_tile::index_t N, + ck_tile::index_t K, + uint32_t num_sk_blocks = 0xffffffff, + ck_tile::StreamKReductionStrategy reduction_strategy = + ck_tile::StreamKReductionStrategy::Atomic, + int occupancy = 1, + int num_cu = 104, + ck_tile::index_t stride_A = 0, + ck_tile::index_t stride_B = 0, + ck_tile::index_t stride_C = 0) + { + + using namespace ck_tile::literals; + + if(reduction_strategy == ck_tile::StreamKReductionStrategy::Reduction) + { + throw std::runtime_error("Reduction Strategy is current unsupported!\n"); + } + + auto f_host_tensor_descriptor = [](std::size_t row, + std::size_t col, + std::size_t stride, + auto layout) { + if constexpr(std::is_same_v) + { + return ck_tile::HostTensorDescriptor({row, col}, {stride, 1_uz}); + } + else + { + return ck_tile::HostTensorDescriptor({row, col}, {1_uz, stride}); + } + }; + + auto f_get_default_stride = + [](std::size_t row, std::size_t col, std::size_t stride, auto layout) { + if(stride == 0) + { + if constexpr(std::is_same_v) + { + return col; + } + else + { + return row; + } + } + else + return stride; + }; + + stride_A = f_get_default_stride(M, K, stride_A, ALayout{}); + stride_B = f_get_default_stride(K, N, stride_B, BLayout{}); + stride_C = f_get_default_stride(M, N, stride_C, CLayout{}); + + ck_tile::HostTensor a_m_k(f_host_tensor_descriptor(M, K, stride_A, ALayout{})); + ck_tile::HostTensor b_k_n(f_host_tensor_descriptor(K, N, stride_B, BLayout{})); + ck_tile::HostTensor c_m_n_dev_result( + f_host_tensor_descriptor(M, N, stride_C, CLayout{})); + + // TODO: Add randomized number generation ranges for different datatypes + ck_tile::FillUniformDistributionIntegerValue{-3, 3, /*seed*/ 11939}(a_m_k); + ck_tile::FillUniformDistributionIntegerValue{-3, 3, /*seed*/ 11940}(b_k_n); + + ck_tile::DeviceMem a_m_k_dev_buf(a_m_k.get_element_space_size_in_bytes()); + ck_tile::DeviceMem b_k_n_dev_buf(b_k_n.get_element_space_size_in_bytes()); + ck_tile::DeviceMem c_m_n_dev_buf(c_m_n_dev_result.get_element_space_size_in_bytes()); + + a_m_k_dev_buf.ToDevice(a_m_k.data()); + b_k_n_dev_buf.ToDevice(b_k_n.data()); + c_m_n_dev_buf.SetZero(); + c_m_n_dev_result.SetZero(); + + ck_tile::StreamKHostArgs args{a_m_k_dev_buf.GetDeviceBuffer(), + b_k_n_dev_buf.GetDeviceBuffer(), + c_m_n_dev_buf.GetDeviceBuffer(), + M, + N, + K, + stride_A, + stride_B, + stride_C, + reduction_strategy, + num_sk_blocks}; + + if(!invoke_streamk( + args, ck_tile::stream_config{nullptr, false, 0, 0, 1}, num_cu, occupancy)) + { + GTEST_SKIP() << "Skipping this test: The kernel cannot solve the problem\n"; + } + + c_m_n_dev_buf.FromDevice(c_m_n_dev_result.data()); + + ck_tile::HostTensor c_m_n_host_ref( + f_host_tensor_descriptor(M, N, stride_C, CLayout{})); + c_m_n_host_ref.SetZero(); + + ck_tile::reference_gemm( + a_m_k, b_k_n, c_m_n_host_ref); + + const float max_accumulated_value = + *std::max_element(c_m_n_host_ref.mData.begin(), c_m_n_host_ref.mData.end()); + const auto rtol_atol = calculate_rtol_atol( + K, /*kbatch*/ 1, max_accumulated_value); + + bool pass = ck_tile::check_err(c_m_n_dev_result, + c_m_n_host_ref, + "Error: Incorrect results!", + rtol_atol.at(ck_tile::number<0>{}), + rtol_atol.at(ck_tile::number<1>{})); + + EXPECT_TRUE(pass); + }; +}; diff --git a/test/ck_tile/gemm_streamk/test_gemm_streamk_cases.inc b/test/ck_tile/gemm_streamk/test_gemm_streamk_cases.inc index 1db7ef0fb0..ff597d5015 100644 --- a/test/ck_tile/gemm_streamk/test_gemm_streamk_cases.inc +++ b/test/ck_tile/gemm_streamk/test_gemm_streamk_cases.inc @@ -3,27 +3,149 @@ #pragma once -TYPED_TEST(TEST_SUITE_NAME, StreamK_M256_N256_K256_DP) -{ +// Ensure that we have the required macros defined before proceeding +#ifndef TEST_SUITE_NAME +#error "TEST_SUITE_NAME must be defined before including this file" +#endif +#ifndef TEST_SUITE_PARAMS +#error "TEST_SUITE_PARAMS must be defined before including this file" +#endif - ck_tile::index_t M = 256; - ck_tile::index_t N = 256; - ck_tile::index_t K = 256; - uint32_t num_sk_blocks = 0; +// Macros to help generate test names from the parameters given +// Concatenate is able to stitch the template parameters symbol together with the runtime args +// values +#define CONCATENATE_TEST_NAME(SIZE_M, SIZE_N, SIZE_K, NUM_SK_BLOCKS) \ + M##SIZE_M##_N##SIZE_N##_K##SIZE_K##_SKBlocks##NUM_SK_BLOCKS +// Helper macro to expand the arguments before passing them to CONCATENATE_TEST_NAME +#define MAKE_TEST_NAME(SIZE_M, SIZE_N, SIZE_K, NUM_SK_BLOCKS) \ + CONCATENATE_TEST_NAME(SIZE_M, SIZE_N, SIZE_K, NUM_SK_BLOCKS) - this->Run(M, N, K, num_sk_blocks); -} +// Macro to add a test TEST_NAME to the TEST_SUITE_NAME with the given parameters +#define STREAM_K_TEST_INTERNAL(SIZE_M, SIZE_N, SIZE_K, NUM_SK_BLOCKS, TEST_NAME) \ + TYPED_TEST(TEST_SUITE_NAME, TEST_NAME) \ + { \ + ck_tile::index_t M = SIZE_M; \ + ck_tile::index_t N = SIZE_N; \ + ck_tile::index_t K = SIZE_K; \ + uint32_t num_sk_blocks = NUM_SK_BLOCKS; \ + \ + this->Run(M, N, K, num_sk_blocks); \ + } -TYPED_TEST(TEST_SUITE_NAME, StreamK_M256_N256_K256_SKBlocks4) -{ +// Macro that generates a test name from the TEST_SUITE_TPARAMS symbol and the given parameters, +// then adds that test to test suite TEST_SUITE_NAME +#define STREAM_K_TEST(SIZE_M, SIZE_N, SIZE_K, NUM_SK_BLOCKS) \ + STREAM_K_TEST_INTERNAL(SIZE_M, \ + SIZE_N, \ + SIZE_K, \ + NUM_SK_BLOCKS, \ + MAKE_TEST_NAME(SIZE_M, SIZE_N, SIZE_K, NUM_SK_BLOCKS)) - ck_tile::index_t M = 256; - ck_tile::index_t N = 256; - ck_tile::index_t K = 256; - uint32_t num_sk_blocks = 4; +STREAM_K_TEST(1, 1, 1, 0) +STREAM_K_TEST(1, 1, 1, 1) - this->Run(M, N, K, num_sk_blocks); -} +// TODO: fails for <= wave tile +// STREAM_K_TEST(16, 16, 16, 0) +// STREAM_K_TEST(16, 16, 16, 1) +// STREAM_K_TEST(32, 32, 16, 0) +// STREAM_K_TEST(32, 32, 16, 1) + +STREAM_K_TEST(32, 32, 32, 0) +STREAM_K_TEST(32, 32, 32, 1) +STREAM_K_TEST(32, 32, 32, 2) +STREAM_K_TEST(32, 32, 32, 3) + +/// Prime number odd offsets +STREAM_K_TEST(37, 32, 32, 0) +STREAM_K_TEST(37, 32, 32, 1) +STREAM_K_TEST(37, 32, 32, 2) +STREAM_K_TEST(37, 32, 32, 3) + +STREAM_K_TEST(32, 37, 32, 0) +STREAM_K_TEST(32, 37, 32, 1) +STREAM_K_TEST(32, 37, 32, 2) +STREAM_K_TEST(32, 37, 32, 3) + +// TODO: Fails +// STREAM_K_TEST(32, 32, 37, 0) +// STREAM_K_TEST(32, 32, 37, 1) +// STREAM_K_TEST(32, 32, 37, 2) +// STREAM_K_TEST(32, 32, 37, 3) + +// TODO: Fails +STREAM_K_TEST(37, 32, 37, 0) +STREAM_K_TEST(37, 32, 37, 1) +STREAM_K_TEST(37, 32, 37, 2) +STREAM_K_TEST(37, 32, 37, 3) + +STREAM_K_TEST(37, 37, 37, 0) +STREAM_K_TEST(37, 37, 37, 1) +STREAM_K_TEST(37, 37, 37, 2) +STREAM_K_TEST(37, 37, 37, 3) + +/// Cubed sizes +STREAM_K_TEST(256, 256, 256, 0) +STREAM_K_TEST(256, 256, 256, 4) +STREAM_K_TEST(256, 256, 256, 8) + +// TODO: Fails +// STREAM_K_TEST(272, 272, 272, 0) +// STREAM_K_TEST(272, 272, 272, 8) +// STREAM_K_TEST(272, 272, 272, 16) + +STREAM_K_TEST(288, 288, 288, 0) +STREAM_K_TEST(288, 288, 288, 4) +STREAM_K_TEST(288, 288, 288, 8) + +STREAM_K_TEST(512, 512, 512, 0) +STREAM_K_TEST(512, 512, 512, 8) +STREAM_K_TEST(512, 512, 512, 16) + +// TODO: Fails +// STREAM_K_TEST(528, 528, 528, 0) +// STREAM_K_TEST(528, 528, 528, 8) +// STREAM_K_TEST(528, 528, 528, 16) + +STREAM_K_TEST(544, 544, 544, 0) +STREAM_K_TEST(544, 544, 544, 8) +STREAM_K_TEST(544, 544, 544, 16) + +/// Long M skinny N and K +STREAM_K_TEST(512, 1, 1, 0) +STREAM_K_TEST(512, 1, 1, 8) +STREAM_K_TEST(512, 1, 1, 16) + +STREAM_K_TEST(512, 32, 32, 0) +STREAM_K_TEST(512, 32, 32, 8) +STREAM_K_TEST(512, 32, 32, 16) + +/// Long M and N and skinny K +// TODO: Fails with core dump +// STREAM_K_TEST(512, 512, 1, 0) +// STREAM_K_TEST(512, 512, 1, 8) +// STREAM_K_TEST(512, 512, 1, 16) + +STREAM_K_TEST(512, 512, 32, 0) +STREAM_K_TEST(512, 512, 32, 8) +STREAM_K_TEST(512, 512, 32, 16) + +/// Long M and K and skinny N +STREAM_K_TEST(512, 1, 512, 0) +STREAM_K_TEST(512, 1, 512, 8) +STREAM_K_TEST(512, 1, 512, 16) + +STREAM_K_TEST(512, 32, 512, 0) +STREAM_K_TEST(512, 32, 512, 8) +STREAM_K_TEST(512, 32, 512, 16) + +/// Long K and skinny M and N +STREAM_K_TEST(1, 1, 512, 0) +STREAM_K_TEST(1, 1, 512, 8) +STREAM_K_TEST(1, 1, 512, 16) + +STREAM_K_TEST(32, 32, 512, 0) +STREAM_K_TEST(32, 32, 512, 8) +STREAM_K_TEST(32, 32, 512, 16) // TODO: Renable this test once reduction is implemented TYPED_TEST(TEST_SUITE_NAME, StreamK_M256_N256_K256_SKBlocks12) @@ -39,72 +161,6 @@ TYPED_TEST(TEST_SUITE_NAME, StreamK_M256_N256_K256_SKBlocks12) this->Run(M, N, K, num_sk_blocks); } -TYPED_TEST(TEST_SUITE_NAME, StreamK_M256_N256_K256_SKBlocks8) -{ - - ck_tile::index_t M = 256; - ck_tile::index_t N = 256; - ck_tile::index_t K = 256; - uint32_t num_sk_blocks = 8; - - this->Run(M, N, K, num_sk_blocks); -} - -TYPED_TEST(TEST_SUITE_NAME, StreamK_M512_N512_K512_DP) -{ - - ck_tile::index_t M = 512; - ck_tile::index_t N = 512; - ck_tile::index_t K = 512; - uint32_t num_sk_blocks = 0; - - this->Run(M, N, K, num_sk_blocks); -} - -TYPED_TEST(TEST_SUITE_NAME, StreamK_M512_N512_K512_SKBlocks16) -{ - - ck_tile::index_t M = 512; - ck_tile::index_t N = 512; - ck_tile::index_t K = 512; - uint32_t num_sk_blocks = 16; - - this->Run(M, N, K, num_sk_blocks); -} - -TYPED_TEST(TEST_SUITE_NAME, StreamK_M512_N512_K512_SKBlocks8) -{ - - ck_tile::index_t M = 512; - ck_tile::index_t N = 512; - ck_tile::index_t K = 512; - uint32_t num_sk_blocks = 8; - - this->Run(M, N, K, num_sk_blocks); -} - -TYPED_TEST(TEST_SUITE_NAME, StreamK_M3840_N4096_K4096_DP) -{ - - ck_tile::index_t M = 3840; - ck_tile::index_t N = 4096; - ck_tile::index_t K = 4096; - uint32_t num_sk_blocks = 0; - - this->Run(M, N, K, num_sk_blocks); -} - -TYPED_TEST(TEST_SUITE_NAME, StreamK_M3840_N4096_K4096_SKBlocks64) -{ - - ck_tile::index_t M = 3840; - ck_tile::index_t N = 4096; - ck_tile::index_t K = 4096; - uint32_t num_sk_blocks = 64; - - this->Run(M, N, K, num_sk_blocks); -} - TYPED_TEST(TEST_SUITE_NAME, StreamK_Unsupported_Reduction) { diff --git a/test/ck_tile/gemm_streamk/test_gemm_streamk.cpp b/test/ck_tile/gemm_streamk/test_gemm_streamk_common_includes.hpp similarity index 54% rename from test/ck_tile/gemm_streamk/test_gemm_streamk.cpp rename to test/ck_tile/gemm_streamk/test_gemm_streamk_common_includes.hpp index 99c3fb397f..b1faf3848b 100644 --- a/test/ck_tile/gemm_streamk/test_gemm_streamk.cpp +++ b/test/ck_tile/gemm_streamk/test_gemm_streamk_common_includes.hpp @@ -1,14 +1,8 @@ // Copyright © Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT +#pragma once +#include "test_gemm_streamk.hpp" #include "test_gemm_streamk_types.hpp" #include "test_gemm_streamk_util.hpp" #include "gtest/gtest.h" - -#define TEST_SUITE_NAME TestCkTileStreamK - -TYPED_TEST_SUITE(TestCkTileStreamK, KernelTypesStreamK); - -#include "test_gemm_streamk_cases.inc" - -#undef TEST_SUITE_NAME diff --git a/test/ck_tile/gemm_streamk/test_gemm_streamk_types.hpp b/test/ck_tile/gemm_streamk/test_gemm_streamk_types.hpp index 399f3f11e8..578eb31189 100644 --- a/test/ck_tile/gemm_streamk/test_gemm_streamk_types.hpp +++ b/test/ck_tile/gemm_streamk/test_gemm_streamk_types.hpp @@ -1,12 +1,15 @@ // Copyright © Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT +#pragma once + #include #include #include "gtest/gtest.h" #include "ck_tile/host.hpp" +#include "test_gemm_streamk_util.hpp" using F16 = ck_tile::half_t; using F32 = float; @@ -15,11 +18,106 @@ using BF16 = ck_tile::bf16_t; using Row = ck_tile::tensor_layout::gemm::RowMajor; using Col = ck_tile::tensor_layout::gemm::ColumnMajor; -// clang-format off -using KernelTypesStreamK = ::testing::Types< -// ALayout BLayout CLayout ADataType BDataType AccDataType CDataType - std::tuple< Row, Col, Row, F16, F16, F32, F16>, - std::tuple< Row, Col, Row, BF16, BF16, F32, BF16> ->; +using Mem = ck_tile::integral_constant; +using CompV3 = ck_tile::integral_constant; +using CompV4 = ck_tile::integral_constant; -// clang-format on +using Persistent = std::true_type; +using NonPersistent = std::false_type; + +using I1 = ck_tile::number<1>; +using I2 = ck_tile::number<2>; +using I4 = ck_tile::number<4>; +using I8 = ck_tile::number<8>; +using I16 = ck_tile::number<16>; +using I32 = ck_tile::number<32>; +using I64 = ck_tile::number<64>; +using I128 = ck_tile::number<128>; +using I256 = ck_tile::number<256>; + +template +struct Layouts +{ + // clang-format off + // Create all combinations of A, B, Acc, C layouts + // ALayout, BLayout, CLayout, ADataType, BDataType, AccDataType, CDataType, M_MacroTile, N_MacroTile, K_MacroTile, M_Warps, N_Warps, K_Warps, M_MmaTile, N_MmaTile, K_MmaTile, PipelineType, Persistent + using RRR = ::testing::Types>; + using RRC = ::testing::Types>; + using RCR = ::testing::Types>; + using RCC = ::testing::Types>; + using CRR = ::testing::Types>; + using CRC = ::testing::Types>; + using CCR = ::testing::Types>; + using CCC = ::testing::Types>; + // clang-format on +}; + +// clang-format off +// Here we use macros to generate a large number of parameter sets for different test configurations. +// One parameter set is intended to be be implemented per .cpp file to keep the compile time down. +// The naming convention is as follows: +// __________________________________________________ ____________________________________________________________________________________ +// | Parameter Name | | Parameter Value Type | +// using F16_RRR_Mem_128x128x32_2x2x1_32x32x16_NonPersistent = F16Layouts::RRR; +// / | \ \ \ \ \ | | | | \ \ \ \ \ \ \ \ \ +// DATA LAYOUT PIPELINE MACRO WARPS MMA PERSISTENT LAYOUT MACRO MACRO MACRO WARPS WARPS WARPS MMA MMA MMA PIPELINE PERSISTENT LAYOUT +// TYPE TYPE TILE MxNxK TILE TYPE CLASS TILE TILE TILE M N K TILE TILE TILE TYPE TYPE +// MxNxK MxNxK M N K M N K +// +// The example options for each field are: +// - DATA_TYPE: F16, BF16 +// - LAYOUT: RRR, RRC, RCR, RCC, CRR, CRC, CCR, CCC +// - PIPELINE_TYPE: Mem, CompV3, CompV4 +// - M_MACRO_TILE: 128, 256, etc +// - N_MACRO_TILE: 128, 256, etc +// - K_MACRO_TILE: 32, 64, 128, etc +// - M_WARPS: 2, 4, 1 +// - N_WARPS: 2, 1, 4 +// - K_WARPS: 1 +// - M_MMA_TILE: 32, 16 +// - N_MMA_TILE: 32, 16 +// - K_MMA_TILE: 16 +// - PERSISTENT_TYPE: NonPersistent, Persistent + +// Macro to concatenate the parameter name +// E.g. F16_RRR_Mem_128x128x32_2x2x1_32x32x16_NonPersistent +#define CONCATENATE_PARAM_NAME(DATA_TYPE, LAYOUT, PIPELINE_TYPE, M_MACRO_TILE, N_MACRO_TILE, K_MACRO_TILE, M_WARPS, N_WARPS, K_WARPS, M_MMA_TILE, N_MMA_TILE, K_MMA_TILE, PERSISTENT) \ + DATA_TYPE##_##LAYOUT##_##PIPELINE_TYPE##_##M_MACRO_TILE##x##N_MACRO_TILE##x##K_MACRO_TILE##_##M_WARPS##x##N_WARPS##x##K_WARPS##_##M_MMA_TILE##x##N_MMA_TILE##x##K_MMA_TILE##_##PERSISTENT + +// Macro to get the parameter value type +// E.g. F16Layouts::RRR +#define CONCATENATE_PARAM_VALUE(LAYOUTS_CLASS, M_MACRO_TILE, N_MACRO_TILE, K_MACRO_TILE, M_WARPS, N_WARPS, K_WARPS, M_MMA_TILE, N_MMA_TILE, K_MMA_TILE, PIPELINE_TYPE, PERSISTENT, LAYOUT) \ + LAYOUTS_CLASS::LAYOUT + +// Macro to declare a single parameter set, consisting of a parameter name and value type +#define DECLARE_PARAM(LAYOUTS_CLASS, DATA_TYPE, LAYOUT, PIPELINE_TYPE, M_MACRO_TILE, N_MACRO_TILE, K_MACRO_TILE, M_WARPS, N_WARPS, K_WARPS, M_MMA_TILE, N_MMA_TILE, K_MMA_TILE, PERSISTENT) \ + using CONCATENATE_PARAM_NAME(DATA_TYPE, LAYOUT, PIPELINE_TYPE, M_MACRO_TILE, N_MACRO_TILE, K_MACRO_TILE, M_WARPS, N_WARPS, K_WARPS, M_MMA_TILE, N_MMA_TILE, K_MMA_TILE, PERSISTENT) = \ + CONCATENATE_PARAM_VALUE(LAYOUTS_CLASS, M_MACRO_TILE, N_MACRO_TILE, K_MACRO_TILE, M_WARPS, N_WARPS, K_WARPS, M_MMA_TILE, N_MMA_TILE, K_MMA_TILE, PIPELINE_TYPE, PERSISTENT, LAYOUT); + +// Macro to declare all layout combinations for a given set of parameters +#define DECLARE_PARAMS_ALL_LAYOUTS(LAYOUTS_CLASS, DATA_TYPE, PIPELINE_TYPE, M_MACRO_TILE, N_MACRO_TILE, K_MACRO_TILE, M_WARPS, N_WARPS, K_WARPS, M_MMA_TILE, N_MMA_TILE, K_MMA_TILE, PERSISTENT) \ + DECLARE_PARAM(LAYOUTS_CLASS, DATA_TYPE, RRR, PIPELINE_TYPE, M_MACRO_TILE, N_MACRO_TILE, K_MACRO_TILE, M_WARPS, N_WARPS, K_WARPS, M_MMA_TILE, N_MMA_TILE, K_MMA_TILE, PERSISTENT) \ + DECLARE_PARAM(LAYOUTS_CLASS, DATA_TYPE, RRC, PIPELINE_TYPE, M_MACRO_TILE, N_MACRO_TILE, K_MACRO_TILE, M_WARPS, N_WARPS, K_WARPS, M_MMA_TILE, N_MMA_TILE, K_MMA_TILE, PERSISTENT) \ + DECLARE_PARAM(LAYOUTS_CLASS, DATA_TYPE, RCR, PIPELINE_TYPE, M_MACRO_TILE, N_MACRO_TILE, K_MACRO_TILE, M_WARPS, N_WARPS, K_WARPS, M_MMA_TILE, N_MMA_TILE, K_MMA_TILE, PERSISTENT) \ + DECLARE_PARAM(LAYOUTS_CLASS, DATA_TYPE, RCC, PIPELINE_TYPE, M_MACRO_TILE, N_MACRO_TILE, K_MACRO_TILE, M_WARPS, N_WARPS, K_WARPS, M_MMA_TILE, N_MMA_TILE, K_MMA_TILE, PERSISTENT) \ + DECLARE_PARAM(LAYOUTS_CLASS, DATA_TYPE, CRR, PIPELINE_TYPE, M_MACRO_TILE, N_MACRO_TILE, K_MACRO_TILE, M_WARPS, N_WARPS, K_WARPS, M_MMA_TILE, N_MMA_TILE, K_MMA_TILE, PERSISTENT) \ + DECLARE_PARAM(LAYOUTS_CLASS, DATA_TYPE, CRC, PIPELINE_TYPE, M_MACRO_TILE, N_MACRO_TILE, K_MACRO_TILE, M_WARPS, N_WARPS, K_WARPS, M_MMA_TILE, N_MMA_TILE, K_MMA_TILE, PERSISTENT) \ + DECLARE_PARAM(LAYOUTS_CLASS, DATA_TYPE, CCR, PIPELINE_TYPE, M_MACRO_TILE, N_MACRO_TILE, K_MACRO_TILE, M_WARPS, N_WARPS, K_WARPS, M_MMA_TILE, N_MMA_TILE, K_MMA_TILE, PERSISTENT) \ + DECLARE_PARAM(LAYOUTS_CLASS, DATA_TYPE, CCC, PIPELINE_TYPE, M_MACRO_TILE, N_MACRO_TILE, K_MACRO_TILE, M_WARPS, N_WARPS, K_WARPS, M_MMA_TILE, N_MMA_TILE, K_MMA_TILE, PERSISTENT) + +#include "test_gemm_streamk_types_fp16.hpp" +#include "test_gemm_streamk_types_bf16.hpp" diff --git a/test/ck_tile/gemm_streamk/test_gemm_streamk_types_bf16.hpp b/test/ck_tile/gemm_streamk/test_gemm_streamk_types_bf16.hpp new file mode 100644 index 0000000000..07aa1e0f04 --- /dev/null +++ b/test/ck_tile/gemm_streamk/test_gemm_streamk_types_bf16.hpp @@ -0,0 +1,76 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#pragma once + +#include "test_gemm_streamk_types.hpp" + +template +struct BF16Layouts +{ + // clang-format off + // For CDNA, we support [A, B, Acc, C] = [bf16, bf16, f32, bf16] and [bf16, bf16, f32, f32]: + using BF16_BF16_F32_BF16 = Layouts; + using BF16_BF16_F32_F32 = Layouts; + using RRR = detail::combine_t; + using RRC = detail::combine_t; + using RCR = detail::combine_t; + using RCC = detail::combine_t; + using CRR = detail::combine_t; + using CRC = detail::combine_t; + using CCR = detail::combine_t; + using CCC = detail::combine_t; + // clang-format on +}; +// clang-format off + +// Macro to declare all layout combinations for BF16 data type +#define DECLARE_BF16_PARAMS_ALL_LAYOUTS(PIPELINE_TYPE, M_MACRO_TILE, N_MACRO_TILE, K_MACRO_TILE, M_WARPS, N_WARPS, K_WARPS, M_MMA_TILE, N_MMA_TILE, K_MMA_TILE, PERSISTENT) \ + DECLARE_PARAMS_ALL_LAYOUTS(BF16Layouts, BF16, PIPELINE_TYPE, M_MACRO_TILE, N_MACRO_TILE, K_MACRO_TILE, M_WARPS, N_WARPS, K_WARPS, M_MMA_TILE, N_MMA_TILE, K_MMA_TILE, PERSISTENT) + +// Macro to declare all layout combinations for BF16 data type and a variety of sizes +#define DECLARE_BF16_PARAMS_ALL_LAYOUTS_ALL_SIZES(PIPELINE_TYPE, PERSISTENT) \ + DECLARE_BF16_PARAMS_ALL_LAYOUTS(PIPELINE_TYPE, 128, 128, 32, 2, 2, 1, 32, 32, 16, PERSISTENT) \ + DECLARE_BF16_PARAMS_ALL_LAYOUTS(PIPELINE_TYPE, 128, 128, 64, 2, 2, 1, 32, 32, 16, PERSISTENT) \ + DECLARE_BF16_PARAMS_ALL_LAYOUTS(PIPELINE_TYPE, 128, 128, 128, 2, 2, 1, 32, 32, 16, PERSISTENT) \ + DECLARE_BF16_PARAMS_ALL_LAYOUTS(PIPELINE_TYPE, 256, 128, 32, 2, 2, 1, 32, 32, 16, PERSISTENT) \ + DECLARE_BF16_PARAMS_ALL_LAYOUTS(PIPELINE_TYPE, 256, 128, 64, 2, 2, 1, 32, 32, 16, PERSISTENT) \ + DECLARE_BF16_PARAMS_ALL_LAYOUTS(PIPELINE_TYPE, 128, 256, 32, 2, 2, 1, 32, 32, 16, PERSISTENT) \ + DECLARE_BF16_PARAMS_ALL_LAYOUTS(PIPELINE_TYPE, 128, 256, 64, 2, 2, 1, 32, 32, 16, PERSISTENT) \ + DECLARE_BF16_PARAMS_ALL_LAYOUTS(PIPELINE_TYPE, 256, 256, 32, 2, 2, 1, 32, 32, 16, PERSISTENT) \ + DECLARE_BF16_PARAMS_ALL_LAYOUTS(PIPELINE_TYPE, 256, 256, 64, 2, 2, 1, 32, 32, 16, PERSISTENT) + +// Declare all BF16 parameter sets for different pipeline types and persistence options +DECLARE_BF16_PARAMS_ALL_LAYOUTS_ALL_SIZES(Mem, NonPersistent) +DECLARE_BF16_PARAMS_ALL_LAYOUTS_ALL_SIZES(CompV3, NonPersistent) +DECLARE_BF16_PARAMS_ALL_LAYOUTS_ALL_SIZES(CompV4, NonPersistent) + +// Here, we have a combination of parameter set symbols that we can use to compile into test cases +// __________________________________________________ +// | Parameter Name | +// using BF16_RRR_Mem_128x128x32_2x2x1_32x32x16_NonPersistent = ... +// / | \ \ \ \ \ +// DATA LAYOUT PIPELINE MACRO WARPS MMA PERSISTENT +// TYPE TYPE TILE MxNxK TILE TYPE +// MxNxK MxNxK +// +// The options for each field are: +// - DATA TYPE: BF16 +// - LAYOUT: RRR, RRC, RCR, RCC, CRR, CRC, CCR, CCC +// - PIPELINE_TYPE: Mem, CompV3, CompV4 +// - Macro Tile: 128x128x32, 128x128x64, 128x128x128, 256x128x32, 256x128x64, 128x256x32, 128x256x64, 256x256x32, 256x256x64 +// - Warps: 2x2x1 +// - MMA Tile: 32x32x16 +// - PERSISTENT_TYPE: NonPersistent + +// clang-format on diff --git a/test/ck_tile/gemm_streamk/test_gemm_streamk_types_fp16.hpp b/test/ck_tile/gemm_streamk/test_gemm_streamk_types_fp16.hpp new file mode 100644 index 0000000000..80dfdf99b3 --- /dev/null +++ b/test/ck_tile/gemm_streamk/test_gemm_streamk_types_fp16.hpp @@ -0,0 +1,77 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#pragma once + +#include "test_gemm_streamk_types.hpp" + +template +struct F16Layouts +{ + // clang-format off + // For CDNA, we support [A, B, Acc, C] = [f16, f16, f32, f16] and [f16, f16, f32, f32]: + using F16_F16_F32_F16 = Layouts; + using F16_F16_F32_F32 = Layouts; + using RRR = detail::combine_t; + using RRC = detail::combine_t; + using RCR = detail::combine_t; + using RCC = detail::combine_t; + using CRR = detail::combine_t; + using CRC = detail::combine_t; + using CCR = detail::combine_t; + using CCC = detail::combine_t; + // clang-format on +}; + +// clang-format off + +// Macro to declare all layout combinations for FP16 data type +#define DECLARE_F16_PARAMS_ALL_LAYOUTS(PIPELINE_TYPE, M_MACRO_TILE, N_MACRO_TILE, K_MACRO_TILE, M_WARPS, N_WARPS, K_WARPS, M_MMA_TILE, N_MMA_TILE, K_MMA_TILE, PERSISTENT) \ + DECLARE_PARAMS_ALL_LAYOUTS(F16Layouts, F16, PIPELINE_TYPE, M_MACRO_TILE, N_MACRO_TILE, K_MACRO_TILE, M_WARPS, N_WARPS, K_WARPS, M_MMA_TILE, N_MMA_TILE, K_MMA_TILE, PERSISTENT) + +// Macro to declare all layout combinations for FP16 data type and a variety of sizes +#define DECLARE_F16_PARAMS_ALL_LAYOUTS_ALL_SIZES(PIPELINE_TYPE, PERSISTENT) \ + DECLARE_F16_PARAMS_ALL_LAYOUTS(PIPELINE_TYPE, 128, 128, 32, 2, 2, 1, 32, 32, 16, PERSISTENT) \ + DECLARE_F16_PARAMS_ALL_LAYOUTS(PIPELINE_TYPE, 128, 128, 64, 2, 2, 1, 32, 32, 16, PERSISTENT) \ + DECLARE_F16_PARAMS_ALL_LAYOUTS(PIPELINE_TYPE, 128, 128, 128, 2, 2, 1, 32, 32, 16, PERSISTENT) \ + DECLARE_F16_PARAMS_ALL_LAYOUTS(PIPELINE_TYPE, 256, 128, 32, 2, 2, 1, 32, 32, 16, PERSISTENT) \ + DECLARE_F16_PARAMS_ALL_LAYOUTS(PIPELINE_TYPE, 256, 128, 64, 2, 2, 1, 32, 32, 16, PERSISTENT) \ + DECLARE_F16_PARAMS_ALL_LAYOUTS(PIPELINE_TYPE, 128, 256, 32, 2, 2, 1, 32, 32, 16, PERSISTENT) \ + DECLARE_F16_PARAMS_ALL_LAYOUTS(PIPELINE_TYPE, 128, 256, 64, 2, 2, 1, 32, 32, 16, PERSISTENT) \ + DECLARE_F16_PARAMS_ALL_LAYOUTS(PIPELINE_TYPE, 256, 256, 32, 2, 2, 1, 32, 32, 16, PERSISTENT) \ + DECLARE_F16_PARAMS_ALL_LAYOUTS(PIPELINE_TYPE, 256, 256, 64, 2, 2, 1, 32, 32, 16, PERSISTENT) + +// Declare all FP16 parameter sets for different pipeline types and persistence options +DECLARE_F16_PARAMS_ALL_LAYOUTS_ALL_SIZES(Mem, NonPersistent) +DECLARE_F16_PARAMS_ALL_LAYOUTS_ALL_SIZES(CompV3, NonPersistent) +DECLARE_F16_PARAMS_ALL_LAYOUTS_ALL_SIZES(CompV4, NonPersistent) + +// Here, we have a combination of parameter set symbols that we can use to compile into test cases +// __________________________________________________ +// | Parameter Name | +// using F16_RRR_Mem_128x128x32_2x2x1_32x32x16_NonPersistent = ... +// / | \ \ \ \ \ +// DATA LAYOUT PIPELINE MACRO WARPS MMA PERSISTENT +// TYPE TYPE TILE MxNxK TILE TYPE +// MxNxK MxNxK +// +// The options for each field are: +// - DATA TYPE: F16 +// - LAYOUT: RRR, RRC, RCR, RCC, CRR, CRC, CCR, CCC +// - PIPELINE_TYPE: Mem, CompV3, CompV4 +// - Macro Tile: 128x128x32, 128x128x64, 128x128x128, 256x128x32, 256x128x64, 128x256x32, 128x256x64, 256x256x32, 256x256x64 +// - Warps: 2x2x1 +// - MMA Tile: 32x32x16 +// - PERSISTENT_TYPE: NonPersistent + +// clang-format on diff --git a/test/ck_tile/gemm_streamk/test_gemm_streamk_util.hpp b/test/ck_tile/gemm_streamk/test_gemm_streamk_util.hpp index b8a55b024d..1384bfc35b 100644 --- a/test/ck_tile/gemm_streamk/test_gemm_streamk_util.hpp +++ b/test/ck_tile/gemm_streamk/test_gemm_streamk_util.hpp @@ -1,6 +1,8 @@ // Copyright © Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT +#pragma once + #include #include #include @@ -37,246 +39,75 @@ auto calculate_rtol_atol(const ck_tile::index_t K, return ck_tile::make_tuple(std::max(rtol, rtol_split_k), std::max(atol, atol_split_k)); } -template -class TestCkTileStreamK : public ::testing::Test +enum struct GemmPipelineType { - protected: - using ALayout = std::tuple_element_t<0, Tuple>; - using BLayout = std::tuple_element_t<1, Tuple>; - using CLayout = std::tuple_element_t<2, Tuple>; - using ADataType = std::tuple_element_t<3, Tuple>; - using BDataType = std::tuple_element_t<4, Tuple>; - using AccDataType = std::tuple_element_t<5, Tuple>; - using CDataType = std::tuple_element_t<6, Tuple>; - using DsLayout = ck_tile::tuple<>; - using DsDataType = ck_tile::tuple<>; - - template - void invoke_streamk(const ck_tile::StreamKHostArgs& args, - const ck_tile::stream_config& s, - int num_cu, - int occupancy) - { - - constexpr ck_tile::index_t M_Tile = 128; - constexpr ck_tile::index_t N_Tile = 128; - constexpr ck_tile::index_t K_Tile = 32; - - constexpr ck_tile::index_t M_Warp = 2; - constexpr ck_tile::index_t N_Warp = 2; - constexpr ck_tile::index_t K_Warp = 1; - - constexpr ck_tile::index_t M_Warp_Tile = 32; - constexpr ck_tile::index_t N_Warp_Tile = 32; - constexpr ck_tile::index_t K_Warp_Tile = 16; - - constexpr bool kPadM = PadM; - constexpr bool kPadN = PadN; - constexpr bool kPadK = PadK; - constexpr bool preshuffle = Preshuffle; - - constexpr bool DoubleSmemBuffer = false; - constexpr int kBlockPerCu = 1; - constexpr bool StructuredSparsity = false; - constexpr bool NumWaveGroup = 1; - - using GemmShape = - ck_tile::TileGemmShape, - ck_tile::sequence, - ck_tile::sequence>; - - using TilePartitioner = ck_tile::StreamKTilePartitioner; - - using GemmUniversalTraits = ck_tile::TileGemmUniversalTraits; - - const auto Run = [&](const auto memory_operation_) { - constexpr auto memory_operation = memory_operation_.value; - constexpr auto scheduler = ck_tile::GemmPipelineScheduler::Intrawave; - - // We create the GEMM pipeline without specifying has_hot_loop or tail_num. - // This is because num_loop can vary (a) per WG and (b) per iteration of the Stream-K - // while loop. Instead, has_hot_loop and tail_num are determined in the Stream-K - // Kernel's RunGemm function. This is a similar pattern used by grouped GEMM. - using UniversalGemmProblem = ck_tile::UniversalGemmPipelineProblem; - // For initial testing, we will just test with one pipeline. - // More extensive testing is coming later and will test other pipelines. - using GemmPipeline = ck_tile::GemmPipelineAgBgCrMem; - - using GemmEpilogue = ck_tile::CShuffleEpilogue< - ck_tile::CShuffleEpilogueProblem, - AccDataType, - CDataType, - ck_tile::tuple<>, - CLayout, - ck_tile::element_wise::PassThrough, - TilePartitioner::MPerBlock, - TilePartitioner::NPerBlock, - M_Warp, - N_Warp, - M_Warp_Tile, - N_Warp_Tile, - K_Warp_Tile, - UniversalGemmProblem::TransposeC, - memory_operation>>; - - using Kernel = ck_tile::StreamKKernel; - - auto kargs = Kernel::MakeKernelArgs(args, num_cu, occupancy); - - if(!Kernel::IsSupportedArgument(kargs)) - { - EXPECT_TRUE(false); - } - - dim3 grid_dims = Kernel::GridSize(kargs.tile_partitioner); - dim3 block_dims = Kernel::BlockSize(); - - ck_tile::launch_kernel( - s, ck_tile::make_kernel(Kernel{}, grid_dims, block_dims, 0, kargs)); - }; - - Run(ck_tile::integral_constant{}); - } - - public: - // Since Stream-K is build on gfx9, the lower bound for CUs is 104. Thus, we default num_cu to - // 104 and occupancy to 1 to ensure tests are reproducible on different architectures. - void Run(ck_tile::index_t M, - ck_tile::index_t N, - ck_tile::index_t K, - uint32_t num_sk_blocks = 0xffffffff, - ck_tile::StreamKReductionStrategy reduction_strategy = - ck_tile::StreamKReductionStrategy::Atomic, - int occupancy = 1, - int num_cu = 104, - ck_tile::index_t stride_A = 0, - ck_tile::index_t stride_B = 0, - ck_tile::index_t stride_C = 0) - { - - using namespace ck_tile::literals; - - if(reduction_strategy == ck_tile::StreamKReductionStrategy::Reduction) - { - throw std::runtime_error("Reduction Strategy is current unsupported!\n"); - } - - auto f_host_tensor_descriptor = [](std::size_t row, - std::size_t col, - std::size_t stride, - auto layout) { - if constexpr(std::is_same_v) - { - return ck_tile::HostTensorDescriptor({row, col}, {stride, 1_uz}); - } - else - { - return ck_tile::HostTensorDescriptor({row, col}, {1_uz, stride}); - } - }; - - auto f_get_default_stride = - [](std::size_t row, std::size_t col, std::size_t stride, auto layout) { - if(stride == 0) - { - if constexpr(std::is_same_v) - { - return col; - } - else - { - return row; - } - } - else - return stride; - }; - - stride_A = f_get_default_stride(M, K, stride_A, ALayout{}); - stride_B = f_get_default_stride(K, N, stride_B, BLayout{}); - stride_C = f_get_default_stride(M, N, stride_C, CLayout{}); - - ck_tile::HostTensor a_m_k(f_host_tensor_descriptor(M, K, stride_A, ALayout{})); - ck_tile::HostTensor b_k_n(f_host_tensor_descriptor(K, N, stride_B, BLayout{})); - ck_tile::HostTensor c_m_n_dev_result( - f_host_tensor_descriptor(M, N, stride_C, CLayout{})); - - ck_tile::FillUniformDistributionIntegerValue{-5, 5, /*seed*/ 11939}(a_m_k); - ck_tile::FillUniformDistributionIntegerValue{-5, 5, /*seed*/ 11940}(b_k_n); - - ck_tile::DeviceMem a_m_k_dev_buf(a_m_k.get_element_space_size_in_bytes()); - ck_tile::DeviceMem b_k_n_dev_buf(b_k_n.get_element_space_size_in_bytes()); - ck_tile::DeviceMem c_m_n_dev_buf(c_m_n_dev_result.get_element_space_size_in_bytes()); - - a_m_k_dev_buf.ToDevice(a_m_k.data()); - b_k_n_dev_buf.ToDevice(b_k_n.data()); - c_m_n_dev_buf.SetZero(); - c_m_n_dev_result.SetZero(); - - ck_tile::StreamKHostArgs args{a_m_k_dev_buf.GetDeviceBuffer(), - b_k_n_dev_buf.GetDeviceBuffer(), - c_m_n_dev_buf.GetDeviceBuffer(), - M, - N, - K, - stride_A, - stride_B, - stride_C, - reduction_strategy, - num_sk_blocks}; - - invoke_streamk( - args, ck_tile::stream_config{nullptr, false, 0, 0, 1}, num_cu, occupancy); - - c_m_n_dev_buf.FromDevice(c_m_n_dev_result.data()); - - ck_tile::HostTensor c_m_n_host_ref( - f_host_tensor_descriptor(M, N, stride_C, CLayout{})); - c_m_n_host_ref.SetZero(); - - ck_tile::reference_gemm( - a_m_k, b_k_n, c_m_n_host_ref); - - const float max_accumulated_value = - *std::max_element(c_m_n_host_ref.mData.begin(), c_m_n_host_ref.mData.end()); - const auto rtol_atol = calculate_rtol_atol( - K, /*kbatch*/ 1, max_accumulated_value); - - bool pass = ck_tile::check_err(c_m_n_dev_result, - c_m_n_host_ref, - "Error: Incorrect results!", - rtol_atol.at(ck_tile::number<0>{}), - rtol_atol.at(ck_tile::number<1>{})); - - EXPECT_TRUE(pass); - }; + Mem, + CompV3, + CompV4 }; + +template +struct GemmPipelineTypeSelector; + +template +struct GemmPipelineTypeSelector +{ + using base_pipeline = ck_tile::BaseGemmPipelineAgBgCrMem; + using pipeline = ck_tile::GemmPipelineAgBgCrMem; + + static constexpr auto GetName() { return "GemmPipelineAgBgCrMem"; } +}; + +template +struct GemmPipelineTypeSelector +{ + using base_pipeline = ck_tile::BaseGemmPipelineAgBgCrCompV3; + using pipeline = ck_tile::GemmPipelineAgBgCrCompV3; + + static constexpr auto GetName() { return "GemmPipelineAgBgCrCompV3"; } +}; + +template +struct GemmPipelineTypeSelector +{ + using base_pipeline = ck_tile::BaseGemmPipelineAgBgCrCompV4; + using pipeline = ck_tile::GemmPipelineAgBgCrCompV4; + + static constexpr auto GetName() { return "GemmPipelineAgBgCrCompV4"; } +}; + +namespace detail { +template +struct combine; + +template +struct combine<::testing::Types, ::testing::Types> +{ + using type = ::testing::Types; +}; + +template +using combine_t = typename combine::type; +} // namespace detail + +// This is the base class for all stream-k tests +#define STREAM_K_TEST_CLASS_BASE TestCkTileStreamK + +// Macros to help generate test suite names from the parameters given +#define CONCATENATE_TEST_SUITE_NAME(PREFIX, TEST_PARAMS) PREFIX##_##TEST_PARAMS +// Helper macro to expand the arguments before passing them to CONCATENATE_TEST_SUITE_NAME +#define MAKE_TEST_SUITE_NAME_INTERNAL(TEST_BASE_NAME, TEST_PARAMS) \ + CONCATENATE_TEST_SUITE_NAME(TEST_BASE_NAME, TEST_PARAMS) + +// Final macro to be used to create the test suite name from the base class name and the test +// parameters +#define MAKE_TEST_SUITE_NAME(TEST_PARAMS) \ + MAKE_TEST_SUITE_NAME_INTERNAL(STREAM_K_TEST_CLASS_BASE, TEST_PARAMS) + +// Macro to declare a test suite with the given name and parameters, based on the base test class +#define DECLARE_STREAM_K_TEST(TEST_SUITE_NAME, TEST_SUITE_PARAMS) \ + template \ + class TEST_SUITE_NAME : public STREAM_K_TEST_CLASS_BASE \ + { \ + }; \ + TYPED_TEST_SUITE(TEST_SUITE_NAME, TEST_SUITE_PARAMS); From 95bdc7410c99096652618759ff2ef3586951a0d0 Mon Sep 17 00:00:00 2001 From: Yi DING Date: Mon, 13 Oct 2025 15:03:46 +0800 Subject: [PATCH 010/262] [CK_TILE] FMHA BWD Add Instance for D48 on GFX950 (#2866) Co-authored-by: asleepzzz --- example/ck_tile/01_fmha/codegen/ops/fmha_bwd.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/example/ck_tile/01_fmha/codegen/ops/fmha_bwd.py b/example/ck_tile/01_fmha/codegen/ops/fmha_bwd.py index 7319ef7ea1..059be0e490 100644 --- a/example/ck_tile/01_fmha/codegen/ops/fmha_bwd.py +++ b/example/ck_tile/01_fmha/codegen/ops/fmha_bwd.py @@ -388,8 +388,12 @@ def get_dq_dk_dv_tiles(dtype : str, tr_load: str) -> List[FmhaBwdDQDKDVTileSize] ] elif (dtype == 'fp16' or dtype == 'bf16') and tr_load == 't': return [ + FmhaBwdDQDKDVTileSize( 32, 128, 64, 32, 64, 32, 32, 64, 64, 1, 4, 1, 4, 1, 1, 1, 4, 1, 16, 16, 32, 16, 16, 32, 1), FmhaBwdDQDKDVTileSize( 32, 128, 128, 32, 128, 32, 32, 128, 128, 1, 4, 1, 4, 1, 1, 1, 4, 1, 16, 16, 32, 16, 16, 32, 1), FmhaBwdDQDKDVTileSize( 16, 192, 128, 16, 128, 16, 32, 128, 128, 1, 4, 1, 4, 1, 1, 1, 4, 1, 16, 16, 32, 16, 16, 16, 1), + + # FmhaBwdDQDKDVTileSize( 32, 32, 64, 32, 64, 32, 32, 64, 64, 1, 1, 1, 1, 1, 1, 1, 1, 1, 16, 16, 32, 16, 16, 32, 1, 32), + FmhaBwdDQDKDVTileSize( 32, 16, 64, 32, 64, 32, 16, 64, 64, 1, 1, 1, 1, 1, 1, 1, 1, 1, 16, 16, 32, 16, 16, 16, 2, 32), # FmhaBwdDQDKDVTileSize( 16, 32, 128, 16, 128, 16, 32, 128, 128, 1, 1, 1, 1, 1, 1, 1, 1, 1, 16, 16, 32, 16, 16, 16, 1, 16), FmhaBwdDQDKDVTileSize( 16, 16, 128, 16, 128, 16, 16, 128, 128, 1, 1, 1, 1, 1, 1, 1, 1, 1, 16, 16, 32, 16, 16, 16, 2, 16), ] @@ -812,7 +816,9 @@ def get_bwd_blobs(filter_list: str, receipt, mask_impl, optdim_list) -> Tuple[Fm if ("wg32" in dropout): continue if tr_load == "t": - continue # tr_load cannot work with dpad or dvpad + # tr_load can only work with 8 pad + if dpad != dvpad or dpad == 1: + continue else: # tr_load == "f" # do not generate instance with only 1 of dpad/dvpad being 8 if dpad != dvpad and dpad == 8: From e9f0cc83a8f3f94ad8462e50a9d9a92d8dca3388 Mon Sep 17 00:00:00 2001 From: msaffari-amd Date: Mon, 13 Oct 2025 12:30:28 +0200 Subject: [PATCH 011/262] [CK Tile] contraction multi d - kernel & example (#2901) * Initial commit. create batched_contraction_kernel file * initial problem definition * implement initial example to launch kernel * add universal gemm to contraction. initial phase * complete implementation for special case all Dims are 1 and no Ds * clean code * initial changes to support multi dimensional G * more progress in implementing multiple G * tmp commit * manage dynamic NumDimG in kernel * improving example for multi M,N,K,G handling. start generalizing kernel. it is a temporary commit * implement the example for general Multi dimension G M N K and test different reference calculation algorithms * 2 functions for reference using multi dimensional and flat indexing * clean the code for muti dimentional G, M, N, K contraction and add some logs * Add Make descriptor function in kernel for merging Ms, Ns, Ks for A, B, E * some cleaning on kernel * clean the code for calculating the offsets from flatten batch number * Start adding MultiD support to kernel and example * more changes to manage multi D in kernel and example * manage passing multi d to kernel and testing. * complete multi D support in kernel. modify example code to support it * Correct algorithm to calc the correct offset values for D tensor batches and some code cleaning * Minor fix * Generalize example code for variable NumD tensors and apply cleanup based on review feedback * Refactored code and addressed review feedback * refactoring, cleaning, add documents, in kernel side and example codes * Optimize batch offset calculation in kernel * Inline CalculateBatchOffset in batched contraction kernel, update CHANGELOG.md --------- Co-authored-by: Adam Osewski <19374865+aosewski@users.noreply.github.com> --- CHANGELOG.md | 1 + .../41_batched_contraction/CMakeLists.txt | 7 + .../batched_contraction.cpp | 245 ++++++++ .../contraction_utils.hpp | 146 +++++ .../run_batched_contraction_example.inc | 405 ++++++++++++++ example/ck_tile/CMakeLists.txt | 1 + .../reference_batched_contraction.hpp | 265 +++++++++ include/ck_tile/ops/batched_contraction.hpp | 9 + .../kernel/batched_contraction_kernel.hpp | 522 ++++++++++++++++++ .../pipeline/batched_contraction_problem.hpp | 32 ++ .../utils/tensor_descriptor_utils.hpp | 169 ++++++ 11 files changed, 1802 insertions(+) create mode 100644 example/ck_tile/41_batched_contraction/CMakeLists.txt create mode 100644 example/ck_tile/41_batched_contraction/batched_contraction.cpp create mode 100644 example/ck_tile/41_batched_contraction/contraction_utils.hpp create mode 100644 example/ck_tile/41_batched_contraction/run_batched_contraction_example.inc create mode 100644 include/ck_tile/host/reference/reference_batched_contraction.hpp create mode 100644 include/ck_tile/ops/batched_contraction.hpp create mode 100644 include/ck_tile/ops/batched_contraction/kernel/batched_contraction_kernel.hpp create mode 100644 include/ck_tile/ops/batched_contraction/pipeline/batched_contraction_problem.hpp create mode 100644 include/ck_tile/ops/batched_contraction/utils/tensor_descriptor_utils.hpp diff --git a/CHANGELOG.md b/CHANGELOG.md index a8fe7b4afb..9de78f3043 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,7 @@ Documentation for Composable Kernel available at [https://rocm.docs.amd.com/proj * Added the row-wise column-wise quantization for CK_TILE GEMM & CK_TILE Grouped GEMM. * Added support for f32 to FMHA (fwd/bwd). * Added tensor-wise quantization for CK_TILE GEMM. +* Added support for batched contraction kernel. * Added pooling kernel in CK_TILE ### Optimized diff --git a/example/ck_tile/41_batched_contraction/CMakeLists.txt b/example/ck_tile/41_batched_contraction/CMakeLists.txt new file mode 100644 index 0000000000..10b2e48cbf --- /dev/null +++ b/example/ck_tile/41_batched_contraction/CMakeLists.txt @@ -0,0 +1,7 @@ +add_executable(tile_example_batched_contraction EXCLUDE_FROM_ALL batched_contraction.cpp) +set(EXAMPLE_CONTRACTION_COMPILE_OPTIONS) +if(CK_USE_OCP_FP8) + list(APPEND EXAMPLE_CONTRACTION_COMPILE_OPTIONS -DCK_TILE_USE_OCP_FP8) +endif() + +target_compile_options(tile_example_batched_contraction PRIVATE ${EXAMPLE_CONTRACTION_COMPILE_OPTIONS}) diff --git a/example/ck_tile/41_batched_contraction/batched_contraction.cpp b/example/ck_tile/41_batched_contraction/batched_contraction.cpp new file mode 100644 index 0000000000..ea78f09dff --- /dev/null +++ b/example/ck_tile/41_batched_contraction/batched_contraction.cpp @@ -0,0 +1,245 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2025, Advanced Micro Devices, Inc. All rights reserved. + +#include + +#include +#include +#include +#include +#include + +#include "ck_tile/core.hpp" +#include "ck_tile/ops/epilogue.hpp" +#include "ck_tile/ops/gemm.hpp" +#include "ck_tile/host.hpp" + +#include "ck_tile/ops/batched_contraction.hpp" +#include "contraction_utils.hpp" + +template + +float batched_contraction_impl(const ck_tile::BatchedContractionHostArgs& args, + const ck_tile::stream_config& s) +{ + constexpr ck_tile::index_t M_Tile = 256; + constexpr ck_tile::index_t N_Tile = 256; + constexpr ck_tile::index_t K_Tile = 64; + + constexpr ck_tile::index_t M_Warp = 2; + constexpr ck_tile::index_t N_Warp = 2; + constexpr ck_tile::index_t K_Warp = 1; + + constexpr ck_tile::index_t M_Warp_Tile = 32; + constexpr ck_tile::index_t N_Warp_Tile = 32; + constexpr ck_tile::index_t K_Warp_Tile = 16; + + constexpr bool DoubleSmemBuffer = false; + + constexpr bool kPadM = false; + constexpr bool kPadN = false; + constexpr bool kPadK = false; + + constexpr bool TransposeC = false; + + constexpr int kBlockPerCu = 1; + constexpr ck_tile::index_t TileParitionerGroupNum = 8; + constexpr ck_tile::index_t TileParitionerM01 = 4; + + using GemmShape = + ck_tile::TileGemmShape, + ck_tile::sequence, + ck_tile::sequence>; + using TilePartitioner = ck_tile:: + GemmSpatiallyLocalTilePartitioner; + + using Traits = ck_tile::TileGemmTraits; + using GemmUniversalTraits = ck_tile::TileGemmUniversalTraits; + + using Problem = ck_tile::BatchedContractionProblem; + + using GemmPipelineProblem = + ck_tile::GemmPipelineProblem; + + using BaseGemmPipeline = UNIVERSAL_GEMM_PIPELINE; + + ck_tile::index_t K_total = 1; + for(ck_tile::index_t i = NumDimG + NumDimM; i < NumDimG + NumDimM + NumDimK; ++i) + { + K_total *= args.A_dims[i]; + } + + const ck_tile::index_t num_loop = TilePartitioner::GetLoopNum(K_total); + const bool has_hot_loop = BaseGemmPipeline::BlockHasHotloop(num_loop); + const ck_tile::TailNumber tail_num = BaseGemmPipeline::GetBlockLoopTailNum(num_loop); + + float ave_time{0}; + + const auto Run = [&](const auto has_hot_loop_, const auto tail_number_) { + constexpr bool has_hot_loop_v = has_hot_loop_.value; + constexpr auto tail_number_v = tail_number_.value; + constexpr auto scheduler = GEMM_PIPELINE_SCHEDULER; + constexpr auto memory_operation = + ck_tile::memory_operation_enum::set; // Always set (no atomic_add) + + using UniversalGemmProblem = ck_tile::UniversalGemmPipelineProblem; + + using GemmPipeline = GEMM_PIPELINE; + + using GemmEpilogue = ck_tile::CShuffleEpilogue< + ck_tile::CShuffleEpilogueProblem>; + + using Kernel = + ck_tile::BatchedContractionKernel; + auto kargs = Kernel::MakeKernelArgs(args); + + const dim3 grids = Kernel::GridSize(kargs); + const dim3 blocks = Kernel::GetBlockSize(); + + if(!Kernel::IsSupportedArguments(kargs)) + { + throw std::runtime_error("Wrong! Arguments not supported! Skipping contraction!\n"); + } + + if(s.log_level_ > 0) + { + std::cout << "Launching kernel with args: " << Kernel::GetKernelName() << '\n' + << "shape: " << GemmShape::GetName() << '\n' + << "problem: " << GemmPipelineProblem::GetName() << '\n' + << "pipeline: " << GemmPipeline::GetName() << '\n' + << "grid: {" << grids.x << ", " << grids.y << ", " << grids.z << "}" + << ", blocks: {" << blocks.x << ", " << blocks.y << ", " << blocks.z << "}" + << std::endl; + } + + auto kernel = ck_tile::make_kernel(Kernel{}, grids, blocks, 0, kargs); + + ave_time = ck_tile::launch_kernel(s, kernel); + + return ave_time; + }; + + BaseGemmPipeline::TailHandler(Run, has_hot_loop, tail_num); + + return ave_time; +} + +#define HANDLE_CASE(G, M, N, K) \ + if(num_g_dims == G && num_m_dims == M && num_n_dims == N && num_k_dims == K) \ + { \ + return batched_contraction_impl(args, s); \ + } + +template +float batched_contraction(const ck_tile::BatchedContractionHostArgs& args, + const ck_tile::stream_config& s, + ck_tile::index_t num_g_dims, + ck_tile::index_t num_m_dims, + ck_tile::index_t num_n_dims, + ck_tile::index_t num_k_dims) +{ + std::cout << "Dimensions: G=" << num_g_dims << ", M=" << num_m_dims << ", N=" << num_n_dims + << ", K=" << num_k_dims << std::endl; + + HANDLE_CASE(1, 1, 1, 1); + HANDLE_CASE(2, 1, 1, 1); + HANDLE_CASE(2, 2, 2, 1); + HANDLE_CASE(1, 2, 1, 1); + HANDLE_CASE(1, 1, 1, 2); + HANDLE_CASE(2, 2, 2, 2); + HANDLE_CASE(4, 4, 4, 4); + + throw std::runtime_error( + "Unsupported dimension combination: G=" + std::to_string(num_g_dims) + + ", M=" + std::to_string(num_m_dims) + ", N=" + std::to_string(num_n_dims) + + ", K=" + std::to_string(num_k_dims) + ". Please add this combination to the kernel."); +} + +#include "run_batched_contraction_example.inc" + +int main(int argc, char* argv[]) +{ + try + { + return !run_batched_contraction_example(argc, argv); + } + catch(const std::runtime_error& e) + { + std::cerr << "Runtime error: " << e.what() << '\n'; + return EXIT_FAILURE; + } +} diff --git a/example/ck_tile/41_batched_contraction/contraction_utils.hpp b/example/ck_tile/41_batched_contraction/contraction_utils.hpp new file mode 100644 index 0000000000..6a75f1c04e --- /dev/null +++ b/example/ck_tile/41_batched_contraction/contraction_utils.hpp @@ -0,0 +1,146 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2025, Advanced Micro Devices, Inc. All rights reserved. + +#pragma once + +#include + +#include "ck_tile/core.hpp" +#include "ck_tile/host/kernel_launch.hpp" + +struct AddDs +{ + template + CK_TILE_HOST_DEVICE auto operator()(E& e, const C& c, const Ds&... ds) const -> void + { + const float x0_f = + ck_tile::type_convert(c) + (ck_tile::type_convert(ds) + ...); + + e = ck_tile::type_convert(x0_f); + } +}; + +#define GEMM_PIPELINE ck_tile::GemmPipelineAgBgCrCompV3 +#define UNIVERSAL_GEMM_PIPELINE ck_tile::BaseGemmPipelineAgBgCrCompV3 +#define GEMM_PIPELINE_SCHEDULER ck_tile::GemmPipelineScheduler::Intrawave + +template +struct BatchedContractionTypeConfig +{ + using ADataType = DataType; + using BDataType = DataType; + using AccDataType = float; + using EDataType = DataType; + using DDataType = DataType; +}; + +using ContractionTypes = BatchedContractionTypeConfig; + +using ADataType = ContractionTypes::ADataType; +using BDataType = ContractionTypes::BDataType; +using AccDataType = ContractionTypes::AccDataType; +using EDataType = ContractionTypes::EDataType; +using DDataType = ContractionTypes::DDataType; + +auto create_args(int argc, char* argv[]) +{ + ck_tile::ArgParser arg_parser; + arg_parser.insert("m_dims", "4,256", "M dimensions separated by comma (e.g., '16,32' for 2D M)") + .insert("n_dims", "16,128", "N dimensions separated by comma (e.g., '32,32' for 2D N)") + .insert("k_dims", "64", "K dimensions separated by comma (e.g., '64,32' for 2D K)") + .insert( + "g_dims", "1,2", "G dimensions separated by comma (e.g., '4,2' for 2D, '2,3,4' for 3D)") + .insert("stride_a", "0", "Custom A tensor leading dimension stride (0 = auto)") + .insert("stride_b", "0", "Custom B tensor leading dimension stride (0 = auto)") + .insert("stride_e", "0", "Custom E tensor leading dimension stride (0 = auto)") + .insert("a_layout", "R", "A tensor data layout - Row by default") + .insert("b_layout", "C", "B tensor data layout - Col by default") + .insert("e_layout", "R", "E tensor data layout - Row by default") + .insert("v", "1", "0. No validation, 1. Validation on CPU") + .insert("prec", "fp16", "data type. fp32/fp16/bf16") + .insert("warmup", "5", "number of iterations before benchmark the kernel") + .insert("repeat", "10", "number of iterations to benchmark the kernel") + .insert("timer", "gpu", "gpu:gpu timer, cpu:cpu timer") + .insert("split_k", "1", "splitK value") + .insert("log", "1", "log level for debugging"); + + bool result = arg_parser.parse(argc, argv); + return std::make_tuple(result, arg_parser); +} + +// Helper function to parse G, M, N, K dimensions from string +std::vector parse_dimensions(const std::string& dims_str) +{ + std::vector dims; + std::stringstream ss(dims_str); + std::string token; + + while(std::getline(ss, token, ',')) + { + dims.push_back(std::stoi(token)); + } + + if(dims.empty()) + { + throw std::invalid_argument("Dimensions cannot be empty"); + } + + return dims; +} + +// Helper function to Calculate total elements from multi-dimensional vector +ck_tile::index_t calculate_total_elements(const std::vector& dims) +{ + ck_tile::index_t total = 1; + for(auto dim : dims) + { + total *= dim; + } + return total; +} + +/** + * @brief Flattens a list of tensor dimension components into a single dimension vector. + * + * This function takes a list of dimension vectors (e.g., representing different components + * such as G, M, N, or K dimensions) and concatenates them into a single vector. + * + * Example: + * Input: {{G0, G1}, {M0, M1}, {K0}} + * Output: {G0, G1, M0, M1, K0} + * + * @param dim_components A vector of vectors, where each inner vector represents a set of tensor + * dimensions. + * @return A single vector containing all dimensions concatenated in order. + */ +std::vector +concatenate_dim_components(const std::vector>& dim_components) +{ + std::vector result; + + // Concatenate all dimension components into a single vector + for(const auto& component : dim_components) + { + result.insert(result.end(), component.begin(), component.end()); + } + + return result; +} + +// Helper function for printing dimensions +void print_dims(const std::string& name, + const std::vector& dims, + ck_tile::index_t total) +{ + std::cout << name << ": ["; + for(size_t i = 0; i < dims.size(); ++i) + { + std::cout << dims[i]; + if(i < dims.size() - 1) + std::cout << ","; + } + std::cout << "] "; + if(total != 0) + std::cout << "(total=" << total << ")"; + std::cout << std::endl; +} diff --git a/example/ck_tile/41_batched_contraction/run_batched_contraction_example.inc b/example/ck_tile/41_batched_contraction/run_batched_contraction_example.inc new file mode 100644 index 0000000000..9bc09a6c9c --- /dev/null +++ b/example/ck_tile/41_batched_contraction/run_batched_contraction_example.inc @@ -0,0 +1,405 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2025, Advanced Micro Devices, Inc. All rights reserved. + +#pragma once + +#include +#include +#include +#include "contraction_utils.hpp" +#include "ck_tile/host/reference/reference_batched_contraction.hpp" + +template +auto calculate_rtol_atol(const ck_tile::index_t K, + const ck_tile::index_t kbatch, + const float max_accumulated_value) +{ + using ComputeType = + std::conditional_t; + + const auto rtol = ck_tile::get_relative_threshold( + ck_tile::integer_divide_ceil(K, kbatch)); + const auto atol = ck_tile::get_absolute_threshold( + max_accumulated_value / kbatch, ck_tile::integer_divide_ceil(K, kbatch)); + + const auto rtol_split_k = + ck_tile::get_relative_threshold(kbatch); + const auto atol_split_k = ck_tile::get_absolute_threshold( + max_accumulated_value, kbatch); + + return ck_tile::make_tuple(std::max(rtol, rtol_split_k), std::max(atol, atol_split_k)); +} + +template +float invoke_batched_contraction_kernel( + const void* a_full_dims_dev_buf, + const void* b_full_dims_dev_buf, + const std::array& ds_dev_buf, + void* e_full_dims_dev_buf, + const std::vector& G_dims, + const std::vector& M_dims, + const std::vector& N_dims, + const std::vector& K_dims, + const std::vector& A_dims, // [G0,G1,..,M0,M1,..,K0,K1,..] + const std::vector& B_dims, // [G0,G1,..,N0,N1,..,K0,K1,..] + const std::array, DsDataType::size()>& + Ds_dims, // [G0, G1, ..., M0, M1, ... , N0, N1, ...][NumDTensor] + const std::vector& E_dims, // [G0,G1,..,M0,M1,..,N0,N1,..] + const std::vector& A_strides, // [G0,G1,..,M0,M1,..,K0,K1,..] + const std::vector& B_strides, // [G0,G1,..,N0,N1,..,K0,K1,..] + const std::array, DsDataType::size()>& Ds_strides, + const std::vector& E_strides, // [G0,G1,..,M0,M1,..,N0,N1,..] + ck_tile::index_t kbatch, + int n_warmup, + int n_repeat) +{ + std::cout << "Creating BatchedContractionHostArgs..." << std::endl; + + ck_tile::BatchedContractionHostArgs args(a_full_dims_dev_buf, // a_ptr + b_full_dims_dev_buf, // b_ptr + ds_dev_buf, // ds_ptr + e_full_dims_dev_buf, // e_ptr + kbatch, // k_batch + A_dims, // A_dims + B_dims, // B_dims + Ds_dims, // Ds_dims + E_dims, // E_dims + A_strides, // A_strides + B_strides, // B_strides + Ds_strides, // Ds_strides + E_strides // E_strides + ); + + std::cout << "Calling batched_contraction with dimensions: G=" << G_dims.size() + << ", M=" << M_dims.size() << ", N=" << N_dims.size() << ", K=" << K_dims.size() + << std::endl; + + float ave_time = batched_contraction( + args, + ck_tile::stream_config{nullptr, true, 1, n_warmup, n_repeat}, + G_dims.size(), // num_g_dims + M_dims.size(), // num_m_dims + N_dims.size(), // num_n_dims + K_dims.size() // num_k_dims + ); + + return ave_time; +} + +template +int run_batched_contraction_example_with_layouts( + int argc, + char* argv[], + [[maybe_unused]] const ALayout a_layout = ALayout{}, + [[maybe_unused]] const BLayout b_layout = BLayout{}, + [[maybe_unused]] const DLayout d_layout = DLayout{}, + [[maybe_unused]] const ELayout e_layout = ELayout{}) +{ + auto [result, arg_parser] = create_args(argc, argv); + if(!result) + return -1; + + std::vector G_dims = parse_dimensions(arg_parser.get_str("g_dims")); + std::vector M_dims = parse_dimensions(arg_parser.get_str("m_dims")); + std::vector N_dims = parse_dimensions(arg_parser.get_str("n_dims")); + std::vector K_dims = parse_dimensions(arg_parser.get_str("k_dims")); + + constexpr ck_tile::index_t NumDTensor = 2; + + ck_tile::index_t G_total = calculate_total_elements(G_dims); + ck_tile::index_t M_total = calculate_total_elements(M_dims); + ck_tile::index_t N_total = calculate_total_elements(N_dims); + ck_tile::index_t K_total = calculate_total_elements(K_dims); + + std::vector A_dims = + concatenate_dim_components({G_dims, M_dims, K_dims}); // [G0,G1,..,M0,M1,..,K0,K1,..] + std::vector B_dims = + concatenate_dim_components({G_dims, N_dims, K_dims}); // [G0,G1,..,N0,N1,..,K0,K1,..] + std::vector E_dims = + concatenate_dim_components({G_dims, M_dims, N_dims}); // [G0,G1,..,M0,M1,..,N0,N1,..] + + std::array, NumDTensor> Ds_dims; + for(ck_tile::index_t d = 0; d < NumDTensor; ++d) + { + Ds_dims[d] = E_dims; + } + + auto convert_strides = [](const std::vector& strides) { + std::vector converted(strides.size()); + std::copy(strides.begin(), strides.end(), converted.begin()); + return converted; + }; + + ck_tile::HostTensorDescriptor a_desc(A_dims); + ck_tile::HostTensorDescriptor b_desc(B_dims); + ck_tile::HostTensorDescriptor e_desc(E_dims); + std::array ds_descs; + for(ck_tile::index_t d = 0; d < NumDTensor; ++d) + { + ds_descs[d] = ck_tile::HostTensorDescriptor(Ds_dims[d], e_desc.get_strides()); + } + + std::vector A_strides = convert_strides(a_desc.get_strides()); + std::vector B_strides = convert_strides(b_desc.get_strides()); + std::vector E_strides = convert_strides(e_desc.get_strides()); + + std::array, NumDTensor> Ds_strides; + for(ck_tile::index_t d = 0; d < NumDTensor; ++d) + { + Ds_strides[d] = convert_strides(ds_descs[d].get_strides()); + } + + ck_tile::index_t kbatch = arg_parser.get_int("split_k"); + int n_warmup = arg_parser.get_int("warmup"); + int n_repeat = arg_parser.get_int("repeat"); + + print_dims("G_dims", G_dims, G_total); + print_dims("M_dims", M_dims, M_total); + print_dims("N_dims", N_dims, N_total); + print_dims("K_dims", K_dims, K_total); + + std::cout << "NumDTensor: " << NumDTensor << std::endl; + std::cout << "\n=== Tensor Shapes for Kernel ===" << std::endl; + print_dims("A_dims", A_dims, 0); + print_dims("B_dims", B_dims, 0); + print_dims("E_dims", E_dims, 0); + for(ck_tile::index_t d = 0; d < NumDTensor; ++d) + { + print_dims("Ds[" + std::to_string(d) + "]_dims", Ds_dims[d], 0); + } + + std::cout << "\n=== Tensor Strides ===" << std::endl; + print_dims("A_strides", A_strides, 0); + print_dims("B_strides", B_strides, 0); + print_dims("E_strides", E_strides, 0); + for(ck_tile::index_t d = 0; d < NumDTensor; ++d) + { + print_dims("Ds[" + std::to_string(d) + "]_strides", Ds_strides[d], 0); + } + + std::cout << "===============================================\n" << std::endl; + + ck_tile::HostTensor<::ADataType> a_full_dims_host(a_desc); + ck_tile::HostTensor<::BDataType> b_full_dims_host(b_desc); + ck_tile::HostTensor<::EDataType> e_full_dims_host(e_desc); + + std::vector> ds_full_dims_host; + for(int d = 0; d < NumDTensor; ++d) + { + ds_full_dims_host.emplace_back(ck_tile::HostTensor<::DDataType>(ds_descs[d])); + } + + ck_tile::FillUniformDistribution<::ADataType>{-5.f, 5.f, std::nullopt}(a_full_dims_host); + ck_tile::FillUniformDistribution<::BDataType>{-5.f, 5.f, std::nullopt}(b_full_dims_host); + + ck_tile::DeviceMem a_full_dims_dev_buf(a_full_dims_host.get_element_space_size_in_bytes()); + ck_tile::DeviceMem b_full_dims_dev_buf(b_full_dims_host.get_element_space_size_in_bytes()); + ck_tile::DeviceMem e_full_dims_dev_buf(e_full_dims_host.get_element_space_size_in_bytes()); + + a_full_dims_dev_buf.ToDevice(a_full_dims_host.data()); + b_full_dims_dev_buf.ToDevice(b_full_dims_host.data()); + + for(int d = 0; d < NumDTensor; ++d) + { + ck_tile::FillUniformDistribution<::DDataType>{-2.f, 2.f, std::nullopt}( + ds_full_dims_host[d]); + } + + std::vector> ds_full_dims_dev_buf; + for(int d = 0; d < NumDTensor; ++d) + { + ds_full_dims_dev_buf.push_back(std::make_unique( + ds_full_dims_host[d].get_element_space_size_in_bytes())); + ds_full_dims_dev_buf[d]->ToDevice(ds_full_dims_host[d].data()); + } + std::array ds_ptr_buf; + for(int d = 0; d < NumDTensor; ++d) + { + ds_ptr_buf[d] = ds_full_dims_dev_buf[d]->GetDeviceBuffer(); + } + + e_full_dims_dev_buf.SetZero(); + e_full_dims_host.SetZero(); + + std::cout << "\n=== Running GPU Kernel ===" << std::endl; + + using DsDataType = ck_tile::tuple_array<::DDataType, NumDTensor>; + using DsLayout = ck_tile::tuple_array; + using CDEElementWise = + std::conditional_t; + + float ave_time = + invoke_batched_contraction_kernel<::ADataType, + ::BDataType, + DsDataType, + ::AccDataType, + ::EDataType, + ALayout, + BLayout, + DsLayout, + ELayout, + CDEElementWise>(a_full_dims_dev_buf.GetDeviceBuffer(), + b_full_dims_dev_buf.GetDeviceBuffer(), + ds_ptr_buf, + e_full_dims_dev_buf.GetDeviceBuffer(), + G_dims, + M_dims, + N_dims, + K_dims, + A_dims, + B_dims, + Ds_dims, + E_dims, + A_strides, + B_strides, + Ds_strides, + E_strides, + kbatch, + n_warmup, + n_repeat); + + std::string op_name{ + "Multi-Dimensional Batched Contraction : G: " + std::to_string(G_dims.size()) + + "D, M: " + std::to_string(M_dims.size()) + "D, N: " + std::to_string(N_dims.size()) + + "D, K: " + std::to_string(K_dims.size()) + "D"}; + + std::size_t flop = std::size_t(2) * G_total * M_total * N_total * K_total + + NumDTensor * K_total * M_total * N_total; // Number of operations + std::size_t num_byte = + sizeof(::ADataType) * G_total * M_total * K_total + // A tensor size + sizeof(::BDataType) * G_total * N_total * K_total + // B tensor size + sizeof(::DDataType) * NumDTensor * G_total * M_total * N_total + // D tensors + sizeof(::EDataType) * G_total * M_total * N_total; // E tensor size + + float tflops = static_cast(flop) / 1.E9 / ave_time; // TFlops calculation + float gb_per_sec = num_byte / 1.E6 / ave_time; // GB/s calculation + print_dims("G_dims", G_dims, G_total); + print_dims("M_dims", M_dims, M_total); + print_dims("N_dims", N_dims, N_total); + print_dims("K_dims", K_dims, K_total); + + std::cout << " Performance: " << ave_time << " ms, " << tflops << " TFlops, " << gb_per_sec + << " GB/s" << std::endl; + + std::cout << "===============================================" << std::endl; + + e_full_dims_dev_buf.FromDevice(e_full_dims_host.data()); + std::cout << "GPU results retrieved from device." << std::endl; + + bool pass = true; + if(arg_parser.get_int("v") == 1) + { + + std::cout << "Computing CPU reference..." << std::endl; + + ck_tile::HostTensor<::EDataType> e_full_dims_host_ref( + ck_tile::HostTensorDescriptor(E_dims, E_strides)); + e_full_dims_host_ref.SetZero(); + + auto start_time = std::chrono::high_resolution_clock::now(); + + calculate_reference_flat_indexing(a_full_dims_host, + b_full_dims_host, + ds_full_dims_host, + e_full_dims_host_ref, + G_total, + M_total, + N_total, + K_total, + CDEElementWise{}); + + auto end_time = std::chrono::high_resolution_clock::now(); + auto duration = + std::chrono::duration_cast(end_time - start_time); + + std::cout << "CPU reference completed in " << duration.count() << "ms" << std::endl; + + const float max_accumulated_value = + *std::max_element(e_full_dims_host_ref.mData.begin(), e_full_dims_host_ref.mData.end()); + + const auto rtol_atol = + calculate_rtol_atol<::ADataType, ::BDataType, ::EDataType, ::AccDataType>( + K_total, kbatch, max_accumulated_value); + + pass = ck_tile::check_err(e_full_dims_host, + e_full_dims_host_ref, + "Error: Incorrect results!", + rtol_atol.at(ck_tile::number<0>{}), + rtol_atol.at(ck_tile::number<1>{})); + + std::cout << "The CPU verification result is: " << (pass ? "correct" : "fail") << std::endl; + + std::cout << "===============================================" << std::endl; + + std::cout << "\n=== Random Samples of Reference and Result ===" << std::endl; + + // Generate 10 random indices + std::vector random_indices; + std::size_t total_elements = e_full_dims_host_ref.mData.size(); + std::mt19937 rng(std::random_device{}()); + std::uniform_int_distribution dist(0, total_elements - 1); + + for(int i = 0; i < 10; ++i) + { + random_indices.push_back(dist(rng)); + } + + // Print the values at the random indices + for(std::size_t idx : random_indices) + { + std::cout << "Index " << idx << ": " + << "ref=" << static_cast(e_full_dims_host_ref.mData[idx]) << ", " + << "GPU=" << static_cast(e_full_dims_host.mData[idx]) << std::endl; + } + + std::cout << "===============================================" << std::endl; + } + + return pass; +} + +int run_batched_contraction_example(int argc, char* argv[]) +{ + auto [result, arg_parser] = create_args(argc, argv); + if(!result) + return -1; + + using Row = ck_tile::tensor_layout::gemm::RowMajor; + using Col = ck_tile::tensor_layout::gemm::ColumnMajor; + + std::string a_layout = arg_parser.get_str("a_layout"); + std::string b_layout = arg_parser.get_str("b_layout"); + + if(a_layout == "R" && b_layout == "C") + { + return run_batched_contraction_example_with_layouts(argc, argv, Row{}, Col{}, Row{}, Row{}); + } + else + { + throw std::runtime_error("Unsupported data layout configuration for A,B and E tensors! " + "Only R-C-R supported for now."); + } +} diff --git a/example/ck_tile/CMakeLists.txt b/example/ck_tile/CMakeLists.txt index 7a8ae065db..5e178e3669 100644 --- a/example/ck_tile/CMakeLists.txt +++ b/example/ck_tile/CMakeLists.txt @@ -27,3 +27,4 @@ add_subdirectory(36_pooling) add_subdirectory(38_block_scale_gemm) add_subdirectory(39_copy) add_subdirectory(40_streamk_gemm) +add_subdirectory(41_batched_contraction) diff --git a/include/ck_tile/host/reference/reference_batched_contraction.hpp b/include/ck_tile/host/reference/reference_batched_contraction.hpp new file mode 100644 index 0000000000..1ce071969c --- /dev/null +++ b/include/ck_tile/host/reference/reference_batched_contraction.hpp @@ -0,0 +1,265 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2018-2025, Advanced Micro Devices, Inc. All rights reserved. + +#pragma once + +#include +#include + +#include "ck_tile/core.hpp" +#include "ck_tile/host/host_tensor.hpp" + +namespace ck_tile { + +template + +void calculate_reference_flat_indexing( + const ck_tile::HostTensor& a_full_dims, + const ck_tile::HostTensor& b_full_dims, + const std::vector>& ds_full_dims_host, + ck_tile::HostTensor& e_full_dims_host_ref, + ck_tile::index_t G_total, + ck_tile::index_t M_total, + ck_tile::index_t N_total, + ck_tile::index_t K_total, + const CDEElementWise& cde_elementwise) +{ + std::cout << "Calculating reference using optimized flat indexing with parallel processing..." + << std::endl; + + // Parallel computation over G and M dimensions using pattern from reference_batched_gemm.hpp + auto f_gm = [&](auto g_flat, auto m_flat) { + for(ck_tile::index_t n_flat = 0; n_flat < N_total; ++n_flat) + { + AccDataType sum = 0; + + // Compute dot product over K dimension + for(ck_tile::index_t k_flat = 0; k_flat < K_total; ++k_flat) + { + auto a_val = + a_full_dims.mData[g_flat * M_total * K_total + m_flat * K_total + k_flat]; + auto b_val = + b_full_dims.mData[g_flat * N_total * K_total + n_flat * K_total + k_flat]; + sum += static_cast(a_val) * static_cast(b_val); + } + + // Apply elementwise operation with D tensors + EDataType result = static_cast(sum); + if(ds_full_dims_host.size() == 0) + { + ; + } + else if(ds_full_dims_host.size() == 1) + { + cde_elementwise(result, + ck_tile::type_convert(sum), + ck_tile::type_convert( + ds_full_dims_host[0].mData[g_flat * M_total * N_total + + m_flat * N_total + n_flat])); + } + else if(ds_full_dims_host.size() == 2) + { + cde_elementwise( + result, + ck_tile::type_convert(sum), + ck_tile::type_convert( + ds_full_dims_host[0] + .mData[g_flat * M_total * N_total + m_flat * N_total + n_flat]), + ck_tile::type_convert( + ds_full_dims_host[1] + .mData[g_flat * M_total * N_total + m_flat * N_total + n_flat])); + } + else if(ds_full_dims_host.size() == 3) + { + cde_elementwise( + result, + ck_tile::type_convert(sum), + ck_tile::type_convert( + ds_full_dims_host[0] + .mData[g_flat * M_total * N_total + m_flat * N_total + n_flat]), + ck_tile::type_convert( + ds_full_dims_host[1] + .mData[g_flat * M_total * N_total + m_flat * N_total + n_flat]), + ck_tile::type_convert( + ds_full_dims_host[2] + .mData[g_flat * M_total * N_total + m_flat * N_total + n_flat])); + } + else if(ds_full_dims_host.size() == 4) + { + cde_elementwise( + result, + ck_tile::type_convert(sum), + ck_tile::type_convert( + ds_full_dims_host[0] + .mData[g_flat * M_total * N_total + m_flat * N_total + n_flat]), + ck_tile::type_convert( + ds_full_dims_host[1] + .mData[g_flat * M_total * N_total + m_flat * N_total + n_flat]), + ck_tile::type_convert( + ds_full_dims_host[2] + .mData[g_flat * M_total * N_total + m_flat * N_total + n_flat]), + ck_tile::type_convert( + ds_full_dims_host[3] + .mData[g_flat * M_total * N_total + m_flat * N_total + n_flat])); + } + else + { + throw std::runtime_error("Unsupported NumDTensor for reference calculation"); + } + + // Store result + e_full_dims_host_ref.mData[g_flat * M_total * N_total + m_flat * N_total + n_flat] = + static_cast(result); + } + }; + + // Execute parallel computation using hardware concurrency + // Parallelize over G_total and M_total dimensions for optimal CPU utilization + make_ParallelTensorFunctor(f_gm, G_total, M_total)(std::thread::hardware_concurrency()); +} + +template +void calculate_reference_multi_dimensional( + const HostTensor& a_full_dims, + const HostTensor& b_full_dims, + const std::vector>& ds_full_dims_host, + HostTensor& e_full_dims_host_ref, + const std::vector& G_dims, + const std::vector& M_dims, + const std::vector& N_dims, + const std::vector& K_dims, + const std::vector& A_dims, + const std::vector& B_dims, + const std::vector& E_dims, + const CDEElementWise& cde_elementwise) +{ + std::cout << "Calculating reference using multi-dimensional indexing..." << std::endl; + + std::vector g_idx(G_dims.size()); + std::vector m_idx(M_dims.size()); + std::vector n_idx(N_dims.size()); + std::vector k_idx(K_dims.size()); + std::vector a_idx, b_idx, e_idx; + + a_idx.reserve(A_dims.size()); + b_idx.reserve(B_dims.size()); + e_idx.reserve(E_dims.size()); + + for(ck_tile::index_t g_flat = 0; g_flat < calculate_total_elements(G_dims); ++g_flat) + { + ck_tile::index_t temp = g_flat; + for(int i = G_dims.size() - 1; i >= 0; --i) + { + g_idx[i] = temp % G_dims[i]; + temp /= G_dims[i]; + } + + for(ck_tile::index_t m_flat = 0; m_flat < calculate_total_elements(M_dims); ++m_flat) + { + temp = m_flat; + for(int i = M_dims.size() - 1; i >= 0; --i) + { + m_idx[i] = temp % M_dims[i]; + temp /= M_dims[i]; + } + + for(ck_tile::index_t n_flat = 0; n_flat < calculate_total_elements(N_dims); ++n_flat) + { + temp = n_flat; + for(int i = N_dims.size() - 1; i >= 0; --i) + { + n_idx[i] = temp % N_dims[i]; + temp /= N_dims[i]; + } + + AccDataType sum = 0; + + for(ck_tile::index_t k_flat = 0; k_flat < calculate_total_elements(K_dims); + ++k_flat) + { + temp = k_flat; + for(int i = K_dims.size() - 1; i >= 0; --i) + { + k_idx[i] = temp % K_dims[i]; + temp /= K_dims[i]; + } + + a_idx.clear(); + b_idx.clear(); + + a_idx.insert(a_idx.end(), g_idx.begin(), g_idx.end()); + a_idx.insert(a_idx.end(), m_idx.begin(), m_idx.end()); + a_idx.insert(a_idx.end(), k_idx.begin(), k_idx.end()); + + b_idx.insert(b_idx.end(), g_idx.begin(), g_idx.end()); + b_idx.insert(b_idx.end(), n_idx.begin(), n_idx.end()); + b_idx.insert(b_idx.end(), k_idx.begin(), k_idx.end()); + + auto a_val = a_full_dims(a_idx); + auto b_val = b_full_dims(b_idx); + + sum += static_cast(a_val) * static_cast(b_val); + } + + e_idx.clear(); + e_idx.insert(e_idx.end(), g_idx.begin(), g_idx.end()); + e_idx.insert(e_idx.end(), m_idx.begin(), m_idx.end()); + e_idx.insert(e_idx.end(), n_idx.begin(), n_idx.end()); + + EDataType result = static_cast(sum); + if(ds_full_dims_host.size() == 0) + { + ; + } + else if(ds_full_dims_host.size() == 1) + { + cde_elementwise(result, + ck_tile::type_convert(sum), + ck_tile::type_convert(ds_full_dims_host[0](e_idx))); + } + else if(ds_full_dims_host.size() == 2) + { + cde_elementwise(result, + ck_tile::type_convert(sum), + ck_tile::type_convert(ds_full_dims_host[0](e_idx)), + ck_tile::type_convert(ds_full_dims_host[1](e_idx))); + } + else if(ds_full_dims_host.size() == 3) + { + cde_elementwise(result, + ck_tile::type_convert(sum), + ck_tile::type_convert(ds_full_dims_host[0](e_idx)), + ck_tile::type_convert(ds_full_dims_host[1](e_idx)), + ck_tile::type_convert(ds_full_dims_host[2](e_idx))); + } + else if(ds_full_dims_host.size() == 4) + { + cde_elementwise(result, + ck_tile::type_convert(sum), + ck_tile::type_convert(ds_full_dims_host[0](e_idx)), + ck_tile::type_convert(ds_full_dims_host[1](e_idx)), + ck_tile::type_convert(ds_full_dims_host[2](e_idx)), + ck_tile::type_convert(ds_full_dims_host[3](e_idx))); + } + else + { + throw std::runtime_error("Unsupported NumDTensor for reference calculation"); + } + + e_full_dims_host_ref(e_idx) = static_cast(result); + } + } + } +} + +} // namespace ck_tile diff --git a/include/ck_tile/ops/batched_contraction.hpp b/include/ck_tile/ops/batched_contraction.hpp new file mode 100644 index 0000000000..9162f421d1 --- /dev/null +++ b/include/ck_tile/ops/batched_contraction.hpp @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2018-2025, Advanced Micro Devices, Inc. All rights reserved. + +#pragma once + +#include "ck_tile/ops/batched_contraction/kernel/batched_contraction_kernel.hpp" +#include "ck_tile/ops/batched_contraction/pipeline/batched_contraction_problem.hpp" +#include "ck_tile/ops/common/tensor_layout.hpp" +#include "ck_tile/ops/common/utils.hpp" diff --git a/include/ck_tile/ops/batched_contraction/kernel/batched_contraction_kernel.hpp b/include/ck_tile/ops/batched_contraction/kernel/batched_contraction_kernel.hpp new file mode 100644 index 0000000000..6d8f9f3f0e --- /dev/null +++ b/include/ck_tile/ops/batched_contraction/kernel/batched_contraction_kernel.hpp @@ -0,0 +1,522 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2025, Advanced Micro Devices, Inc. All rights reserved. + +#pragma once + +#include "ck_tile/core.hpp" +#include "ck_tile/ops/batched_contraction/pipeline/batched_contraction_problem.hpp" +#include "ck_tile/ops/gemm/kernel/universal_gemm_kernel.hpp" + +/** + * @file batched_contraction_kernel.hpp + * @brief Batched Tensor Contraction Operations + * + * @section batched_contraction_overview What is Batched Tensor Contraction with Multiple D? + * + * Tensor contraction is a fundamental operation that generalizes matrix multiplication to + * multi-dimensional tensors. It performs element-wise multiplication and summation over + * shared dimensions + * + * **Beyond pure contraction, this kernel supports multiple auxiliary input tensors (D tensors)** + * that are fused with the contraction result through configurable epilogue operations, enabling + * efficient computation of complex tensor expressions in a single kernel launch. + * + * @subsection mathematical_formulation Mathematical Formulation + * + * For tensors A and B with arbitrary dimensionalities, the complete operation computes: + * + * **E[G₀,G₁,...,M₀,M₁,...,N₀,N₁,...] = epilogue_op(C, D₀, D₁, D₂, ...)** + * + * Where: + * **C[G₀,G₁,...,M₀,M₁,...,N₀,N₁,...] = Σ_{K₀,K₁,...} A[G₀,G₁,...,M₀,M₁,...,K₀,K₁,...] × + * B[G₀,G₁,...,N₀,N₁,...,K₀,K₁,...]** + * + * Where: + * - **G dimensions**: Batch dimensions (shared across A, B, and output E) + * - **M dimensions**: Row dimensions of the output matrix (from tensor A) + * - **N dimensions**: Column dimensions of the output matrix (from tensor B) + * - **K dimensions**: Contraction dimensions (summed over, present in both A and B) + * + * @subsection why_gemm_implementation Why Tensor Contraction Can Be Implemented Using GEMM + * + * **Mathematical Equivalence**: Tensor contraction is fundamentally equivalent to matrix + * multiplication when dimensions are appropriately flattened. The key insight is that the summation + * operation over shared dimensions (K dimensions) in tensor contraction is mathematically identical + * to the dot product computation in matrix multiplication. + * + * **Dimension Flattening Strategy**: + * - **M dimensions** (from tensor A) → Flattened into matrix rows (M_total) + * - **N dimensions** (from tensor B) → Flattened into matrix columns (N_total) + * - **K dimensions** (contraction dims) → Flattened into inner dimension (K_total) + * - **G dimensions** (batch dims) → Handled through batch processing + * + * **Mathematical Transformation**: + * ``` + * Original: E[g,m₀,m₁,n₀,n₁] = Σ_{k₀,k₁} A[g,m₀,m₁,k₀,k₁] × B[g,n₀,n₁,k₀,k₁] + * Flattened: E[g,M,N] = Σ_K A[g,M,K] × B[g,N,K] (where M=m₀×m₁, N=n₀×n₁, K=k₀×k₁) + * GEMM Form: E = A × Bᵀ + * + * **Why This Approach Is Optimal**: + * Rather than implementing tensor contraction from scratch, this kernel leverages the highly + * optimized `UniversalGemmKernel` as its computational backend. + * + * @subsection current_limitations Current Kernel Limitations + * + * **Layout Restrictions:** + * - **Row-Major Only**: All tensors must use row-major memory layout + * - **Packed Tensors**: Only contiguous/packed tensor layouts supported + * - **Hardcoded Strides**: stride_A = K_total, stride_B = K_total, stride_E = N_total + * - **D Tensor Layout**: All D tensors must match E tensor layout (stride_Ds = N_total) + * + * **Implementation Constraints:** + * - **Fixed Stride Calculation**: Strides are automatically calculated and cannot be customized + * - **No Column-Major**: Column-major or custom stride patterns not supported + * - **No Strided Access**: Non-contiguous tensor slicing not supported + * + * **Future Enhancements:** + * - Support for arbitrary stride patterns + * - Column-major and mixed layout support + * - Non-contiguous tensor operation support + */ + +namespace ck_tile { + +/// @brief Host arguments for batched tensor contraction operations. +/// +/// @par Overview +/// This structure encapsulates all host-side arguments required for batched tensor contraction. +/// It supports arbitrary number of batch dimensions (G), M dimensions, N dimensions, and K +/// dimensions. +/// +/// @par Tensor Layout Assumptions +/// - A tensor: [G0, G1, ..., M0, M1, M2, ..., K0, K1, K2, ...] +/// - B tensor: [G0, G1, ..., N0, N1, N2, ..., K0, K1, K2, ...] +/// - D tensors: [G0, G1, ..., M0, M1, M2, ..., N0, N1, N2, ...] (auxiliary input tensors) +/// - E tensor: [G0, G1, ..., M0, M1, M2, ..., N0, N1, N2, ...] (output tensor) +/// +/// @tparam NumDTensor Number of D (auxiliary input) tensors. Default is 0. +template +struct BatchedContractionHostArgs +{ + /// @brief Constructor for batched contraction host arguments. + /// + /// @param a_ptr_ Pointer to input tensor A + /// @param b_ptr_ Pointer to input tensor B + /// @param ds_ptr_ Array of pointers to auxiliary input tensors D + /// @param e_ptr_ Pointer to output tensor E + /// @param k_batch_ Number of k-splits for split-K batching + /// @param A_dims_ Dimension vector for tensor A: [G0, G1, ..., M0, M1, ..., K0, K1, ...] + /// @param B_dims_ Dimension vector for tensor B: [G0, G1, ..., N0, N1, ..., K0, K1, ...] + /// @param Ds_dims_ Dimension vectors for D tensors: [G0, G1, ..., M0, M1, ..., N0, N1, ...] + /// @param E_dims_ Dimension vector for tensor E: [G0, G1, ..., M0, M1, ..., N0, N1, ...] + /// @param A_strides_ Stride vector for tensor A: [G0, G1, ..., M0, M1, ..., K0, K1, ...] + /// @param B_strides_ Stride vector for tensor B: [G0, G1, ..., N0, N1, ..., K0, K1, ...] + /// @param Ds_strides_ Stride vectors for D tensors: [G0, G1, ..., M0, M1, ..., N0, N1, ...] + /// @param E_strides_ Stride vector for tensor E: [G0, G1, ..., M0, M1, ..., N0, N1, ...] + CK_TILE_HOST + BatchedContractionHostArgs( + const void* a_ptr_, + const void* b_ptr_, + const std::array& ds_ptr_, + void* e_ptr_, + ck_tile::index_t k_batch_, + const std::vector& A_dims_, // [G0, G1, ..., M0, M1, ... , K0, K1, ...] + const std::vector& B_dims_, // [G0, G1, ..., N0, N1, ... , K0, K1, ...] + const std::array, NumDTensor>& + Ds_dims_, // [G0, G1, ..., M0, M1, ... , N0, N1, ...][NumDTensor] + const std::vector& E_dims_, // [G0, G1, ..., M0, M1, ... , N0, N1, ...] + + const std::vector& A_strides_, // [G0, G1, ..., M0, M1, ...,K0, K1, ...] + const std::vector& B_strides_, // [G0, G1, ..., N0, N1, ...,K0, K1, ...] + const std::array, NumDTensor>& + Ds_strides_, // [G0, G1, ..., M0, M1, ...,N0, N1, ...] + const std::vector& + E_strides_) // [G0, G1, ..., M0, M1, ...,N0, N1, ...][NumDTensor] + + : a_ptr(a_ptr_), + b_ptr(b_ptr_), + ds_ptr(ds_ptr_), + e_ptr(e_ptr_), + k_batch(k_batch_), + A_dims(A_dims_), + B_dims(B_dims_), + Ds_dims(Ds_dims_), + E_dims(E_dims_), + A_strides(A_strides_), + B_strides(B_strides_), + Ds_strides(Ds_strides_), + E_strides(E_strides_) + { + } + + const void* a_ptr; ///< Pointer to input tensor A + const void* b_ptr; ///< Pointer to input tensor B + std::array ds_ptr; ///< Array of pointers to auxiliary input tensors D + void* e_ptr; ///< Pointer to output tensor E + ck_tile::index_t k_batch; ///< Number of k-splits for split-K batching + const std::vector + A_dims; ///< Dimension vector for tensor A: [G0, G1, ..., M0, M1, ..., K0, K1, ...] + const std::vector + B_dims; ///< Dimension vector for tensor B: [G0, G1, ..., N0, N1, ..., K0, K1, ...] + const std::array, NumDTensor> + Ds_dims; ///< Dimension vectors for D tensors: [G0, G1, ..., M0, M1, ..., N0, N1, ...] + const std::vector + E_dims; ///< Dimension vector for tensor E: [G0, G1, ..., M0, M1, ..., N0, N1, ...] + const std::vector + A_strides; ///< Stride vector for tensor A: [G0, G1, ..., M0, M1, ..., K0, K1, ...] + const std::vector + B_strides; ///< Stride vector for tensor B: [G0, G1, ..., N0, N1, ..., K0, K1, ...] + const std::array, NumDTensor> + Ds_strides; ///< Stride vectors for D tensors: [G0, G1, ..., M0, M1, ..., N0, N1, ...] + const std::vector + E_strides; ///< Stride vector for tensor E: [G0, G1, ..., M0, M1, ..., N0, N1, ...] +}; + +/// @brief Kernel arguments for batched tensor contraction operations. +/// +/// @tparam NumDimG Number of batch dimensions +/// @tparam NumDimM Number of M (output row) dimensions +/// @tparam NumDimN Number of N (output column) dimensions +/// @tparam NumDimK Number of K (contraction) dimensions +/// @tparam NumDTensor Number of auxiliary input D tensors. Default is 0. + +template +struct BatchedContractionKernelArgs +{ + const void* a_ptr; ///< Pointer to input tensor A + const void* b_ptr; ///< Pointer to input tensor B + std::array ds_ptr; ///< Array of pointers to auxiliary input tensors D + void* e_ptr; ///< Pointer to output tensor E + ck_tile::index_t k_batch; ///< Number of k-splits for split-K batching + + ck_tile::index_t M_dims[NumDimM]; ///< M dimension sizes: [M0, M1, M2, ..., M_{NumDimM-1}] + ck_tile::index_t N_dims[NumDimN]; ///< N dimension sizes: [N0, N1, N2, ..., N_{NumDimN-1}] + ck_tile::index_t K_dims[NumDimK]; ///< K dimension sizes: [K0, K1, K2, ..., K_{NumDimK-1}] + ck_tile::index_t + G_dims[NumDimG]; ///< G (batch) dimension sizes: [G0, G1, G2, ..., G_{NumDimG-1}] + + // Batch strides for efficient offset calculation + ck_tile::index_t batch_stride_A; ///< Batch stride for tensor A + ck_tile::index_t batch_stride_B; ///< Batch stride for tensor B + ck_tile::index_t batch_stride_E; ///< Batch stride for tensor E + std::array batch_stride_Ds; ///< Batch strides for D tensors + + ck_tile::index_t G_total; ///< Total batch size: G0 * G1 * ... * G_{NumDimG-1} + ck_tile::index_t M_total; ///< Total M dimension: M0 * M1 * ... * M_{NumDimM-1} + ck_tile::index_t N_total; ///< Total N dimension: N0 * N1 * ... * N_{NumDimN-1} + ck_tile::index_t K_total; ///< Total K dimension: K0 * K1 * ... * K_{NumDimK-1} + + ck_tile::index_t stride_A; ///< Leading dimension stride for tensor A (row-major: K_total) + ck_tile::index_t stride_B; ///< Leading dimension stride for tensor B (row-major: K_total) + std::array + stride_Ds; ///< Leading dimension strides for D tensors (row-major: N_total) + ck_tile::index_t stride_E; ///< Leading dimension stride for tensor E (row-major: N_total) +}; + +/// @brief GPU kernel for batched tensor contraction operations. +/// +/// @par Overview +/// This kernel performs batched tensor contraction operations using the underlying +/// UniversalGemmKernel. It supports arbitrary tensor dimensionalities (G, M, N, K) and +/// processes multiple batch instances in parallel. Each batch performs: E = +/// epilogue_op(contraction(A, B), D0, D1, ...). +/// +/// @tparam Problem_ Tensor contraction problem specification defining data types and dimensions +/// @tparam TilePartitioner_ Tile partitioning strategy for workload distribution +/// @tparam GemmPipeline_ GEMM computation pipeline for core matrix operations +/// @tparam EpiloguePipeline_ Epilogue pipeline for post-GEMM operations and tensor fusion + +template +struct BatchedContractionKernel +{ + // Type aliases for cleaner code and better readability + using Problem = ck_tile::remove_cvref_t; ///< Tensor contraction problem specification + using ADataType = + ck_tile::remove_cvref_t; ///< Data type for input tensor A + using BDataType = + ck_tile::remove_cvref_t; ///< Data type for input tensor B + using DsDataType = + ck_tile::remove_cvref_t; ///< Data types for auxiliary input + ///< tensors D + using EDataType = + ck_tile::remove_cvref_t; ///< Data type for output tensor E + + // Compile-time dimension constants extracted from problem specification + static constexpr ck_tile::index_t NumDimG = Problem::NumDimG; ///< Number of batch dimensions + static constexpr ck_tile::index_t NumDimM = + Problem::NumDimM; ///< Number of M (output row) dimensions + static constexpr ck_tile::index_t NumDimN = + Problem::NumDimN; ///< Number of N (output column) dimensions + static constexpr ck_tile::index_t NumDimK = + Problem::NumDimK; ///< Number of K (contraction) dimensions + static constexpr ck_tile::index_t NumDTensor = + Problem::NumDTensor; ///< Number of auxiliary input D tensors + + // Pipeline and partitioning strategy types + using TilePartitioner = + ck_tile::remove_cvref_t; ///< Tile partitioning strategy for workload + ///< distribution + using GemmPipeline = ck_tile::remove_cvref_t; ///< GEMM computation pipeline + using EpiloguePipeline = + ck_tile::remove_cvref_t; ///< Epilogue pipeline for post-GEMM operations + + // Underlying GEMM kernel that performs the actual computation + using UniversalGemmKernel = + ck_tile::UniversalGemmKernel; + + static constexpr ck_tile::index_t kBlockSize = + UniversalGemmKernel::kBlockSize; ///< GPU block size inherited from GEMM kernel + + using KernelArgs = + BatchedContractionKernelArgs; ///< Kernel + ///< argument + ///< structure + + /// @brief Returns the kernel name for debugging and profiling purposes. + /// @return Constant string identifier for this kernel + CK_TILE_HOST static constexpr auto GetKernelName() { return "batched_contraction_kernel"; } + + /// @brief Validates whether the given kernel arguments are supported. + /// @param kargs Kernel arguments to validate + /// @return True if arguments are supported, false otherwise + /// @details Checks underlying GEMM kernel support and ensures valid batch dimensions + CK_TILE_HOST static constexpr bool IsSupportedArguments(const KernelArgs& kargs) + { + typename UniversalGemmKernel::KernelArgs gemm_kargs{{kargs.a_ptr}, + {kargs.b_ptr}, + kargs.ds_ptr, + kargs.e_ptr, + kargs.M_total, + kargs.N_total, + kargs.K_total, + {kargs.stride_A}, + {kargs.stride_B}, + kargs.stride_Ds, + kargs.stride_E, + kargs.k_batch}; + + return UniversalGemmKernel::IsSupportedArgument(gemm_kargs) && kargs.G_total > 0; + } + + /// @brief Returns the shared memory size required by the kernel. + /// @return Shared memory size in bytes + /// @details Delegates to underlying GEMM kernel's shared memory requirements + CK_TILE_HOST static constexpr ck_tile::index_t GetSmemSize() + { + return UniversalGemmKernel::GetSmemSize(); + } + + /// @brief Returns the GPU block size for kernel launch. + /// @return 3D block dimensions for GPU kernel execution + CK_TILE_HOST static constexpr auto GetBlockSize() + { + return dim3(UniversalGemmKernel::kBlockSize); + } + + CK_TILE_HOST static constexpr auto GridSize(const KernelArgs& kargs) + { + return dim3( + TilePartitioner::GridSize(kargs.M_total, kargs.N_total), kargs.G_total, kargs.k_batch); + } + + CK_TILE_HOST static constexpr KernelArgs + MakeKernelArgs(const BatchedContractionHostArgs& host_args) + { + const auto expected_A_dims = NumDimG + NumDimM + NumDimK; + const auto expected_B_dims = NumDimG + NumDimN + NumDimK; + const auto expected_E_dims = NumDimG + NumDimM + NumDimN; + + if(host_args.A_dims.size() != expected_A_dims || + host_args.A_strides.size() != expected_A_dims) + { + throw std::invalid_argument("A dimension size mismatch"); + } + if(host_args.B_dims.size() != expected_B_dims || + host_args.B_strides.size() != expected_B_dims) + { + throw std::invalid_argument("B dimension size mismatch"); + } + if(host_args.E_dims.size() != expected_E_dims || + host_args.E_strides.size() != expected_E_dims) + { + throw std::invalid_argument("E dimension size mismatch"); + } + + for(ck_tile::index_t d = 0; d < NumDTensor; ++d) + { + if(host_args.Ds_dims[d].size() != expected_E_dims || + host_args.Ds_strides[d].size() != expected_E_dims) + { + throw std::invalid_argument("D dimension size mismatch"); + } + } + + KernelArgs kargs; + kargs.a_ptr = host_args.a_ptr; + kargs.b_ptr = host_args.b_ptr; + kargs.ds_ptr = host_args.ds_ptr; + kargs.e_ptr = host_args.e_ptr; + kargs.k_batch = host_args.k_batch; + + // Validate and set G dimensions (must be identical across all tensors) + for(ck_tile::index_t i = 0; i < NumDimG; ++i) + { + // All tensors must have same G dimensions for valid contraction + if(host_args.A_dims[i] != host_args.B_dims[i] || + host_args.A_dims[i] != host_args.E_dims[i]) + { + throw std::invalid_argument( + "All tensors must have identical G dimensions for valid contraction"); + } + + // Store G dimensions (same for all tensors) + kargs.G_dims[i] = host_args.A_dims[i]; + } + + // Set batch strides from the stride of last G dimension + kargs.batch_stride_A = host_args.A_strides[NumDimG - 1]; + kargs.batch_stride_B = host_args.B_strides[NumDimG - 1]; + kargs.batch_stride_E = host_args.E_strides[NumDimG - 1]; + + for(ck_tile::index_t i = 0; i < NumDimM; ++i) + { + kargs.M_dims[i] = host_args.A_dims[NumDimG + i]; + if(kargs.M_dims[i] != host_args.E_dims[NumDimG + i]) + { + throw std::invalid_argument("M dimension mismatch between A and E tensors"); + } + } + for(ck_tile::index_t i = 0; i < NumDimN; ++i) + { + kargs.N_dims[i] = host_args.B_dims[NumDimG + i]; + if(kargs.N_dims[i] != host_args.E_dims[NumDimG + NumDimM + i]) + { + throw std::invalid_argument("N dimension mismatch between B and E tensors"); + } + } + for(ck_tile::index_t i = 0; i < NumDimK; ++i) + { + kargs.K_dims[i] = host_args.A_dims[NumDimG + NumDimM + i]; + if(kargs.K_dims[i] != host_args.B_dims[NumDimG + NumDimN + i]) + { + throw std::invalid_argument("K dimension mismatch between A and B tensors"); + } + } + + // Calculate total dimensions from individual dimension arrays + kargs.G_total = 1; + for(ck_tile::index_t i = 0; i < NumDimG; ++i) + { + kargs.G_total *= kargs.G_dims[i]; + } + + kargs.M_total = 1; + for(ck_tile::index_t i = 0; i < NumDimM; ++i) + { + kargs.M_total *= kargs.M_dims[i]; + } + + kargs.N_total = 1; + for(ck_tile::index_t i = 0; i < NumDimN; ++i) + { + kargs.N_total *= kargs.N_dims[i]; + } + + kargs.K_total = 1; + for(ck_tile::index_t i = 0; i < NumDimK; ++i) + { + kargs.K_total *= kargs.K_dims[i]; + } + + kargs.stride_A = kargs.K_total; + kargs.stride_B = kargs.K_total; + kargs.stride_E = kargs.N_total; + + // Validate D tensors have same G dimensions and set their batch strides + for(ck_tile::index_t d = 0; d < NumDTensor; ++d) + { + for(ck_tile::index_t i = 0; i < NumDimG; ++i) + { + if(host_args.Ds_dims[d][i] != host_args.A_dims[i]) + { + throw std::invalid_argument( + "D tensor G dimensions must match A/B/E tensor G dimensions"); + } + } + // Set batch stride for D tensor + kargs.batch_stride_Ds[d] = host_args.Ds_strides[d][NumDimG - 1]; + kargs.stride_Ds[d] = kargs.N_total; // D tensors same shape as E + } + + return kargs; + } + + CK_TILE_DEVICE void operator()(const KernelArgs& kargs) const + { + + const auto [iM, iN] = + TilePartitioner{kargs.M_total, kargs.N_total}.GetOutputTileIndex(blockIdx.x); + const ck_tile::index_t i_m = + __builtin_amdgcn_readfirstlane(iM * TilePartitioner::MPerBlock); + const ck_tile::index_t i_n = + __builtin_amdgcn_readfirstlane(iN * TilePartitioner::NPerBlock); + + const auto i_batch_flat = __builtin_amdgcn_readfirstlane(blockIdx.y); + const auto i_splitk = __builtin_amdgcn_readfirstlane(blockIdx.z); + + // Calculate batch offsets for each tensor + const auto batch_offset_A = i_batch_flat * kargs.batch_stride_A; + const auto batch_offset_B = i_batch_flat * kargs.batch_stride_B; + const auto batch_offset_E = i_batch_flat * kargs.batch_stride_E; + + const ADataType* a_ptr = static_cast(kargs.a_ptr) + batch_offset_A; + const BDataType* b_ptr = static_cast(kargs.b_ptr) + batch_offset_B; + EDataType* e_ptr = static_cast(kargs.e_ptr) + batch_offset_E; + + std::array ds_batch_ptr; + static_for<0, NumDTensor, 1>{}([&](auto i) { + using DDataType = typename std::tuple_element::type; + const auto batch_offset_D = i_batch_flat * kargs.batch_stride_Ds[i]; + ds_batch_ptr[i] = static_cast(kargs.ds_ptr[i]) + batch_offset_D; + }); + + typename UniversalGemmKernel::KernelArgs gemm_kargs{{a_ptr}, + {b_ptr}, + ds_batch_ptr, + e_ptr, + kargs.M_total, + kargs.N_total, + kargs.K_total, + {kargs.stride_A}, + {kargs.stride_B}, + kargs.stride_Ds, + kargs.stride_E, + kargs.k_batch}; + + const typename UniversalGemmKernel::SplitKBatchOffset splitk_batch_offset(gemm_kargs, + i_splitk); + + const ADataType* a_ptr_final = a_ptr + splitk_batch_offset.as_k_split_offset[0]; + const BDataType* b_ptr_final = b_ptr + splitk_batch_offset.bs_k_split_offset[0]; + __shared__ char smem_ptr[GetSmemSize()]; + + UniversalGemmKernel::RunGemm({a_ptr_final}, + {b_ptr_final}, + ds_batch_ptr, + e_ptr, + smem_ptr, + gemm_kargs, + splitk_batch_offset, + i_m, + i_n); + } +}; + +} // namespace ck_tile diff --git a/include/ck_tile/ops/batched_contraction/pipeline/batched_contraction_problem.hpp b/include/ck_tile/ops/batched_contraction/pipeline/batched_contraction_problem.hpp new file mode 100644 index 0000000000..9ebaae3c97 --- /dev/null +++ b/include/ck_tile/ops/batched_contraction/pipeline/batched_contraction_problem.hpp @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2025, Advanced Micro Devices, Inc. All rights reserved. +#pragma once + +#include "ck_tile/core.hpp" + +namespace ck_tile { + +template +struct BatchedContractionProblem +{ + using ADataType = ck_tile::remove_cvref_t; + using BDataType = ck_tile::remove_cvref_t; + using DsDataType = ck_tile::remove_cvref_t; + using EDataType = ck_tile::remove_cvref_t; + + static constexpr ck_tile::index_t NumDimG = NumDimG_; + static constexpr ck_tile::index_t NumDimM = NumDimM_; + static constexpr ck_tile::index_t NumDimN = NumDimN_; + static constexpr ck_tile::index_t NumDimK = NumDimK_; + static constexpr ck_tile::index_t NumDTensor = NumDTensor_; +}; + +} // namespace ck_tile diff --git a/include/ck_tile/ops/batched_contraction/utils/tensor_descriptor_utils.hpp b/include/ck_tile/ops/batched_contraction/utils/tensor_descriptor_utils.hpp new file mode 100644 index 0000000000..6d3286ce09 --- /dev/null +++ b/include/ck_tile/ops/batched_contraction/utils/tensor_descriptor_utils.hpp @@ -0,0 +1,169 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2025, Advanced Micro Devices, Inc. All rights reserved. + +#pragma once + +#include "ck_tile/core.hpp" + +/** + * @file tensor_descriptor_utils.hpp + * @brief Utility functions for creating tensor descriptors in batched contraction operations + * + * @details This file contains utility functions for creating tensor descriptors with flattened + * dimensions for GEMM operations. These functions transform multi-dimensional tensors into + * 2D matrix descriptors by removing batch dimensions and flattening the remaining dimensions. + * + * These utilities are currently not used in the main batched contraction kernel but are preserved + * for future implementations that may require explicit tensor descriptor creation. + */ + +namespace ck_tile { + +/** + * @brief Utility class for creating tensor descriptors in batched contraction operations + * + * @tparam NumDimG Number of batch dimensions + * @tparam NumDimM Number of M (output row) dimensions + * @tparam NumDimN Number of N (output column) dimensions + * @tparam NumDimK Number of K (contraction) dimensions + */ +template +struct TensorDescriptorUtils +{ + /// @brief Creates a tensor descriptor for input tensor A with batch dimensions removed. + /// @param A_dims Dimension vector for tensor A: [G0, G1, ..., M0, M1, M2, ..., K0, K1, K2, ...] + /// @param A_strides Stride vector for tensor A: [G0, G1, ..., M0, M1, M2, ..., K0, K1, K2, ...] + /// @return Flattened tensor descriptor: [M_total, K_total] for GEMM computation + /// @details Removes batch dimensions and flattens M and K dimensions for efficient GEMM + /// execution + CK_TILE_HOST static constexpr auto + Make_A_GridDescriptor_M_K(const std::vector& A_dims = {}, + const std::vector& A_strides = {}) + { + const auto to_tuple = [&](auto& vec, auto start, auto end) { + return generate_tuple([&](auto i) { return vec[start + i]; }, number{}); + }; + + // Remove G Dimensions + const auto A_dims_M_K = + to_tuple(A_dims, number{}, number{}); + const auto A_strides_M_K = + to_tuple(A_strides, number{}, number{}); + + // dimension Ids for M and K + constexpr auto A_dims_M_ids = typename arithmetic_sequence_gen<0, NumDimM, 1>::type{}; + constexpr auto A_dims_K_ids = + typename arithmetic_sequence_gen::type{}; + + // Dimensions for M [M0, M1, ...] and K [K0, K1, ...] + const auto dims_M = get_container_subset(A_dims_M_K, A_dims_M_ids); + const auto dims_K = get_container_subset(A_dims_M_K, A_dims_K_ids); + + // naive tensor A[M0, M1, M2, ..., K0, K1, K2...] Discriptor + const auto A_grid_desc_Ms_Ks = + ck_tile::make_naive_tensor_descriptor(A_dims_M_K, A_strides_M_K); + + // transformed tensor to flatten M and K dimensions [M_total = M0 * M1 * M2 * ... , K_total + // = K0 * K1 * K2 * ...] + const auto A_grid_desc_Mflat_Kflat = ck_tile::transform_tensor_descriptor( + A_grid_desc_Ms_Ks, + make_tuple(make_merge_transform(dims_M), make_merge_transform(dims_K)), + make_tuple(A_dims_M_ids, A_dims_K_ids), + make_tuple(sequence<0>{}, sequence<1>{})); + + return A_grid_desc_Mflat_Kflat; + } + + /// @brief Creates a tensor descriptor for input tensor B with batch dimensions removed. + /// @param B_dims Dimension vector for tensor B: [G0, G1, ..., N0, N1, N2, ..., K0, K1, K2, ...] + /// @param B_strides Stride vector for tensor B: [G0, G1, ..., N0, N1, N2, ..., K0, K1, K2, ...] + /// @return Flattened tensor descriptor: [N_total, K_total] for GEMM computation + /// @details Removes batch dimensions and flattens N and K dimensions for efficient GEMM + /// execution + CK_TILE_HOST static constexpr auto + Make_B_GridDescriptor_N_K(const std::vector& B_dims = {}, + const std::vector& B_strides = {}) + { + const auto to_tuple = [&](auto& vec, auto start, auto end) { + return generate_tuple([&](auto i) { return vec[start + i]; }, number{}); + }; + + // Remove G Dimensions + const auto B_dims_N_K = + to_tuple(B_dims, number{}, number{}); + const auto B_strides_N_K = + to_tuple(B_strides, number{}, number{}); + + // dimension Ids for N and K + constexpr auto B_dims_N_ids = typename arithmetic_sequence_gen<0, NumDimN, 1>::type{}; + constexpr auto B_dims_K_ids = + typename arithmetic_sequence_gen::type{}; + + // Dimensions for N [N0, N1, ...] and K [K0, K1, ...] + const auto dims_N = get_container_subset(B_dims_N_K, B_dims_N_ids); + const auto dims_K = get_container_subset(B_dims_N_K, B_dims_K_ids); + + // naive tensor B[N0, N1, N2, ..., K0, K1, K2...] Discriptor + const auto B_grid_desc_Ns_Ks = + ck_tile::make_naive_tensor_descriptor(B_dims_N_K, B_strides_N_K); + + // transformed tensor to flatten N and K dimensions [N_total = N0 * N1 * N2 * ... , K_total + // = K0 * K1 * K2 * ...] + const auto B_grid_desc_Nflat_Kflat = ck_tile::transform_tensor_descriptor( + B_grid_desc_Ns_Ks, + make_tuple(make_merge_transform(dims_N), make_merge_transform(dims_K)), + make_tuple(B_dims_N_ids, B_dims_K_ids), + make_tuple(sequence<0>{}, sequence<1>{})); + + return B_grid_desc_Nflat_Kflat; + } + + /// @brief Creates a tensor descriptor for output tensor E with batch dimensions removed. + /// @param E_dims Dimension vector for tensor E: [G0, G1, ..., M0, M1, M2, ..., N0, N1, N2, ...] + /// @param E_strides Stride vector for tensor E: [G0, G1, ..., M0, M1, M2, ..., N0, N1, N2, ...] + /// @return Flattened tensor descriptor: [M_total, N_total] for GEMM computation + /// @details Removes batch dimensions and flattens M and N dimensions for efficient GEMM + /// execution + CK_TILE_HOST static constexpr auto + Make_E_GridDescriptor_M_N(const std::vector& E_dims = {}, + const std::vector& E_strides = {}) + { + const auto to_tuple = [&](auto& vec, auto start, auto end) { + return generate_tuple([&](auto i) { return vec[start + i]; }, number{}); + }; + + // Remove G dimensions + const auto E_dims_M_N = + to_tuple(E_dims, number{}, number{}); + const auto E_strides_M_N = + to_tuple(E_strides, number{}, number{}); + + // dimension Ids for M and N + constexpr auto E_dims_M_ids = typename arithmetic_sequence_gen<0, NumDimM, 1>::type{}; + constexpr auto E_dims_N_ids = + typename arithmetic_sequence_gen::type{}; + + // Dimensions for M and N + const auto dims_M = get_container_subset(E_dims_M_N, E_dims_M_ids); + const auto dims_N = get_container_subset(E_dims_M_N, E_dims_N_ids); + + // naive tensor E[M0, M1, M2, ..., N0, N1, N2...] Discriptor + const auto E_grid_desc_Ms_Ns = + ck_tile::make_naive_tensor_descriptor(E_dims_M_N, E_strides_M_N); + + // transformed tensor to flatten M and N dimensions [M_total = M0 * M1 * M2 * ... , + // N_total = N0 * N1 * N2 * ...] + const auto E_grid_desc_Mflat_Nflat = ck_tile::transform_tensor_descriptor( + E_grid_desc_Ms_Ns, + make_tuple(make_merge_transform(dims_M), make_merge_transform(dims_N)), + make_tuple(E_dims_M_ids, E_dims_N_ids), + make_tuple(sequence<0>{}, sequence<1>{})); + + return E_grid_desc_Mflat_Nflat; + } +}; + +} // namespace ck_tile From 46c10c316db0b4e987ff69b2804d97a98bb01c1a Mon Sep 17 00:00:00 2001 From: damien-lejeune <31985270+damien-lejeune@users.noreply.github.com> Date: Mon, 13 Oct 2025 13:24:47 +0200 Subject: [PATCH 012/262] Update include path to break the remod's cyclic dep issue (#2978) * Update include path to break the cyclic dep issue * Use ck_tile::permute_vectors_i4x4_b in tile engine --------- Co-authored-by: Damien Lejeune Co-authored-by: Adam Osewski <19374865+aosewski@users.noreply.github.com> --- include/ck_tile/host.hpp | 1 + include/ck_tile/ops/add_rmsnorm2d_rdquant.hpp | 2 + include/ck_tile/ops/batched_transpose.hpp | 2 + include/ck_tile/ops/common.hpp | 3 +- .../ops/common/load_interleaved_pk_type.hpp | 2 +- include/ck_tile/ops/elementwise.hpp | 2 + include/ck_tile/ops/epilogue.hpp | 2 + include/ck_tile/ops/flatmm.hpp | 2 + include/ck_tile/ops/fmha.hpp | 2 + include/ck_tile/ops/fused_moe.hpp | 2 + include/ck_tile/ops/gemm.hpp | 8 +-- include/ck_tile/ops/gemm_quant.hpp | 8 +-- include/ck_tile/ops/grouped_convolution.hpp | 2 + include/ck_tile/ops/image_to_column.hpp | 2 + include/ck_tile/ops/layernorm2d.hpp | 2 + include/ck_tile/ops/norm_reduce.hpp | 2 + include/ck_tile/ops/permute.hpp | 2 + include/ck_tile/ops/reduce.hpp | 2 + include/ck_tile/ops/rmsnorm2d.hpp | 2 + include/ck_tile/ops/smoothquant.hpp | 2 + include/ck_tile/ops/softmax.hpp | 2 + include/ck_tile/ops/topk.hpp | 2 + include/ck_tile/ops/topk_softmax.hpp | 2 + tile_engine/ops/gemm/gemm_common.hpp | 52 ------------------- tile_engine/ops/gemm/gemm_profiler.hpp | 2 +- 25 files changed, 51 insertions(+), 61 deletions(-) diff --git a/include/ck_tile/host.hpp b/include/ck_tile/host.hpp index 86110d57ec..d815b1db40 100644 --- a/include/ck_tile/host.hpp +++ b/include/ck_tile/host.hpp @@ -16,6 +16,7 @@ #include "ck_tile/host/host_tensor.hpp" #include "ck_tile/host/joinable_thread.hpp" #include "ck_tile/host/kernel_launch.hpp" +#include "ck_tile/host/permute_pk_int4.hpp" #include "ck_tile/host/ranges.hpp" #include "ck_tile/host/reference/reference_batched_dropout.hpp" #include "ck_tile/host/reference/reference_batched_dropout_randval.hpp" diff --git a/include/ck_tile/ops/add_rmsnorm2d_rdquant.hpp b/include/ck_tile/ops/add_rmsnorm2d_rdquant.hpp index 1768c802d5..6c0972e10a 100644 --- a/include/ck_tile/ops/add_rmsnorm2d_rdquant.hpp +++ b/include/ck_tile/ops/add_rmsnorm2d_rdquant.hpp @@ -9,5 +9,7 @@ #include "ck_tile/ops/add_rmsnorm2d_rdquant/pipeline/add_rmsnorm2d_rdquant_fwd_pipeline_problem.hpp" #include "ck_tile/ops/add_rmsnorm2d_rdquant/pipeline/add_rmsnorm2d_rdquant_fwd_pipeline_three_pass.hpp" #include "ck_tile/ops/common/generic_2d_block_shape.hpp" +#include "ck_tile/ops/common/load_interleaved_pk_type.hpp" +#include "ck_tile/ops/common/streamk_common.hpp" #include "ck_tile/ops/common/tensor_layout.hpp" #include "ck_tile/ops/common/utils.hpp" diff --git a/include/ck_tile/ops/batched_transpose.hpp b/include/ck_tile/ops/batched_transpose.hpp index ca0088c812..5822d7b91b 100644 --- a/include/ck_tile/ops/batched_transpose.hpp +++ b/include/ck_tile/ops/batched_transpose.hpp @@ -12,5 +12,7 @@ #include "ck_tile/ops/batched_transpose/pipeline/batched_transpose_policy.hpp" #include "ck_tile/ops/batched_transpose/pipeline/batched_transpose_problem.hpp" #include "ck_tile/ops/common/generic_2d_block_shape.hpp" +#include "ck_tile/ops/common/load_interleaved_pk_type.hpp" +#include "ck_tile/ops/common/streamk_common.hpp" #include "ck_tile/ops/common/tensor_layout.hpp" #include "ck_tile/ops/common/utils.hpp" diff --git a/include/ck_tile/ops/common.hpp b/include/ck_tile/ops/common.hpp index 7c6adc3ec2..eff2d625b3 100644 --- a/include/ck_tile/ops/common.hpp +++ b/include/ck_tile/ops/common.hpp @@ -4,6 +4,7 @@ #pragma once #include "ck_tile/ops/common/generic_2d_block_shape.hpp" +#include "ck_tile/ops/common/load_interleaved_pk_type.hpp" +#include "ck_tile/ops/common/streamk_common.hpp" #include "ck_tile/ops/common/tensor_layout.hpp" #include "ck_tile/ops/common/utils.hpp" -#include "ck_tile/ops/common/streamk_common.hpp" diff --git a/include/ck_tile/ops/common/load_interleaved_pk_type.hpp b/include/ck_tile/ops/common/load_interleaved_pk_type.hpp index f8432b9da0..fb7a05044f 100644 --- a/include/ck_tile/ops/common/load_interleaved_pk_type.hpp +++ b/include/ck_tile/ops/common/load_interleaved_pk_type.hpp @@ -4,7 +4,7 @@ #pragma once #include "ck_tile/core/config.hpp" -#include "ck_tile/ops/elementwise.hpp" +#include "ck_tile/ops/elementwise/unary_element_wise_operation.hpp" namespace ck_tile { diff --git a/include/ck_tile/ops/elementwise.hpp b/include/ck_tile/ops/elementwise.hpp index 4858245ec4..7f2303932e 100644 --- a/include/ck_tile/ops/elementwise.hpp +++ b/include/ck_tile/ops/elementwise.hpp @@ -10,5 +10,7 @@ #include "ck_tile/ops/elementwise/pipeline/elementwise_shape.hpp" #include "ck_tile/ops/elementwise/unary_element_wise_operation.hpp" #include "ck_tile/ops/common/generic_2d_block_shape.hpp" +#include "ck_tile/ops/common/load_interleaved_pk_type.hpp" +#include "ck_tile/ops/common/streamk_common.hpp" #include "ck_tile/ops/common/tensor_layout.hpp" #include "ck_tile/ops/common/utils.hpp" diff --git a/include/ck_tile/ops/epilogue.hpp b/include/ck_tile/ops/epilogue.hpp index 6cc0fa8540..ec5a8ef445 100644 --- a/include/ck_tile/ops/epilogue.hpp +++ b/include/ck_tile/ops/epilogue.hpp @@ -8,5 +8,7 @@ #include "ck_tile/ops/epilogue/default_2d_epilogue.hpp" #include "ck_tile/ops/epilogue/dynamic_quant_epilogue.hpp" #include "ck_tile/ops/common/generic_2d_block_shape.hpp" +#include "ck_tile/ops/common/load_interleaved_pk_type.hpp" +#include "ck_tile/ops/common/streamk_common.hpp" #include "ck_tile/ops/common/tensor_layout.hpp" #include "ck_tile/ops/common/utils.hpp" diff --git a/include/ck_tile/ops/flatmm.hpp b/include/ck_tile/ops/flatmm.hpp index 1714789e63..41463e6a2d 100644 --- a/include/ck_tile/ops/flatmm.hpp +++ b/include/ck_tile/ops/flatmm.hpp @@ -14,5 +14,7 @@ #include "ck_tile/ops/flatmm/pipeline/flatmm_pipeline_agmem_bgmem_creg_v1_policy.hpp" #include "ck_tile/ops/flatmm/pipeline/tile_flatmm_shape.hpp" #include "ck_tile/ops/common/generic_2d_block_shape.hpp" +#include "ck_tile/ops/common/load_interleaved_pk_type.hpp" +#include "ck_tile/ops/common/streamk_common.hpp" #include "ck_tile/ops/common/tensor_layout.hpp" #include "ck_tile/ops/common/utils.hpp" diff --git a/include/ck_tile/ops/fmha.hpp b/include/ck_tile/ops/fmha.hpp index 31de21a726..6b25c089bd 100644 --- a/include/ck_tile/ops/fmha.hpp +++ b/include/ck_tile/ops/fmha.hpp @@ -60,5 +60,7 @@ #include "ck_tile/ops/fmha/pipeline/tile_fmha_shape.hpp" #include "ck_tile/ops/fmha/pipeline/tile_fmha_traits.hpp" #include "ck_tile/ops/common/generic_2d_block_shape.hpp" +#include "ck_tile/ops/common/load_interleaved_pk_type.hpp" +#include "ck_tile/ops/common/streamk_common.hpp" #include "ck_tile/ops/common/tensor_layout.hpp" #include "ck_tile/ops/common/utils.hpp" diff --git a/include/ck_tile/ops/fused_moe.hpp b/include/ck_tile/ops/fused_moe.hpp index ddb64a2189..71721f3408 100644 --- a/include/ck_tile/ops/fused_moe.hpp +++ b/include/ck_tile/ops/fused_moe.hpp @@ -16,5 +16,7 @@ #include "ck_tile/ops/fused_moe/pipeline/moe_sorting_pipeline.hpp" #include "ck_tile/ops/fused_moe/pipeline/moe_sorting_policy.hpp" #include "ck_tile/ops/common/generic_2d_block_shape.hpp" +#include "ck_tile/ops/common/load_interleaved_pk_type.hpp" +#include "ck_tile/ops/common/streamk_common.hpp" #include "ck_tile/ops/common/tensor_layout.hpp" #include "ck_tile/ops/common/utils.hpp" diff --git a/include/ck_tile/ops/gemm.hpp b/include/ck_tile/ops/gemm.hpp index 5edde31cd9..204d67a0ff 100644 --- a/include/ck_tile/ops/gemm.hpp +++ b/include/ck_tile/ops/gemm.hpp @@ -30,18 +30,18 @@ #include "ck_tile/ops/gemm/block/block_wp_asmem_bsmem_creg_v1_custom_policy.hpp" #include "ck_tile/ops/gemm/kernel/batched_gemm_kernel.hpp" #include "ck_tile/ops/gemm/kernel/gemm_kernel.hpp" -#include "ck_tile/ops/gemm/kernel/gemm_multi_d_kernel.hpp" #include "ck_tile/ops/gemm/kernel/gemm_multi_abd_kernel.hpp" +#include "ck_tile/ops/gemm/kernel/gemm_multi_d_kernel.hpp" #include "ck_tile/ops/gemm/kernel/gemm_tile_partitioner.hpp" #include "ck_tile/ops/gemm/kernel/grouped_gemm_kernel.hpp" #include "ck_tile/ops/gemm/kernel/streamk_gemm_kernel.hpp" #include "ck_tile/ops/gemm/kernel/universal_gemm_kernel.hpp" #include "ck_tile/ops/gemm/pipeline/gemm_pipeline_ag_bg_cr_base.hpp" +#include "ck_tile/ops/gemm/pipeline/gemm_pipeline_ag_bg_cr_comp_async.hpp" +#include "ck_tile/ops/gemm/pipeline/gemm_pipeline_ag_bg_cr_comp_async_default_policy.hpp" #include "ck_tile/ops/gemm/pipeline/gemm_pipeline_ag_bg_cr_comp_v3.hpp" #include "ck_tile/ops/gemm/pipeline/gemm_pipeline_ag_bg_cr_comp_v4.hpp" #include "ck_tile/ops/gemm/pipeline/gemm_pipeline_ag_bg_cr_comp_v4_default_policy.hpp" -#include "ck_tile/ops/gemm/pipeline/gemm_pipeline_ag_bg_cr_comp_async.hpp" -#include "ck_tile/ops/gemm/pipeline/gemm_pipeline_ag_bg_cr_comp_async_default_policy.hpp" #include "ck_tile/ops/gemm/pipeline/gemm_pipeline_ag_bg_cr_comp_v5.hpp" #include "ck_tile/ops/gemm/pipeline/gemm_pipeline_ag_bg_cr_comp_v5_default_policy.hpp" #include "ck_tile/ops/gemm/pipeline/gemm_pipeline_ag_bg_cr_mem.hpp" @@ -72,5 +72,7 @@ #include "ck_tile/ops/gemm/warp/warp_gemm_smfmac_impl.hpp" #include "ck_tile/ops/gemm/warp/warp_wmma_gemm.hpp" #include "ck_tile/ops/common/generic_2d_block_shape.hpp" +#include "ck_tile/ops/common/load_interleaved_pk_type.hpp" +#include "ck_tile/ops/common/streamk_common.hpp" #include "ck_tile/ops/common/tensor_layout.hpp" #include "ck_tile/ops/common/utils.hpp" diff --git a/include/ck_tile/ops/gemm_quant.hpp b/include/ck_tile/ops/gemm_quant.hpp index cde0b6833f..61cb96c8f4 100644 --- a/include/ck_tile/ops/gemm_quant.hpp +++ b/include/ck_tile/ops/gemm_quant.hpp @@ -3,8 +3,8 @@ #pragma once -#include "ck_tile/ops/gemm_quant/block/block_universal_gemm_as_aquant_bs_cr.hpp" #include "ck_tile/ops/gemm_quant/block/block_universal_gemm_ar_flatbr_bquant_cr.hpp" +#include "ck_tile/ops/gemm_quant/block/block_universal_gemm_as_aquant_bs_cr.hpp" #include "ck_tile/ops/gemm_quant/block/block_universal_gemm_as_bs_bquant_cr.hpp" #include "ck_tile/ops/gemm_quant/kernel/gemm_quant_kernel.hpp" #include "ck_tile/ops/gemm_quant/kernel/grouped_gemm_quant_kernel.hpp" @@ -15,11 +15,13 @@ #include "ck_tile/ops/gemm_quant/pipeline/gemm_bquant_pipeline_ag_bg_cr_base.hpp" #include "ck_tile/ops/gemm_quant/pipeline/gemm_bquant_pipeline_ag_bg_cr_policy.hpp" #include "ck_tile/ops/gemm_quant/pipeline/gemm_bquant_pipeline_ag_bg_cr_v3.hpp" -#include "ck_tile/ops/gemm_quant/pipeline/gemm_wp_bquant_pipeline_ag_bg_cr_base_policy.hpp" -#include "ck_tile/ops/gemm_quant/pipeline/gemm_wp_bquant_pipeline_ag_bg_cr_v2.hpp" #include "ck_tile/ops/gemm_quant/pipeline/gemm_group_quant_utils.hpp" #include "ck_tile/ops/gemm_quant/pipeline/gemm_quant_pipeline_problem.hpp" +#include "ck_tile/ops/gemm_quant/pipeline/gemm_wp_bquant_pipeline_ag_bg_cr_base_policy.hpp" +#include "ck_tile/ops/gemm_quant/pipeline/gemm_wp_bquant_pipeline_ag_bg_cr_v2.hpp" #include "ck_tile/ops/gemm_quant/pipeline/tile_gemm_quant_traits.hpp" #include "ck_tile/ops/common/generic_2d_block_shape.hpp" +#include "ck_tile/ops/common/load_interleaved_pk_type.hpp" +#include "ck_tile/ops/common/streamk_common.hpp" #include "ck_tile/ops/common/tensor_layout.hpp" #include "ck_tile/ops/common/utils.hpp" diff --git a/include/ck_tile/ops/grouped_convolution.hpp b/include/ck_tile/ops/grouped_convolution.hpp index 09b50f26b0..1dd13b6246 100644 --- a/include/ck_tile/ops/grouped_convolution.hpp +++ b/include/ck_tile/ops/grouped_convolution.hpp @@ -12,5 +12,7 @@ #include "ck_tile/ops/grouped_convolution/utils/transform_conv_bwd_weight_to_gemm.hpp" #include "ck_tile/ops/grouped_convolution/utils/transform_conv_fwd_to_gemm.hpp" #include "ck_tile/ops/common/generic_2d_block_shape.hpp" +#include "ck_tile/ops/common/load_interleaved_pk_type.hpp" +#include "ck_tile/ops/common/streamk_common.hpp" #include "ck_tile/ops/common/tensor_layout.hpp" #include "ck_tile/ops/common/utils.hpp" diff --git a/include/ck_tile/ops/image_to_column.hpp b/include/ck_tile/ops/image_to_column.hpp index 93664ea138..2307b05190 100644 --- a/include/ck_tile/ops/image_to_column.hpp +++ b/include/ck_tile/ops/image_to_column.hpp @@ -7,5 +7,7 @@ #include "ck_tile/ops/image_to_column/pipeline/block_image_to_column_problem.hpp" #include "ck_tile/ops/image_to_column/pipeline/tile_image_to_column_shape.hpp" #include "ck_tile/ops/common/generic_2d_block_shape.hpp" +#include "ck_tile/ops/common/load_interleaved_pk_type.hpp" +#include "ck_tile/ops/common/streamk_common.hpp" #include "ck_tile/ops/common/tensor_layout.hpp" #include "ck_tile/ops/common/utils.hpp" diff --git a/include/ck_tile/ops/layernorm2d.hpp b/include/ck_tile/ops/layernorm2d.hpp index afbb817db1..9ce22137bf 100644 --- a/include/ck_tile/ops/layernorm2d.hpp +++ b/include/ck_tile/ops/layernorm2d.hpp @@ -10,5 +10,7 @@ #include "ck_tile/ops/layernorm2d/pipeline/layernorm2d_fwd_pipeline_two_pass.hpp" #include "ck_tile/ops/layernorm2d/pipeline/layernorm2d_fwd_traits.hpp" #include "ck_tile/ops/common/generic_2d_block_shape.hpp" +#include "ck_tile/ops/common/load_interleaved_pk_type.hpp" +#include "ck_tile/ops/common/streamk_common.hpp" #include "ck_tile/ops/common/tensor_layout.hpp" #include "ck_tile/ops/common/utils.hpp" diff --git a/include/ck_tile/ops/norm_reduce.hpp b/include/ck_tile/ops/norm_reduce.hpp index 7dc3e8b7e7..aa074b7f9f 100644 --- a/include/ck_tile/ops/norm_reduce.hpp +++ b/include/ck_tile/ops/norm_reduce.hpp @@ -7,5 +7,7 @@ #include "ck_tile/ops/norm_reduce/block/block_norm_reduce_problem.hpp" #include "ck_tile/ops/norm_reduce/thread/thread_welford.hpp" #include "ck_tile/ops/common/generic_2d_block_shape.hpp" +#include "ck_tile/ops/common/load_interleaved_pk_type.hpp" +#include "ck_tile/ops/common/streamk_common.hpp" #include "ck_tile/ops/common/tensor_layout.hpp" #include "ck_tile/ops/common/utils.hpp" diff --git a/include/ck_tile/ops/permute.hpp b/include/ck_tile/ops/permute.hpp index 1cc3d9cbc3..46512c57fe 100644 --- a/include/ck_tile/ops/permute.hpp +++ b/include/ck_tile/ops/permute.hpp @@ -6,5 +6,7 @@ #include "ck_tile/ops/permute/kernel/generic_permute_kernel.hpp" #include "ck_tile/ops/permute/pipeline/generic_petmute_problem.hpp" #include "ck_tile/ops/common/generic_2d_block_shape.hpp" +#include "ck_tile/ops/common/load_interleaved_pk_type.hpp" +#include "ck_tile/ops/common/streamk_common.hpp" #include "ck_tile/ops/common/tensor_layout.hpp" #include "ck_tile/ops/common/utils.hpp" diff --git a/include/ck_tile/ops/reduce.hpp b/include/ck_tile/ops/reduce.hpp index a6721c9305..d628e9c945 100644 --- a/include/ck_tile/ops/reduce.hpp +++ b/include/ck_tile/ops/reduce.hpp @@ -11,5 +11,7 @@ #include "ck_tile/ops/reduce/pipeline/reduce2d_problem.hpp" #include "ck_tile/ops/reduce/pipeline/reduce2d_shape.hpp" #include "ck_tile/ops/common/generic_2d_block_shape.hpp" +#include "ck_tile/ops/common/load_interleaved_pk_type.hpp" +#include "ck_tile/ops/common/streamk_common.hpp" #include "ck_tile/ops/common/tensor_layout.hpp" #include "ck_tile/ops/common/utils.hpp" diff --git a/include/ck_tile/ops/rmsnorm2d.hpp b/include/ck_tile/ops/rmsnorm2d.hpp index 610541b2e4..00afcf4aed 100644 --- a/include/ck_tile/ops/rmsnorm2d.hpp +++ b/include/ck_tile/ops/rmsnorm2d.hpp @@ -11,5 +11,7 @@ #include "ck_tile/ops/rmsnorm2d/pipeline/rmsnorm2d_fwd_pipeline_two_pass.hpp" #include "ck_tile/ops/rmsnorm2d/pipeline/rmsnorm2d_fwd_traits.hpp" #include "ck_tile/ops/common/generic_2d_block_shape.hpp" +#include "ck_tile/ops/common/load_interleaved_pk_type.hpp" +#include "ck_tile/ops/common/streamk_common.hpp" #include "ck_tile/ops/common/tensor_layout.hpp" #include "ck_tile/ops/common/utils.hpp" diff --git a/include/ck_tile/ops/smoothquant.hpp b/include/ck_tile/ops/smoothquant.hpp index dc164dc1a0..1aa14c69e1 100644 --- a/include/ck_tile/ops/smoothquant.hpp +++ b/include/ck_tile/ops/smoothquant.hpp @@ -10,5 +10,7 @@ #include "ck_tile/ops/smoothquant/pipeline/smoothquant_pipeline_problem.hpp" #include "ck_tile/ops/smoothquant/pipeline/smoothquant_pipeline_two_pass.hpp" #include "ck_tile/ops/common/generic_2d_block_shape.hpp" +#include "ck_tile/ops/common/load_interleaved_pk_type.hpp" +#include "ck_tile/ops/common/streamk_common.hpp" #include "ck_tile/ops/common/tensor_layout.hpp" #include "ck_tile/ops/common/utils.hpp" diff --git a/include/ck_tile/ops/softmax.hpp b/include/ck_tile/ops/softmax.hpp index b23e869d81..d559dc15e2 100644 --- a/include/ck_tile/ops/softmax.hpp +++ b/include/ck_tile/ops/softmax.hpp @@ -6,5 +6,7 @@ #include "ck_tile/ops/softmax/block/block_softmax_2d.hpp" #include "ck_tile/ops/softmax/block/block_softmax_2d_problem.hpp" #include "ck_tile/ops/common/generic_2d_block_shape.hpp" +#include "ck_tile/ops/common/load_interleaved_pk_type.hpp" +#include "ck_tile/ops/common/streamk_common.hpp" #include "ck_tile/ops/common/tensor_layout.hpp" #include "ck_tile/ops/common/utils.hpp" diff --git a/include/ck_tile/ops/topk.hpp b/include/ck_tile/ops/topk.hpp index 1dc563f757..040c6b8ddc 100644 --- a/include/ck_tile/ops/topk.hpp +++ b/include/ck_tile/ops/topk.hpp @@ -6,5 +6,7 @@ #include "ck_tile/ops/topk/block/block_topk_stream_2d.hpp" #include "ck_tile/ops/topk/block/block_topk_stream_2d_problem.hpp" #include "ck_tile/ops/common/generic_2d_block_shape.hpp" +#include "ck_tile/ops/common/load_interleaved_pk_type.hpp" +#include "ck_tile/ops/common/streamk_common.hpp" #include "ck_tile/ops/common/tensor_layout.hpp" #include "ck_tile/ops/common/utils.hpp" diff --git a/include/ck_tile/ops/topk_softmax.hpp b/include/ck_tile/ops/topk_softmax.hpp index d0a810de4f..d9657a9764 100644 --- a/include/ck_tile/ops/topk_softmax.hpp +++ b/include/ck_tile/ops/topk_softmax.hpp @@ -8,5 +8,7 @@ #include "ck_tile/ops/topk_softmax/pipeline/topk_softmax_warp_per_row_policy.hpp" #include "ck_tile/ops/topk_softmax/pipeline/topk_softmax_warp_per_row_problem.hpp" #include "ck_tile/ops/common/generic_2d_block_shape.hpp" +#include "ck_tile/ops/common/load_interleaved_pk_type.hpp" +#include "ck_tile/ops/common/streamk_common.hpp" #include "ck_tile/ops/common/tensor_layout.hpp" #include "ck_tile/ops/common/utils.hpp" diff --git a/tile_engine/ops/gemm/gemm_common.hpp b/tile_engine/ops/gemm/gemm_common.hpp index 5188915f1a..179aeb7307 100644 --- a/tile_engine/ops/gemm/gemm_common.hpp +++ b/tile_engine/ops/gemm/gemm_common.hpp @@ -74,58 +74,6 @@ constexpr auto is_row_major(Layout) return ck_tile::bool_constant>{}; } -// Permutation function for pk_int4_t -template -void permute_vectors_i4x4_b(Tensor& tensor) -{ - const ck_tile::index_t K = tensor.get_length(0); - const ck_tile::index_t N = tensor.get_length(1); - // vector pk_i4x4 permute - for(int i = 0; i < N; i++) - { - for(int j = 0; j < K; j += 8) - { - int8_t input[8]; - - for(int k = 0; k < 4; k++) - { - int8_t i4x2 = tensor(j + k * 2, i).data; - input[k * 2 + 0] = (i4x2 >> 4) & 0xf; - input[k * 2 + 1] = (i4x2 >> 0) & 0xf; - } - - // permute 01234567->20643175 - { - int8_t hi = input[2]; - int8_t lo = input[0]; - int8_t i4x2 = (hi << 4) | lo; - tensor(j + 0, i) = i4x2; - } - - { - int8_t hi = input[6]; - int8_t lo = input[4]; - int8_t i4x2 = (hi << 4) | lo; - tensor(j + 2, i) = i4x2; - } - - { - int8_t hi = input[3]; - int8_t lo = input[1]; - int8_t i4x2 = (hi << 4) | lo; - tensor(j + 4, i) = i4x2; - } - - { - int8_t hi = input[7]; - int8_t lo = input[5]; - int8_t i4x2 = (hi << 4) | lo; - tensor(j + 6, i) = i4x2; - } - } - } -} - // Structure to hold kernel traits for dispatcher struct KernelTraits { diff --git a/tile_engine/ops/gemm/gemm_profiler.hpp b/tile_engine/ops/gemm/gemm_profiler.hpp index bbf0c92e67..1298c78d18 100644 --- a/tile_engine/ops/gemm/gemm_profiler.hpp +++ b/tile_engine/ops/gemm/gemm_profiler.hpp @@ -96,7 +96,7 @@ class GemmProfiler // Permute vector pk_i4x4 data for device implementation ck_tile::HostTensor b_k_n_dev = b_k_n; // permute_tensor_b(b_k_n_dev); - permute_vectors_i4x4_b(b_k_n_dev); + ck_tile::permute_vectors_i4x4_b(b_k_n_dev); b_k_n_dev_buf.ToDevice(b_k_n_dev.data()); } else From 3021604213750fc5acb02dad50e60ea8b0176b91 Mon Sep 17 00:00:00 2001 From: aledudek Date: Mon, 13 Oct 2025 13:55:23 +0200 Subject: [PATCH 013/262] [CK_TILE] Batched Gemm Kernel IsSupported function checks (#2860) * Add valid check batched gemm part1 * [CK_TILE] Add batched gemm kernel IsSupported func checks * revert broken pre-commit hook changes * revert broken pre-commit hook changes v2 * Clarify error messages --- .../ops/gemm/kernel/batched_gemm_kernel.hpp | 37 ++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/include/ck_tile/ops/gemm/kernel/batched_gemm_kernel.hpp b/include/ck_tile/ops/gemm/kernel/batched_gemm_kernel.hpp index 6f9d53467f..806a471397 100644 --- a/include/ck_tile/ops/gemm/kernel/batched_gemm_kernel.hpp +++ b/include/ck_tile/ops/gemm/kernel/batched_gemm_kernel.hpp @@ -161,8 +161,43 @@ struct BatchedGemmKernel } CK_TILE_HOST static auto - IsSupportedArgument(const typename UniversalGemmKernel::KernelArgs& kargs) -> bool + IsSupportedArgument(const typename BatchedGemmKernel::KernelArgs& kargs) -> bool { + if(kargs.batch_count < 1) + { + if(ck_tile::EnvIsEnabled(CK_TILE_ENV(CK_TILE_LOGGING))) + { + CK_TILE_ERROR("Conditions not met: batch_count must be at least 1 !"); + } + return false; + } + if(kargs.batch_stride_A < 0 || kargs.batch_stride_A < kargs.M * kargs.K) + { + if(ck_tile::EnvIsEnabled(CK_TILE_ENV(CK_TILE_LOGGING))) + { + CK_TILE_ERROR( + "Conditions not met: batch_stride_A must be non-negative and at least K * M!"); + } + return false; + } + if(kargs.batch_stride_B < 0 || kargs.batch_stride_B < kargs.K * kargs.N) + { + if(ck_tile::EnvIsEnabled(CK_TILE_ENV(CK_TILE_LOGGING))) + { + CK_TILE_ERROR( + "Conditions not met: batch_stride_B must be non-negative and at least K * N!"); + } + return false; + } + if(kargs.batch_stride_E < 0 || kargs.batch_stride_E < kargs.M * kargs.N) + { + if(ck_tile::EnvIsEnabled(CK_TILE_ENV(CK_TILE_LOGGING))) + { + CK_TILE_ERROR( + "Conditions not met: batch_stride_E must be non-negative and at least M * N!"); + } + return false; + } return UniversalGemmKernel::IsSupportedArgument(kargs); } From 634634f5c09a3b42f5f838a5af9c948602e246db Mon Sep 17 00:00:00 2001 From: aledudek Date: Mon, 13 Oct 2025 13:57:37 +0200 Subject: [PATCH 014/262] [CK_TILE] Blockwise GEMM pipeline v6 - port of v5 from old CK (#2955) * First checkpoint * Second checkpoint - hot loop scheduler * Third checkpoint - init main operator * Fourth checkpoint - main loop ready * Fifth checkpoint - main loop fix * Sixth checkpoint - ReadWritecompFunc * Seventh checkpoint - Tail finished * [CK_TILE] Blockwise gemm pipeline v5 complete * Working * Working fixes 2 * Rename v5 to v77 temporarily * Data type adjustment * Data type adjustment 2 * [CK_TILE] Blockwise Gemm pipeline v5 add tests * [CK_TILE] Fix calculation error * TEMP: check pipeline * Fix name to V6 * naming and documentation changes * WIP dump * Try fixing v1 * Failing tests v5 * Debugging * Changes v2 * F16 tests working great * Working BlockwiseGemmPipelineV5 as V6 * Cleanup and format * Merging changes part1 * [CK_TILE] Blockwise Gemm Pipeline Comp V5/V6 * Remove commented code * Fix gfx950 build issues * Fix file formatting * Review changes, more concat info, add bf16 bf8 tests * Fix formatting * Add bf16 and bf8 tests --------- Co-authored-by: Adam Osewski --- example/ck_tile/03_gemm/gemm_utils.hpp | 40 +- include/ck_tile/ops/gemm.hpp | 2 + .../gemm_pipeline_ag_bg_cr_comp_v6.hpp | 770 ++++++++++++++++++ ...peline_ag_bg_cr_comp_v6_default_policy.hpp | 56 ++ test/ck_tile/gemm/CMakeLists.txt | 10 +- .../gemm/test_gemm_pipeline_compv6.cpp | 17 + .../gemm/test_gemm_pipeline_kernel_types.hpp | 23 + test/ck_tile/gemm/test_gemm_pipeline_util.hpp | 18 +- 8 files changed, 924 insertions(+), 12 deletions(-) create mode 100644 include/ck_tile/ops/gemm/pipeline/gemm_pipeline_ag_bg_cr_comp_v6.hpp create mode 100644 include/ck_tile/ops/gemm/pipeline/gemm_pipeline_ag_bg_cr_comp_v6_default_policy.hpp create mode 100644 test/ck_tile/gemm/test_gemm_pipeline_compv6.cpp diff --git a/example/ck_tile/03_gemm/gemm_utils.hpp b/example/ck_tile/03_gemm/gemm_utils.hpp index 07b925d0eb..a831a4f26c 100644 --- a/example/ck_tile/03_gemm/gemm_utils.hpp +++ b/example/ck_tile/03_gemm/gemm_utils.hpp @@ -16,8 +16,9 @@ #define CK_TILE_PIPELINE_MEMORY 2 #define CK_TILE_PIPELINE_COMPUTE_V4 3 #define CK_TILE_PIPELINE_COMPUTE_V5 4 -#define CK_TILE_PIPELINE_PRESHUFFLE_V1 5 -#define CK_TILE_PIPELINE_PRESHUFFLE_V2 6 +#define CK_TILE_PIPELINE_COMPUTE_V6 5 +#define CK_TILE_PIPELINE_PRESHUFFLE_V1 6 +#define CK_TILE_PIPELINE_PRESHUFFLE_V2 7 template constexpr ck_tile::index_t get_k_warp_tile() @@ -251,9 +252,29 @@ struct GemmConfigComputeV5 : public GemmConfigBase static constexpr ck_tile::index_t N_Warp_Tile = 32; static constexpr ck_tile::index_t K_Warp_Tile = get_k_warp_tile(); - static constexpr bool DoubleSmemBuffer = false; - static constexpr ck_tile::index_t Pipeline = CK_TILE_PIPELINE_COMPUTE_V5; - static constexpr ck_tile::index_t NumWaNumWaveGroups = 2; + static constexpr bool DoubleSmemBuffer = false; + static constexpr ck_tile::index_t Pipeline = CK_TILE_PIPELINE_COMPUTE_V5; + static constexpr ck_tile::index_t NumWaveGroups = 2; +}; + +template +struct GemmConfigComputeV6 : public GemmConfigBase +{ + static constexpr ck_tile::index_t M_Tile = 256; + static constexpr ck_tile::index_t N_Tile = 256; + static constexpr ck_tile::index_t K_Tile = 32; + + static constexpr ck_tile::index_t M_Warp = 2; + static constexpr ck_tile::index_t N_Warp = 2; + static constexpr ck_tile::index_t K_Warp = 1; + + static constexpr ck_tile::index_t M_Warp_Tile = 32; + static constexpr ck_tile::index_t N_Warp_Tile = 32; + static constexpr ck_tile::index_t K_Warp_Tile = 16; + + static constexpr bool DoubleSmemBuffer = false; + static constexpr ck_tile::index_t Pipeline = CK_TILE_PIPELINE_COMPUTE_V6; + static constexpr ck_tile::index_t NumWaveGroups = 1; }; template @@ -484,6 +505,15 @@ struct PipelineTypeTraits using UniversalGemmPipeline = ck_tile::BaseGemmPipelineAgBgCrCompV5; }; +template <> +struct PipelineTypeTraits +{ + template + using GemmPipeline = ck_tile::GemmPipelineAgBgCrCompV6; + template + using UniversalGemmPipeline = ck_tile::BaseGemmPipelineAgBgCrCompV6; +}; + template <> struct PipelineTypeTraits { diff --git a/include/ck_tile/ops/gemm.hpp b/include/ck_tile/ops/gemm.hpp index 204d67a0ff..2a4f9d21e3 100644 --- a/include/ck_tile/ops/gemm.hpp +++ b/include/ck_tile/ops/gemm.hpp @@ -44,6 +44,8 @@ #include "ck_tile/ops/gemm/pipeline/gemm_pipeline_ag_bg_cr_comp_v4_default_policy.hpp" #include "ck_tile/ops/gemm/pipeline/gemm_pipeline_ag_bg_cr_comp_v5.hpp" #include "ck_tile/ops/gemm/pipeline/gemm_pipeline_ag_bg_cr_comp_v5_default_policy.hpp" +#include "ck_tile/ops/gemm/pipeline/gemm_pipeline_ag_bg_cr_comp_v6.hpp" +#include "ck_tile/ops/gemm/pipeline/gemm_pipeline_ag_bg_cr_comp_v6_default_policy.hpp" #include "ck_tile/ops/gemm/pipeline/gemm_pipeline_ag_bg_cr_mem.hpp" #include "ck_tile/ops/gemm/pipeline/gemm_pipeline_ag_bg_cr_scheduler.hpp" #include "ck_tile/ops/gemm/pipeline/gemm_pipeline_agmem_bgmem_creg_v1.hpp" diff --git a/include/ck_tile/ops/gemm/pipeline/gemm_pipeline_ag_bg_cr_comp_v6.hpp b/include/ck_tile/ops/gemm/pipeline/gemm_pipeline_ag_bg_cr_comp_v6.hpp new file mode 100644 index 0000000000..2ae9001098 --- /dev/null +++ b/include/ck_tile/ops/gemm/pipeline/gemm_pipeline_ag_bg_cr_comp_v6.hpp @@ -0,0 +1,770 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2025, Advanced Micro Devices, Inc. All rights reserved. +#pragma once +#include "ck_tile/core.hpp" +#include "ck_tile/ops/gemm/pipeline/gemm_pipeline_ag_bg_cr_scheduler.hpp" +#include "ck_tile/ops/gemm/pipeline/gemm_pipeline_ag_bg_cr_base.hpp" +#include "ck_tile/ops/gemm/pipeline/gemm_pipeline_ag_bg_cr_comp_v6_default_policy.hpp" + +namespace ck_tile { + +// A Tile Window: global memory +// B Tile Window: global memory +// C Distributed tensor: register +template +struct BaseGemmPipelineAgBgCrCompV6 +{ + static constexpr index_t PrefetchStages = 3; + static constexpr index_t PrefillStages = 1; + static constexpr index_t GlobalBufferNum = 2; + static constexpr index_t HotloopUnroll = 2; + + CK_TILE_HOST_DEVICE static constexpr auto TransposeC() { return Problem::TransposeC; } + + CK_TILE_HOST static constexpr bool BlockHasHotloop(index_t num_loop) + { + return num_loop > PrefetchStages; + } + + CK_TILE_HOST static constexpr TailNumber GetBlockLoopTailNum(index_t num_loop) + { + if(num_loop % HotloopUnroll == 1) + { + return TailNumber::Odd; + } + else + { + return TailNumber::Even; + } + } + + template + CK_TILE_HOST_DEVICE static auto + TailHandler(const RunFunction& run_func, bool has_hot_loop, TailNumber tail_number) + { + // Handle all the valid cases. + if(has_hot_loop) + { + if(tail_number == TailNumber::Odd) + { + return run_func(bool_constant{}, + integral_constant{}); + } + else if(tail_number == TailNumber::Even) + { + return run_func(bool_constant{}, + integral_constant{}); + } + } + else + { + if(tail_number == TailNumber::Odd) + { + return run_func(bool_constant{}, + integral_constant{}); + } + else if(tail_number == TailNumber::Even) + { + return run_func(bool_constant{}, + integral_constant{}); + } + } + // If execution reaches here, it's an invalid tail_number because it wasn't handled above. +#if defined(__HIP_DEVICE_COMPILE__) + __builtin_unreachable(); +#else + throw std::logic_error("Invalid TailNumber: Only TailNumber::Odd and TailNumber::Even are " + "supported in this pipeline context."); +#endif + } +}; + +// Compute optimized pipeline +// GlobalPrefetchStages: 3 +// LocalPreFillStages: 1 +// LocalPreFetchStages: 1 +// LocalSharedMemoryBuffer: 2 +template +struct GemmPipelineAgBgCrCompV6 : public BaseGemmPipelineAgBgCrCompV6 +{ + using Base = BaseGemmPipelineAgBgCrCompV6; + using BasePImpl = GemmPipelineAgBgCrImplBase; + + using AsDataType = remove_cvref_t; + using BsDataType = remove_cvref_t; + using CDataType = remove_cvref_t; + using BlockGemmShape = remove_cvref_t; + + using AElementWise = remove_cvref_t; + using BElementWise = remove_cvref_t; + + using AsLayout = remove_cvref_t; + using BsLayout = remove_cvref_t; + using CLayout = remove_cvref_t; + + using ALayout = remove_cvref_t>; + using BLayout = remove_cvref_t>; + + using ADataType = remove_cvref_t>; + using BDataType = remove_cvref_t>; + + static_assert(!std::is_same_v, "Not implemented"); + + static constexpr index_t APackedSize = + ck_tile::numeric_traits>::PackedSize; + static constexpr index_t BPackedSize = + ck_tile::numeric_traits>::PackedSize; + + static constexpr index_t NumWaveGroups = Problem::NumWaveGroups; + + using BlockGemm = remove_cvref_t())>; + static constexpr auto I0 = number<0>{}; + static constexpr auto I1 = number<1>{}; + static constexpr auto I2 = number<2>{}; + + static constexpr index_t BlockSize = Problem::kBlockSize; + + static constexpr index_t MPerBlock = BlockGemmShape::kM; + static constexpr index_t NPerBlock = BlockGemmShape::kN; + static constexpr index_t KPerBlock = BlockGemmShape::kK; + + template + static constexpr index_t GetVectorSizeA() + { + return Policy::template GetVectorSizeA(); + } + template + static constexpr index_t GetVectorSizeB() + { + return Policy::template GetVectorSizeB(); + } + static constexpr index_t GetVectorSizeC() { return Policy::template GetVectorSizeC(); } + + static constexpr index_t GetSmemPackA() { return Policy::template GetSmemPackA(); } + static constexpr index_t GetSmemPackB() { return Policy::template GetSmemPackB(); } + + static constexpr index_t KRepeat = BlockGemm::WarpGemm::kKPerThread / GetSmemPackA(); + + static constexpr bool kPadM = Problem::kPadM; + static constexpr bool kPadN = Problem::kPadN; + static constexpr bool kPadK = Problem::kPadK; + + static constexpr bool DoubleSmemBuffer = Problem::DoubleSmemBuffer; + static constexpr index_t Preshuffle = Problem::Preshuffle; + + static constexpr bool HasHotLoop = Problem::HasHotLoop; + static constexpr auto TailNum = Problem::TailNum; + static constexpr auto Scheduler = Problem::Scheduler; + + static constexpr auto is_a_load_tr_v = bool_constant{}; + static constexpr auto is_b_load_tr_v = bool_constant{}; + + [[nodiscard]] CK_TILE_HOST static const std::string GetName() + { + // clang-format off + return concat('_', "pipeline_AgBgCrCompV6", BlockSize, + concat('x', GetVectorSizeA(), GetVectorSizeB(), GetVectorSizeC()), + concat('x', kPadM, kPadN, kPadK), + concat('x', TailNum), + concat('_', KRepeat), + concat('_', DoubleSmemBuffer), + concat('_', Preshuffle), + concat('_', HasHotLoop)); + // clang-format on + } + + CK_TILE_HOST_DEVICE static constexpr index_t GetSmemSize() + { + return Policy::template GetSmemSize(); + } + + CK_TILE_HOST_DEVICE static constexpr auto IsTransposeC() + { + return Policy::template IsTransposeC(); + } + + template + struct PipelineImpl : public BasePImpl + { + }; + + template <> + struct PipelineImpl : public BasePImpl + { + CK_TILE_DEVICE static constexpr auto HotLoopScheduler() + { + constexpr index_t MPerXDL = BlockGemmShape::WarpTile::at(I0); + constexpr index_t NPerXDL = BlockGemmShape::WarpTile::at(I1); + constexpr index_t KPerXDL = BlockGemmShape::WarpTile::at(I2); + + constexpr index_t WaveSize = 64; + constexpr index_t WaveNumM = BlockGemmShape::BlockWarps::at(I0); + constexpr index_t WaveNumN = BlockGemmShape::BlockWarps::at(I1); + + constexpr index_t A_LDS_Read_Width = KPerXDL; + constexpr index_t B_LDS_Read_Width = KPerXDL; + + constexpr index_t A_Buffer_Load_Inst_Num = + MPerBlock * KPerBlock / (BlockSize * GetVectorSizeA()); + constexpr index_t B_Buffer_Load_Inst_Num = + NPerBlock * KPerBlock / (BlockSize * GetVectorSizeB()); + + constexpr index_t A_LDS_Write_Inst_Num = MPerBlock * KPerBlock / (BlockSize * KPerXDL); + constexpr index_t B_LDS_Write_Inst_Num = NPerBlock * KPerBlock / (BlockSize * KPerXDL); + + constexpr index_t A_LDS_Read_Inst_Num = + WaveNumN * MPerBlock * KPerBlock / (BlockSize * KPerXDL); + constexpr index_t B_LDS_Read_Inst_Num = + WaveNumM * NPerBlock * KPerBlock / (BlockSize * KPerXDL); + + constexpr index_t C_MFMA_Inst_Num = MPerBlock * NPerBlock * KPerBlock / + (BlockSize / WaveSize) / + (MPerXDL * NPerXDL * KPerXDL); + + constexpr auto num_ds_read_inst_a = + A_LDS_Read_Width * sizeof(ADataType) / APackedSize == 16 ? A_LDS_Read_Inst_Num + : A_LDS_Read_Inst_Num / 2; + constexpr auto num_ds_read_inst_b = + B_LDS_Read_Width * sizeof(BDataType) / BPackedSize == 16 ? B_LDS_Read_Inst_Num + : B_LDS_Read_Inst_Num / 2; + + constexpr auto mfma_cycle = NPerXDL == 16 ? 16 : 32; + + constexpr auto ds_read_a_issue_cycle = + A_LDS_Read_Width * sizeof(ADataType) / APackedSize == 16 ? 8 : 4; + constexpr auto ds_read_b_issue_cycle = + B_LDS_Read_Width * sizeof(BDataType) / BPackedSize == 16 ? 8 : 4; + + constexpr auto ds_read_a_mfma_rate = + (mfma_cycle - 4 + 2 * ds_read_a_issue_cycle - 1) / (2 * ds_read_a_issue_cycle); + constexpr auto ds_read_b_mfma_rate = + (mfma_cycle - 4 + 2 * ds_read_b_issue_cycle - 1) / (2 * ds_read_b_issue_cycle); + + constexpr auto num_dsread_stage1_a = num_ds_read_inst_a / KRepeat * (KRepeat - 1); + constexpr auto num_dsread_stage1_b = num_ds_read_inst_b / KRepeat * (KRepeat - 1); + constexpr auto num_dsread_stage3_a = num_ds_read_inst_a / KRepeat; + constexpr auto num_dsread_stage3_b = num_ds_read_inst_b / KRepeat; + + constexpr auto num_dsread_stage1_a_mfma = + (num_dsread_stage1_a + ds_read_a_mfma_rate - 1) / ds_read_a_mfma_rate; + constexpr auto num_dsread_stage1_b_mfma = + (num_dsread_stage1_b + ds_read_b_mfma_rate - 1) / ds_read_b_mfma_rate; + constexpr auto num_dsread_stage3_a_mfma = + (num_dsread_stage3_a + ds_read_a_mfma_rate - 1) / ds_read_a_mfma_rate; + constexpr auto num_dsread_stage3_b_mfma = + (num_dsread_stage3_b + ds_read_b_mfma_rate - 1) / ds_read_b_mfma_rate; + + constexpr auto num_mfma_stage2 = C_MFMA_Inst_Num - + num_ds_read_inst_a / ds_read_a_mfma_rate - + num_ds_read_inst_b / ds_read_b_mfma_rate; + constexpr auto num_mfma_per_issue = + num_mfma_stage2 / (A_Buffer_Load_Inst_Num + B_Buffer_Load_Inst_Num); + constexpr auto num_dswrite_per_issue_a = A_LDS_Write_Inst_Num / A_Buffer_Load_Inst_Num; + constexpr auto num_dswrite_per_issue_b = B_LDS_Write_Inst_Num / B_Buffer_Load_Inst_Num; + + // stage 1 + static_for<0, num_dsread_stage1_a_mfma, 1>{}([&](auto i) { + ignore = i; + if constexpr((num_dsread_stage1_a - (i + 1) * ds_read_a_mfma_rate) >= + ds_read_a_mfma_rate) + { + __builtin_amdgcn_sched_group_barrier(0x100, ds_read_a_mfma_rate, 0); // DS read + } + else + { + __builtin_amdgcn_sched_group_barrier( + 0x100, + num_dsread_stage1_a - (num_dsread_stage1_a_mfma - 1) * ds_read_a_mfma_rate, + 0); // DS read + } + __builtin_amdgcn_sched_group_barrier(0x008, 1, 0); // MFMA + }); + static_for<0, num_dsread_stage1_b_mfma, 1>{}([&](auto i) { + ignore = i; + if constexpr((num_dsread_stage1_b - (i + 1) * ds_read_b_mfma_rate) >= + ds_read_b_mfma_rate) + { + __builtin_amdgcn_sched_group_barrier(0x100, ds_read_b_mfma_rate, 0); // DS read + } + else + { + __builtin_amdgcn_sched_group_barrier( + 0x100, + num_dsread_stage1_b - (num_dsread_stage1_b_mfma - 1) * ds_read_b_mfma_rate, + 0); // DS read + } + __builtin_amdgcn_sched_group_barrier(0x008, 1, 0); // MFMA + }); + + // stage 2 + static_for<0, A_Buffer_Load_Inst_Num, 1>{}([&](auto i) { + ignore = i; + static_for<0, num_dswrite_per_issue_a, 1>{}([&](auto idswrite) { + ignore = idswrite; + __builtin_amdgcn_sched_group_barrier(0x200, 1, 0); // DS write + __builtin_amdgcn_sched_group_barrier(0x008, 1, 0); // MFMA + }); + __builtin_amdgcn_sched_group_barrier(0x020, 1, 0); // VMEM read + __builtin_amdgcn_sched_group_barrier( + 0x008, num_mfma_per_issue - num_dswrite_per_issue_a, 0); // MFMA + }); + static_for<0, B_Buffer_Load_Inst_Num, 1>{}([&](auto i) { + ignore = i; + static_for<0, num_dswrite_per_issue_b, 1>{}([&](auto idswrite) { + ignore = idswrite; + __builtin_amdgcn_sched_group_barrier(0x200, 1, 0); // DS write + __builtin_amdgcn_sched_group_barrier(0x008, 1, 0); // MFMA + }); + __builtin_amdgcn_sched_group_barrier(0x020, 1, 0); // VMEM read + __builtin_amdgcn_sched_group_barrier( + 0x008, num_mfma_per_issue - num_dswrite_per_issue_b, 0); // MFMA + }); + + // stage 3 + static_for<0, num_dsread_stage3_a_mfma, 1>{}([&](auto i) { + ignore = i; + if constexpr((num_dsread_stage3_a - (i + 1) * ds_read_a_mfma_rate) >= + ds_read_a_mfma_rate) + { + __builtin_amdgcn_sched_group_barrier(0x100, ds_read_a_mfma_rate, 0); // DS read + } + else + { + __builtin_amdgcn_sched_group_barrier( + 0x100, + num_dsread_stage3_a - (num_dsread_stage3_a_mfma - 1) * ds_read_a_mfma_rate, + 0); // DS read + } + __builtin_amdgcn_sched_group_barrier(0x008, 1, 0); // MFMA + }); + static_for<0, num_dsread_stage3_b_mfma, 1>{}([&](auto i) { + ignore = i; + if constexpr((num_dsread_stage3_b - (i + 1) * ds_read_b_mfma_rate) >= + ds_read_b_mfma_rate) + { + __builtin_amdgcn_sched_group_barrier(0x100, ds_read_b_mfma_rate, 0); // DS read + } + else + { + __builtin_amdgcn_sched_group_barrier( + 0x100, + num_dsread_stage3_b - (num_dsread_stage3_b_mfma - 1) * ds_read_b_mfma_rate, + 0); // DS read + } + __builtin_amdgcn_sched_group_barrier(0x008, 1, 0); // MFMA + }); + __builtin_amdgcn_sched_barrier(0); + } + + template ::value && + is_detected::value, + bool>* = nullptr> + CK_TILE_DEVICE auto operator()(const AsDramBlockWindowTmp& a_dram_block_window_tmp, + const AElementFunction& a_element_func, + const BsDramBlockWindowTmp& b_dram_block_window_tmp, + const BElementFunction& b_element_func, + index_t num_loop, + void* __restrict__ p_smem) const + { + // TODO: Add Multi A/B support + static_assert(std::tuple_size>::value == 1, + "Multi A/B is not yet supported for this pipeline."); + static_assert(std::tuple_size>::value == 1, + "Multi A/B is not yet supported for this pipeline."); + + using ADramBlockWindowTmp = + remove_cvref_t{}, AsDramBlockWindowTmp>>; + using BDramBlockWindowTmp = + remove_cvref_t{}, BsDramBlockWindowTmp>>; + static_assert( + std::is_same_v> && + std::is_same_v>, + "Data Type conflict on A and B matrix input data type."); + + constexpr bool is_a_col_major = + std::is_same_v; + constexpr bool is_b_row_major = std::is_same_v; + + static_assert(is_a_col_major + ? (KPerBlock == ADramBlockWindowTmp{}.get_window_lengths()[I0] && + MPerBlock == ADramBlockWindowTmp{}.get_window_lengths()[I1]) + : (MPerBlock == ADramBlockWindowTmp{}.get_window_lengths()[I0] && + KPerBlock == ADramBlockWindowTmp{}.get_window_lengths()[I1]), + "A block window has incorrect lengths for defined ALayout!"); + static_assert(is_b_row_major + ? (KPerBlock == BDramBlockWindowTmp{}.get_window_lengths()[I0] && + NPerBlock == BDramBlockWindowTmp{}.get_window_lengths()[I1]) + : (NPerBlock == BDramBlockWindowTmp{}.get_window_lengths()[I0] && + KPerBlock == BDramBlockWindowTmp{}.get_window_lengths()[I1]), + "B block window has incorrect lengths for defined BLayout!"); + + ////////////// LDS desc, window & register ///////////////// + using ALdsType = + remove_cvref_t; + using BLdsType = + remove_cvref_t; + auto&& ABLdsTensorViews = BasePImpl::GetABLdsTensorViews(p_smem); + ALdsType& a_lds_block = ABLdsTensorViews.at(I0); + BLdsType& b_lds_block = ABLdsTensorViews.at(I1); + + // Tile distribution for load from lds + constexpr auto a_lds_load_tile_distr = + make_static_tile_distribution(BlockGemm::MakeABlockDistributionEncode()); + constexpr auto b_lds_load_tile_distr = + make_static_tile_distribution(BlockGemm::MakeBBlockDistributionEncode()); + + using acopy_dram_type = + remove_cvref_t; + using bcopy_dram_type = + remove_cvref_t; + + using a_copy_lds_window_type = + remove_cvref_t; + using b_copy_lds_window_type = + remove_cvref_t; + + using a_lds_load_tile_distr_type = + remove_cvref_t; + using b_lds_load_tile_distr_type = + remove_cvref_t; + + auto&& aWindows = + BasePImpl::GetAWindows(a_dram_block_window_tmp, a_lds_block, a_lds_load_tile_distr); + auto&& bWindows = + BasePImpl::GetBWindows(b_dram_block_window_tmp, b_lds_block, b_lds_load_tile_distr); + + // A DRAM tile window for load + // A LDS tile window for store + // A LDS tile for block GEMM + acopy_dram_type& a_copy_dram_window = aWindows.at(I0); + a_copy_lds_window_type& a_copy_lds_window = aWindows.at(I1); + a_lds_load_tile_distr_type& a_lds_gemm_window = aWindows.at(I2); + + // B DRAM tile window for load + // B LDS tile window for store + // B LDS tile for block GEMM + bcopy_dram_type& b_copy_dram_window = bWindows.at(I0); + b_copy_lds_window_type& b_copy_lds_window = bWindows.at(I1); + b_lds_load_tile_distr_type& b_lds_gemm_window = bWindows.at(I2); + + // Block GEMM + auto block_gemm = BlockGemm(); + auto c_block_tile = block_gemm.MakeCBlockTile(); + + using ABlockTileDistr = + decltype(a_copy_dram_window[number<0>{}].get_tile_distribution()); + using BBlockTileDistr = + decltype(b_copy_dram_window[number<0>{}].get_tile_distribution()); + + using ABlockTile = + decltype(make_static_distributed_tensor(ABlockTileDistr{})); + using BBlockTile = + decltype(make_static_distributed_tensor(BBlockTileDistr{})); + + ABlockTile a_block_tile[Base::GlobalBufferNum]; + BBlockTile b_block_tile[Base::GlobalBufferNum]; + + using ADramTileWindowStep = typename ADramBlockWindowTmp::BottomTensorIndex; + using BDramTileWindowStep = typename BDramBlockWindowTmp::BottomTensorIndex; + + constexpr ADramTileWindowStep a_dram_tile_window_step = + is_a_col_major ? make_array(KPerBlock, 0) : make_array(0, KPerBlock); + constexpr BDramTileWindowStep b_dram_tile_window_step = + is_b_row_major ? make_array(KPerBlock, 0) : make_array(0, KPerBlock); + + constexpr auto ALdsTileDistr = decltype(make_static_tile_distribution( + BlockGemm::MakeABlockDistributionEncode())){}; + constexpr auto BLdsTileDistr = decltype(make_static_tile_distribution( + BlockGemm::MakeBBlockDistributionEncode())){}; + + using ALdsTile = decltype(make_static_distributed_tensor(ALdsTileDistr)); + using BLdsTile = decltype(make_static_distributed_tensor(BLdsTileDistr)); + + ALdsTile a_lds_tile; + BLdsTile b_lds_tile; + // ----------------------------------------------------------------------------------------- + // Gemm pipeline start + + // Global prefetch 1 + a_block_tile[I0] = load_tile_with_elementwise(a_copy_dram_window, a_element_func); + move_tile_window(a_copy_dram_window, a_dram_tile_window_step); + b_block_tile[I0] = load_tile_with_elementwise(b_copy_dram_window, b_element_func); + move_tile_window(b_copy_dram_window, b_dram_tile_window_step); + + // initialize C + tile_elementwise_inout([](auto& c) { c = 0; }, c_block_tile); + + // Local prefill 1 + if constexpr(is_a_col_major && !is_a_load_tr_v()) + { + auto a_shuffle_tmp = make_static_distributed_tensor( + Policy::template MakeShuffledARegTileDistribution()); + transpose_tile2d(a_shuffle_tmp, a_block_tile[I0]); + BasePImpl::LocalPrefill(a_copy_lds_window, a_shuffle_tmp); + } + else + { + BasePImpl::LocalPrefill(a_copy_lds_window, a_block_tile[I0]); + } + if constexpr(is_b_row_major && !is_b_load_tr_v()) + { + auto b_shuffle_tmp = make_static_distributed_tensor( + Policy::template MakeShuffledBRegTileDistribution()); + transpose_tile2d(b_shuffle_tmp, b_block_tile[I0]); + BasePImpl::LocalPrefill(b_copy_lds_window, b_shuffle_tmp); + } + else + { + BasePImpl::LocalPrefill(b_copy_lds_window, b_block_tile[I0]); + } + + // Global prefetch 2 + a_block_tile[I0] = load_tile_with_elementwise(a_copy_dram_window, a_element_func); + move_tile_window(a_copy_dram_window, a_dram_tile_window_step); + b_block_tile[I0] = load_tile_with_elementwise(b_copy_dram_window, b_element_func); + move_tile_window(b_copy_dram_window, b_dram_tile_window_step); + + // Global prefetch 3 + a_block_tile[I1] = load_tile_with_elementwise(a_copy_dram_window, a_element_func); + move_tile_window(a_copy_dram_window, a_dram_tile_window_step); + b_block_tile[I1] = load_tile_with_elementwise(b_copy_dram_window, b_element_func); + move_tile_window(b_copy_dram_window, b_dram_tile_window_step); + + block_sync_lds(); + + // Local prefetch 1 + BasePImpl::LocalPrefetch(a_lds_tile, a_lds_gemm_window, is_a_load_tr_v); + BasePImpl::LocalPrefetch(b_lds_tile, b_lds_gemm_window, is_b_load_tr_v); + + if(HasHotLoop) + { + index_t i = 0; + do + { + auto LoopFunc = [&](auto vmem_buf_idx) { + static_for<0, KRepeat, 1>{}([&](auto k0) { + if constexpr(k0 == (KRepeat - 1)) + { + block_sync_lds(); + + // Local prefill 2 + if constexpr(is_a_col_major && !is_a_load_tr_v()) + { + auto a_shuffle_tmp = make_static_distributed_tensor( + Policy::template MakeShuffledARegTileDistribution< + Problem>()); + transpose_tile2d(a_shuffle_tmp, a_block_tile[vmem_buf_idx]); + BasePImpl::LocalPrefill(a_copy_lds_window, a_shuffle_tmp); + } + else + { + BasePImpl::LocalPrefill(a_copy_lds_window, + a_block_tile[vmem_buf_idx]); + } + if constexpr(is_b_row_major && !is_b_load_tr_v()) + { + auto b_shuffle_tmp = make_static_distributed_tensor( + Policy::template MakeShuffledBRegTileDistribution< + Problem>()); + transpose_tile2d(b_shuffle_tmp, b_block_tile[vmem_buf_idx]); + BasePImpl::LocalPrefill(b_copy_lds_window, b_shuffle_tmp); + } + else + { + BasePImpl::LocalPrefill(b_copy_lds_window, + b_block_tile[vmem_buf_idx]); + } + + // Global prefetch 4 + a_block_tile[vmem_buf_idx] = + load_tile_with_elementwise(a_copy_dram_window, a_element_func); + move_tile_window(a_copy_dram_window, a_dram_tile_window_step); + b_block_tile[vmem_buf_idx] = + load_tile_with_elementwise(b_copy_dram_window, b_element_func); + move_tile_window(b_copy_dram_window, b_dram_tile_window_step); + + block_sync_lds(); + } + block_gemm(c_block_tile, a_lds_tile, b_lds_tile); + + // Local prefetch 2 + BasePImpl::LocalPrefetch(a_lds_tile, a_lds_gemm_window, is_a_load_tr_v); + BasePImpl::LocalPrefetch(b_lds_tile, b_lds_gemm_window, is_b_load_tr_v); + }); + + HotLoopScheduler(); + }; + + LoopFunc(I0); + LoopFunc(I1); + + i += Base::HotloopUnroll; + } while(i < (num_loop - Base::PrefetchStages)); + } + + auto ReadWriteCompFunc = [&](auto vmem_buf_idx) { + static_for<0, KRepeat, 1>{}([&](auto k0) { + if constexpr(k0 == (KRepeat - 1)) + { + block_sync_lds(); + + // Local prefill 3 + if constexpr(is_a_col_major && !is_a_load_tr_v()) + { + auto a_shuffle_tmp = make_static_distributed_tensor( + Policy::template MakeShuffledARegTileDistribution()); + transpose_tile2d(a_shuffle_tmp, a_block_tile[vmem_buf_idx]); + BasePImpl::LocalPrefill(a_copy_lds_window, a_shuffle_tmp); + } + else + { + BasePImpl::LocalPrefill(a_copy_lds_window, a_block_tile[vmem_buf_idx]); + } + if constexpr(is_b_row_major && !is_b_load_tr_v()) + { + auto b_shuffle_tmp = make_static_distributed_tensor( + Policy::template MakeShuffledBRegTileDistribution()); + transpose_tile2d(b_shuffle_tmp, b_block_tile[vmem_buf_idx]); + BasePImpl::LocalPrefill(b_copy_lds_window, b_shuffle_tmp); + } + else + { + BasePImpl::LocalPrefill(b_copy_lds_window, b_block_tile[vmem_buf_idx]); + } + + block_sync_lds(); + } + + block_gemm(c_block_tile, a_lds_tile, b_lds_tile); + + BasePImpl::LocalPrefetch(a_lds_tile, a_lds_gemm_window, is_a_load_tr_v); + BasePImpl::LocalPrefetch(b_lds_tile, b_lds_gemm_window, is_b_load_tr_v); + }); + + HotLoopScheduler(); + }; + + auto ReadCompFunc = [&]() { + static_for<0, KRepeat - 1, 1>{}([&]() { + __syncthreads(); + block_gemm(c_block_tile, a_lds_tile, b_lds_tile); + + // Local prefetch 4 + BasePImpl::LocalPrefetch(a_lds_tile, a_lds_gemm_window, is_a_load_tr_v); + BasePImpl::LocalPrefetch(b_lds_tile, b_lds_gemm_window, is_b_load_tr_v); + + __syncthreads(); + }); + + block_gemm(c_block_tile, a_lds_tile, b_lds_tile); + + HotLoopScheduler(); + }; + + if constexpr(TailNum == TailNumber::Odd) + { + ReadWriteCompFunc(I0); + ReadWriteCompFunc(I1); + ReadCompFunc(); + } + else if constexpr(TailNum == TailNumber::Even) + { + ReadWriteCompFunc(I0); + ReadCompFunc(); + } + + return c_block_tile; + } + }; + + public: + template ::value && + is_detected::value, + bool>* = nullptr> + CK_TILE_DEVICE auto operator()(const AsDramBlockWindowTmp& a_dram_block_window_tmp, + const AElementFunction& a_element_func, + const BsDramBlockWindowTmp& b_dram_block_window_tmp, + const BElementFunction& b_element_func, + index_t num_loop, + void* __restrict__ p_smem) const + { + return PipelineImpl{}.template operator()( + a_dram_block_window_tmp, + a_element_func, + b_dram_block_window_tmp, + b_element_func, + num_loop, + p_smem); + } + + template ::value && + is_detected::value, + bool>* = nullptr> + CK_TILE_DEVICE auto operator()(const AsDramBlockWindowTmp& a_dram_block_window_tmp, + const BsDramBlockWindowTmp& b_dram_block_window_tmp, + const index_t num_loop, + void* __restrict__ p_smem) const + { + return PipelineImpl{}.template operator()( + a_dram_block_window_tmp, + [](auto& e, const ADataType& a) { e = a; }, + b_dram_block_window_tmp, + [](auto& e, const BDataType& b) { e = b; }, + num_loop, + p_smem); + } + + template ::value && + !is_detected::value, + bool>* = nullptr> + CK_TILE_DEVICE auto operator()(const ADramBlockWindowTmp& a_dram_block_window_tmp, + const AElementFunction& a_element_func, + const BDramBlockWindowTmp& b_dram_block_window_tmp, + const BElementFunction& b_element_func, + index_t num_loop, + void* __restrict__ p_smem) const + { + return operator()(ck_tile::make_tuple(a_dram_block_window_tmp), + a_element_func, + ck_tile::make_tuple(b_dram_block_window_tmp), + b_element_func, + num_loop, + p_smem); + } +}; +} // namespace ck_tile diff --git a/include/ck_tile/ops/gemm/pipeline/gemm_pipeline_ag_bg_cr_comp_v6_default_policy.hpp b/include/ck_tile/ops/gemm/pipeline/gemm_pipeline_ag_bg_cr_comp_v6_default_policy.hpp new file mode 100644 index 0000000000..6ac702d38b --- /dev/null +++ b/include/ck_tile/ops/gemm/pipeline/gemm_pipeline_ag_bg_cr_comp_v6_default_policy.hpp @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2025, Advanced Micro Devices, Inc. All rights reserved. + +#pragma once + +#include "ck_tile/core.hpp" +#include "ck_tile/ops/gemm/warp/warp_gemm_dispatcher.hpp" +#include "ck_tile/ops/common/tensor_layout.hpp" +#include "ck_tile/ops/gemm/pipeline/gemm_universal_pipeline_ag_bg_cr_policy.hpp" + +namespace ck_tile { +// Default policy for GemmPipelineAGmemBGmemCregComputeV6, except the block gemm method, it shares +// the same vector size implementation, SmemSize, Global memory tile distiribution as the +// UniversalGemm Pipeline Policy. +// Default policy class should not be templated, put template on +// member functions instead. +struct GemmPipelineAgBgCrCompV6DefaultPolicy + : public UniversalGemmBasePolicy +{ + template + CK_TILE_HOST_DEVICE static constexpr auto GetBlockGemm() + { + using BlockWarps = typename Problem::BlockGemmShape::BlockWarps; + using WarpTile = typename Problem::BlockGemmShape::WarpTile; + + constexpr index_t vector_size = + DS_READ_TR_SIZE() / sizeof(typename Problem::ComputeDataType); + constexpr index_t thread_elements = WarpTile::at(I1) * WarpTile::at(I2) / get_warp_size(); + constexpr auto wg_attr_num_access = + !(is_a_load_tr || is_b_load_tr) ? WGAttrNumAccessEnum::Single + : vector_size == thread_elements ? WGAttrNumAccessEnum::Single + : vector_size * 2 == thread_elements ? WGAttrNumAccessEnum::Double + : vector_size * 4 == thread_elements ? WGAttrNumAccessEnum::Quad + : WGAttrNumAccessEnum::Invalid; + + using WarpGemm = WarpGemmDispatcher; + + using BlockGemmPolicy = BlockGemmARegBRegCRegV1CustomPolicy; + + return BlockGemmARegBRegCRegV1{}; + } +}; +} // namespace ck_tile diff --git a/test/ck_tile/gemm/CMakeLists.txt b/test/ck_tile/gemm/CMakeLists.txt index 1ca7f4fc7d..24cc1bc5ab 100644 --- a/test/ck_tile/gemm/CMakeLists.txt +++ b/test/ck_tile/gemm/CMakeLists.txt @@ -24,12 +24,13 @@ endif() if(GPU_TARGETS MATCHES "gfx94|gfx95|gfx12") add_test_executable(test_ck_tile_gemm_pipeline_universal_fp8 test_gemm_pipeline_universal_fp8.cpp) - target_compile_options(test_ck_tile_gemm_pipeline_universal_fp8 PRIVATE ${EXAMPLE_GEMM_COMPILE_OPTIONS}) add_test_executable(test_ck_tile_gemm_pipeline_universal_bf8 test_gemm_pipeline_universal_bf8.cpp) - target_compile_options(test_ck_tile_gemm_pipeline_universal_bf8 PRIVATE ${EXAMPLE_GEMM_COMPILE_OPTIONS}) add_test_executable(test_ck_tile_gemm_pipeline_basic_fp8 test_gemm_pipeline_basic_fp8.cpp) - target_compile_options(test_ck_tile_gemm_pipeline_basic_fp8 PRIVATE ${EXAMPLE_GEMM_COMPILE_OPTIONS}) add_test_executable(test_ck_tile_gemm_pipeline_basic_bf8 test_gemm_pipeline_basic_bf8.cpp) + + target_compile_options(test_ck_tile_gemm_pipeline_basic_fp8 PRIVATE ${EXAMPLE_GEMM_COMPILE_OPTIONS}) + target_compile_options(test_ck_tile_gemm_pipeline_universal_bf8 PRIVATE ${EXAMPLE_GEMM_COMPILE_OPTIONS}) + target_compile_options(test_ck_tile_gemm_pipeline_universal_fp8 PRIVATE ${EXAMPLE_GEMM_COMPILE_OPTIONS}) target_compile_options(test_ck_tile_gemm_pipeline_basic_bf8 PRIVATE ${EXAMPLE_GEMM_COMPILE_OPTIONS}) else() message(DEBUG "Skipping ck_tile_gemm tests for current target") @@ -55,10 +56,13 @@ if(GPU_TARGETS MATCHES "gfx94|gfx95|gfx90a|gfx11|gfx12") add_gtest_executable(test_ck_tile_gemm_pipeline_compv3 test_gemm_pipeline_compv3.cpp) add_gtest_executable(test_ck_tile_gemm_pipeline_compv4 test_gemm_pipeline_compv4.cpp) add_gtest_executable(test_ck_tile_gemm_pipeline_persistent test_gemm_pipeline_persistent.cpp) + add_gtest_executable(test_ck_tile_gemm_pipeline_compv6 test_gemm_pipeline_compv6.cpp) + target_compile_options(test_ck_tile_gemm_pipeline_mem PRIVATE ${EXAMPLE_GEMM_COMPILE_OPTIONS}) target_compile_options(test_ck_tile_gemm_pipeline_compv3 PRIVATE ${EXAMPLE_GEMM_COMPILE_OPTIONS}) target_compile_options(test_ck_tile_gemm_pipeline_compv4 PRIVATE ${EXAMPLE_GEMM_COMPILE_COMPUTE_V4_OPTIONS}) target_compile_options(test_ck_tile_gemm_pipeline_persistent PRIVATE ${EXAMPLE_GEMM_COMPILE_OPTIONS}) + target_compile_options(test_ck_tile_gemm_pipeline_compv6 PRIVATE ${EXAMPLE_GEMM_COMPILE_OPTIONS}) endif() if(GPU_TARGETS MATCHES "gfx95") diff --git a/test/ck_tile/gemm/test_gemm_pipeline_compv6.cpp b/test/ck_tile/gemm/test_gemm_pipeline_compv6.cpp new file mode 100644 index 0000000000..a72ff98055 --- /dev/null +++ b/test/ck_tile/gemm/test_gemm_pipeline_compv6.cpp @@ -0,0 +1,17 @@ +#include "test_gemm_pipeline_kernel_types.hpp" +#include "test_gemm_pipeline_util.hpp" +#include "gtest/gtest.h" + +template +class TestCkTileGemmPipelineCompV6 + : public TestCkTileGemmPipeline> +{ +}; + +#define TEST_SUITE_NAME TestCkTileGemmPipelineCompV6 + +TYPED_TEST_SUITE(TestCkTileGemmPipelineCompV6, KernelTypesCompV6); + +#include "test_gemm_pipeline_ut_cases.inc" + +#undef TEST_SUITE_NAME diff --git a/test/ck_tile/gemm/test_gemm_pipeline_kernel_types.hpp b/test/ck_tile/gemm/test_gemm_pipeline_kernel_types.hpp index bba106174c..aa1f610022 100644 --- a/test/ck_tile/gemm/test_gemm_pipeline_kernel_types.hpp +++ b/test/ck_tile/gemm/test_gemm_pipeline_kernel_types.hpp @@ -29,6 +29,7 @@ using Interwave = ck_tile::integral_constant; using CompV3 = ck_tile::integral_constant; using CompV4 = ck_tile::integral_constant; +using CompV6 = ck_tile::integral_constant; using CompAsync = ck_tile::integral_constant; using Persistent = std::true_type; @@ -130,6 +131,28 @@ using KernelTypesCompV4 = ::testing::Types< std::tuple< Col, Col, Row, F16, F16, F32, F16, I256, I256, I32, I32, I32, I16, Intrawave, CompV4> >; +using KernelTypesCompV6 = ::testing::Types< + std::tuple< Row, Row, Row, F16, F16, F32, F16, I256, I256, I64, I32, I32, I16, Intrawave, CompV6>, + std::tuple< Col, Row, Row, F16, F16, F32, F16, I256, I256, I64, I32, I32, I16, Intrawave, CompV6>, + std::tuple< Col, Col, Row, F16, F16, F32, F16, I256, I256, I64, I32, I32, I16, Intrawave, CompV6>, + std::tuple< Row, Col, Row, F16, F16, F32, F16, I256, I256, I64, I32, I32, I16, Intrawave, CompV6>, + std::tuple< Row, Row, Row, BF16, BF16, F32, F16, I256, I256, I64, I32, I32, I16, Intrawave, CompV6>, + std::tuple< Col, Row, Row, BF16, BF16, F32, F16, I256, I256, I64, I32, I32, I16, Intrawave, CompV6>, + std::tuple< Col, Col, Row, BF16, BF16, F32, F16, I256, I256, I64, I32, I32, I16, Intrawave, CompV6>, + std::tuple< Row, Col, Row, BF16, BF16, F32, F16, I256, I256, I64, I32, I32, I16, Intrawave, CompV6>, + std::tuple< Row, Row, Row, F8, F8, F32, F16, I256, I256, I64, I32, I32, I16, Intrawave, CompV6>, + std::tuple< Row, Col, Row, F8, F8, F32, F16, I256, I256, I64, I32, I32, I16, Intrawave, CompV6>, + std::tuple< Col, Row, Row, F8, F8, F32, F16, I256, I256, I64, I32, I32, I16, Intrawave, CompV6>, + std::tuple< Col, Col, Row, F8, F8, F32, F16, I256, I256, I64, I32, I32, I16, Intrawave, CompV6>, + std::tuple< Row, Row, Row, BF8, BF8, F32, F16, I256, I256, I64, I32, I32, I16, Intrawave, CompV6>, + std::tuple< Row, Col, Row, BF8, BF8, F32, F16, I256, I256, I64, I32, I32, I16, Intrawave, CompV6>, + std::tuple< Col, Row, Row, BF8, BF8, F32, F16, I256, I256, I64, I32, I32, I16, Intrawave, CompV6>, + std::tuple< Col, Col, Row, BF8, BF8, F32, F16, I256, I256, I64, I32, I32, I16, Intrawave, CompV6>, + std::tuple< Row, Row, Row, INT8, INT8, INT32, INT32, I256, I256, I64, I32, I32, I16, Intrawave, CompV6>, + std::tuple< Row, Col, Row, INT8, INT8, INT32, INT32, I256, I256, I64, I32, I32, I16, Intrawave, CompV6>, + std::tuple< Col, Row, Row, INT8, INT8, INT32, INT32, I256, I256, I64, I32, I32, I16, Intrawave, CompV6>, + std::tuple< Col, Col, Row, INT8, INT8, INT32, INT32, I256, I256, I64, I32, I32, I16, Intrawave, CompV6> +>; using KernelTypesCompAsync = ::testing::Types< std::tuple< Row, Row, Row, F16, F16, F32, F16, I256, I256, I32, I32, I32, I16, Intrawave, CompAsync>, std::tuple< Row, Col, Row, F16, F16, F32, F16, I256, I256, I32, I32, I32, I16, Intrawave, CompAsync>, diff --git a/test/ck_tile/gemm/test_gemm_pipeline_util.hpp b/test/ck_tile/gemm/test_gemm_pipeline_util.hpp index 01bc3d7522..994510c060 100644 --- a/test/ck_tile/gemm/test_gemm_pipeline_util.hpp +++ b/test/ck_tile/gemm/test_gemm_pipeline_util.hpp @@ -38,6 +38,7 @@ enum struct GemmPipelineType Mem, CompV3, CompV4, + CompV6, CompAsync }; @@ -71,6 +72,15 @@ struct GemmPipelineTypeSelector static constexpr auto GetName() { return "GemmPipelineAgBgCrCompV4"; } }; +template +struct GemmPipelineTypeSelector +{ + using base_pipeline = ck_tile::BaseGemmPipelineAgBgCrCompV6; + using pipeline = ck_tile::GemmPipelineAgBgCrCompV6; + + static constexpr auto GetName() { return "GemmPipelineAgBgCrCompV6"; } +}; + template struct GemmPipelineTypeSelector { @@ -120,11 +130,13 @@ class TestCkTileGemmPipeline : public ::testing::Test constexpr bool kPadK = PadK; constexpr bool preshuffle = Preshuffle; - constexpr bool DoubleSmemBuffer = (PipelineType == GemmPipelineType::CompV4 || + constexpr bool DoubleSmemBuffer = (PipelineType == GemmPipelineType::CompV4 || PipelineType == GemmPipelineType::CompAsync); + constexpr bool TransposeC = false; + static constexpr bool StructuredSparsity = false; + static constexpr bool NumWaveGroup = 1; // TODO: For now - but this should also be a test parameter - constexpr bool TransposeC = false; constexpr int kBlockPerCu = 1; constexpr ck_tile::index_t TileParitionerGroupNum = 8; @@ -140,8 +152,6 @@ class TestCkTileGemmPipeline : public ::testing::Test GemmSpatiallyLocalTilePartitioner; using Traits = ck_tile::TileGemmTraits; - static constexpr bool StructuredSparsity = false; - static constexpr bool NumWaveGroup = 1; using GemmUniversalTraits = ck_tile::TileGemmUniversalTraits Date: Mon, 13 Oct 2025 13:27:02 +0100 Subject: [PATCH 015/262] [CK_TILE] Non-K Major from old CK to CK-Tile (#2442) * Enable the adapted LDS B layout for Row-Major * fix formatting * Implement specialized col-major A LDS block descriptor * Fix formatting * Use VecLoadSize for AK1/BK1 * Fix some thread access pattern values * Use GetVectorSizeA for A * Fix formatting * Add extra condition to avoid division by zero * disable layout for wave32 * remove extra else * fix formatting * Fix formatting * Rename one remaining TileDistributionEncodingPattern2D * Use integer ceil division * revert remod.py changes * also revert utility.hpp * use getA/BTileAccessPattern everywhere * use integer_divide_ceil for AK0 too --------- Co-authored-by: Adam Osewski <19374865+aosewski@users.noreply.github.com> Co-authored-by: Adam Osewski --- ...emm_universal_pipeline_ag_bg_cr_policy.hpp | 520 +++++++++++------- 1 file changed, 318 insertions(+), 202 deletions(-) diff --git a/include/ck_tile/ops/gemm/pipeline/gemm_universal_pipeline_ag_bg_cr_policy.hpp b/include/ck_tile/ops/gemm/pipeline/gemm_universal_pipeline_ag_bg_cr_policy.hpp index 4030783ecc..89e0346961 100644 --- a/include/ck_tile/ops/gemm/pipeline/gemm_universal_pipeline_ag_bg_cr_policy.hpp +++ b/include/ck_tile/ops/gemm/pipeline/gemm_universal_pipeline_ag_bg_cr_policy.hpp @@ -73,10 +73,14 @@ struct UniversalGemmBasePolicy template CK_TILE_HOST_DEVICE static constexpr auto MakeALdsBlockDescriptor() { + using ALayout = remove_cvref_t; + using ADataType = remove_cvref_t; using ADataType = remove_cvref_t; constexpr index_t MPerBlock = Problem::BlockGemmShape::kM; constexpr index_t KPerBlock = Problem::BlockGemmShape::kK; + constexpr index_t KPack = GetSmemPackA(); + constexpr auto DataTypeSize = sizeof(ADataType); if constexpr(is_a_load_tr) { @@ -90,47 +94,168 @@ struct UniversalGemmBasePolicy } else { - constexpr index_t KPack = GetSmemPackA(); + // Only use this ColumnMajor layout for Wave64 mode (gfx9) + constexpr auto Wave64 = get_warp_size() == 64; + if constexpr(Wave64 && + std::is_same_v) + { + // kfold and mpair dimension is not always required. + // more dimension in merge_transform increase the difficulty of generating immarg + // offset for compiler. + constexpr index_t BlockSize = Problem::kBlockSize; + constexpr index_t VecLoadSize = GetVectorSizeA(); + using TileEncodingPattern = + tile_distribution_encoding_pattern_2d; + // AK1 + constexpr auto AK1 = number{}; + constexpr auto AK0 = number{}; + // How the M dimension is split across threads + constexpr auto M0 = TileEncodingPattern::X0; // # of threads in M dim + constexpr auto M1 = number{}; - constexpr auto DataTypeSize = sizeof(ADataType); - constexpr auto MLdsLayer = - (32 * 4 / KPerBlock / DataTypeSize) < 1 ? 1 : (32 * 4 / KPerBlock / DataTypeSize); + // Get the warp tile size + using WarpTile = typename Problem::BlockGemmShape::WarpTile; + constexpr auto MPerXdl = number{}; - constexpr auto a_lds_block_desc_0 = make_naive_tensor_descriptor( - make_tuple(number{}, - number{}, - number{}), - make_tuple(number{}, number{}, number<1>{}), - number{}, - number<1>{}); + // How many elements we can write by single thread to LDS, + // the transposed / shuffled tile dstr has size: + constexpr auto KThreadWrite = TileEncodingPattern::Y2; + constexpr auto K0PerThreadWrite = integer_divide_ceil(AK0, KThreadWrite); + constexpr auto KThreadRead = get_warp_size() / MPerXdl; + constexpr auto K0PerThreadRead = integer_divide_ceil(AK0, KThreadRead); - constexpr auto a_lds_block_desc_permuted = transform_tensor_descriptor( - a_lds_block_desc_0, - make_tuple(make_xor_transform(make_tuple(number{}, - number{})), - make_pass_through_transform(number{})), - make_tuple(sequence<1, 0>{}, sequence<2>{}), - make_tuple(sequence<1, 0>{}, sequence<2>{})); + constexpr auto LdsBanksWidth = 128; + constexpr auto kfold = (AK1 * M0 * sizeof(ADataType) > LdsBanksWidth) + ? 1 + : LdsBanksWidth / (AK1 * M0 * sizeof(ADataType)); + constexpr auto KThreadReadPerm = + ((kfold * K0PerThreadWrite / K0PerThreadRead) > 1 && + (kfold * K0PerThreadWrite / K0PerThreadRead) < KThreadRead) + ? KThreadRead / (kfold * K0PerThreadWrite / K0PerThreadRead) + : KThreadRead; - constexpr auto a_lds_block_desc_xk0_mnldslayer_mn_xk1 = transform_tensor_descriptor( - a_lds_block_desc_permuted, - make_tuple(make_unmerge_transform( - make_tuple(number{}, number{})), - make_pass_through_transform(number{}), - make_pass_through_transform(number{})), - make_tuple(sequence<0>{}, sequence<1>{}, sequence<2>{}), - make_tuple(sequence<0, 2>{}, sequence<1>{}, sequence<3>{})); + // 1<=mpair<=n0 + constexpr auto mpair = + (AK1 * MPerXdl * sizeof(ADataType) > LdsBanksWidth) + ? 1 + : ((LdsBanksWidth / (AK1 * MPerXdl * sizeof(ADataType))) > M0 + ? M0 + : LdsBanksWidth / (AK1 * MPerXdl * sizeof(ADataType))); - constexpr auto a_lds_block_desc = transform_tensor_descriptor( - a_lds_block_desc_xk0_mnldslayer_mn_xk1, - make_tuple(make_merge_transform_v3_division_mod( - make_tuple(number{}, number{})), - make_merge_transform_v3_division_mod( - make_tuple(number{}, number{}))), - make_tuple(sequence<1, 0>{}, sequence<2, 3>{}), - make_tuple(sequence<0>{}, sequence<1>{})); + constexpr auto a_lds_block_desc = make_naive_tensor_descriptor_packed( + make_tuple(number{}, + number{}, + number{}, + number{}, + number{}, + AK1), + AK1); - return a_lds_block_desc; + constexpr auto a_lds_block_desc_permuted = transform_tensor_descriptor( + a_lds_block_desc, + make_tuple(make_pass_through_transform( + number{}), + make_pass_through_transform(number{}), + make_xor_transform(make_tuple(number{}, + number{})), + make_pass_through_transform(number{}), + make_pass_through_transform(AK1)), + make_tuple(sequence<0>{}, + sequence<1>{}, + sequence<2, 3>{}, + sequence<4>{}, + sequence<5>{}), + make_tuple(sequence<0>{}, + sequence<1>{}, + sequence<2, 3>{}, + sequence<4>{}, + sequence<5>{})); + + constexpr auto a_lds_block_desc_unmerged = transform_tensor_descriptor( + a_lds_block_desc_permuted, + make_tuple( + make_pass_through_transform( + number{}), + make_pass_through_transform(number{}), + make_unmerge_transform(make_tuple(number{}, number{})), + make_unmerge_transform(make_tuple(number{}, number{})), + make_pass_through_transform(number{}), + make_pass_through_transform(AK1)), + make_tuple(sequence<0>{}, + sequence<1>{}, + sequence<2>{}, + sequence<3>{}, + sequence<4>{}, + sequence<5>{}), + make_tuple(sequence<1>{}, + sequence<2>{}, + sequence<0, 3>{}, + sequence<4, 5>{}, + sequence<6>{}, + sequence<7>{})); + + constexpr auto a_lds_block_desc_ak0_m_ak1 = transform_tensor_descriptor( + a_lds_block_desc_unmerged, + make_tuple(make_merge_transform_v3_division_mod( + make_tuple(number{}, + number{}, + number{}, + number{}, + AK1)), + make_merge_transform_v3_division_mod(make_tuple( + number{}, number{}, number{}))), + make_tuple(sequence<0, 1, 4, 2, 7>{}, sequence<5, 6, 3>{}), + make_tuple(sequence<1>{}, sequence<0>{})); + + return a_lds_block_desc_ak0_m_ak1; + } + else // A is in RowMajor + { + constexpr auto MLdsLayer = (32 * 4 / KPerBlock / DataTypeSize) < 1 + ? 1 + : (32 * 4 / KPerBlock / DataTypeSize); + + constexpr auto a_lds_block_desc_0 = make_naive_tensor_descriptor( + make_tuple(number{}, + number{}, + number{}), + make_tuple(number{}, number{}, number<1>{}), + number{}, + number<1>{}); + + constexpr auto a_lds_block_desc_permuted = transform_tensor_descriptor( + a_lds_block_desc_0, + make_tuple( + make_xor_transform(make_tuple(number{}, + number{})), + make_pass_through_transform(number{})), + make_tuple(sequence<1, 0>{}, sequence<2>{}), + make_tuple(sequence<1, 0>{}, sequence<2>{})); + + constexpr auto a_lds_block_desc_xk0_mnldslayer_mn_xk1 = transform_tensor_descriptor( + a_lds_block_desc_permuted, + make_tuple(make_unmerge_transform( + make_tuple(number{}, number{})), + make_pass_through_transform(number{}), + make_pass_through_transform(number{})), + make_tuple(sequence<0>{}, sequence<1>{}, sequence<2>{}), + make_tuple(sequence<0, 2>{}, sequence<1>{}, sequence<3>{})); + + constexpr auto a_lds_block_desc = transform_tensor_descriptor( + a_lds_block_desc_xk0_mnldslayer_mn_xk1, + make_tuple(make_merge_transform_v3_division_mod(make_tuple( + number{}, number{})), + make_merge_transform_v3_division_mod( + make_tuple(number{}, number{}))), + make_tuple(sequence<1, 0>{}, sequence<2, 3>{}), + make_tuple(sequence<0>{}, sequence<1>{})); + + return a_lds_block_desc; + } } } @@ -143,12 +268,12 @@ struct UniversalGemmBasePolicy template CK_TILE_HOST_DEVICE static constexpr auto MakeBLdsBlockDescriptor() { + using BLayout = remove_cvref_t; using BDataType = remove_cvref_t; constexpr index_t NPerBlock = Problem::BlockGemmShape::kN; constexpr index_t KPerBlock = Problem::BlockGemmShape::kK; -#if 1 if constexpr(is_b_load_tr) { // TODO: better lds descriptor for performance @@ -160,178 +285,169 @@ struct UniversalGemmBasePolicy return b_lds_block_desc_0; } else - // else if constexpr(std::is_same_v) { - constexpr index_t KPack = GetSmemPackB(); - constexpr auto BK0 = number{}; - constexpr auto DataTypeSize = sizeof(BDataType); - constexpr auto NLdsLayer = - (32 * 4 / KPerBlock / DataTypeSize) < 1 ? 1 : (32 * 4 / KPerBlock / DataTypeSize); + // Only use this RowMajor layout for Wave64 mode (gfx9) + constexpr auto Wave64 = get_warp_size() == 64; + if constexpr(Wave64 && std::is_same_v) + { + constexpr index_t BlockSize = Problem::kBlockSize; + constexpr index_t VecLoadSize = GetVectorSizeB(); + using TileEncodingPattern = + tile_distribution_encoding_pattern_2d; + // BK1 + constexpr auto BK1 = number{}; + constexpr auto BK0 = number{}; - constexpr auto b_lds_block_desc_0 = make_naive_tensor_descriptor( - make_tuple( - BK0 * number{}, number{}, number{}), - make_tuple(number{}, number{}, number<1>{}), - number{}, - number<1>{}); + // How threads access data on N dim + constexpr auto N0 = TileEncodingPattern::X0; // # of threads in N dim + constexpr auto N1 = number{}; - constexpr auto b_lds_block_desc_permuted = transform_tensor_descriptor( - b_lds_block_desc_0, - make_tuple(make_xor_transform(make_tuple(number{}, - BK0 * number{})), - make_pass_through_transform(number{})), - make_tuple(sequence<1, 0>{}, sequence<2>{}), - make_tuple(sequence<1, 0>{}, sequence<2>{})); + // Get NPerXdl, the warp tile size + using WarpTile = typename Problem::BlockGemmShape::WarpTile; + constexpr auto NPerXdl = number{}; - constexpr auto b_lds_block_desc_bk0_nldslayer_n_bk1 = transform_tensor_descriptor( - b_lds_block_desc_permuted, - make_tuple(make_unmerge_transform(make_tuple(number{}, BK0)), - make_pass_through_transform(number{}), - make_pass_through_transform(number{})), - make_tuple(sequence<0>{}, sequence<1>{}, sequence<2>{}), - make_tuple(sequence<0, 2>{}, sequence<1>{}, sequence<3>{})); + // How many elements we can write by single thread to LDS, + // the transposed / shuffled tile dstr has size: + constexpr auto KThreadWrite = TileEncodingPattern::Y2; + constexpr auto K0PerThreadWrite = integer_divide_ceil(BK0, KThreadWrite); + constexpr auto KThreadRead = get_warp_size() / NPerXdl; + constexpr auto K0PerThreadRead = integer_divide_ceil(BK0, KThreadRead); - constexpr auto b_lds_block_desc = transform_tensor_descriptor( - b_lds_block_desc_bk0_nldslayer_n_bk1, - make_tuple(make_merge_transform_v3_division_mod( - make_tuple(number{}, number{})), - make_merge_transform_v3_division_mod(make_tuple(BK0, number{}))), - make_tuple(sequence<1, 0>{}, sequence<2, 3>{}), - make_tuple(sequence<0>{}, sequence<1>{})); - return b_lds_block_desc; + // check if we exceed all 32banks width - (32x4B) + constexpr auto LdsBanksWidth = 128; + constexpr auto kfold = (BK1 * N0 * sizeof(BDataType) > LdsBanksWidth) + ? 1 + : LdsBanksWidth / (BK1 * N0 * sizeof(BDataType)); + constexpr auto KThreadReadPerm = + ((kfold * K0PerThreadWrite / K0PerThreadRead) > 1 && + (kfold * K0PerThreadWrite / K0PerThreadRead) < KThreadRead) + ? KThreadRead / (kfold * K0PerThreadWrite / K0PerThreadRead) + : KThreadRead; + + // 1<=npair<=n0 + constexpr auto npair = + (BK1 * NPerXdl * sizeof(BDataType) > LdsBanksWidth) + ? 1 + : ((LdsBanksWidth / (BK1 * NPerXdl * sizeof(BDataType))) > N0 + ? N0 + : LdsBanksWidth / (BK1 * NPerXdl * sizeof(BDataType))); + + constexpr auto b_lds_block_desc = make_naive_tensor_descriptor_packed( + make_tuple(number{}, + number{}, + number{}, + number{}, + number{}, + BK1), + BK1); + + constexpr auto b_lds_block_desc_permuted = transform_tensor_descriptor( + b_lds_block_desc, + make_tuple(make_pass_through_transform( + number{}), + make_pass_through_transform(number{}), + make_xor_transform(make_tuple(number{}, + number{})), + make_pass_through_transform(number{}), + make_pass_through_transform(BK1)), + make_tuple(sequence<0>{}, + sequence<1>{}, + sequence<2, 3>{}, + sequence<4>{}, + sequence<5>{}), + make_tuple(sequence<0>{}, + sequence<1>{}, + sequence<2, 3>{}, + sequence<4>{}, + sequence<5>{})); + + constexpr auto b_lds_block_desc_unmerged = transform_tensor_descriptor( + b_lds_block_desc_permuted, + make_tuple( + make_pass_through_transform( + number{}), + make_pass_through_transform(number{}), + make_unmerge_transform(make_tuple(number{}, number{})), + make_unmerge_transform(make_tuple(number{}, number{})), + make_pass_through_transform(number{}), + make_pass_through_transform(BK1)), + make_tuple(sequence<0>{}, + sequence<1>{}, + sequence<2>{}, + sequence<3>{}, + sequence<4>{}, + sequence<5>{}), + make_tuple( + sequence<1>{}, // 0: K0PerThreadWrite + sequence<2>{}, // 1: KThreadReadPerm + sequence<0, 3>{}, // 2: KThreadWrite / kfold / KThreadReadPerm, 3: N1 + sequence<4, 5>{}, // 4: kfold, 5: N0 / npair + sequence<6>{}, // 6: npair + sequence<7>{})); // 7: BK1 + + constexpr auto b_lds_block_desc_nk = transform_tensor_descriptor( + b_lds_block_desc_unmerged, + make_tuple(make_merge_transform_v3_division_mod( + make_tuple(number{}, + number{}, + number{}, + number{}, + BK1)), + make_merge_transform_v3_division_mod(make_tuple( + number{}, number{}, number{}))), + make_tuple(sequence<0, 1, 4, 2, 7>{}, sequence<5, 6, 3>{}), + make_tuple(sequence<1>{}, sequence<0>{})); + + return b_lds_block_desc_nk; + } + else // B is Column Major + { + constexpr index_t KPack = GetSmemPackB(); + constexpr auto BK0 = number{}; + constexpr auto DataTypeSize = sizeof(BDataType); + constexpr auto NLdsLayer = (32 * 4 / KPerBlock / DataTypeSize) < 1 + ? 1 + : (32 * 4 / KPerBlock / DataTypeSize); + + constexpr auto b_lds_block_desc_0 = make_naive_tensor_descriptor( + make_tuple(BK0 * number{}, + number{}, + number{}), + make_tuple(number{}, number{}, number<1>{}), + number{}, + number<1>{}); + + constexpr auto b_lds_block_desc_permuted = transform_tensor_descriptor( + b_lds_block_desc_0, + make_tuple(make_xor_transform(make_tuple(number{}, + BK0 * number{})), + make_pass_through_transform(number{})), + make_tuple(sequence<1, 0>{}, sequence<2>{}), + make_tuple(sequence<1, 0>{}, sequence<2>{})); + + constexpr auto b_lds_block_desc_bk0_nldslayer_n_bk1 = transform_tensor_descriptor( + b_lds_block_desc_permuted, + make_tuple(make_unmerge_transform(make_tuple(number{}, BK0)), + make_pass_through_transform(number{}), + make_pass_through_transform(number{})), + make_tuple(sequence<0>{}, sequence<1>{}, sequence<2>{}), + make_tuple(sequence<0, 2>{}, sequence<1>{}, sequence<3>{})); + + constexpr auto b_lds_block_desc = transform_tensor_descriptor( + b_lds_block_desc_bk0_nldslayer_n_bk1, + make_tuple( + make_merge_transform_v3_division_mod( + make_tuple(number{}, number{})), + make_merge_transform_v3_division_mod(make_tuple(BK0, number{}))), + make_tuple(sequence<1, 0>{}, sequence<2, 3>{}), + make_tuple(sequence<0>{}, sequence<1>{})); + return b_lds_block_desc; + } } -#else - else // B is Row Major - { - constexpr index_t BlockSize = Problem::kBlockSize; - constexpr index_t VecLoadSize = GetVectorSizeB(); - using TileEncodingPattern = - tile_distribution_encoding_pattern_2d; - - constexpr auto BK0 = number{}; - constexpr auto BK1 = number{}; - // constexpr auto N0 = BBlockTransferThreadClusterLengths_BK0_N_BK1{}.At(I1); - constexpr auto N0 = TileEncodingPattern::X0; - constexpr auto N1 = NPerBlock / N0; - - using WarpTile = typename Problem::BlockGemmShape::WarpTile; - constexpr auto NPerXdl = number{}; - - // constexpr auto KThreadWrite = - // BBlockTransferThreadClusterLengths_BK0_N_BK1{}.At(I0); - constexpr auto KThreadWrite = TileEncodingPattern::Y2; - constexpr auto K0PerThreadWrite = BK0 / KThreadWrite; - constexpr auto KThreadRead = 64 / NPerXdl; - constexpr auto K0PerThreadRead = BK0 / KThreadRead; - - constexpr auto kfold = - (BK1 * N0 * sizeof(BDataType) > 128) ? 1 : 128 / (BK1 * N0 * sizeof(BDataType)); - constexpr auto KThreadReadPerm = - (kfold * K0PerThreadWrite / K0PerThreadRead) > 1 - ? KThreadRead / (kfold * K0PerThreadWrite / K0PerThreadRead) - : KThreadRead; - - // 1<=npair<=n0 - constexpr auto npair = (BK1 * NPerXdl * sizeof(BDataType) > 128) - ? 1 - : ((128 / (BK1 * NPerXdl * sizeof(BDataType))) > N0 - ? N0 - : 128 / (BK1 * NPerXdl * sizeof(BDataType))); - - constexpr auto b_lds_block_desc = make_naive_tensor_descriptor_packed( - make_tuple(number{}, - number{}, - number{}, - number{}, - number{}, - BK1)); - - constexpr auto b_lds_block_desc_permuted = transform_tensor_descriptor( - b_lds_block_desc, - make_tuple( - make_pass_through_transform(number{}), - make_pass_through_transform(number{}), - make_xor_transform( - make_tuple(number{}, number{})), - make_pass_through_transform(number{}), - make_pass_through_transform(BK1)), - make_tuple( - sequence<0>{}, sequence<1>{}, sequence<2, 3>{}, sequence<4>{}, sequence<5>{}), - make_tuple( - sequence<0>{}, sequence<1>{}, sequence<2, 3>{}, sequence<4>{}, sequence<5>{})); - - constexpr auto b_lds_block_desc_unmerged = transform_tensor_descriptor( - b_lds_block_desc_permuted, - make_tuple( - make_pass_through_transform(number{}), - make_pass_through_transform(number{}), - make_unmerge_transform(make_tuple(number{}, number{})), - make_unmerge_transform(make_tuple(number{}, number{})), - make_pass_through_transform(number{}), - make_pass_through_transform(BK1)), - make_tuple(sequence<0>{}, - sequence<1>{}, - sequence<2>{}, - sequence<3>{}, - sequence<4>{}, - sequence<5>{}), - make_tuple(sequence<1>{}, - sequence<2>{}, - sequence<0, 3>{}, - sequence<4, 5>{}, - sequence<6>{}, - sequence<7>{})); - - // constexpr auto b_lds_block_desc_bk0_n_bk1 = transform_tensor_descriptor( - // b_lds_block_desc_unmerged, - // make_tuple(make_merge_transform_v3_division_mod( - // make_tuple(number{}, - // number{}, - // number{}, - // number{})), - // make_merge_transform_v3_division_mod( - // make_tuple(number{}, number{}, number{})), - // make_pass_through_transform(BK1)), - // make_tuple(sequence<0, 1, 4, 2>{}, sequence<5, 6, 3>{}, sequence<7>{}), - // make_tuple(sequence<0>{}, sequence<1>{}, sequence<2>{})); - - constexpr auto b_lds_block_desc_kn = transform_tensor_descriptor( - b_lds_block_desc_unmerged, - make_tuple(make_merge_transform_v3_division_mod( - make_tuple(number{}, - number{}, - number{}, - number{}, - BK1)), - make_merge_transform_v3_division_mod( - make_tuple(number{}, number{}, number{}))), - make_tuple(sequence<0, 1, 4, 2, 7>{}, sequence<5, 6, 3>{}), - make_tuple(sequence<1>{}, sequence<0>{})); - - // return b_lds_block_desc_bk0_n_bk1; - return b_lds_block_desc_kn; - - // constexpr auto b_lds_block_desc_bk0_n_bk1 = make_naive_tensor_descriptor( - // make_tuple(BK0, number{}, number{}), - // make_tuple(number{}, number{}, number<1>{}), - // number{}, - // number<1>{}); - - // constexpr auto b_lds_block_desc = transform_tensor_descriptor( - // b_lds_block_desc_bk0_n_bk1, - // make_tuple(make_pass_through_transform(number{}), - // make_merge_transform_v3_division_mod(make_tuple(BK0, - // number{}))), - // make_tuple(sequence<1>{}, sequence<0, 2>{}), - // make_tuple(sequence<0>{}, sequence<1>{})); - - // return b_lds_block_desc; - } -#endif } /** From fc2a121c4446b4ca939e977563528019b30e6114 Mon Sep 17 00:00:00 2001 From: John Shumway Date: Mon, 13 Oct 2025 08:11:51 -0700 Subject: [PATCH 016/262] Enable GMock and improve gtest configuration (#2976) Our current cmake/gtest.cmake file does not enable gmock. Gmock is needed for matchers that are needed for more readable unit tests. This PR enables gmock and does a little cleanup in gtest.cmake: * Enable BUILD_GMOCK by default (was previously disabled) * Patch gtest-src/googlemock/CMakeLists.txt for broken include path. * Add configuration to gmock if the target is used. No other changes in this PR, but I've verified I can use gmock matchers correctly once I include these changes in other code. --- cmake/gtest.cmake | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/cmake/gtest.cmake b/cmake/gtest.cmake index 6587f4c4be..41e2fa2cc0 100644 --- a/cmake/gtest.cmake +++ b/cmake/gtest.cmake @@ -12,6 +12,17 @@ FetchContent_Declare( GIT_TAG f8d7d77c06936315286eb55f8de22cd23c188571 ) +FetchContent_Populate(GTest) + +# Patch googlemock/CMakeLists.txt to fix invalid include path +set(GMOCK_CMAKE "${gtest_SOURCE_DIR}/googlemock/CMakeLists.txt") +file(READ "${GMOCK_CMAKE}" GMOCK_CMAKE_CONTENT) +string(REPLACE [[gtest_SOURCE_DIR}/include]] + [[gtest_SOURCE_DIR}/googletest/include]] + GMOCK_CMAKE_CONTENT + "${GMOCK_CMAKE_CONTENT}") +file(WRITE "${GMOCK_CMAKE}" "${GMOCK_CMAKE_CONTENT}") + # Suppress ROCMChecks WARNING on GoogleTests set(ROCM_DISABLE_CHECKS FALSE) macro(rocm_check_toolchain_var var access value list_file) @@ -24,7 +35,7 @@ if(WIN32) set(gtest_force_shared_crt ON CACHE_INTERNAL "") endif() -set(BUILD_GMOCK OFF CACHE INTERNAL "") +set(BUILD_GMOCK ON CACHE INTERNAL "") set(INSTALL_GTEST OFF CACHE INTERNAL "") # Store the current value of BUILD_SHARED_LIBS @@ -32,15 +43,12 @@ set(__build_shared_libs ${BUILD_SHARED_LIBS}) set(BUILD_SHARED_LIBS OFF CACHE INTERNAL "") set(ROCM_DISABLE_CHECKS TRUE) -FetchContent_MakeAvailable(GTest) +add_subdirectory(${gtest_SOURCE_DIR} ${gtest_BINARY_DIR}) set(ROCM_DISABLE_CHECKS FALSE) # Restore the old value of BUILD_SHARED_LIBS set(BUILD_SHARED_LIBS ${__build_shared_libs} CACHE BOOL "Type of libraries to build" FORCE) -set(BUILD_GMOCK OFF CACHE INTERNAL "") -set(INSTALL_GTEST OFF CACHE INTERNAL "") - set(GTEST_CXX_FLAGS -Wno-undef -Wno-reserved-identifier @@ -71,3 +79,12 @@ target_compile_options(gtest_main PRIVATE ${GTEST_CXX_FLAGS}) target_compile_definitions(gtest PRIVATE GTEST_HAS_SEH=0) target_compile_definitions(gtest_main PRIVATE GTEST_HAS_SEH=0) +if(TARGET gmock) + target_compile_options(gmock PRIVATE ${GTEST_CXX_FLAGS}) + target_compile_definitions(gmock PRIVATE GTEST_HAS_SEH=0) +endif() + +if(TARGET gmock_main) + target_compile_options(gmock_main PRIVATE ${GTEST_CXX_FLAGS}) + target_compile_definitions(gmock_main PRIVATE GTEST_HAS_SEH=0) +endif() From e1b0bdfbfa92f47006fdbced627c7470eacdea2b Mon Sep 17 00:00:00 2001 From: ClementLinCF <162283536+ClementLinCF@users.noreply.github.com> Date: Tue, 14 Oct 2025 02:52:37 +0800 Subject: [PATCH 017/262] [CK_TILE] Correct BlockWarps calculation and fix smoke-test in rmsnorm (#2540) * [CK_TILE] Correct BlockWarps calculation and fix smoke-test in rmsnorm * Update rmsnorm host reference * Update tree reduction of rmsnorm for reference host * Fix cross warp for m > 1 cases * Add RMSNorm model selectable option for host reference * Fix save_unquant cases * Update reference rmsnorm forward function to use enum for model sensitivity * Update reference rmsnorm calculation for model sensitivity * Fix m warp for layernorm * Adjust parameter of reference for twoPass * Fix clang format * Run clang-format-overwrite.sh to fix formating issue * fix clang format --------- Co-authored-by: MHYang Co-authored-by: illsilin_amdeng Co-authored-by: ThomasNing --- example/ck_tile/02_layernorm2d/generate.py | 33 +++++ example/ck_tile/10_rmsnorm2d/generate.py | 39 +++++- .../ck_tile/10_rmsnorm2d/rmsnorm2d_fwd.cpp | 47 +++++-- .../ck_tile/10_rmsnorm2d/script/smoke_test.sh | 124 +++++++++++------- .../reference/reference_rmsnorm2d_fwd.hpp | 31 ++++- .../ops/reduce/block/block_reduce2d.hpp | 4 +- ...rm2d_fwd_pipeline_model_sensitive_pass.hpp | 6 +- 7 files changed, 217 insertions(+), 67 deletions(-) diff --git a/example/ck_tile/02_layernorm2d/generate.py b/example/ck_tile/02_layernorm2d/generate.py index b7512b2999..5f589db8d0 100644 --- a/example/ck_tile/02_layernorm2d/generate.py +++ b/example/ck_tile/02_layernorm2d/generate.py @@ -75,6 +75,39 @@ struct layernorm2d_fwd_traits_ using SmoothScaleDataType = ck_tile::remove_cvref_t; using YScaleDataType = ck_tile::remove_cvref_t; + static constexpr bool is_warp_per_row = ThreadPerBlock_N_ <= ck_tile::get_warp_size(); + static_assert((ThreadPerBlock_M_ * ThreadPerBlock_N_) % ck_tile::get_warp_size() == 0); + static constexpr ck_tile::index_t total_warps = + (ThreadPerBlock_M_ * ThreadPerBlock_N_) / ck_tile::get_warp_size(); + + // num of warps along m + static constexpr ck_tile::index_t BlockWarps_M = []() { + if constexpr(is_warp_per_row) + { + static_assert(ck_tile::get_warp_size() % ThreadPerBlock_N_ == 0); + return total_warps; + } + else + { + // static_assert(ck_tile::get_warp_size() % ThreadPerBlock_M_ == 0); + return total_warps / (ThreadPerBlock_N_ / ck_tile::get_warp_size()); + } + }(); + + // num of warps along n + static constexpr ck_tile::index_t BlockWarps_N = []() { + if constexpr(is_warp_per_row) + { + static_assert(ck_tile::get_warp_size() % ThreadPerBlock_N_ == 0); + return 1; + } + else + { + static_assert(ThreadPerBlock_N_ % ck_tile::get_warp_size() == 0); + return ThreadPerBlock_N_ / ck_tile::get_warp_size(); + } + }(); + static constexpr ck_tile::index_t Repeat_M = Repeat_M_; static constexpr ck_tile::index_t Repeat_N = Repeat_N_; diff --git a/example/ck_tile/10_rmsnorm2d/generate.py b/example/ck_tile/10_rmsnorm2d/generate.py index 0e948322a2..75d7abd0ad 100644 --- a/example/ck_tile/10_rmsnorm2d/generate.py +++ b/example/ck_tile/10_rmsnorm2d/generate.py @@ -75,6 +75,39 @@ struct rmsnorm2d_fwd_traits_ using YScaleDataType = ck_tile::remove_cvref_t; using UnquantYDataType = ck_tile::remove_cvref_t; + static constexpr bool is_warp_per_row = ThreadPerBlock_N_ <= ck_tile::get_warp_size(); + static_assert((ThreadPerBlock_M_ * ThreadPerBlock_N_) % ck_tile::get_warp_size() == 0); + static constexpr ck_tile::index_t total_warps = + (ThreadPerBlock_M_ * ThreadPerBlock_N_) / ck_tile::get_warp_size(); + + // num of warps along m + static constexpr ck_tile::index_t BlockWarps_M = []() { + if constexpr(is_warp_per_row) + { + static_assert(ck_tile::get_warp_size() % ThreadPerBlock_N_ == 0); + return total_warps; + } + else + { + // static_assert(ck_tile::get_warp_size() % ThreadPerBlock_M_ == 0); + return total_warps / (ThreadPerBlock_N_ / ck_tile::get_warp_size()); + } + }(); + + // num of warps along n + static constexpr ck_tile::index_t BlockWarps_N = []() { + if constexpr(is_warp_per_row) + { + static_assert(ck_tile::get_warp_size() % ThreadPerBlock_N_ == 0); + return 1; + } + else + { + static_assert(ThreadPerBlock_N_ % ck_tile::get_warp_size() == 0); + return ThreadPerBlock_N_ / ck_tile::get_warp_size(); + } + }(); + static constexpr ck_tile::index_t Repeat_M = Repeat_M_; static constexpr ck_tile::index_t Repeat_N = Repeat_N_; @@ -605,15 +638,15 @@ float rmsnorm2d_fwd(rmsnorm2d_fwd_traits t, h_traits('x', 'y', 'xs', 'ys', 'uqy', 1, 4, 1, 256, 4, True, False, False, True, 0, 0, 1), h_traits('x', 'y', 'xs', 'ys', 'uqy', 1, 12, 1, 256, 2, True, False, False, True, 0, 0, 1), h_traits('x', 'y', 'xs', 'ys', 'uqy', 1, 4, 1,1024, 1, True, False, False, True, 0, 0, 1)] - } + } } - + total_blob = list() for model_sensitive_flag in [0, 1]: # 0: default; 1: model sensitive current_trait_dict = h_trait_dicts[model_sensitive_flag] for hs_key in current_trait_dict: - hs = current_trait_dict[hs_key] + hs = current_trait_dict[hs_key] current_n = hs_key for dtype, scale_type, fused_add, fused_quant, save_unquant in itertools.product(dtype_list, scale_list, fused_add_list, fused_sweep_list, bool_list): prec_i, prec_o = dtype.split(',') diff --git a/example/ck_tile/10_rmsnorm2d/rmsnorm2d_fwd.cpp b/example/ck_tile/10_rmsnorm2d/rmsnorm2d_fwd.cpp index 6e2664e9ba..8518b5ddc7 100644 --- a/example/ck_tile/10_rmsnorm2d/rmsnorm2d_fwd.cpp +++ b/example/ck_tile/10_rmsnorm2d/rmsnorm2d_fwd.cpp @@ -70,16 +70,16 @@ template bool run(const ck_tile::ArgParser& arg_parser) { - ck_tile::index_t m = arg_parser.get_int("m"); - ck_tile::index_t n = arg_parser.get_int("n"); - float epsilon = arg_parser.get_float("e"); - int kname = arg_parser.get_int("kname"); - int do_validation = arg_parser.get_int("v"); - int fused_add = arg_parser.get_int("fadd"); - int fused_quant = arg_parser.get_int("fquant"); - int warmup = arg_parser.get_int("warmup"); - int repeat = arg_parser.get_int("repeat"); - const int use_model_sensitive_rmsnorm = arg_parser.get_int("s"); + ck_tile::index_t m = arg_parser.get_int("m"); + ck_tile::index_t n = arg_parser.get_int("n"); + float epsilon = arg_parser.get_float("e"); + int kname = arg_parser.get_int("kname"); + int do_validation = arg_parser.get_int("v"); + int fused_add = arg_parser.get_int("fadd"); + int fused_quant = arg_parser.get_int("fquant"); + int warmup = arg_parser.get_int("warmup"); + int repeat = arg_parser.get_int("repeat"); + int use_model_sensitive_rmsnorm = arg_parser.get_int("s"); ck_tile::index_t x_stride = arg_parser.get_int("x_stride"); if(x_stride < 0) @@ -196,6 +196,11 @@ bool run(const ck_tile::ArgParser& arg_parser) return base_str; }(); + if(n > 8192) + { + use_model_sensitive_rmsnorm = 0; + } + std::cout << "[" << prec_str << "]" << " m:" << m << ", n:" << n << ", x_stride:" << x_stride << ", xr_stride:" << xr_stride << ", y_stride:" << y_stride << ", yr_stride:" << yr_stride << ", s:" << use_model_sensitive_rmsnorm << std::flush; @@ -297,7 +302,7 @@ bool run(const ck_tile::ArgParser& arg_parser) const int N = acc_.mDesc.get_lengths()[1]; for(int n_ = 0; n_ < N; ++n_) { - o_unquant_(m_, n_) = ck_tile::type_convert(acc_(m_, n_)); + o_unquant_(m_, n_) = ck_tile::type_convert(acc_(m_, n_)); } dquant_functor(m_, o_, acc_); @@ -316,7 +321,8 @@ bool run(const ck_tile::ArgParser& arg_parser) invRms_host_ref, unquant_y_host_ref, epsilon, - default_and_dquant_functor); + default_and_dquant_functor, + use_model_sensitive_rmsnorm); } else { @@ -331,7 +337,8 @@ bool run(const ck_tile::ArgParser& arg_parser) invRms_host_ref, unquant_y_host_ref, epsilon, - dquant_functor); + dquant_functor, + use_model_sensitive_rmsnorm); } } else @@ -343,7 +350,14 @@ bool run(const ck_tile::ArgParser& arg_parser) YDataType, InvRmsDataType, ck_tile::null_type>( - x_host, gamma_host, y_host_ref, invRms_host_ref, unquant_y_null, epsilon); + x_host, + gamma_host, + y_host_ref, + invRms_host_ref, + unquant_y_null, + epsilon, + ck_tile::reference_rmsnorm2d_default_epilogue{}, + use_model_sensitive_rmsnorm); } y_buf.FromDevice(y_host_dev.data()); @@ -354,6 +368,11 @@ bool run(const ck_tile::ArgParser& arg_parser) y_residual_buf.FromDevice(y_residual_host_dev.data()); } + if constexpr(SaveUnquant) + { + unquant_y_buf.FromDevice(unquant_y_host_dev.data()); + } + auto [rtol, atol] = get_elimit(); if(x_stride == n) { diff --git a/example/ck_tile/10_rmsnorm2d/script/smoke_test.sh b/example/ck_tile/10_rmsnorm2d/script/smoke_test.sh index 1c79dafadd..3a0f7dbb66 100755 --- a/example/ck_tile/10_rmsnorm2d/script/smoke_test.sh +++ b/example/ck_tile/10_rmsnorm2d/script/smoke_test.sh @@ -1,49 +1,85 @@ -#!/bin/sh +#!/bin/bash + EXE="$(find . -name tile_rmsnorm2d_fwd -type f | head -n 1)" -for fquant in "" "-fquant=1 -prec_o=int8" "-fquant=2 -prec_o=int8" "-fquant=1 -prec_o=fp8" "-fquant=2 -prec_o=fp8"\ - "-fquant=1 -prec_o=int8 -save_unquant=1" "-fquant=2 -prec_o=int8 -save_unquant=1" "-fquant=1 -prec_o=fp8 -save_unquant=1" "-fquant=2 -prec_o=fp8 -save_unquant=1"; do -for pr_i in "fp16" "bf16" ; do -for fadd in "0" "1"; do -# 0: for no specific RMSNorm; 1: for T-5 like RMSNorm -for s in "0" "1"; do -$EXE -prec_i=$pr_i -fadd=$fadd -s=$s $fquant -m=99 -n=13 -$EXE -prec_i=$pr_i -fadd=$fadd -s=$s $fquant -m=17 -n=16 -$EXE -prec_i=$pr_i -fadd=$fadd -s=$s $fquant -m=1 -n=100 -$EXE -prec_i=$pr_i -fadd=$fadd -s=$s $fquant -m=4 -n=128 -$EXE -prec_i=$pr_i -fadd=$fadd -s=$s $fquant -m=80 -n=127 -# $EXE -prec_i=$pr_i -fadd=$fadd -s=$s $fquant -m=22 -n=255 -stride=256 -$EXE -prec_i=$pr_i -fadd=$fadd -s=$s $fquant -m=7 -n=599 -$EXE -prec_i=$pr_i -fadd=$fadd -s=$s $fquant -m=19 -n=512 -# $EXE -prec_i=$pr_i -fadd=$fadd -s=$s $fquant -m=33 -n=313 -stride=1000 -$EXE -prec_i=$pr_i -fadd=$fadd -s=$s $fquant -m=11 -n=510 -# $EXE -prec_i=$pr_i -fadd=$fadd -s=$s $fquant -m=171 -n=676 -stride=818 -$EXE -prec_i=$pr_i -fadd=$fadd -s=$s $fquant -m=91 -n=636 -# $EXE -prec_i=$pr_i -fadd=$fadd -s=$s $fquant -m=12 -n=768 -stride=800 -# $EXE -prec_i=$pr_i -fadd=$fadd -s=$s $fquant -m=100 -n=766 -stride=812 -$EXE -prec_i=$pr_i -fadd=$fadd -s=$s $fquant -m=31 -n=1024 -# $EXE -prec_i=$pr_i -fadd=$fadd -s=$s $fquant -m=64 -n=1000 -stride=1004 -$EXE -prec_i=$pr_i -fadd=$fadd -s=$s $fquant -m=8 -n=1501 -$EXE -prec_i=$pr_i -fadd=$fadd -s=$s $fquant -m=3 -n=1826 -$EXE -prec_i=$pr_i -fadd=$fadd -s=$s $fquant -m=5 -n=2040 -$EXE -prec_i=$pr_i -fadd=$fadd -s=$s $fquant -m=7 -n=2734 -$EXE -prec_i=$pr_i -fadd=$fadd -s=$s $fquant -m=1 -n=3182 -$EXE -prec_i=$pr_i -fadd=$fadd -s=$s $fquant -m=9 -n=4096 -$EXE -prec_i=$pr_i -fadd=$fadd -s=$s $fquant -m=3 -n=8192 -done -done -done +total=0 +valid=0 + +run_case() { + cmd="$EXE -prec_i=$1 -fadd=$2 -s=$3 $4 -m=$5 -n=$6 $7" + echo "[CMD] $cmd" + output=$($cmd 2>&1) + echo "$output" + if echo "$output" | grep -q "valid:y"; then + valid=$((valid + 1)) + fi + total=$((total + 1)) +} + +fquant_list=( + "" + "-fquant=1 -prec_o=int8" + "-fquant=2 -prec_o=int8" + "-fquant=1 -prec_o=fp8" + "-fquant=2 -prec_o=fp8" + "-fquant=1 -prec_o=int8 -save_unquant=1" + "-fquant=2 -prec_o=int8 -save_unquant=1" + "-fquant=1 -prec_o=fp8 -save_unquant=1" + "-fquant=2 -prec_o=fp8 -save_unquant=1" +) + +m_n_list=( + "99 13" "17 16" "1 100" "4 128" "80 127" + "7 599" "19 512" "11 510" "91 636" + "31 1024" "8 1501" "3 1826" "5 2040" + "7 2734" "1 3182" "9 4096" "3 8192" +) + +### Add special stride test ### +m_n_stride_list=( + "22 255 -x_stride=256 -xr_stride=256 -y_stride=256 -yr_stride=256" + "33 313 -x_stride=1000 -xr_stride=1000 -y_stride=1000 -yr_stride=1000" + "171 676 -x_stride=818 -xr_stride=818 -y_stride=818 -yr_stride=818" + "12 768 -x_stride=800 -xr_stride=800 -y_stride=800 -yr_stride=800" + "100 766 -x_stride=812 -xr_stride=812 -y_stride=812 -yr_stride=812" + "64 1000 -x_stride=1004 -xr_stride=1004 -y_stride=1004 -yr_stride=1004" +) + +for fquant in "${fquant_list[@]}"; do + for pr_i in "fp16" "bf16"; do + for fadd in "0" "1"; do + for s in "0" "1"; do + for pair in "${m_n_list[@]}"; do + m=$(echo $pair | cut -d ' ' -f1) + n=$(echo $pair | cut -d ' ' -f2) + run_case "$pr_i" "$fadd" "$s" "$fquant" "$m" "$n" "" + done + + ### Running tests with stride ### + for triple in "${m_n_stride_list[@]}"; do + m=$(echo $triple | cut -d ' ' -f1) + n=$(echo $triple | cut -d ' ' -f2) + stride_args=$(echo $triple | cut -d ' ' -f3-) + run_case "$pr_i" "$fadd" "$s" "$fquant" "$m" "$n" "$stride_args" + done + done + done + done done -# The following cases uses two pass pipeline which doesn't support quant epilogue. -for fquant in "" -for pr_i in "fp16" "bf16" ; do -for fadd in "0" "1"; do -# 0: for no specific RMSNorm; 1: for T-5 like RMSNorm -for s in "0" "1"; do -$EXE -prec_i=$pr_i -fadd=$fadd -s=$s $fquant -m=1 -n=10547 -#$EXE -prec_i=$pr_i -fadd=$fadd $fquant -m=3 -n=17134 -done -done -done +# Special two-pass only +for pr_i in "fp16" "bf16"; do + for fadd in "0" "1"; do + for s in "0" "1"; do + run_case "$pr_i" "$fadd" "$s" "" "1" "10547" "" + done + done done + +# Summary +echo "==============================" +echo "Total cases: $total" +echo "Valid cases: $valid" +accuracy=$(awk "BEGIN {printf \"%.2f\", ($valid / $total) * 100}") +echo "Accuracy: $accuracy%" +echo "==============================" diff --git a/include/ck_tile/host/reference/reference_rmsnorm2d_fwd.hpp b/include/ck_tile/host/reference/reference_rmsnorm2d_fwd.hpp index 070168b51d..424fff4470 100644 --- a/include/ck_tile/host/reference/reference_rmsnorm2d_fwd.hpp +++ b/include/ck_tile/host/reference/reference_rmsnorm2d_fwd.hpp @@ -5,6 +5,7 @@ #include "ck_tile/core.hpp" #include "ck_tile/host/host_tensor.hpp" +#include "ck_tile/ops/rmsnorm2d/pipeline/rmsnorm2d_fwd_traits.hpp" namespace ck_tile { @@ -43,7 +44,9 @@ void reference_rmsnorm2d_fwd(const HostTensor& x_m_n, HostTensor& invRms_m, HostTensor& unquant_y_m_n, ComputeDataType epsilon, - Epilogue epilogue_functor = {}) + Epilogue epilogue_functor = {}, + const int use_model_sensitive_rmsnorm = + static_cast(Rmsnorm2dSensitiveEnum::NO_SPECIFIC_MODEL)) { auto rmsnorm2d_fwd_func = [&](auto m) { const int N = x_m_n.mDesc.get_lengths()[1]; @@ -68,7 +71,30 @@ void reference_rmsnorm2d_fwd(const HostTensor& x_m_n, { ComputeDataType x = ck_tile::type_convert(x_m_n(m, n)); ComputeDataType gamma = ck_tile::type_convert(gamma_n(n)); - acc(m, n) = x * divisor * gamma; + if(use_model_sensitive_rmsnorm == + static_cast( + Rmsnorm2dSensitiveEnum::NO_SPECIFIC_MODEL)) // 0: for no specific model + { + acc(m, n) = x * divisor * gamma; + } + else if(use_model_sensitive_rmsnorm == + static_cast(Rmsnorm2dSensitiveEnum::T5_MODEL_LIKE)) // 1: for T5-like model + { + if constexpr(std::is_same_v) + { + const auto tmp0 = float_to_bf16(x * divisor); + const auto tmp1 = float_to_bf16( + type_convert(tmp0) * gamma); + const auto rmsn_ = type_convert(tmp1); + acc(m, n) = rmsn_; + } + else + { + const auto tmp = type_convert(x * divisor); + const auto rmsn_ = type_convert(tmp) * gamma; + acc(m, n) = rmsn_; + } + } } if constexpr(!std::is_same_v) @@ -84,4 +110,5 @@ void reference_rmsnorm2d_fwd(const HostTensor& x_m_n, make_ParallelTensorFunctor(rmsnorm2d_fwd_func, invRms_m.mDesc.get_lengths()[0])( std::thread::hardware_concurrency()); } + } // namespace ck_tile diff --git a/include/ck_tile/ops/reduce/block/block_reduce2d.hpp b/include/ck_tile/ops/reduce/block/block_reduce2d.hpp index b72657b785..b97a66a3ec 100644 --- a/include/ck_tile/ops/reduce/block/block_reduce2d.hpp +++ b/include/ck_tile/ops/reduce/block/block_reduce2d.hpp @@ -400,11 +400,13 @@ struct BlockReduce2dTreeCrossWarpSync block_sync_lds(); // We let each warp holds a duplication to do reduction. + const index_t local_warp_id = warp_id / num_reduce_warps; + const index_t local_smem_os = local_warp_id * num_reduce_warps; static_for<0, thread_buf_size, 1>{}([&](auto i) { DataType v = 0; if(lane_id < num_reduce_warps) { - v = smem_ptr[lane_id + i * num_warps]; + v = smem_ptr[i * num_warps + local_smem_os + lane_id]; } // cross-lane reduce for replication diff --git a/include/ck_tile/ops/rmsnorm2d/pipeline/rmsnorm2d_fwd_pipeline_model_sensitive_pass.hpp b/include/ck_tile/ops/rmsnorm2d/pipeline/rmsnorm2d_fwd_pipeline_model_sensitive_pass.hpp index c5923ba10d..1d5467b459 100644 --- a/include/ck_tile/ops/rmsnorm2d/pipeline/rmsnorm2d_fwd_pipeline_model_sensitive_pass.hpp +++ b/include/ck_tile/ops/rmsnorm2d/pipeline/rmsnorm2d_fwd_pipeline_model_sensitive_pass.hpp @@ -146,7 +146,7 @@ struct Rmsnorm2dFwdPipelineModelSensitiveT5Pass // compute mean square each-thread->cross-lane->cross-warp auto square_sum = block_reduce2d.template MakeYBlockTile(); set_tile(square_sum, 0); - if constexpr(Problem::BlockShape::Vector_N % 2 == 0) + if constexpr((Problem::BlockShape::Repeat_N * Problem::BlockShape::Vector_N) % 2 == 0) { sweep_tile( acc, @@ -179,7 +179,7 @@ struct Rmsnorm2dFwdPipelineModelSensitiveT5Pass const auto gamma_ = type_convert(gamma[j_idx]); - if constexpr(std::is_same_v) + if constexpr(std::is_same_v) { const auto tmp0 = float_to_bf16(acc[idx] * inv_rms_[i_idx]); @@ -190,7 +190,7 @@ struct Rmsnorm2dFwdPipelineModelSensitiveT5Pass } else { - const auto tmp = type_convert(acc[idx] * inv_rms_[i_idx]); + const auto tmp = type_convert(acc[idx] * inv_rms_[i_idx]); const auto rmsn_ = type_convert(tmp) * gamma_; rmsn(idx) = rmsn_; } From 589e242eda730958b36c4f78bfad1991c499b0d2 Mon Sep 17 00:00:00 2001 From: msaffari-amd Date: Tue, 14 Oct 2025 13:20:25 +0200 Subject: [PATCH 018/262] Fix: Handle JSON boolean values (pad_m, pad_n, pad_k and persistent) in gemm_instance_builder (#3008) --- tile_engine/ops/gemm/gemm_instance_builder.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tile_engine/ops/gemm/gemm_instance_builder.py b/tile_engine/ops/gemm/gemm_instance_builder.py index c2214da613..0dc9fffedb 100644 --- a/tile_engine/ops/gemm/gemm_instance_builder.py +++ b/tile_engine/ops/gemm/gemm_instance_builder.py @@ -450,11 +450,11 @@ struct SelectedKernel {{ static constexpr ck_tile::index_t WarpTileK = {tile_config["warp_tile_k"]}; // Traits - static constexpr bool kPadM = {"true" if pad_m == "true" else "false"}; - static constexpr bool kPadN = {"true" if pad_n == "true" else "false"}; - static constexpr bool kPadK = {"true" if pad_k == "true" else "false"}; + static constexpr bool kPadM = {"true" if pad_m in [True, "true"] else "false"}; + static constexpr bool kPadN = {"true" if pad_n in [True, "true"] else "false"}; + static constexpr bool kPadK = {"true" if pad_k in [True, "true"] else "false"}; static constexpr bool TransposeC = false; - static constexpr bool UsePersistentKernel = {"true" if persistent == "true" else "false"}; + static constexpr bool UsePersistentKernel = {"true" if persistent in [True, "true"] else "false"}; static constexpr bool DoubleSmemBuffer = {"true" if pipeline == "compv4" else "false"}; static constexpr bool UseStructuredSparsity = false; static constexpr bool Preshuffle = false; @@ -576,7 +576,7 @@ struct SelectedKernel {{ }} // Get grid and block sizes - const dim3 grids = {"GemmKernel::MaxOccupancyGridSize(stream)" if persistent == "true" else "GemmKernel::GridSize(args.M, args.N, args.k_batch)"}; + const dim3 grids = {"GemmKernel::MaxOccupancyGridSize(stream)" if persistent in [True, "true"] else "GemmKernel::GridSize(args.M, args.N, args.k_batch)"}; const dim3 blocks = GemmKernel::BlockSize(); if(stream.log_level_ > 0) {{ From 6deaaa92cc561f5bc29d956d6f6de903db19a079 Mon Sep 17 00:00:00 2001 From: jakpiase Date: Tue, 14 Oct 2025 16:09:16 +0200 Subject: [PATCH 019/262] [CK_TILE] Switch into universal gemms for conv bwds (#2981) * switch into universal gemms for conv bwds * some fixes and support universal gemm in conv fwd * add reviewer comments --- .../20_grouped_convolution/gemm_configs.hpp | 303 ++++++++++++++++++ .../grouped_convolution_backward_data.cpp | 10 +- ...uped_convolution_backward_data_invoker.hpp | 239 ++++++++------ .../grouped_convolution_backward_weight.cpp | 10 +- ...ed_convolution_backward_weight_invoker.hpp | 182 +++++++---- ..._convolution_backward_weight_two_stage.cpp | 11 +- ...tion_backward_weight_two_stage_invoker.hpp | 180 +++++++---- .../grouped_convolution_forward.cpp | 14 +- .../grouped_convolution_forward_invoker.hpp | 228 ++++++++----- .../grouped_convolution_utils.hpp | 6 +- ...n_grouped_convolution_bwd_data_example.inc | 16 +- ...grouped_convolution_bwd_weight_example.inc | 16 +- .../run_grouped_convolution_fwd_example.inc | 16 +- ...ouped_convolution_backward_data_kernel.hpp | 91 +++--- ...ped_convolution_backward_weight_kernel.hpp | 144 ++++----- .../grouped_convolution_forward_kernel.hpp | 84 ++--- .../utils/grouped_convolution_utils.hpp | 10 +- .../utils/transform_conv_bwd_data_to_gemm.hpp | 14 +- .../transform_conv_bwd_weight_to_gemm.hpp | 19 +- 19 files changed, 1043 insertions(+), 550 deletions(-) create mode 100644 example/ck_tile/20_grouped_convolution/gemm_configs.hpp diff --git a/example/ck_tile/20_grouped_convolution/gemm_configs.hpp b/example/ck_tile/20_grouped_convolution/gemm_configs.hpp new file mode 100644 index 0000000000..37a63cd65c --- /dev/null +++ b/example/ck_tile/20_grouped_convolution/gemm_configs.hpp @@ -0,0 +1,303 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2025, Advanced Micro Devices, Inc. All rights reserved. + +#pragma once + +#include +#include + +#include "ck_tile/core.hpp" +#include "ck_tile/host/kernel_launch.hpp" +#include "ck_tile/ops/epilogue.hpp" +#include "ck_tile/ops/gemm.hpp" +#include "ck_tile/utility/json_dump.hpp" + +#define CK_TILE_PIPELINE_COMPUTE_V3 1 +#define CK_TILE_PIPELINE_MEMORY 2 +#define CK_TILE_PIPELINE_COMPUTE_V4 3 +#define CK_TILE_PIPELINE_COMPUTE_V5 4 + +struct GemmConfigBase +{ + static constexpr bool kPadM = true; + static constexpr bool kPadN = true; + static constexpr bool kPadK = true; + + static constexpr bool PermuteA = false; + static constexpr bool PermuteB = false; + + static constexpr bool TransposeC = false; + static constexpr bool UseStructuredSparsity = false; + + static constexpr int kBlockPerCu = 1; + static constexpr ck_tile::index_t TileParitionerGroupNum = 8; + static constexpr ck_tile::index_t TileParitionerM01 = 4; + static constexpr auto Scheduler = ck_tile::GemmPipelineScheduler::Intrawave; + static constexpr ck_tile::index_t Pipeline = CK_TILE_PIPELINE_COMPUTE_V3; + static constexpr ck_tile::index_t NumWaveGroups = 1; + static constexpr bool Preshuffle = false; + static constexpr bool TiledMMAPermuteN = false; +}; + +template +struct GemmConfigMemoryInterwave : public GemmConfigBase +{ + // Memory friendly for Interwave scheduler + static constexpr ck_tile::index_t M_Tile = 128; + static constexpr ck_tile::index_t N_Tile = 32; + static constexpr ck_tile::index_t K_Tile = 128 / sizeof(PrecType); + + static constexpr ck_tile::index_t M_Warp = 4; + static constexpr ck_tile::index_t N_Warp = 1; + static constexpr ck_tile::index_t K_Warp = 1; + + static constexpr ck_tile::index_t M_Warp_Tile = 32; + static constexpr ck_tile::index_t N_Warp_Tile = 32; + static constexpr ck_tile::index_t K_Warp_Tile = 16; + + static constexpr bool DoubleSmemBuffer = false; + static constexpr ck_tile::index_t Pipeline = CK_TILE_PIPELINE_MEMORY; + static constexpr auto Scheduler = ck_tile::GemmPipelineScheduler::Interwave; +}; + +template +struct GemmConfigMemoryIntrawave : public GemmConfigBase +{ + static constexpr ck_tile::index_t M_Tile = 128; + static constexpr ck_tile::index_t N_Tile = 32; + static constexpr ck_tile::index_t K_Tile = 128 / sizeof(PrecType); + + static constexpr ck_tile::index_t M_Warp = 4; + static constexpr ck_tile::index_t N_Warp = 1; + static constexpr ck_tile::index_t K_Warp = 1; + + static constexpr ck_tile::index_t M_Warp_Tile = 32; + static constexpr ck_tile::index_t N_Warp_Tile = 32; + static constexpr ck_tile::index_t K_Warp_Tile = 16; + + static constexpr bool DoubleSmemBuffer = false; + static constexpr ck_tile::index_t Pipeline = CK_TILE_PIPELINE_MEMORY; +}; + +template +struct GemmConfigComputeV3 : public GemmConfigBase +{ + // Compute V3 only support Intrawave scheduler + static constexpr ck_tile::index_t M_Tile = 16; + static constexpr ck_tile::index_t N_Tile = 64; + static constexpr ck_tile::index_t K_Tile = 64; + + static constexpr ck_tile::index_t M_Warp = 1; + static constexpr ck_tile::index_t N_Warp = 4; + static constexpr ck_tile::index_t K_Warp = 1; + + static constexpr ck_tile::index_t M_Warp_Tile = 16; + static constexpr ck_tile::index_t N_Warp_Tile = 16; + static constexpr ck_tile::index_t K_Warp_Tile = 32; + + static constexpr bool DoubleSmemBuffer = false; + static constexpr ck_tile::index_t Pipeline = CK_TILE_PIPELINE_COMPUTE_V3; +}; + +template +struct GemmConfigComputeV3_1 : public GemmConfigBase +{ + static constexpr ck_tile::index_t M_Tile = 256; + static constexpr ck_tile::index_t N_Tile = 256; + static constexpr ck_tile::index_t K_Tile = 128 / sizeof(PrecType); + + static constexpr ck_tile::index_t M_Warp = 2; + static constexpr ck_tile::index_t N_Warp = 2; + static constexpr ck_tile::index_t K_Warp = 1; + + static constexpr ck_tile::index_t M_Warp_Tile = 32; + static constexpr ck_tile::index_t N_Warp_Tile = 32; + static constexpr ck_tile::index_t K_Warp_Tile = 16; + + static constexpr bool DoubleSmemBuffer = false; + static constexpr ck_tile::index_t Pipeline = CK_TILE_PIPELINE_COMPUTE_V3; +}; + +template +struct GemmConfigComputeV3_2 : public GemmConfigBase +{ + static constexpr ck_tile::index_t M_Tile = 128; + static constexpr ck_tile::index_t N_Tile = 128; + static constexpr ck_tile::index_t K_Tile = 128 / sizeof(PrecType); + + static constexpr ck_tile::index_t M_Warp = 2; + static constexpr ck_tile::index_t N_Warp = 2; + static constexpr ck_tile::index_t K_Warp = 1; + + static constexpr ck_tile::index_t M_Warp_Tile = 16; + static constexpr ck_tile::index_t N_Warp_Tile = 16; + static constexpr ck_tile::index_t K_Warp_Tile = 32; + + static constexpr bool DoubleSmemBuffer = false; + static constexpr ck_tile::index_t Pipeline = CK_TILE_PIPELINE_COMPUTE_V3; + + static constexpr int kBlockPerCu = 2; +}; + +template +struct GemmConfigComputeV3_WMMA : public GemmConfigBase +{ + static constexpr ck_tile::index_t M_Tile = 128; + static constexpr ck_tile::index_t N_Tile = 128; + static constexpr ck_tile::index_t K_Tile = 64 / sizeof(PrecType); + + static constexpr ck_tile::index_t M_Warp = 4; + static constexpr ck_tile::index_t N_Warp = 2; + static constexpr ck_tile::index_t K_Warp = 1; + + static constexpr ck_tile::index_t M_Warp_Tile = 16; + static constexpr ck_tile::index_t N_Warp_Tile = 16; + static constexpr ck_tile::index_t K_Warp_Tile = 16; + + static constexpr bool DoubleSmemBuffer = false; + static constexpr ck_tile::index_t Pipeline = CK_TILE_PIPELINE_COMPUTE_V3; + + static constexpr int kBlockPerCu = 2; +}; + +template +struct GemmConfigComputeV4 : public GemmConfigBase +{ + // Compute V4 only support Intrawave scheduler + // Using the ping pong reader in the lds level + static constexpr ck_tile::index_t M_Tile = 256; + static constexpr ck_tile::index_t N_Tile = 256; + static constexpr ck_tile::index_t K_Tile = 64 / sizeof(PrecType); + + static constexpr ck_tile::index_t M_Warp = 2; + static constexpr ck_tile::index_t N_Warp = 2; + static constexpr ck_tile::index_t K_Warp = 1; + + static constexpr ck_tile::index_t M_Warp_Tile = 32; + static constexpr ck_tile::index_t N_Warp_Tile = 32; + static constexpr ck_tile::index_t K_Warp_Tile = 16; + + static constexpr bool DoubleSmemBuffer = true; + static constexpr ck_tile::index_t Pipeline = CK_TILE_PIPELINE_COMPUTE_V4; +}; + +template +struct GemmConfigComputeV4_1 : public GemmConfigBase +{ + static constexpr ck_tile::index_t M_Tile = 256; + static constexpr ck_tile::index_t N_Tile = 256; + static constexpr ck_tile::index_t K_Tile = 128 / sizeof(PrecType); + + static constexpr ck_tile::index_t M_Warp = 2; + static constexpr ck_tile::index_t N_Warp = 2; + static constexpr ck_tile::index_t K_Warp = 1; + + static constexpr ck_tile::index_t M_Warp_Tile = 32; + static constexpr ck_tile::index_t N_Warp_Tile = 32; + static constexpr ck_tile::index_t K_Warp_Tile = 16; + + static constexpr bool DoubleSmemBuffer = true; + static constexpr ck_tile::index_t Pipeline = CK_TILE_PIPELINE_COMPUTE_V4; +}; + +template +struct GemmConfigComputeV5 : public GemmConfigBase +{ + static constexpr ck_tile::index_t M_Tile = 128; + static constexpr ck_tile::index_t N_Tile = 128; + static constexpr ck_tile::index_t K_Tile = 64 / sizeof(PrecType); + + static constexpr ck_tile::index_t M_Warp = 1; + static constexpr ck_tile::index_t N_Warp = 1; + static constexpr ck_tile::index_t K_Warp = 2; + + static constexpr ck_tile::index_t M_Warp_Tile = 32; + static constexpr ck_tile::index_t N_Warp_Tile = 32; + static constexpr ck_tile::index_t K_Warp_Tile = 16; + + static constexpr bool DoubleSmemBuffer = false; + static constexpr ck_tile::index_t Pipeline = CK_TILE_PIPELINE_COMPUTE_V5; + static constexpr ck_tile::index_t NumWaNumWaveGroups = 2; +}; + +template +struct ConvTypeConfig; + +template <> +struct ConvTypeConfig +{ + using InDataType = ck_tile::half_t; + using WeiDataType = ck_tile::half_t; + using AccDataType = float; + using OutDataType = ck_tile::half_t; + // ToDo: Add more bias config to support different categories of GEMM. +}; + +template <> +struct ConvTypeConfig +{ + using InDataType = ck_tile::bf16_t; + using WeiDataType = ck_tile::bf16_t; + using AccDataType = float; + using OutDataType = ck_tile::bf16_t; +}; + +template +struct DataTypeTraits; + +template <> +struct DataTypeTraits +{ + static constexpr const char* name = "fp32"; +}; + +template <> +struct DataTypeTraits +{ + static constexpr const char* name = "fp16"; +}; + +template <> +struct DataTypeTraits +{ + static constexpr const char* name = "bf16"; +}; + +template +struct PipelineTypeTraits; + +template <> +struct PipelineTypeTraits +{ + template + using GemmPipeline = ck_tile::GemmPipelineAgBgCrMem; + template + using UniversalGemmPipeline = ck_tile::BaseGemmPipelineAgBgCrMem; +}; + +template <> +struct PipelineTypeTraits +{ + template + using GemmPipeline = ck_tile::GemmPipelineAgBgCrCompV3; + template + using UniversalGemmPipeline = ck_tile::BaseGemmPipelineAgBgCrCompV3; +}; + +template <> +struct PipelineTypeTraits +{ + template + using GemmPipeline = ck_tile::GemmPipelineAgBgCrCompV4; + template + using UniversalGemmPipeline = ck_tile::BaseGemmPipelineAgBgCrCompV4; +}; + +template <> +struct PipelineTypeTraits +{ + template + using GemmPipeline = ck_tile::GemmPipelineAgBgCrCompV5; + template + using UniversalGemmPipeline = ck_tile::BaseGemmPipelineAgBgCrCompV5; +}; diff --git a/example/ck_tile/20_grouped_convolution/grouped_convolution_backward_data.cpp b/example/ck_tile/20_grouped_convolution/grouped_convolution_backward_data.cpp index fa914a7119..6f3bedc32a 100644 --- a/example/ck_tile/20_grouped_convolution/grouped_convolution_backward_data.cpp +++ b/example/ck_tile/20_grouped_convolution/grouped_convolution_backward_data.cpp @@ -14,7 +14,7 @@ #include "grouped_convolution_backward_data_invoker.hpp" #include "run_grouped_convolution_bwd_data_example.inc" -template +template