diff --git a/library/include/ck/library/tensor_operation_instance/gpu/gemm_add.hpp b/library/include/ck/library/tensor_operation_instance/gpu/gemm_add.hpp index 4b7bdeb5af..3832a9f6d4 100644 --- a/library/include/ck/library/tensor_operation_instance/gpu/gemm_add.hpp +++ b/library/include/ck/library/tensor_operation_instance/gpu/gemm_add.hpp @@ -1,5 +1,5 @@ // SPDX-License-Identifier: MIT -// Copyright (c) 2025, Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2024-2025, Advanced Micro Devices, Inc. All rights reserved. #pragma once @@ -15,6 +15,7 @@ namespace ck { namespace tensor_operation { namespace device { namespace instance { + #ifdef CK_USE_XDL void add_device_gemm_add_xdl_c_shuffle_f16_i8_f16_f16_mk_kn_mn_mn_instances( std::vector>>&); -void add_device_gemm_add_xdl_c_shuffle_bf16_i8_bf16_bf16_mk_kn_mn_mn_instances( +void add_device_gemm_add_c_shuffle_bf16_i8_bf16_bf16_mk_kn_mn_mn_instances( std::vector>>&); + std::vector>>&); void add_device_gemm_add_wmma_c_shuffle_bf16_bf16_bf16_bf16_mk_kn_mn_mn_instances( - std::vector>>&); + std::vector>>&); #endif -// GEMM + Add + + +// GEMM + Add template struct DeviceOperationInstanceFactory< - ck::tensor_operation::device::DeviceGemmMultipleD, - ELayout, - ADataType, - BDataType, - ck::Tuple, - EDataType, - PassThrough, - PassThrough, - Add>> + ck::tensor_operation::device::DeviceGemmMultipleDSplitK, + ELayout, + ADataType, + BDataType, + ck::Tuple, + EDataType, + PassThrough, + PassThrough, + Add>> +{ + using DeviceOp = DeviceGemmMultipleDSplitK, + ELayout, + ADataType, + BDataType, + ck::Tuple, + EDataType, + PassThrough, + PassThrough, + Add>; + + static auto GetInstances() + { + std::vector> op_ptrs; + +#if defined(CK_USE_XDL) + // No XDL instances for DeviceGemmMultipleDSplitK with Add at the moment +#endif // CK_USE_XDL + +#if defined(CK_USE_WMMA) + +#if defined(CK_ENABLE_FP16) + if constexpr(is_same_v && is_same_v && + is_same_v && is_same_v) + { + if constexpr(is_same_v && is_same_v && + is_same_v && is_same_v) + { + add_device_gemm_add_wmma_c_shuffle_f16_f16_f16_f16_mk_kn_mn_mn_instances(op_ptrs); + } + } +#endif + +#if defined(CK_ENABLE_BF16) + if constexpr(is_same_v && is_same_v && + is_same_v && is_same_v) + { + if constexpr(is_same_v && is_same_v && + is_same_v && is_same_v) + { + add_device_gemm_add_wmma_c_shuffle_bf16_bf16_bf16_bf16_mk_kn_mn_mn_instances( + op_ptrs); + } + } +#endif +#endif + + return op_ptrs; + } +}; + +// GEMM + Add +// DeviceGemmMultipleD specialization +template +struct DeviceOperationInstanceFactory, + ELayout, + ADataType, + BDataType, + ck::Tuple, + EDataType, + PassThrough, + PassThrough, + Add>> { using DeviceOp = DeviceGemmMultipleD> op_ptrs; + #ifdef CK_USE_XDL #if defined(CK_ENABLE_INT8) && defined(CK_ENABLE_FP16) if constexpr(is_same_v && is_same_v && @@ -130,40 +207,33 @@ struct DeviceOperationInstanceFactory< } } #endif -#elif defined(CK_USE_WMMA) -// TODO: -// here for WMMA, currently BDataType and ADataType must be the same -#if defined(CK_ENABLE_FP16) - if constexpr(is_same_v && is_same_v && - is_same_v && is_same_v) - { - if constexpr(is_same_v && is_same_v && - is_same_v && is_same_v) - { - add_device_gemm_add_wmma_c_shuffle_f16_f16_f16_f16_mk_kn_mn_mn_instances(op_ptrs); - } - } -#endif -#if defined(CK_ENABLE_BF16) - // TODO: - // here for WMMA, currently BDataType and ADataType must be the same - if constexpr(is_same_v && is_same_v && - is_same_v && is_same_v) +#endif // CK_USE_XDL + +#if defined(CK_USE_WMMA) + // Reuse DeviceGemmMultipleDSplitK instances + using Wrapper = DeviceGemmMultipleDSplitKWrapper, + ELayout, + ADataType, + BDataType, + ck::Tuple, + EDataType, + PassThrough, + PassThrough, + Add>; + auto new_op_ptrs = + DeviceOperationInstanceFactory::GetInstances(); + for(auto& op_ptr : new_op_ptrs) { - if constexpr(is_same_v && is_same_v && - is_same_v && is_same_v) - { - add_device_gemm_add_wmma_c_shuffle_bf16_bf16_bf16_bf16_mk_kn_mn_mn_instances( - op_ptrs); - } + op_ptrs.emplace_back(std::make_unique(std::move(op_ptr))); } -#endif -#endif +#endif // CK_USE_WMMA + return op_ptrs; } }; - } // namespace instance } // namespace device } // namespace tensor_operation