Merge commit 'b7403809062654e6a0e54abc2623d3f8f9fd0288' into develop

test/gemm_add/CMakeLists.txt

@@ -1,19 +1,71 @@
-add_gtest_executable(test_gemm_add test_gemm_add_xdl.hpp)
+# Implements test instances for MultipleD with xdl and wmma support.
+
+add_gtest_executable(test_gemm_add_xdl test_gemm_add_xdl.cpp)
 if(result EQUAL 0)
-    target_link_libraries(test_gemm_add PRIVATE utility device_gemm_add_instance)
+    target_link_libraries(test_gemm_add_xdl PRIVATE utility device_gemm_add_instance)
 endif()
 
-add_gtest_executable(test_gemm_add_relu test_gemm_add_relu_xdl.cpp)
+add_gtest_executable(test_gemm_add_relu_xdl test_gemm_add_relu_xdl.cpp)
 if(result EQUAL 0)
-    target_link_libraries(test_gemm_add_relu PRIVATE utility device_gemm_add_instance device_gemm_add_relu_instance)
+    target_link_libraries(test_gemm_add_relu_xdl PRIVATE utility device_gemm_add_instance device_gemm_add_relu_instance)
 endif()
 
-add_gtest_executable(test_gemm_add_silu test_gemm_add_silu_xdl.cpp)
+add_gtest_executable(test_gemm_add_silu_xdl test_gemm_add_silu_xdl.cpp)
 if(result EQUAL 0)
-    target_link_libraries(test_gemm_add_silu PRIVATE utility device_gemm_add_instance device_gemm_add_silu_instance)
+    target_link_libraries(test_gemm_add_silu_xdl PRIVATE utility device_gemm_add_instance device_gemm_add_silu_instance)
 endif()
 
-add_gtest_executable(test_gemm_add_fastgelu test_gemm_add_fastgelu_xdl.cpp)
+add_gtest_executable(test_gemm_add_silu_wmma test_gemm_add_silu_wmma.cpp)
 if(result EQUAL 0)
-    target_link_libraries(test_gemm_add_fastgelu PRIVATE utility device_gemm_add_instance device_gemm_add_fastgelu_instance)
+    target_link_libraries(test_gemm_add_silu_wmma PRIVATE utility device_gemm_add_instance device_gemm_add_silu_instance)
 endif()
+
+add_gtest_executable(test_gemm_add_fastgelu_xdl test_gemm_add_fastgelu_xdl.cpp)
+if(result EQUAL 0)
+    target_link_libraries(test_gemm_add_fastgelu_xdl PRIVATE utility device_gemm_add_instance device_gemm_add_fastgelu_instance)
+endif()
+
+add_gtest_executable(test_gemm_fastgelu_wmma test_gemm_fastgelu_wmma.cpp)
+if(result EQUAL 0)
+    target_link_libraries(test_gemm_fastgelu_wmma PRIVATE utility device_gemm_fastgelu_instance)
+endif()
+
+add_gtest_executable(test_gemm_add_fastgelu_wmma test_gemm_add_fastgelu_wmma.cpp)
+if(result EQUAL 0)
+    target_link_libraries(test_gemm_add_fastgelu_wmma PRIVATE utility device_gemm_add_fastgelu_instance)
+endif()
+
+add_gtest_executable(test_gemm_add_wmma test_gemm_add_wmma.cpp)
+if(result EQUAL 0)
+    target_link_libraries(test_gemm_add_wmma PRIVATE utility device_gemm_add_instance)
+endif()
+
+add_gtest_executable(test_gemm_add_add_fastgelu_wmma test_gemm_add_add_fastgelu_wmma.cpp)
+if(result EQUAL 0)
+    target_link_libraries(test_gemm_add_add_fastgelu_wmma PRIVATE utility device_gemm_add_add_fastgelu_instance)
+endif()
+
+add_gtest_executable(test_gemm_multiply_multiply_wmma test_gemm_multiply_multiply_wmma.cpp)
+if(result EQUAL 0)
+    target_link_libraries(test_gemm_multiply_multiply_wmma PRIVATE utility device_gemm_multiply_multiply_instance)
+endif()
+
+add_gtest_executable(test_gemm_add_multiply_wmma test_gemm_add_multiply_wmma.cpp)
+if(result EQUAL 0)
+    target_link_libraries(test_gemm_add_multiply_wmma PRIVATE utility device_gemm_add_multiply_instance)
+endif()
+
+add_gtest_executable(test_gemm_multiply_add_wmma test_gemm_multiply_add_wmma.cpp)
+if(result EQUAL 0)
+    target_link_libraries(test_gemm_multiply_add_wmma PRIVATE utility device_gemm_multiply_add_instance)
+endif()
+
+add_gtest_executable(test_gemm_bilinear_wmma test_gemm_bilinear_wmma.cpp)
+if(result EQUAL 0)
+    target_link_libraries(test_gemm_bilinear_wmma PRIVATE utility device_gemm_bilinear_instance)
+endif()
+
+add_gtest_executable(test_gemm_add_relu_wmma test_gemm_add_relu_wmma.cpp)
+if(result EQUAL 0)
+    target_link_libraries(test_gemm_add_relu_wmma PRIVATE utility device_gemm_add_relu_instance)
+endif()

test/gemm_add/test_gemm_add_add_fastgelu_wmma.cpp

@@ -0,0 +1,39 @@
+// SPDX-License-Identifier: MIT
+// Copyright (c) 2024-2025, Advanced Micro Devices, Inc. All rights reserved.
+
+#include "gtest/gtest.h"
+#include "ck/ck.hpp"
+#include "profiler/profile_gemm_add_add_fastgelu_impl.hpp"
+#include "test_gemm_common.hpp"
+
+template <typename Tuple>
+class TestGemmAddAddFastgelu : public TestGemmD0D1Common<Tuple>
+{
+    using ProfileCall = typename TestGemmD0D1Common<Tuple>::ProfileCall;
+
+    public:
+    ProfileCall GetImpl() override
+    {
+        return ck::profiler::profile_gemm_add_add_fastgelu_impl<
+            typename TestGemmD0D1Common<Tuple>::ADataType,
+            typename TestGemmD0D1Common<Tuple>::BDataType,
+            typename TestGemmD0D1Common<Tuple>::AccDataType,
+            typename TestGemmD0D1Common<Tuple>::D0DataType,
+            typename TestGemmD0D1Common<Tuple>::D1DataType,
+            typename TestGemmD0D1Common<Tuple>::EDataType,
+            typename TestGemmD0D1Common<Tuple>::ALayout,
+            typename TestGemmD0D1Common<Tuple>::BLayout,
+            typename TestGemmD0D1Common<Tuple>::D0Layout,
+            typename TestGemmD0D1Common<Tuple>::D1Layout,
+            typename TestGemmD0D1Common<Tuple>::ELayout>;
+    }
+};
+
+using KernelTypes =
+    ::testing::Types<std::tuple<F16, F16, F32, F16, F16, F16, Row, Row, Row, Row, Row>,
+                     std::tuple<F16, F16, F32, F16, F16, F16, Row, Col, Row, Row, Row>,
+                     std::tuple<F16, F16, F32, F16, F16, F16, Col, Row, Row, Row, Row>,
+                     std::tuple<F16, F16, F32, F16, F16, F16, Col, Col, Row, Row, Row>>;
+
+TYPED_TEST_SUITE(TestGemmAddAddFastgelu, KernelTypes);
+TYPED_TEST(TestGemmAddAddFastgelu, Test_FP16FP16) { this->Run(); }

test/gemm_add/test_gemm_add_fastgelu_wmma.cpp

@@ -0,0 +1,35 @@
+// SPDX-License-Identifier: MIT
+// Copyright (c) 2024-2025, Advanced Micro Devices, Inc. All rights reserved.
+
+#include "gtest/gtest.h"
+#include "ck/ck.hpp"
+#include "profiler/profile_gemm_add_fastgelu_impl.hpp"
+#include "test_gemm_common.hpp"
+
+template <typename Tuple>
+class TestGemmAddFastgelu : public TestGemmD0Common<Tuple>
+{
+    using ProfileCall = typename TestGemmD0Common<Tuple>::ProfileCall;
+
+    ProfileCall GetImpl() override
+    {
+        return ck::profiler::profile_gemm_add_fastgelu_impl<
+            typename TestGemmD0Common<Tuple>::ADataType,
+            typename TestGemmD0Common<Tuple>::BDataType,
+            typename TestGemmD0Common<Tuple>::AccDataType,
+            typename TestGemmD0Common<Tuple>::D0DataType,
+            typename TestGemmD0Common<Tuple>::EDataType,
+            typename TestGemmD0Common<Tuple>::ALayout,
+            typename TestGemmD0Common<Tuple>::BLayout,
+            typename TestGemmD0Common<Tuple>::D0Layout,
+            typename TestGemmD0Common<Tuple>::ELayout>;
+    }
+};
+
+using KernelTypes = ::testing::Types<std::tuple<F16, F16, F32, F16, F16, Row, Row, Row, Row>,
+                                     std::tuple<F16, F16, F32, F16, F16, Row, Col, Row, Row>,
+                                     std::tuple<F16, F16, F32, F16, F16, Col, Row, Row, Row>,
+                                     std::tuple<F16, F16, F32, F16, F16, Col, Col, Row, Row>>;
+
+TYPED_TEST_SUITE(TestGemmAddFastgelu, KernelTypes);
+TYPED_TEST(TestGemmAddFastgelu, Test_FP16FP16) { this->Run(); }

test/gemm_add/test_gemm_add_fastgelu_xdl.cpp

@@ -1,37 +1,29 @@
 // SPDX-License-Identifier: MIT
-// Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved.
+// Copyright (c) 2024-2025, Advanced Micro Devices, Inc. All rights reserved.
 
 #include "gtest/gtest.h"
 #include "ck/ck.hpp"
 #include "profiler/profile_gemm_add_fastgelu_impl.hpp"
-#include "test_gemm_add_xdl.hpp"
+#include "test_gemm_common.hpp"
 
 template <typename Tuple>
-class TestGemmAddFastgelu : public TestGemmAdd<Tuple>
+class TestGemmAddFastgelu : public TestGemmD0Common<Tuple>
 {
-    private:
-    using ADataType   = std::tuple_element_t<0, Tuple>;
-    using BDataType   = std::tuple_element_t<1, Tuple>;
-    using AccDataType = std::tuple_element_t<2, Tuple>;
-    using D0DataType  = std::tuple_element_t<3, Tuple>;
-    using EDataType   = std::tuple_element_t<4, Tuple>;
-    using ALayout     = std::tuple_element_t<5, Tuple>;
-    using BLayout     = std::tuple_element_t<6, Tuple>;
-    using D0Layout    = std::tuple_element_t<7, Tuple>;
-    using ELayout     = std::tuple_element_t<8, Tuple>;
+    using ProfileCall = typename TestGemmD0Common<Tuple>::ProfileCall;
 
-    constexpr static auto ProfileGemmAddFastgeluImpl =
-        ck::profiler::profile_gemm_add_fastgelu_impl<ADataType,
-                                                     BDataType,
-                                                     AccDataType,
-                                                     D0DataType,
-                                                     EDataType,
-                                                     ALayout,
-                                                     BLayout,
-                                                     D0Layout,
-                                                     ELayout>;
-
-    decltype(ProfileGemmAddFastgeluImpl) GetImpl() override { return ProfileGemmAddFastgeluImpl; }
+    ProfileCall GetImpl() override
+    {
+        return ck::profiler::profile_gemm_add_fastgelu_impl<
+            typename TestGemmD0Common<Tuple>::ADataType,
+            typename TestGemmD0Common<Tuple>::BDataType,
+            typename TestGemmD0Common<Tuple>::AccDataType,
+            typename TestGemmD0Common<Tuple>::D0DataType,
+            typename TestGemmD0Common<Tuple>::EDataType,
+            typename TestGemmD0Common<Tuple>::ALayout,
+            typename TestGemmD0Common<Tuple>::BLayout,
+            typename TestGemmD0Common<Tuple>::D0Layout,
+            typename TestGemmD0Common<Tuple>::ELayout>;
+    }
 };
 
 using KernelTypes = ::testing::Types<std::tuple<F16, I8, F32, F16, F16, Row, Row, Row, Row>,

test/gemm_add/test_gemm_add_multiply_wmma.cpp

@@ -0,0 +1,39 @@
+// SPDX-License-Identifier: MIT
+// Copyright (c) 2025, Advanced Micro Devices, Inc. All rights reserved.
+
+#include "gtest/gtest.h"
+#include "ck/ck.hpp"
+#include "test_gemm_common.hpp"
+#include "profiler/profile_gemm_add_multiply_impl.hpp"
+
+template <typename Tuple>
+class TestGemmAddMultiply : public TestGemmD0D1Common<Tuple>
+{
+    using ProfileCall = typename TestGemmD0D1Common<Tuple>::ProfileCall;
+
+    ProfileCall GetImpl() override
+    {
+        return ck::profiler::profile_gemm_add_multiply_impl<
+            typename TestGemmD0D1Common<Tuple>::ADataType,
+            typename TestGemmD0D1Common<Tuple>::BDataType,
+            typename TestGemmD0D1Common<Tuple>::AccDataType,
+            typename TestGemmD0D1Common<Tuple>::D0DataType,
+            typename TestGemmD0D1Common<Tuple>::D1DataType,
+            typename TestGemmD0D1Common<Tuple>::EDataType,
+            typename TestGemmD0D1Common<Tuple>::ALayout,
+            typename TestGemmD0D1Common<Tuple>::BLayout,
+            typename TestGemmD0D1Common<Tuple>::D0Layout,
+            typename TestGemmD0D1Common<Tuple>::D1Layout,
+            typename TestGemmD0D1Common<Tuple>::ELayout>;
+    }
+};
+
+using KernelTypes =
+    ::testing::Types<std::tuple<F16, F16, F32, F16, F16, F16, Row, Col, Row, Row, Row>,
+                     std::tuple<F16, F16, F32, F16, F16, F16, Row, Row, Row, Row, Row>,
+                     std::tuple<F16, F16, F32, F16, F16, F16, Col, Col, Row, Row, Row>,
+                     std::tuple<F16, F16, F32, F16, F16, F16, Col, Row, Row, Row, Row>>;
+
+TYPED_TEST_SUITE(TestGemmAddMultiply, KernelTypes);
+// Due to F16 shuffle data type tests has to run with limited K size. Change instances to FP32?
+TYPED_TEST(TestGemmAddMultiply, Test) { this->Run({{16, 32, 64}, {2048, 1024, 256}}); }

test/gemm_add/test_gemm_add_relu_wmma.cpp

@@ -0,0 +1,33 @@
+// SPDX-License-Identifier: MIT
+// Copyright (c) 2025, Advanced Micro Devices, Inc. All rights reserved.
+
+#include "gtest/gtest.h"
+#include "ck/ck.hpp"
+#include "profiler/profile_gemm_add_relu_impl.hpp"
+#include "test_gemm_common.hpp"
+
+template <typename Tuple>
+class TestGemmAddRelu : public TestGemmD0Common<Tuple>
+{
+    using ProfileCall = typename TestGemmD0Common<Tuple>::ProfileCall;
+
+    ProfileCall GetImpl() override
+    {
+        return ck::profiler::profile_gemm_add_relu_impl<
+            typename TestGemmD0Common<Tuple>::ADataType,
+            typename TestGemmD0Common<Tuple>::BDataType,
+            typename TestGemmD0Common<Tuple>::AccDataType,
+            typename TestGemmD0Common<Tuple>::D0DataType,
+            typename TestGemmD0Common<Tuple>::EDataType,
+            typename TestGemmD0Common<Tuple>::ALayout,
+            typename TestGemmD0Common<Tuple>::BLayout,
+            typename TestGemmD0Common<Tuple>::D0Layout,
+            typename TestGemmD0Common<Tuple>::ELayout>;
+    }
+};
+
+using KernelTypes = ::testing::Types<std::tuple<F16, F16, F32, F16, F16, Row, Row, Row, Row>,
+                                     std::tuple<BF16, BF16, F32, BF16, BF16, Row, Row, Row, Row>>;
+
+TYPED_TEST_SUITE(TestGemmAddRelu, KernelTypes);
+TYPED_TEST(TestGemmAddRelu, Test_BF16FP16) { this->Run(); }

test/gemm_add/test_gemm_add_relu_xdl.cpp

@@ -1,37 +1,29 @@
 // SPDX-License-Identifier: MIT
-// Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved.
+// Copyright (c) 2024-2025, Advanced Micro Devices, Inc. All rights reserved.
 
 #include "gtest/gtest.h"
 #include "ck/ck.hpp"
 #include "profiler/profile_gemm_add_relu_impl.hpp"
-#include "test_gemm_add_xdl.hpp"
+#include "test_gemm_common.hpp"
 
 template <typename Tuple>
-class TestGemmAddRelu : public TestGemmAdd<Tuple>
+class TestGemmAddRelu : public TestGemmD0Common<Tuple>
 {
-    private:
-    using ADataType   = std::tuple_element_t<0, Tuple>;
-    using BDataType   = std::tuple_element_t<1, Tuple>;
-    using AccDataType = std::tuple_element_t<2, Tuple>;
-    using D0DataType  = std::tuple_element_t<3, Tuple>;
-    using EDataType   = std::tuple_element_t<4, Tuple>;
-    using ALayout     = std::tuple_element_t<5, Tuple>;
-    using BLayout     = std::tuple_element_t<6, Tuple>;
-    using D0Layout    = std::tuple_element_t<7, Tuple>;
-    using ELayout     = std::tuple_element_t<8, Tuple>;
+    using ProfileCall = typename TestGemmD0Common<Tuple>::ProfileCall;
 
-    constexpr static auto ProfileGemmAddReluImpl =
-        ck::profiler::profile_gemm_add_relu_impl<ADataType,
-                                                 BDataType,
-                                                 AccDataType,
-                                                 D0DataType,
-                                                 EDataType,
-                                                 ALayout,
-                                                 BLayout,
-                                                 D0Layout,
-                                                 ELayout>;
-
-    decltype(ProfileGemmAddReluImpl) GetImpl() override { return ProfileGemmAddReluImpl; }
+    ProfileCall GetImpl() override
+    {
+        return ck::profiler::profile_gemm_add_relu_impl<
+            typename TestGemmD0Common<Tuple>::ADataType,
+            typename TestGemmD0Common<Tuple>::BDataType,
+            typename TestGemmD0Common<Tuple>::AccDataType,
+            typename TestGemmD0Common<Tuple>::D0DataType,
+            typename TestGemmD0Common<Tuple>::EDataType,
+            typename TestGemmD0Common<Tuple>::ALayout,
+            typename TestGemmD0Common<Tuple>::BLayout,
+            typename TestGemmD0Common<Tuple>::D0Layout,
+            typename TestGemmD0Common<Tuple>::ELayout>;
+    }
 };
 
 using KernelTypes = ::testing::Types<std::tuple<F16, I8, F32, F16, F16, Row, Row, Row, Row>,

test/gemm_add/test_gemm_add_silu_wmma.cpp

@@ -0,0 +1,34 @@
+
+// SPDX-License-Identifier: MIT
+// Copyright (c) 2025, Advanced Micro Devices, Inc. All rights reserved.
+
+#include "gtest/gtest.h"
+#include "ck/ck.hpp"
+#include "profiler/profile_gemm_add_silu_impl.hpp"
+#include "test_gemm_common.hpp"
+
+template <typename Tuple>
+class TestGemmAddSilu : public TestGemmD0Common<Tuple>
+{
+    using ProfileCall = typename TestGemmD0Common<Tuple>::ProfileCall;
+
+    ProfileCall GetImpl() override
+    {
+        return ck::profiler::profile_gemm_add_silu_impl<
+            typename TestGemmD0Common<Tuple>::ADataType,
+            typename TestGemmD0Common<Tuple>::BDataType,
+            typename TestGemmD0Common<Tuple>::AccDataType,
+            typename TestGemmD0Common<Tuple>::D0DataType,
+            typename TestGemmD0Common<Tuple>::EDataType,
+            typename TestGemmD0Common<Tuple>::ALayout,
+            typename TestGemmD0Common<Tuple>::BLayout,
+            typename TestGemmD0Common<Tuple>::D0Layout,
+            typename TestGemmD0Common<Tuple>::ELayout>;
+    }
+};
+
+using KernelTypes = ::testing::Types<std::tuple<F16, F16, F32, F16, F16, Row, Row, Row, Row>,
+                                     std::tuple<BF16, BF16, F32, BF16, BF16, Row, Row, Row, Row>>;
+
+TYPED_TEST_SUITE(TestGemmAddSilu, KernelTypes);
+TYPED_TEST(TestGemmAddSilu, Test_BF16FP16_BF16FP16) { this->Run(); }

test/gemm_add/test_gemm_add_silu_xdl.cpp

@@ -1,37 +1,29 @@
 // SPDX-License-Identifier: MIT
-// Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved.
+// Copyright (c) 2024-2025, Advanced Micro Devices, Inc. All rights reserved.
 
 #include "gtest/gtest.h"
 #include "ck/ck.hpp"
 #include "profiler/profile_gemm_add_silu_impl.hpp"
-#include "test_gemm_add_xdl.hpp"
+#include "test_gemm_common.hpp"
 
 template <typename Tuple>
-class TestGemmAddSilu : public TestGemmAdd<Tuple>
+class TestGemmAddSilu : public TestGemmD0Common<Tuple>
 {
-    private:
-    using ADataType   = std::tuple_element_t<0, Tuple>;
-    using BDataType   = std::tuple_element_t<1, Tuple>;
-    using AccDataType = std::tuple_element_t<2, Tuple>;
-    using D0DataType  = std::tuple_element_t<3, Tuple>;
-    using EDataType   = std::tuple_element_t<4, Tuple>;
-    using ALayout     = std::tuple_element_t<5, Tuple>;
-    using BLayout     = std::tuple_element_t<6, Tuple>;
-    using D0Layout    = std::tuple_element_t<7, Tuple>;
-    using ELayout     = std::tuple_element_t<8, Tuple>;
+    using ProfileCall = typename TestGemmD0Common<Tuple>::ProfileCall;
 
-    constexpr static auto ProfileGemmAddSiluImpl =
-        ck::profiler::profile_gemm_add_silu_impl<ADataType,
-                                                 BDataType,
-                                                 AccDataType,
-                                                 D0DataType,
-                                                 EDataType,
-                                                 ALayout,
-                                                 BLayout,
-                                                 D0Layout,
-                                                 ELayout>;
-
-    decltype(ProfileGemmAddSiluImpl) GetImpl() override { return ProfileGemmAddSiluImpl; }
+    ProfileCall GetImpl() override
+    {
+        return ck::profiler::profile_gemm_add_silu_impl<
+            typename TestGemmD0Common<Tuple>::ADataType,
+            typename TestGemmD0Common<Tuple>::BDataType,
+            typename TestGemmD0Common<Tuple>::AccDataType,
+            typename TestGemmD0Common<Tuple>::D0DataType,
+            typename TestGemmD0Common<Tuple>::EDataType,
+            typename TestGemmD0Common<Tuple>::ALayout,
+            typename TestGemmD0Common<Tuple>::BLayout,
+            typename TestGemmD0Common<Tuple>::D0Layout,
+            typename TestGemmD0Common<Tuple>::ELayout>;
+    }
 };
 
 using KernelTypes = ::testing::Types<std::tuple<F16, I8, F32, F16, F16, Row, Row, Row, Row>,

test/gemm_add/test_gemm_add_wmma.cpp

@@ -0,0 +1,32 @@
+// SPDX-License-Identifier: MIT
+// Copyright (c) 2025, Advanced Micro Devices, Inc. All rights reserved.
+
+#include "gtest/gtest.h"
+#include "ck/ck.hpp"
+#include "profiler/profile_gemm_add_impl.hpp"
+#include "test_gemm_common.hpp"
+
+template <typename Tuple>
+class TestGemmAdd : public TestGemmD0Common<Tuple>
+{
+    using ProfileCall = typename TestGemmD0Common<Tuple>::ProfileCall;
+
+    ProfileCall GetImpl() override
+    {
+        return ck::profiler::profile_gemm_add_impl<typename TestGemmD0Common<Tuple>::ADataType,
+                                                   typename TestGemmD0Common<Tuple>::BDataType,
+                                                   typename TestGemmD0Common<Tuple>::AccDataType,
+                                                   typename TestGemmD0Common<Tuple>::D0DataType,
+                                                   typename TestGemmD0Common<Tuple>::EDataType,
+                                                   typename TestGemmD0Common<Tuple>::ALayout,
+                                                   typename TestGemmD0Common<Tuple>::BLayout,
+                                                   typename TestGemmD0Common<Tuple>::D0Layout,
+                                                   typename TestGemmD0Common<Tuple>::ELayout>;
+    }
+};
+
+using KernelTypes = ::testing::Types<std::tuple<F16, F16, F32, F16, F16, Row, Row, Row, Row>,
+                                     std::tuple<BF16, BF16, F32, BF16, BF16, Row, Row, Row, Row>>;
+
+TYPED_TEST_SUITE(TestGemmAdd, KernelTypes);
+TYPED_TEST(TestGemmAdd, Test_BF16FP16) { this->Run(); }

test/gemm_add/test_gemm_add_xdl.cpp

@@ -0,0 +1,32 @@
+// SPDX-License-Identifier: MIT
+// Copyright (c) 2024-2025, Advanced Micro Devices, Inc. All rights reserved.
+
+#include "gtest/gtest.h"
+#include "ck/ck.hpp"
+#include "profiler/profile_gemm_add_impl.hpp"
+#include "test_gemm_common.hpp"
+
+template <typename Tuple>
+class TestGemmAdd : public TestGemmD0Common<Tuple>
+{
+    using ProfileCall = typename TestGemmD0Common<Tuple>::ProfileCall;
+
+    ProfileCall GetImpl() override
+    {
+        return ck::profiler::profile_gemm_add_impl<typename TestGemmD0Common<Tuple>::ADataType,
+                                                   typename TestGemmD0Common<Tuple>::BDataType,
+                                                   typename TestGemmD0Common<Tuple>::AccDataType,
+                                                   typename TestGemmD0Common<Tuple>::D0DataType,
+                                                   typename TestGemmD0Common<Tuple>::EDataType,
+                                                   typename TestGemmD0Common<Tuple>::ALayout,
+                                                   typename TestGemmD0Common<Tuple>::BLayout,
+                                                   typename TestGemmD0Common<Tuple>::D0Layout,
+                                                   typename TestGemmD0Common<Tuple>::ELayout>;
+    }
+};
+
+using KernelTypes = ::testing::Types<std::tuple<F16, I8, F32, F16, F16, Row, Row, Row, Row>,
+                                     std::tuple<BF16, I8, F32, BF16, BF16, Row, Row, Row, Row>>;
+
+TYPED_TEST_SUITE(TestGemmAdd, KernelTypes);
+TYPED_TEST(TestGemmAdd, Test_BF16FP16_INT8) { this->Run(); }

test/gemm_add/test_gemm_add_xdl.hpp

@@ -1,72 +0,0 @@
-// SPDX-License-Identifier: MIT
-// Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved.
-
-#include "gtest/gtest.h"
-#include "ck/ck.hpp"
-#include "profiler/profile_gemm_add_impl.hpp"
-
-using Row = ck::tensor_layout::gemm::RowMajor;
-using Col = ck::tensor_layout::gemm::ColumnMajor;
-
-using I8   = int8_t;
-using BF16 = ck::bhalf_t;
-using F16  = ck::half_t;
-using F32  = float;
-
-template <typename Tuple>
-class TestGemmAdd : public ::testing::Test
-{
-    protected:
-    using ADataType   = std::tuple_element_t<0, Tuple>;
-    using BDataType   = std::tuple_element_t<1, Tuple>;
-    using AccDataType = std::tuple_element_t<2, Tuple>;
-    using D0DataType  = std::tuple_element_t<3, Tuple>;
-    using EDataType   = std::tuple_element_t<4, Tuple>;
-    using ALayout     = std::tuple_element_t<5, Tuple>;
-    using BLayout     = std::tuple_element_t<6, Tuple>;
-    using D0Layout    = std::tuple_element_t<7, Tuple>;
-    using ELayout     = std::tuple_element_t<8, Tuple>;
-
-    constexpr static auto ProfileGemmAddImpl = ck::profiler::profile_gemm_add_impl<ADataType,
-                                                                                   BDataType,
-                                                                                   AccDataType,
-                                                                                   D0DataType,
-                                                                                   EDataType,
-                                                                                   ALayout,
-                                                                                   BLayout,
-                                                                                   D0Layout,
-                                                                                   ELayout>;
-
-    virtual decltype(ProfileGemmAddImpl) GetImpl() { return ProfileGemmAddImpl; }
-
-    void Run()
-    {
-        std::vector<std::vector<ck::index_t>> lengths = {
-            {16, 32, 64}, {2048, 4096, 8192}, {2048, 1024, 16}};
-
-        bool all_success = true;
-
-        for(auto length : lengths)
-        {
-            int M        = length[0];
-            int N        = length[1];
-            int K        = length[2];
-            int StrideA  = ck::is_same_v<ALayout, Row> ? K : M;
-            int StrideB  = ck::is_same_v<BLayout, Row> ? N : K;
-            int StrideD0 = ck::is_same_v<D0Layout, Row> ? N : M;
-            int StrideE  = ck::is_same_v<ELayout, Row> ? N : M;
-
-            all_success =
-                all_success &
-                GetImpl()(true, 1, false, false, M, N, K, StrideA, StrideB, StrideD0, StrideE);
-        }
-
-        EXPECT_TRUE(all_success);
-    }
-};
-
-using KernelTypes = ::testing::Types<std::tuple<F16, I8, F32, F16, F16, Row, Row, Row, Row>,
-                                     std::tuple<BF16, I8, F32, BF16, BF16, Row, Row, Row, Row>>;
-
-TYPED_TEST_SUITE(TestGemmAdd, KernelTypes);
-TYPED_TEST(TestGemmAdd, Test_BF16FP16_INT8) { this->Run(); }

test/gemm_add/test_gemm_bilinear_wmma.cpp

@@ -0,0 +1,69 @@
+// SPDX-License-Identifier: MIT
+// Copyright (c) 2025, Advanced Micro Devices, Inc. All rights reserved.
+
+#include "gtest/gtest.h"
+#include "ck/ck.hpp"
+#include "profiler/profile_gemm_bilinear_impl.hpp"
+#include "test_gemm_common.hpp"
+
+template <typename Tuple>
+class TestGemmBilinear : public ::testing::Test
+{
+    private:
+    using ADataType   = std::tuple_element_t<0, Tuple>;
+    using BDataType   = std::tuple_element_t<1, Tuple>;
+    using AccDataType = std::tuple_element_t<2, Tuple>;
+    using D0DataType  = std::tuple_element_t<3, Tuple>;
+    using EDataType   = std::tuple_element_t<4, Tuple>;
+    using ALayout     = std::tuple_element_t<5, Tuple>;
+    using BLayout     = std::tuple_element_t<6, Tuple>;
+    using D0Layout    = std::tuple_element_t<7, Tuple>;
+    using ELayout     = std::tuple_element_t<8, Tuple>;
+
+    constexpr static auto ProfileGemmBilinearImpl =
+        ck::profiler::profile_gemm_bilinear_impl<ADataType,
+                                                 BDataType,
+                                                 AccDataType,
+                                                 D0DataType,
+                                                 EDataType,
+                                                 ALayout,
+                                                 BLayout,
+                                                 D0Layout,
+                                                 ELayout>;
+
+    public:
+    void Run(TestMatrixSizes const& lengths)
+    {
+        bool all_success = true;
+
+        for(auto length : lengths)
+        {
+            int M        = length[0];
+            int N        = length[1];
+            int K        = length[2];
+            int StrideA  = ck::is_same_v<ALayout, Row> ? K : M;
+            int StrideB  = ck::is_same_v<BLayout, Row> ? N : K;
+            int StrideD0 = ck::is_same_v<D0Layout, Row> ? N : M;
+            int StrideE  = ck::is_same_v<ELayout, Row> ? N : M;
+
+            all_success =
+                all_success &
+                ProfileGemmBilinearImpl(
+                    1, 1, false, true, M, N, K, StrideA, StrideB, StrideD0, StrideE, 1.F, 1.F);
+        }
+
+        EXPECT_TRUE(all_success);
+    }
+};
+
+using KernelTypes = ::testing::Types<std::tuple<F16, F16, F32, F16, F16, Row, Row, Row, Row>,
+                                     std::tuple<F16, F16, F32, F16, F16, Row, Col, Row, Row>,
+                                     std::tuple<F16, F16, F32, F16, F16, Col, Row, Row, Row>,
+                                     std::tuple<F16, F16, F32, F16, F16, Col, Col, Row, Row>,
+                                     std::tuple<I8, I8, I32, I8, I8, Row, Row, Row, Row>,
+                                     std::tuple<I8, I8, I32, I8, I8, Row, Col, Row, Row>,
+                                     std::tuple<I8, I8, I32, I8, I8, Col, Row, Row, Row>,
+                                     std::tuple<I8, I8, I32, I8, I8, Col, Col, Row, Row>>;
+
+TYPED_TEST_SUITE(TestGemmBilinear, KernelTypes);
+TYPED_TEST(TestGemmBilinear, Test) { this->Run(DefaultTestMatrixSizes); }

test/gemm_add/test_gemm_common.hpp

@@ -0,0 +1,146 @@
+// SPDX-License-Identifier: MIT
+// Copyright (c) 2025, Advanced Micro Devices, Inc. All rights reserved.
+
+#include "gtest/gtest.h"
+#include "ck/ck.hpp"
+#include "ck/tensor_operation/gpu/device/tensor_layout.hpp"
+#include "ck/utility/data_type.hpp"
+
+using Row = ck::tensor_layout::gemm::RowMajor;
+using Col = ck::tensor_layout::gemm::ColumnMajor;
+
+using I8   = int8_t;
+using I32  = int32_t;
+using BF16 = ck::bhalf_t;
+using F16  = ck::half_t;
+using F32  = float;
+using F8   = ck::f8_t;
+
+// M, N, K
+using TestMatrixSizes = std::vector<std::vector<ck::index_t>>;
+
+static const TestMatrixSizes DefaultTestMatrixSizes = {
+    {16, 32, 64}, {512, 2048, 4096}, {2048, 1024, 16}};
+
+template <typename Tuple>
+class TestGemmCommon : public ::testing::Test
+{
+    protected:
+    using ADataType   = std::tuple_element_t<0, Tuple>;
+    using BDataType   = std::tuple_element_t<1, Tuple>;
+    using AccDataType = std::tuple_element_t<2, Tuple>;
+    using EDataType   = std::tuple_element_t<3, Tuple>;
+    using ALayout     = std::tuple_element_t<4, Tuple>;
+    using BLayout     = std::tuple_element_t<5, Tuple>;
+    using ELayout     = std::tuple_element_t<6, Tuple>;
+
+    using ProfileCall = bool (*const)(int, int, bool, bool, int, int, int, int, int, int);
+
+    virtual ProfileCall GetImpl() = 0;
+
+    void Run(const TestMatrixSizes& lengths = DefaultTestMatrixSizes)
+    {
+        bool all_success = true;
+
+        for(auto length : lengths)
+        {
+            int M       = length[0];
+            int N       = length[1];
+            int K       = length[2];
+            int StrideA = ck::is_same_v<ALayout, Row> ? K : M;
+            int StrideB = ck::is_same_v<BLayout, Row> ? N : K;
+            int StrideE = ck::is_same_v<ELayout, Row> ? N : M;
+
+            all_success =
+                all_success & GetImpl()(1, 1, false, true, M, N, K, StrideA, StrideB, StrideE);
+        }
+
+        EXPECT_TRUE(all_success);
+    }
+};
+
+template <typename Tuple>
+class TestGemmD0Common : public ::testing::Test
+{
+    protected:
+    using ADataType   = std::tuple_element_t<0, Tuple>;
+    using BDataType   = std::tuple_element_t<1, Tuple>;
+    using AccDataType = std::tuple_element_t<2, Tuple>;
+    using D0DataType  = std::tuple_element_t<3, Tuple>;
+    using EDataType   = std::tuple_element_t<4, Tuple>;
+    using ALayout     = std::tuple_element_t<5, Tuple>;
+    using BLayout     = std::tuple_element_t<6, Tuple>;
+    using D0Layout    = std::tuple_element_t<7, Tuple>;
+    using ELayout     = std::tuple_element_t<8, Tuple>;
+
+    using ProfileCall = bool (*const)(int, int, bool, bool, int, int, int, int, int, int, int);
+
+    virtual ProfileCall GetImpl() = 0;
+
+    void Run(const TestMatrixSizes& lengths = DefaultTestMatrixSizes)
+    {
+        bool all_success = true;
+
+        for(auto length : lengths)
+        {
+            int M        = length[0];
+            int N        = length[1];
+            int K        = length[2];
+            int StrideA  = ck::is_same_v<ALayout, Row> ? K : M;
+            int StrideB  = ck::is_same_v<BLayout, Row> ? N : K;
+            int StrideD0 = ck::is_same_v<D0Layout, Row> ? N : M;
+            int StrideE  = ck::is_same_v<ELayout, Row> ? N : M;
+
+            all_success =
+                all_success &
+                GetImpl()(1, 1, false, true, M, N, K, StrideA, StrideB, StrideD0, StrideE);
+        }
+
+        EXPECT_TRUE(all_success);
+    }
+};
+
+template <typename Tuple>
+class TestGemmD0D1Common : public ::testing::Test
+{
+    protected:
+    using ADataType   = std::tuple_element_t<0, Tuple>;
+    using BDataType   = std::tuple_element_t<1, Tuple>;
+    using AccDataType = std::tuple_element_t<2, Tuple>;
+    using D0DataType  = std::tuple_element_t<3, Tuple>;
+    using D1DataType  = std::tuple_element_t<4, Tuple>;
+    using EDataType   = std::tuple_element_t<5, Tuple>;
+    using ALayout     = std::tuple_element_t<6, Tuple>;
+    using BLayout     = std::tuple_element_t<7, Tuple>;
+    using D0Layout    = std::tuple_element_t<8, Tuple>;
+    using D1Layout    = std::tuple_element_t<9, Tuple>;
+    using ELayout     = std::tuple_element_t<10, Tuple>;
+
+    using ProfileCall = bool (*const)(int, int, bool, bool, int, int, int, int, int, int, int, int);
+
+    virtual ProfileCall GetImpl() = 0;
+
+    void Run(const TestMatrixSizes& lengths = DefaultTestMatrixSizes)
+    {
+        bool all_success = true;
+
+        for(auto length : lengths)
+        {
+            int M        = length[0];
+            int N        = length[1];
+            int K        = length[2];
+            int StrideA  = ck::is_same_v<ALayout, Row> ? K : M;
+            int StrideB  = ck::is_same_v<BLayout, Row> ? N : K;
+            int StrideD0 = ck::is_same_v<D0Layout, Row> ? N : M;
+            int StrideD1 = ck::is_same_v<D1Layout, Row> ? N : M;
+            int StrideE  = ck::is_same_v<ELayout, Row> ? N : M;
+
+            all_success =
+                all_success &
+                GetImpl()(
+                    1, 1, false, true, M, N, K, StrideA, StrideB, StrideD0, StrideD1, StrideE);
+        }
+
+        EXPECT_TRUE(all_success);
+    }
+};

test/gemm_add/test_gemm_fastgelu_wmma.cpp

@@ -0,0 +1,32 @@
+// SPDX-License-Identifier: MIT
+// Copyright (c) 2024-2025, Advanced Micro Devices, Inc. All rights reserved.
+
+#include "gtest/gtest.h"
+#include "ck/ck.hpp"
+#include "profiler/profile_gemm_fastgelu_impl.hpp"
+#include "test_gemm_common.hpp"
+
+template <typename Tuple>
+class TestGemmFastgelu : public TestGemmCommon<Tuple>
+{
+    using ProfileCall = typename TestGemmCommon<Tuple>::ProfileCall;
+
+    ProfileCall GetImpl() override
+    {
+        return ck::profiler::profile_gemm_fastgelu_impl<typename TestGemmCommon<Tuple>::ADataType,
+                                                        typename TestGemmCommon<Tuple>::BDataType,
+                                                        typename TestGemmCommon<Tuple>::AccDataType,
+                                                        typename TestGemmCommon<Tuple>::EDataType,
+                                                        typename TestGemmCommon<Tuple>::ALayout,
+                                                        typename TestGemmCommon<Tuple>::BLayout,
+                                                        typename TestGemmCommon<Tuple>::ELayout>;
+    }
+};
+
+using KernelTypes = ::testing::Types<std::tuple<F16, F16, F32, F16, Row, Row, Row>,
+                                     std::tuple<F16, F16, F32, F16, Row, Col, Row>,
+                                     std::tuple<F16, F16, F32, F16, Col, Row, Row>,
+                                     std::tuple<F16, F16, F32, F16, Col, Col, Row>>;
+
+TYPED_TEST_SUITE(TestGemmFastgelu, KernelTypes);
+TYPED_TEST(TestGemmFastgelu, Test_BF16FP16) { this->Run(); }

test/gemm_add/test_gemm_multiply_add_wmma.cpp

@@ -0,0 +1,40 @@
+// SPDX-License-Identifier: MIT
+// Copyright (c) 2025, Advanced Micro Devices, Inc. All rights reserved.
+
+#include "gtest/gtest.h"
+#include "test_gemm_common.hpp"
+#include "profiler/profile_gemm_multiply_add_impl.hpp"
+
+template <typename Tuple>
+class TestGemmMultiplyAdd : public TestGemmD0D1Common<Tuple>
+{
+    using ProfileCall = typename TestGemmD0D1Common<Tuple>::ProfileCall;
+
+    ProfileCall GetImpl() override
+    {
+        return ck::profiler::profile_gemm_multiply_add_impl<
+            typename TestGemmD0D1Common<Tuple>::ADataType,
+            typename TestGemmD0D1Common<Tuple>::BDataType,
+            typename TestGemmD0D1Common<Tuple>::AccDataType,
+            typename TestGemmD0D1Common<Tuple>::D0DataType,
+            typename TestGemmD0D1Common<Tuple>::D1DataType,
+            typename TestGemmD0D1Common<Tuple>::EDataType,
+            typename TestGemmD0D1Common<Tuple>::ALayout,
+            typename TestGemmD0D1Common<Tuple>::BLayout,
+            typename TestGemmD0D1Common<Tuple>::D0Layout,
+            typename TestGemmD0D1Common<Tuple>::D1Layout,
+            typename TestGemmD0D1Common<Tuple>::ELayout>;
+    }
+};
+
+using KernelTypes = ::testing::Types<
+#ifdef CK_USE_WMMA_FP8
+    std::tuple<F16, F8, F32, F32, F32, F16, Row, Col, Row, Row, Row>,
+    std::tuple<F16, F8, F32, F32, F32, F16, Row, Row, Row, Row, Row>,
+#endif
+    std::tuple<F16, F16, F32, F16, F16, F16, Row, Col, Row, Row, Row>,
+    std::tuple<F16, F16, F32, F16, F16, F16, Row, Row, Row, Row, Row>>;
+
+TYPED_TEST_SUITE(TestGemmMultiplyAdd, KernelTypes);
+// Due to F16 shuffle data type tests has to run with limited K size. Change instances to FP32?
+TYPED_TEST(TestGemmMultiplyAdd, Test) { this->Run({{16, 32, 64}, {2048, 1024, 256}}); }

test/gemm_add/test_gemm_multiply_multiply_wmma.cpp

@@ -0,0 +1,99 @@
+// SPDX-License-Identifier: MIT
+// Copyright (c) 2024-2025, Advanced Micro Devices, Inc. All rights reserved.
+
+#include "gtest/gtest.h"
+#include "ck/ck.hpp"
+#include "profiler/profile_gemm_multiply_multiply_impl.hpp"
+
+using Row = ck::tensor_layout::gemm::RowMajor;
+using Col = ck::tensor_layout::gemm::ColumnMajor;
+
+using I8   = int8_t;
+using I32  = int32_t;
+using F8   = ck::f8_t;
+using BF16 = ck::bhalf_t;
+using F16  = ck::half_t;
+using F32  = float;
+
+template <typename Tuple>
+class TestGemmMultiplyMultiply : public ::testing::Test
+{
+    private:
+    using ADataType   = std::tuple_element_t<0, Tuple>;
+    using BDataType   = std::tuple_element_t<1, Tuple>;
+    using AccDataType = std::tuple_element_t<2, Tuple>;
+    using D0DataType  = std::tuple_element_t<3, Tuple>;
+    using D1DataType  = std::tuple_element_t<4, Tuple>;
+    using EDataType   = std::tuple_element_t<5, Tuple>;
+    using ALayout     = std::tuple_element_t<6, Tuple>;
+    using BLayout     = std::tuple_element_t<7, Tuple>;
+    using D0Layout    = std::tuple_element_t<8, Tuple>;
+    using D1Layout    = std::tuple_element_t<9, Tuple>;
+    using ELayout     = std::tuple_element_t<10, Tuple>;
+
+    constexpr static auto ProfileGemmMultiplyMultiplyImpl =
+        ck::profiler::profile_gemm_multiply_multiply_impl<ADataType,
+                                                          BDataType,
+                                                          AccDataType, // ComputeDataType for
+                                                                       // reference gemm
+                                                          AccDataType,
+                                                          D0DataType,
+                                                          D1DataType,
+                                                          EDataType,
+                                                          ALayout,
+                                                          BLayout,
+                                                          D0Layout,
+                                                          D1Layout,
+                                                          ELayout>;
+
+    public:
+    void Run()
+    {
+        std::vector<std::vector<ck::index_t>> lengths = {
+            {16, 32, 64}, {512, 2048, 4096}, {2048, 1024, 16}};
+
+        bool all_success = true;
+
+        for(auto length : lengths)
+        {
+            int M        = length[0];
+            int N        = length[1];
+            int K        = length[2];
+            int StrideA  = ck::is_same_v<ALayout, Row> ? K : M;
+            int StrideB  = ck::is_same_v<BLayout, Row> ? N : K;
+            int StrideD0 = ck::is_same_v<D0Layout, Row> ? N : M;
+            int StrideD1 = ck::is_same_v<D1Layout, Row> ? N : M;
+            int StrideE  = ck::is_same_v<ELayout, Row> ? N : M;
+
+            all_success = all_success & ProfileGemmMultiplyMultiplyImpl(1,
+                                                                        1,
+                                                                        false,
+                                                                        true,
+                                                                        M,
+                                                                        N,
+                                                                        K,
+                                                                        StrideA,
+                                                                        StrideB,
+                                                                        StrideD0,
+                                                                        StrideD1,
+                                                                        StrideE,
+                                                                        1,
+                                                                        1,
+                                                                        1,
+                                                                        0);
+        }
+
+        EXPECT_TRUE(all_success);
+    }
+};
+
+using KernelTypes = ::testing::Types<
+#ifdef CK_USE_WMMA_FP8
+    std::tuple<F8, F8, F32, F32, F32, F16, Row, Col, Row, Col, Row>,
+    std::tuple<F8, F8, F32, F32, F32, BF16, Row, Col, Row, Col, Row>,
+#endif
+    std::tuple<I8, I8, I32, F16, F16, F16, Row, Col, Row, Col, Row>,
+    std::tuple<I8, I8, I32, F32, F32, BF16, Row, Col, Row, Col, Row>>;
+
+TYPED_TEST_SUITE(TestGemmMultiplyMultiply, KernelTypes);
+TYPED_TEST(TestGemmMultiplyMultiply, Test) { this->Run(); }