ROCm/composable_kernel
1
0
Fork 0
mirror of https://github.com/ROCm/composable_kernel.git synced 2026-05-20 12:59:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Reorganize project folders (#6)

Browse Source
This commit is contained in:
Joseph Macaranas
2025-04-30 13:46:39 -04:00
committed by GitHub
commit 1eb2e57380
3952 changed files with 654944 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
test/softmax/CMakeLists.txt Normal file
View File

@@ -0,0 +1,11 @@
add_custom_target(test_softmax)
add_gtest_executable(test_softmax_rank3 test_softmax_rank3.cpp)
add_gtest_executable(test_softmax_rank4 test_softmax_rank4.cpp)
add_gtest_executable(test_softmax_interface test_softmax_interface.cpp)
target_link_libraries(test_softmax_rank3 PRIVATE utility device_softmax_instance)
target_link_libraries(test_softmax_rank4 PRIVATE utility device_softmax_instance)
target_link_libraries(test_softmax_interface PRIVATE utility device_softmax_instance)
add_dependencies(test_softmax test_softmax_rank3)
add_dependencies(test_softmax test_softmax_rank4)
add_dependencies(test_softmax test_softmax_interface)

86
test/softmax/test_softmax_interface.cpp Normal file
View File

@@ -0,0 +1,86 @@
// SPDX-License-Identifier: MIT
// Copyright (c) 2018-2023, Advanced Micro Devices, Inc. All rights reserved.
#include <stdexcept>
#include <vector>
#include "gtest/gtest.h"
#include "test_softmax_util.hpp"
class TestSoftmaxInterface : public ::testing::Test
{
protected:
template <ck::index_t Rank, ck::index_t NumReduceDims>
using SoftmaxInstance =
ck::DeviceSoftmaxInstanceWrapper<Rank, NumReduceDims, 256, 1, 256, 1, 8, 1, 8, 8>;
};
TEST_F(TestSoftmaxInterface, IncorrectReduceDims)
{
std::vector<ck::index_t> lengths{2, 128, 1536};
std::vector<ck::index_t> strides{128 * 1536, 1536, 1};
EXPECT_THROW((SoftmaxInstance<3, 1>{}.IsSupported(lengths, strides, {-1})), std::runtime_error);
EXPECT_THROW((SoftmaxInstance<3, 1>{}.IsSupported(lengths, strides, {3})), std::runtime_error);
EXPECT_THROW((SoftmaxInstance<3, 1>{}.IsSupported(lengths, strides, {0, 1})),
std::runtime_error);
EXPECT_THROW((SoftmaxInstance<3, 1>{}.IsSupported(lengths, strides, {})), std::runtime_error);
EXPECT_THROW((SoftmaxInstance<3, 2>{}.IsSupported(lengths, strides, {2, -1})),
std::runtime_error);
EXPECT_THROW((SoftmaxInstance<3, 2>{}.IsSupported(lengths, strides, {2, 4})),
std::runtime_error);
EXPECT_THROW((SoftmaxInstance<3, 2>{}.IsSupported(lengths, strides, {2})), std::runtime_error);
}
TEST_F(TestSoftmaxInterface, IncorrectLengthsSize)
{
std::vector<ck::index_t> lengths{128, 1536};
std::vector<ck::index_t> strides{128 * 1536, 1536, 1};
std::vector<ck::index_t> reduce_dims{2};
EXPECT_THROW((SoftmaxInstance<3, 1>{}.IsSupported({128, 1536}, strides, reduce_dims)),
std::runtime_error);
EXPECT_THROW((SoftmaxInstance<3, 1>{}.IsSupported({}, strides, reduce_dims)),
std::runtime_error);
EXPECT_THROW((SoftmaxInstance<3, 1>{}.IsSupported({1, 8, 128, 1536}, strides, reduce_dims)),
std::runtime_error);
}
TEST_F(TestSoftmaxInterface, IncorrectStridesSize)
{
std::vector<ck::index_t> lengths{2, 128, 1536};
std::vector<ck::index_t> reduce_dims{2};
EXPECT_THROW((SoftmaxInstance<3, 1>{}.IsSupported(lengths, {1536, 1}, reduce_dims)),
std::runtime_error);
EXPECT_THROW((SoftmaxInstance<3, 1>{}.IsSupported(lengths, {}, reduce_dims)),
std::runtime_error);
EXPECT_THROW((SoftmaxInstance<3, 1>{}.IsSupported(lengths, {1, 2, 3, 4}, reduce_dims)),
std::runtime_error);
}
TEST_F(TestSoftmaxInterface, UnsupportedLengths)
{
using SoftmaxInstance1 = ck::DeviceSoftmaxInstanceWrapper<3, 1, 256, 1, 256, 1, 8, 1, 8, 4>;
EXPECT_FALSE(SoftmaxInstance1{}.IsSupported({2, 128, 1500}, {128 * 1500, 1500, 1}, {2}));
EXPECT_FALSE(SoftmaxInstance1{}.IsSupported({2, 127, 1536}, {127 * 1536, 1536, 1}, {2}));
EXPECT_FALSE(SoftmaxInstance1{}.IsSupported({2, 128, 1537}, {128 * 1537, 1537, 1}, {2}));
// Reduction of middle dimensions
using SoftmaxInstance2 = ck::DeviceSoftmaxInstanceWrapper<3, 3, 256, 8, 32, 8, 8, 0, 8, 4>;
EXPECT_FALSE(SoftmaxInstance2{}.IsSupported({2, 128, 1536}, {128 * 1536, 1536, 1}, {0, 1, 2}));
// Reduction of middle dimensions
using SoftmaxInstance3 = ck::DeviceSoftmaxInstanceWrapper<3, 1, 256, 8, 32, 8, 8, 0, 4, 8>;
EXPECT_FALSE(SoftmaxInstance3{}.IsSupported({2, 128, 1536}, {128 * 1536, 1536, 1}, {2}));
EXPECT_FALSE(SoftmaxInstance3{}.IsSupported({2, 128, 1537}, {128 * 1537, 1537, 1}, {1}));
EXPECT_FALSE(SoftmaxInstance3{}.IsSupported({2, 128, 1540}, {128 * 1540, 1540, 1}, {1}));
EXPECT_FALSE(SoftmaxInstance3{}.IsSupported({2, 127, 1536}, {127 * 1536, 1536, 1}, {1}));
}
TEST_F(TestSoftmaxInterface, UnsupportedInstance)
{
// Instance with InSrcVectorDim = 1, can't reduce middle dims if in/out vec size != 1
using SoftmaxInstance1 = ck::DeviceSoftmaxInstanceWrapper<3, 1, 256, 8, 32, 1, 8, 1, 8, 8>;
EXPECT_FALSE(SoftmaxInstance1{}.IsSupported({2, 128, 1024}, {128 * 1024, 1024, 1}, {0}));
}

35
test/softmax/test_softmax_rank3.cpp Normal file
View File

@@ -0,0 +1,35 @@
// SPDX-License-Identifier: MIT
// Copyright (c) 2018-2023, Advanced Micro Devices, Inc. All rights reserved.
#include <algorithm>
#include <stdexcept>
#include <vector>
#include "gtest/gtest.h"
#include "test_softmax_util.hpp"
template <ck::index_t N>
using I = ck::Number<N>;
#ifdef CK_ENABLE_FP16
using F16 = ck::half_t;
#endif
using F32 = float;
template <typename Tuple>
class TestSoftmax : public ck::TestSoftmax<Tuple>
{
};
// clang-format off
using KernelTypes = ::testing::Types<
// InDataType, AccDataType, OutDataType, Rank
#ifdef CK_ENABLE_FP16
std::tuple< F16, F32, F16, I<3>>,
#endif
std::tuple< F32, F32, F32, I<3>>
>;
// clang-format on
TYPED_TEST_SUITE(TestSoftmax, KernelTypes);
#include "test_softmax_ut_cases.inc"

35
test/softmax/test_softmax_rank4.cpp Normal file
View File

@@ -0,0 +1,35 @@
// SPDX-License-Identifier: MIT
// Copyright (c) 2018-2023, Advanced Micro Devices, Inc. All rights reserved.
#include <algorithm>
#include <stdexcept>
#include <vector>
#include "gtest/gtest.h"
#include "test_softmax_util.hpp"
template <ck::index_t N>
using I = ck::Number<N>;
#ifdef CK_ENABLE_FP16
using F16 = ck::half_t;
#endif
using F32 = float;
template <typename Tuple>
class TestSoftmax : public ck::TestSoftmax<Tuple>
{
};
// clang-format off
using KernelTypes = ::testing::Types<
// InDataType, AccDataType, OutDataType, Rank
#ifdef CK_ENABLE_FP16
std::tuple< F16, F32, F16, I<4>>,
#endif
std::tuple< F32, F32, F32, I<4>>
>;
// clang-format on
TYPED_TEST_SUITE(TestSoftmax, KernelTypes);
#include "test_softmax_ut_cases.inc"

60
test/softmax/test_softmax_ut_cases.inc Normal file
View File

@@ -0,0 +1,60 @@
#pragma once
TYPED_TEST(TestSoftmax, ReduceOutermostDim)
{
std::vector<ck::index_t> reduce_dims{this->Rank - 1};
this->Run(reduce_dims);
}
TYPED_TEST(TestSoftmax, ReduceMiddleDim)
{
for(int dim = 0; dim < this->Rank - 1; ++dim)
{
std::vector<ck::index_t> reduce_dims{dim};
this->Run(reduce_dims);
}
}
TYPED_TEST(TestSoftmax, ReduceMultipleDimsWithOutermost)
{
for(int dim = 0; dim < this->Rank - 1; ++dim)
{
std::vector<ck::index_t> reduce_dims{dim, this->Rank - 1};
this->Run(reduce_dims);
}
}
TYPED_TEST(TestSoftmax, ReduceMultipleMiddleDims)
{
std::vector<ck::index_t> reduce_dims{0, 1};
if(this->Rank >= 3)
{
this->Run(reduce_dims);
}
if(this->Rank >= 4)
{
reduce_dims = std::vector<ck::index_t>{0, 2};
this->Run(reduce_dims);
reduce_dims = std::vector<ck::index_t>{0, 1, 2};
this->Run(reduce_dims);
}
}
TYPED_TEST(TestSoftmax, ReduceAllDims)
{
std::vector<ck::index_t> reduce_dims(this->Rank);
std::iota(std::begin(reduce_dims), std::end(reduce_dims), 0);
this->Run(reduce_dims);
}
TYPED_TEST(TestSoftmax, ReduceOddLengths)
{
this->in_lengths_ = {{3, 63, 1032}};
if(this->Rank >= 4)
{
this->in_lengths_ = {{1, 3, 63, 1032}};
}
this->Run({this->Rank - 1});
this->Run({this->Rank - 2});
}

226
test/softmax/test_softmax_util.hpp Normal file
View File

@@ -0,0 +1,226 @@
// SPDX-License-Identifier: MIT
// Copyright (c) 2018-2023, Advanced Micro Devices, Inc. All rights reserved.
#pragma once
#include <string>
#include <sstream>
#include <tuple>
#include <vector>
#include <gtest/gtest.h>
#include "ck/ck.hpp"
#include "ck/tensor_operation/gpu/device/impl/device_softmax_impl.hpp"
#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp"
#include "include/ck/utility/data_type.hpp"
#include "profiler/profile_softmax_impl.hpp"
namespace ck {
template <typename Range>
std::string serialize_range(const Range& range)
{
std::stringstream ss;
for(auto& r : range)
{
ss << r << ", ";
}
std::string str = ss.str();
return std::string(str.begin(), str.end() - 2);
}
template <typename Tuple>
class TestSoftmax : public ::testing::Test
{
protected:
using InDataType = std::tuple_element_t<0, Tuple>;
using AccDataType = std::tuple_element_t<1, Tuple>;
using OutDataType = std::tuple_element_t<2, Tuple>;
static constexpr index_t Rank = std::tuple_element_t<3, Tuple>{}.value;
public:
std::vector<std::vector<index_t>> in_lengths_ = {{2, 128, 1024}, {4, 16, 8448}, {128, 128, 64}};
std::vector<std::vector<AccDataType>> scales_ = {{2, 0}, {0, 2}, {2, 2}};
bool bench_ = false; // measure kernel performance
bool verify_ = true;
void SetUp() override
{
if constexpr(Rank == 4)
{
in_lengths_ = std::vector<std::vector<index_t>>{
{1, 2, 128, 1024}, {2, 4, 16, 8448}, {1, 128, 128, 64}};
}
}
void RunSingle(std::vector<index_t> in_length,
std::vector<index_t> reduce_dims,
AccDataType alpha,
AccDataType beta)
{
int init_method = 1; // integer value initialization
bool log = false;
std::vector<ck::index_t> strides; // intenionally empty, to get packed layout.
bool pass = false;
if constexpr(Rank == 3)
{
if(reduce_dims.size() == 1)
pass = ck::profiler::
profile_softmax_impl<InDataType, AccDataType, OutDataType, Rank, 1>(verify_,
init_method,
log,
bench_,
in_length,
strides,
reduce_dims,
alpha,
beta);
else if(reduce_dims.size() == 2)
pass = ck::profiler::
profile_softmax_impl<InDataType, AccDataType, OutDataType, Rank, 2>(verify_,
init_method,
log,
bench_,
in_length,
strides,
reduce_dims,
alpha,
beta);
else if(reduce_dims.size() == 3)
pass = ck::profiler::
profile_softmax_impl<InDataType, AccDataType, OutDataType, Rank, 3>(verify_,
init_method,
log,
bench_,
in_length,
strides,
reduce_dims,
alpha,
beta);
}
else if constexpr(Rank == 4)
{
if(reduce_dims.size() == 1)
pass = ck::profiler::
profile_softmax_impl<InDataType, AccDataType, OutDataType, Rank, 1>(verify_,
init_method,
log,
bench_,
in_length,
strides,
reduce_dims,
alpha,
beta);
else if(reduce_dims.size() == 2)
pass = ck::profiler::
profile_softmax_impl<InDataType, AccDataType, OutDataType, Rank, 2>(verify_,
init_method,
log,
bench_,
in_length,
strides,
reduce_dims,
alpha,
beta);
else if(reduce_dims.size() == 3)
pass = ck::profiler::
profile_softmax_impl<InDataType, AccDataType, OutDataType, Rank, 3>(verify_,
init_method,
log,
bench_,
in_length,
strides,
reduce_dims,
alpha,
beta);
else if(reduce_dims.size() == 4)
pass = ck::profiler::
profile_softmax_impl<InDataType, AccDataType, OutDataType, Rank, 4>(verify_,
init_method,
log,
bench_,
in_length,
strides,
reduce_dims,
alpha,
beta);
};
EXPECT_TRUE(pass);
}
void Run(std::vector<index_t> reduce_dims = {})
{
if(reduce_dims.empty())
{
reduce_dims.push_back(Rank - 1);
}
for(auto in_length : this->in_lengths_)
{
for(auto scale : this->scales_)
{
this->RunSingle(in_length, reduce_dims, scale[0], scale[1]);
}
}
}
};
template <index_t Rank,
index_t NumReduceDim,
index_t BlockSize,
index_t MThreadClusterSize,
index_t KThreadClusterSize,
index_t MThreadSliceSize,
index_t KThreadSliceSize,
index_t InSrcVectorDim,
index_t InSrcVectorSize,
index_t OutDstVectorSize>
struct DeviceSoftmaxInstanceWrapper
{
using F16 = half_t;
using F32 = float;
using Pass = tensor_operation::element_wise::PassThrough;
using InDataType = F16;
using AccDataType = F32;
using OutDataType = F16;
using InElementOp = Pass;
using AccElementOp = Pass;
using DeviceSoftmaxInstance = tensor_operation::device::DeviceSoftmaxImpl<InDataType,
AccDataType,
OutDataType,
InElementOp,
AccElementOp,
Rank,
NumReduceDim,
BlockSize,
MThreadClusterSize,
KThreadClusterSize,
MThreadSliceSize,
KThreadSliceSize,
InSrcVectorDim,
InSrcVectorSize,
OutDstVectorSize>;
bool IsSupported(const std::vector<index_t> in_lengths,
const std::vector<index_t> in_strides,
const std::vector<index_t> reduce_dims) const
{
auto softmax = DeviceSoftmaxInstance{};
auto argument = softmax.MakeArgument(in_lengths,
in_strides,
reduce_dims,
1, // alpha
1, // beta
nullptr, // in_dev
nullptr, // in_out
Pass{}, // in elementwise op
Pass{}); // acc elementwise op
return softmax.IsSupportedArgument(argument);
}
};
} // namespace ck