mirror of
https://github.com/ROCm/composable_kernel.git
synced 2026-05-17 19:40:04 +00:00
7e29eecb18
* Convolution ND
* Code unification across dimensions for generating tensor descriptors.
* Example
* Instances
* Move convnd f32 instance file to comply with repo structure.
* Conv 1D tensor layouts.
* Formatting and use ReferenceConv
* Reference ConvFwd supporting 1D and 2D convolution.
* Debug printing TensorLayout name.
* Conv fwd 1D instance f32
* Refactor conv ND example.
Needed to support various conv dimensio.
Needed to support various conv dimensions
* Rename conv nd example director to prevent conflicts.
* Refactor some common utility to single file.
Plus some tests.
* Refactor GetHostTensorDescriptor + UT.
* Add 1D test case.
* Test reference convolution 1d/2d
* Remove some leftovers.
* Fix convolution example error for 1D
* Refactor test check errors utility function.
* Test Conv2D Fwd XDL
* More UT for 1D case.
* Parameterize input & weight initializers.
* Rename example to prevent conflicts.
* Split convnd instance into separate files for 1d/2d
* Address review comments.
* Fix data type for flops/gbytes calculations.
* Assign example number 11.
* 3D cases for convolution utility functions.
* 3D reference convolution.
* Add support for 3D convolution.
* Check for inputs bigger than 2GB.
* Formatting
* Support for bf16/f16/f32/i8 - conv instances + UT.
* Use check_err from test_util.hpp.
* Split convnd test into separate files for each dim.
* Fix data generation and use proper instances.
* Formatting
* Skip tensor initialization if not necessary.
* Fix CMakefiles.
* Remove redundant conv2d_fwd test.
* Lower problem size for conv3D UT.
* 3D case for convnd example.
* Remove leftovers after merge.
* Add Conv Specialization string to GetTypeString
* Skip instance causing numerical errors.
* Small fixes.
* Remove redundant includes.
* Fix namespace name error.
* Script for automatic testing and logging convolution fwd UTs
* Comment out numactl cmd.
* Refine weights initalization and relax rtol for fp16
* Fix weights initialization for int8.
* Add type_convert when store output in ref conv 1D.
* Get back old conv2d_fwd_xdl operation.
* Silence conv debug print.
* format
* clean
* clean
* Fix merge.
* Fix namespace for check_err
Co-authored-by: Adam Osewski <aosewski@amd.com>
Co-authored-by: Chao Liu <chao.liu2@amd.com>
[ROCm/composable_kernel commit: f91579aab6]
345 lines
13 KiB
C++
345 lines
13 KiB
C++
#ifndef GEMM_UTILS_HPP
|
|
#define GEMM_UTILS_HPP
|
|
|
|
#include "config.hpp"
|
|
#include "device.hpp"
|
|
#include "host_tensor.hpp"
|
|
#include "host_tensor_generator.hpp"
|
|
#include "reference_gemm.hpp"
|
|
#include "tensor_layout.hpp"
|
|
#include "test_util.hpp"
|
|
|
|
namespace ck {
|
|
namespace gemm_util {
|
|
|
|
struct GemmParams
|
|
{
|
|
GemmParams()
|
|
: M(1024), N(1024), K(1024), StrideA(1024), StrideB(1024), StrideC(1024), alpha(1), beta(0)
|
|
{
|
|
}
|
|
|
|
ck::index_t M;
|
|
ck::index_t N;
|
|
ck::index_t K;
|
|
|
|
ck::index_t StrideA;
|
|
ck::index_t StrideB;
|
|
ck::index_t StrideC;
|
|
|
|
float alpha;
|
|
float beta;
|
|
};
|
|
|
|
template <typename GemmInstance,
|
|
typename ADataType,
|
|
typename BDataType,
|
|
typename CDataType,
|
|
typename AElementwiseOperation,
|
|
typename BElementwiseOperation,
|
|
typename CElementwiseOperation>
|
|
void RunHostGEMM(const Tensor<ADataType>& A,
|
|
const Tensor<BDataType>& B,
|
|
Tensor<CDataType>& C,
|
|
AElementwiseOperation a_element_op,
|
|
BElementwiseOperation b_element_op,
|
|
CElementwiseOperation c_element_op)
|
|
{
|
|
auto ref_gemm = GemmInstance{};
|
|
auto ref_invoker = ref_gemm.MakeInvoker();
|
|
|
|
auto ref_argument = ref_gemm.MakeArgument(A, B, C, a_element_op, b_element_op, c_element_op);
|
|
|
|
ref_invoker.Run(ref_argument);
|
|
}
|
|
|
|
template <typename DeviceGemmPtr_,
|
|
typename ADataType,
|
|
typename BDataType,
|
|
typename CDataType,
|
|
typename AElementwiseOperation,
|
|
typename BElementwiseOperation,
|
|
typename CElementwiseOperation>
|
|
void RunDeviceGEMM(DeviceGemmPtr_& gemmPtr,
|
|
const ck::gemm_util::GemmParams& params,
|
|
const Tensor<ADataType>& A,
|
|
const Tensor<BDataType>& B,
|
|
Tensor<CDataType>& C,
|
|
AElementwiseOperation a_element_op,
|
|
BElementwiseOperation b_element_op,
|
|
CElementwiseOperation c_element_op)
|
|
{
|
|
DeviceMem a_m_k_device_buf(sizeof(ADataType) * A.mDesc.GetElementSpace());
|
|
DeviceMem b_k_n_device_buf(sizeof(BDataType) * B.mDesc.GetElementSpace());
|
|
DeviceMem c_m_n_device_buf(sizeof(CDataType) * C.mDesc.GetElementSpace());
|
|
|
|
a_m_k_device_buf.ToDevice(A.mData.data());
|
|
b_k_n_device_buf.ToDevice(B.mData.data());
|
|
|
|
auto invoker_ptr = gemmPtr->MakeInvokerPointer();
|
|
auto argument_ptr =
|
|
gemmPtr->MakeArgumentPointer(static_cast<ADataType*>(a_m_k_device_buf.GetDeviceBuffer()),
|
|
static_cast<BDataType*>(b_k_n_device_buf.GetDeviceBuffer()),
|
|
static_cast<CDataType*>(c_m_n_device_buf.GetDeviceBuffer()),
|
|
params.M,
|
|
params.N,
|
|
params.K,
|
|
params.StrideA,
|
|
params.StrideB,
|
|
params.StrideC,
|
|
a_element_op,
|
|
b_element_op,
|
|
c_element_op);
|
|
|
|
if(!gemmPtr->IsSupportedArgument(argument_ptr.get()))
|
|
{
|
|
throw std::runtime_error(
|
|
"wrong! device_gemm with the specified compilation parameters does "
|
|
"not support this GEMM problem");
|
|
}
|
|
|
|
invoker_ptr->Run(argument_ptr.get());
|
|
c_m_n_device_buf.FromDevice(C.mData.data());
|
|
}
|
|
|
|
template <typename DeviceGemmPtr_,
|
|
typename ADataType,
|
|
typename BDataType,
|
|
typename CDataType,
|
|
typename ALayout,
|
|
typename BLayout,
|
|
typename CLayout,
|
|
typename AElementwiseOperation,
|
|
typename BElementwiseOperation,
|
|
typename CElementwiseOperation>
|
|
struct TestGemm
|
|
{
|
|
auto PrepareGemmTensor(const ck::gemm_util::GemmParams& params)
|
|
{
|
|
auto f_host_tensor_descriptor =
|
|
[](std::size_t row, std::size_t col, std::size_t stride, auto layout) {
|
|
if(std::is_same<decltype(layout), ck::tensor_layout::gemm::RowMajor>::value)
|
|
{
|
|
return HostTensorDescriptor(std::vector<std::size_t>({row, col}),
|
|
std::vector<std::size_t>({stride, 1}));
|
|
}
|
|
else
|
|
{
|
|
return HostTensorDescriptor(std::vector<std::size_t>({row, col}),
|
|
std::vector<std::size_t>({1, stride}));
|
|
}
|
|
};
|
|
|
|
Tensor<ADataType> a_m_k(
|
|
f_host_tensor_descriptor(params.M, params.K, params.StrideA, ALayout{}));
|
|
Tensor<BDataType> b_k_n(
|
|
f_host_tensor_descriptor(params.K, params.N, params.StrideB, BLayout{}));
|
|
Tensor<CDataType> c_m_n_host_result(
|
|
f_host_tensor_descriptor(params.M, params.N, params.StrideC, CLayout{}));
|
|
Tensor<CDataType> c_m_n_device_result(
|
|
f_host_tensor_descriptor(params.M, params.N, params.StrideC, CLayout{}));
|
|
|
|
auto f_generate_tensor_value = [](auto desc, auto type) {
|
|
using dataType = decltype(type);
|
|
|
|
if(std::is_same<dataType, int8_t>::value)
|
|
{
|
|
desc.GenerateTensorValue(GeneratorTensor_2<int8_t>{-5, 5});
|
|
}
|
|
else
|
|
{
|
|
desc.GenerateTensorValue(GeneratorTensor_3<dataType>{-0.5, 0.5});
|
|
}
|
|
};
|
|
|
|
f_generate_tensor_value(a_m_k, ADataType{});
|
|
f_generate_tensor_value(b_k_n, BDataType{});
|
|
|
|
return std::make_tuple(a_m_k, b_k_n, c_m_n_host_result, c_m_n_device_result);
|
|
}
|
|
|
|
auto operator()(DeviceGemmPtr_& gemmPtr)
|
|
{
|
|
std::cout << "ALayout = " << ALayout{}.name << ", BLayout = " << BLayout{}.name
|
|
<< ", CLayout = " << CLayout{}.name << std::endl;
|
|
std::cout << gemmPtr->GetTypeString() << std::endl;
|
|
|
|
// Arrange
|
|
ck::gemm_util::GemmParams params;
|
|
params.M = 1024;
|
|
params.N = 1024;
|
|
params.K = 1024;
|
|
params.StrideA = 1024;
|
|
params.StrideB = 1024;
|
|
params.StrideC = 1024;
|
|
|
|
auto host_tensors = PrepareGemmTensor(params);
|
|
|
|
const Tensor<ADataType>& a = std::get<0>(host_tensors);
|
|
const Tensor<BDataType>& b = std::get<1>(host_tensors);
|
|
Tensor<CDataType>& c_host = std::get<2>(host_tensors);
|
|
Tensor<CDataType>& c_device = std::get<3>(host_tensors);
|
|
|
|
auto a_element_op = AElementwiseOperation{};
|
|
auto b_element_op = BElementwiseOperation{};
|
|
auto c_element_op = CElementwiseOperation{};
|
|
|
|
using ReferenceGemmInstance =
|
|
ck::tensor_operation::host::ReferenceGemm<ADataType,
|
|
BDataType,
|
|
CDataType,
|
|
AElementwiseOperation,
|
|
BElementwiseOperation,
|
|
CElementwiseOperation>;
|
|
ck::gemm_util::RunHostGEMM<ReferenceGemmInstance>(
|
|
a, b, c_host, a_element_op, b_element_op, c_element_op);
|
|
|
|
// Act
|
|
ck::gemm_util::RunDeviceGEMM(
|
|
gemmPtr, params, a, b, c_device, a_element_op, b_element_op, c_element_op);
|
|
|
|
// Assert
|
|
bool res = false;
|
|
if(std::is_same<CDataType, float>::value)
|
|
{
|
|
res = test::check_err(c_device.mData, c_host.mData, "Error: incorrect results!");
|
|
|
|
std::cout << (res ? "SUCCESS" : "FAILURE") << std::endl;
|
|
}
|
|
else if(std::is_same<CDataType, ck::half_t>::value)
|
|
{
|
|
res = test::check_err(c_device.mData, c_host.mData, "Error: incorrect results!");
|
|
|
|
std::cout << (res ? "SUCCESS" : "FAILURE") << std::endl;
|
|
}
|
|
else if(std::is_same<CDataType, int8_t>::value)
|
|
{
|
|
res = test::check_err(c_device.mData, c_host.mData, "Error: incorrect results!");
|
|
|
|
std::cout << (res ? "SUCCESS" : "FAILURE") << std::endl;
|
|
}
|
|
|
|
return res;
|
|
}
|
|
};
|
|
|
|
template <typename DeviceGemmPtr_,
|
|
typename ALayout,
|
|
typename BLayout,
|
|
typename CLayout,
|
|
typename AElementwiseOperation,
|
|
typename BElementwiseOperation,
|
|
typename CElementwiseOperation>
|
|
struct TestGemmBF16
|
|
{
|
|
using BF16 = ck::bhalf_t;
|
|
|
|
auto PrepareGemmTensorBF16(const ck::gemm_util::GemmParams& params)
|
|
{
|
|
auto f_host_tensor_descriptor =
|
|
[](std::size_t row, std::size_t col, std::size_t stride, auto layout) {
|
|
if(std::is_same<decltype(layout), ck::tensor_layout::gemm::RowMajor>::value)
|
|
{
|
|
return HostTensorDescriptor(std::vector<std::size_t>({row, col}),
|
|
std::vector<std::size_t>({stride, 1}));
|
|
}
|
|
else
|
|
{
|
|
return HostTensorDescriptor(std::vector<std::size_t>({row, col}),
|
|
std::vector<std::size_t>({1, stride}));
|
|
}
|
|
};
|
|
|
|
// use fp32 host kernel to verify bf16 device kernel
|
|
Tensor<BF16> a_m_k_bf16(
|
|
f_host_tensor_descriptor(params.M, params.K, params.StrideA, ALayout{}));
|
|
Tensor<BF16> b_k_n_bf16(
|
|
f_host_tensor_descriptor(params.K, params.N, params.StrideB, BLayout{}));
|
|
Tensor<BF16> c_m_n_device_bf16(
|
|
f_host_tensor_descriptor(params.M, params.N, params.StrideC, CLayout{}));
|
|
|
|
Tensor<float> a_m_k_fp32(
|
|
f_host_tensor_descriptor(params.M, params.K, params.StrideA, ALayout{}));
|
|
Tensor<float> b_k_n_fp32(
|
|
f_host_tensor_descriptor(params.K, params.N, params.StrideB, BLayout{}));
|
|
Tensor<float> c_m_n_host_fp32(
|
|
f_host_tensor_descriptor(params.M, params.N, params.StrideC, CLayout{}));
|
|
Tensor<float> c_m_n_device_fp32(
|
|
f_host_tensor_descriptor(params.M, params.N, params.StrideC, CLayout{}));
|
|
|
|
a_m_k_bf16.GenerateTensorValue(GeneratorTensor_3<BF16>{-0.5, 0.5});
|
|
b_k_n_bf16.GenerateTensorValue(GeneratorTensor_3<BF16>{-0.5, 0.5});
|
|
|
|
bf16_to_f32_(a_m_k_bf16, a_m_k_fp32);
|
|
bf16_to_f32_(b_k_n_bf16, b_k_n_fp32);
|
|
|
|
return std::make_tuple(a_m_k_bf16,
|
|
b_k_n_bf16,
|
|
c_m_n_device_bf16,
|
|
a_m_k_fp32,
|
|
b_k_n_fp32,
|
|
c_m_n_host_fp32,
|
|
c_m_n_device_fp32);
|
|
}
|
|
|
|
auto operator()(DeviceGemmPtr_& gemmPtr)
|
|
{
|
|
// Arrange
|
|
ck::gemm_util::GemmParams params;
|
|
params.M = 1024;
|
|
params.N = 1024;
|
|
params.K = 1024;
|
|
params.StrideA = 1024;
|
|
params.StrideB = 1024;
|
|
params.StrideC = 1024;
|
|
|
|
auto host_tensors = PrepareGemmTensorBF16(params);
|
|
const Tensor<BF16>& a_bf16 = std::get<0>(host_tensors);
|
|
const Tensor<BF16>& b_bf16 = std::get<1>(host_tensors);
|
|
Tensor<BF16>& c_device_bf16 = std::get<2>(host_tensors);
|
|
Tensor<float>& a_fp32 = std::get<3>(host_tensors);
|
|
Tensor<float>& b_fp32 = std::get<4>(host_tensors);
|
|
Tensor<float>& c_host_fp32 = std::get<5>(host_tensors);
|
|
Tensor<float>& c_device_fp32 = std::get<6>(host_tensors);
|
|
|
|
auto a_element_op = AElementwiseOperation{};
|
|
auto b_element_op = BElementwiseOperation{};
|
|
auto c_element_op = CElementwiseOperation{};
|
|
|
|
// use fp32 host kernel to verify bf16 device kernel
|
|
using ReferenceGemmInstance =
|
|
ck::tensor_operation::host::ReferenceGemm<float,
|
|
float,
|
|
float,
|
|
AElementwiseOperation,
|
|
BElementwiseOperation,
|
|
CElementwiseOperation>;
|
|
ck::gemm_util::RunHostGEMM<ReferenceGemmInstance>(
|
|
a_fp32, b_fp32, c_host_fp32, a_element_op, b_element_op, c_element_op);
|
|
|
|
// Act
|
|
ck::gemm_util::RunDeviceGEMM(gemmPtr,
|
|
params,
|
|
a_bf16,
|
|
b_bf16,
|
|
c_device_bf16,
|
|
a_element_op,
|
|
b_element_op,
|
|
c_element_op);
|
|
|
|
bf16_to_f32_(c_device_bf16, c_device_fp32);
|
|
|
|
// Assert
|
|
bool res = test::check_err(
|
|
c_device_fp32.mData, c_host_fp32.mData, "Error: incorrect results!", 1e-2f, 1e-3f);
|
|
|
|
std::cout << (res ? "SUCCESS" : "FAILURE") << std::endl;
|
|
|
|
return res;
|
|
};
|
|
};
|
|
|
|
} // namespace gemm_util
|
|
} // namespace ck
|
|
#endif
|