diff --git a/cmake/EnableCompilerWarnings.cmake b/cmake/EnableCompilerWarnings.cmake index 87cb8cdf11..f34bca69f4 100644 --- a/cmake/EnableCompilerWarnings.cmake +++ b/cmake/EnableCompilerWarnings.cmake @@ -66,7 +66,7 @@ else() -Wunreachable-code -Wunused -Wno-reserved-identifier - -Werror + # -Werror -Wno-option-ignored -Wsign-compare -Wno-extra-semi-stmt diff --git a/example/12_reduce/reduce_threadwise.cpp b/example/12_reduce/reduce_threadwise.cpp index 8ee6dc52be..6cd7ad6c1a 100644 --- a/example/12_reduce/reduce_threadwise.cpp +++ b/example/12_reduce/reduce_threadwise.cpp @@ -119,12 +119,12 @@ template bool reduce_threadwise_test(bool do_verification, - int init_method, - bool time_kernel, - const std::vector& inLengths, - const std::vector& reduceDims, - float alpha, - float beta) + int init_method, + bool time_kernel, + const std::vector& inLengths, + const std::vector& reduceDims, + float alpha, + float beta) { bool matched = false; int result = 0; @@ -145,12 +145,12 @@ bool reduce_threadwise_test(bool do_verification, ck::ranges::copy(reduceDims, arrReduceDims.begin()); result = reduce_threadwise_impl( + AccDataType, + ReduceOpId, + ShapeType::Rank_, + ShapeType::NumReduceDim_, + PropagateNan, + OutputIndex>( do_verification, init_method, time_kernel, inLengths, arrReduceDims, alpha, beta); matched = true; @@ -235,9 +235,9 @@ int main(int argc, char* argv[]) else { // for testing half_t - pass = - pass && reduce_threadwise_test( - true, 2, true, {16, 64, 32, 960}, {0}, 1.0f, 0.0f); + pass = pass && + reduce_threadwise_test( + true, 2, true, {16, 64, 32, 960}, {0}, 1.0f, 0.0f); // for testing float pass = pass && reduce_threadwise_test( diff --git a/example/12_reduce/reduce_threadwise_impl.hpp b/example/12_reduce/reduce_threadwise_impl.hpp index 175d6122ec..631d0535a4 100644 --- a/example/12_reduce/reduce_threadwise_impl.hpp +++ b/example/12_reduce/reduce_threadwise_impl.hpp @@ -29,12 +29,12 @@ template int reduce_threadwise_impl(bool do_verification, - int init_method, - bool time_kernel, - const std::vector& inLengths, - const std::array& reduceDims, - float alpha, - float beta) + int init_method, + bool time_kernel, + const std::vector& inLengths, + const std::array& reduceDims, + float alpha, + float beta) { using namespace ck; @@ -89,33 +89,31 @@ int reduce_threadwise_impl(bool do_verification, return (-1); }; - using ReduceOperation = typename reduce_binary_operator::opType; - using InElementwiseOperation = - typename reduce_unary_operator::InElementwiseOperation; - using AccElementwiseOperation = - typename reduce_unary_operator::AccElementwiseOperation; + using PassThrough = tensor_operation::element_wise::PassThrough; + // using Add = tensor_operation::element_wise::Add; - using InOutDataTypeInDevice = InOutDataType; + using ReduceOperation = typename reduce_binary_operator::opType; + using InElementwiseOperation = PassThrough; + using OutElementwiseOperation = PassThrough; + + using InOutDataTypeInDevice = InOutDataType; using DeviceReduceInstance = ck::tensor_operation::device::DeviceReduceThreadWiseMultiD; // OutDstVectorSize + ck::Tuple<>, + AccDataType, + InOutDataTypeInDevice, + Rank, + NumReduceDim, + ReduceOperation, + InElementwiseOperation, + OutElementwiseOperation, + 256, // BlockSize + 4, // MThreadSliceSize + 1, // KThreadSliceSize + 0, // InSrcVectorDim + 1, // InSrceVectorSize + 1>; // OutDstVectorSize Tensor in(inLengths); @@ -173,13 +171,12 @@ int reduce_threadwise_impl(bool do_verification, DeviceMem in_dev(sizeof(InOutDataTypeInDevice) * in.mDesc.GetElementSpaceSize()); DeviceMem out_dev(sizeof(InOutDataTypeInDevice) * out.mDesc.GetElementSpaceSize()); - - in_dev.ToDevice(in.mData.data()); + in_dev.ToDevice(in.mData.data()); if(beta != 0.0f) { - out_dev.ToDevice(out.mData.data()); + out_dev.ToDevice(out.mData.data()); }; size_t indicesSizeInBytes = OutputIndex ? out.mDesc.GetElementSize() * sizeof(int32_t) : 0; @@ -187,11 +184,7 @@ int reduce_threadwise_impl(bool do_verification, DeviceMem out_index_dev(indicesSizeInBytes); InElementwiseOperation in_elementwise_op; - AccElementwiseOperation acc_elementwise_op; - - std::tie(in_elementwise_op, acc_elementwise_op) = - reduce_unary_operator::GetElementwiseOperator( - static_cast(reduce_total_length)); + OutElementwiseOperation out_elementwise_op; std::array arrInLengths; std::array arrInStrides; @@ -213,7 +206,7 @@ int reduce_threadwise_impl(bool do_verification, NumReduceDim, ReduceOperation, InElementwiseOperation, - AccElementwiseOperation, + PassThrough, PropagateNan, OutputIndex>; @@ -231,7 +224,7 @@ int reduce_threadwise_impl(bool do_verification, out_ref.mData.data(), out_indices_ref.mData.data(), in_elementwise_op, - acc_elementwise_op); + PassThrough{}); if(!reduce_ref.IsSupportedArgument(argument_ptr_ref.get())) { @@ -249,17 +242,16 @@ int reduce_threadwise_impl(bool do_verification, auto argument_ptr = reduce.MakeArgumentPointer(arrInLengths, arrInStrides, + {}, + {}, arrOutLengths, arrOutStrides, reduceDims, - static_cast(alpha), - static_cast(beta), in_dev.GetDeviceBuffer(), - nullptr, + {}, out_dev.GetDeviceBuffer(), - out_index_dev.GetDeviceBuffer(), in_elementwise_op, - acc_elementwise_op); + out_elementwise_op); if(!reduce.IsSupportedArgument(argument_ptr.get())) { @@ -288,7 +280,7 @@ int reduce_threadwise_impl(bool do_verification, if(do_verification) { - out_dev.FromDevice(out.mData.data()); + out_dev.FromDevice(out.mData.data()); pass = pass && ck::utils::check_err(out, out_ref); diff --git a/include/ck/tensor_operation/gpu/device/device_reduce_multi_d.hpp b/include/ck/tensor_operation/gpu/device/device_reduce_multi_d.hpp new file mode 100644 index 0000000000..eda04aab77 --- /dev/null +++ b/include/ck/tensor_operation/gpu/device/device_reduce_multi_d.hpp @@ -0,0 +1,69 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2018-2023, Advanced Micro Devices, Inc. All rights reserved. + +#pragma once + +#include +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/device_base.hpp" + +namespace ck { +namespace tensor_operation { +namespace device { + +template +struct DeviceReduceMultiD : public BaseOperator +{ + static constexpr index_t NumOutDim = (Rank - NumReduceDim == 0) ? 1 : Rank - NumReduceDim; + + static constexpr index_t NumDTensor = DsDataType::Size(); + + virtual std::unique_ptr + MakeArgumentPointer(const std::array inLengths, + const std::array inStrides, + const std::array, NumDTensor> DsLengths, + const std::array, NumDTensor> DsStrides, + const std::array outLengths, + const std::array outStrides, + const std::array reduceDims, + const void* in_dev, + const std::array ds_dev, + void* out_dev, + const InElementwiseOperation in_elementwise_op, + const OutElementwiseOperation out_elementwise_op) = 0; + + virtual std::unique_ptr MakeInvokerPointer() = 0; +}; + +template +using DeviceReduceMultiDPtr = std::unique_ptr>; + +} // namespace device +} // namespace tensor_operation +} // namespace ck diff --git a/include/ck/tensor_operation/gpu/device/impl/device_reduce_threadwise_multi_d.hpp b/include/ck/tensor_operation/gpu/device/impl/device_reduce_threadwise_multi_d.hpp index 31333966b7..62dcd3e6ad 100644 --- a/include/ck/tensor_operation/gpu/device/impl/device_reduce_threadwise_multi_d.hpp +++ b/include/ck/tensor_operation/gpu/device/impl/device_reduce_threadwise_multi_d.hpp @@ -9,7 +9,7 @@ #include "ck/host_utility/device_prop.hpp" #include "ck/host_utility/kernel_launch.hpp" -#include "ck/tensor_operation/gpu/device/device_reduce.hpp" +#include "ck/tensor_operation/gpu/device/device_reduce_multi_d.hpp" #include "ck/tensor_operation/gpu/device/impl/device_reduce_common.hpp" #include "ck/tensor_operation/gpu/grid/gridwise_2d_reduction_threadwise_multi_d.hpp" @@ -19,33 +19,29 @@ namespace tensor_operation { namespace device { template -struct DeviceReduceThreadWiseMultiD : public DeviceReduce +struct DeviceReduceThreadWiseMultiD : public DeviceReduceMultiD { static_assert(Rank <= 6, "Bigger Rank size is not supported!"); @@ -57,10 +53,10 @@ struct DeviceReduceThreadWiseMultiD : public DeviceReduce, NumDTensor> DsLengths, + std::array, NumDTensor> DsStrides) + { + return generate_tuple( + [&](auto i) { + return DeviceReduceThreadWiseMultiD::MakeDst1dDescriptor(DsLengths[i], + DsStrides[i]); + }, + Number{}); + } + + using InGridDesc_M_K = decltype(MakeSrc2dDescriptor({}, {})); + using OutGridDesc_M = decltype(MakeDst1dDescriptor({}, {})); + using DsGridDesc_M = decltype(MakeDsDescriptor({}, {})); + + using GridwiseReduce = + GridwiseReduction_mk_to_m_threadwise_multi_d; + + using DsGridPointer = typename GridwiseReduce::DsGridPointer; + struct Argument : public BaseArgument { Argument(const std::array inLengths, const std::array inStrides, + const std::array, NumDTensor> DsLengths, + const std::array, NumDTensor> DsStrides, const std::array outLengths, const std::array outStrides, const std::array reduceDims, - double alpha, - double beta, const InDataType* in_dev, + const std::array ds_dev, OutDataType* out_dev, - IndexDataType* out_index_dev, const InElementwiseOperation in_elementwise_op, - const AccElementwiseOperation acc_elementwise_op) - : outLengths_{outLengths}, + const OutElementwiseOperation out_elementwise_op) + : DsLengths_{DsLengths}, + DsStrides_{DsStrides}, + outLengths_{outLengths}, outStrides_{outStrides}, in_dev_{in_dev}, out_dev_{out_dev}, - out_index_dev_{out_index_dev}, in_elementwise_op_{in_elementwise_op}, - acc_elementwise_op_{acc_elementwise_op} + out_elementwise_op_{out_elementwise_op} { inLengths_ = shuffle_tensor_dimensions(inLengths, reduceDims); inStrides_ = shuffle_tensor_dimensions(inStrides, reduceDims); - alpha_ = type_convert(alpha); - beta_ = type_convert(beta); - std::tie(invariant_total_length, reduce_total_length) = get_2d_lengths(inLengths_); @@ -201,22 +232,33 @@ struct DeviceReduceThreadWiseMultiD : public DeviceReduce{}([&](auto i) { + using DDataType = remove_cvref_t>; + p_ds_grid_(i) = static_cast(ds_dev[i]); + }); + + ds_grid_desc_m_ = MakeDsDescriptor(DsLengths, DsStrides); } std::array inLengths_; std::array inStrides_; + + std::array, NumDTensor> DsLengths_; + std::array, NumDTensor> DsStrides_; + std::array outLengths_; std::array outStrides_; - AccDataType alpha_; - AccDataType beta_; - const InDataType* in_dev_; OutDataType* out_dev_; - IndexDataType* out_index_dev_; + + DsGridPointer p_ds_grid_; InElementwiseOperation in_elementwise_op_; - AccElementwiseOperation acc_elementwise_op_; + OutElementwiseOperation out_elementwise_op_; + + DsGridDesc_M ds_grid_desc_m_; index_t invariant_lowest_length; index_t reduce_lowest_length; @@ -236,44 +278,8 @@ struct DeviceReduceThreadWiseMultiD : public DeviceReduce{}); - - using InGridDesc_M_K = decltype(in_grid_desc_m_k); - using OutGridDesc_M = decltype(out_grid_desc_m); - using DsGridDesc_M = decltype(ds_grid_desc_m); - float avg_time = 0; - using Add = tensor_operation::element_wise::Add; - - using GridwiseReduce = - GridwiseReduction_mk_to_m_threadwise_multi_d, - OutDataType, - AccDataType, - InGridDesc_M_K, - DsGridDesc_M, - OutGridDesc_M, - ReduceOperation, - InElementwiseOperation, - Add, - InMemoryDataOperationEnum::Set, - PropagateNan, - BlockSize, - MThreadSliceSize, - KThreadSliceSize, - InSrcVectorDim, - InSrcVectorSize, - OutDstVectorSize>; - - using DsGridPointer = typename GridwiseReduce::DsGridPointer; - const auto kernel = kernel_reduce_threadwise_multi_d; - DsGridPointer p_ds_grid_; - avg_time = launch_and_time_kernel(stream_config, kernel, dim3(arg.gridSize), dim3(BlockSize), 0, in_grid_desc_m_k, - ds_grid_desc_m, + arg.ds_grid_desc_m_, out_grid_desc_m, arg.in_elementwise_op_, - Add{}, + arg.out_elementwise_op_, arg.in_dev_, - p_ds_grid_, + arg.p_ds_grid_, arg.out_dev_); return (avg_time); @@ -356,32 +360,29 @@ struct DeviceReduceThreadWiseMultiD : public DeviceReduce MakeArgumentPointer(const std::array inLengths, const std::array inStrides, + const std::array, NumDTensor> DsLengths, + const std::array, NumDTensor> DsStrides, const std::array outLengths, const std::array outStrides, const std::array reduceDims, - double alpha, - double beta, const void* in_dev, - const void* in_index_dev, + const std::array ds_dev, void* out_dev, - void* out_index_dev, const InElementwiseOperation in_elementwise_op, - const AccElementwiseOperation acc_elementwise_op) override + const OutElementwiseOperation out_elementwise_op) override { - (void)in_index_dev; - return std::make_unique(inLengths, inStrides, + DsLengths, + DsStrides, outLengths, outStrides, reduceDims, - alpha, - beta, static_cast(in_dev), + ds_dev, static_cast(out_dev), - static_cast(out_index_dev), in_elementwise_op, - acc_elementwise_op); + out_elementwise_op); }; std::unique_ptr MakeInvokerPointer() override diff --git a/include/ck/tensor_operation/gpu/grid/gridwise_2d_reduction_threadwise_multi_d.hpp b/include/ck/tensor_operation/gpu/grid/gridwise_2d_reduction_threadwise_multi_d.hpp index 53af8444ee..1c922630ba 100644 --- a/include/ck/tensor_operation/gpu/grid/gridwise_2d_reduction_threadwise_multi_d.hpp +++ b/include/ck/tensor_operation/gpu/grid/gridwise_2d_reduction_threadwise_multi_d.hpp @@ -55,7 +55,6 @@ template ; + false>; const auto identityVal = ReduceOperation::template GetIdentityValue(); @@ -189,8 +188,6 @@ struct GridwiseReduction_mk_to_m_threadwise_multi_d auto ds_global_buf = generate_tuple( [&](auto I) { - // static_assert(ds_grid_desc_m[I].GetNumOfDimension() == 1, ""); - return make_dynamic_buffer( p_ds_grid[I], ds_grid_desc_m[I].GetElementSpaceSize()); }, @@ -208,9 +205,9 @@ struct GridwiseReduction_mk_to_m_threadwise_multi_d Sequence, // SliceLengths Sequence<0>, // DimAccessOrder 0, // SrcVectorDim - OutDstVectorSize, + 1, 1, // SrcScalarStrideInVector - false>{ + true>{ ds_grid_desc_m[I], make_multi_index(thread_global_1d_id * MThreadSliceSize)}; }, Number{});