From 4a27f120ea5ec18796fa9898c9b85264db2b364d Mon Sep 17 00:00:00 2001 From: Chao Liu Date: Fri, 24 Jun 2022 20:51:04 -0500 Subject: [PATCH] Absolute include path (#281) * ad gelu and fast_gelu * added GeLU and fast GeLU * clean up * add gemm+fastgelu example * add gemm+gelu instances * update profiler * clean up * clean up * adding gemm+bias+activation * clean * adding bias * clean * adding gemm multiple d * debugging * add gemm bias add fastgelu * rename, clean * refactoring; add readme * refactor * refactor * refactor * refactor * refactor * refactor * fix * fix * update example * update example * rename * update example * add ckProfiler * clean * clean * clean * clean * add client app example * update readme * delete obselete files * remove old client app * delete old file * cleaning * clean * remove half * fix header path * fix header path * fix header path * fix header path * fix header path * fix header path for all examples * fix header path * fix header path * fix header path * fix header path * fix header path * fix header path * fix header path * fix header path * fix header path * revert client app example * clean build * fix build * temporary disable client test on Jenkins * clean * clean * clean [ROCm/composable_kernel commit: d1db6a0c3ea190996bdae37adda191f746bfc34e] --- CMakeLists.txt | 5 - Jenkinsfile | 34 +- example/01_gemm/gemm_dl_fp16.cpp | 22 +- example/01_gemm/gemm_dl_fp32.cpp | 22 +- example/01_gemm/gemm_dl_int8.cpp | 22 +- example/01_gemm/gemm_xdl_bf16.cpp | 22 +- example/01_gemm/gemm_xdl_fp16.cpp | 24 +- example/01_gemm/gemm_xdl_fp64.cpp | 24 +- example/01_gemm/gemm_xdl_int8.cpp | 23 +- .../gemm_xdl_alpha_beta.cpp | 24 +- .../03_gemm_bias_relu/gemm_xdl_bias_relu.cpp | 22 +- .../gemm_add_add_fastgelu_xdl_fp16.cpp | 22 +- .../conv2d_fwd_xdl_bias_relu.cpp | 24 +- .../conv2d_fwd_xdl_bias_relu_add.cpp | 24 +- example/09_convnd_fwd/convnd_fwd_xdl_fp16.cpp | 22 +- example/09_convnd_fwd/convnd_fwd_xdl_fp32.cpp | 22 +- example/09_convnd_fwd/convnd_fwd_xdl_fp64.cpp | 22 +- example/09_convnd_fwd/convnd_fwd_xdl_int8.cpp | 22 +- .../conv2d_bwd_data_xdl.cpp | 24 +- .../conv2d_bwd_weight_xdl.cpp | 24 +- example/12_reduce/reduce_blockwise.cpp | 23 +- .../12_reduce/reduce_blockwise_two_call.cpp | 23 +- example/13_pool2d_fwd/pool2d_fwd_common.hpp | 22 +- example/13_pool2d_fwd/pool2d_fwd_fp16.cpp | 6 +- example/13_pool2d_fwd/pool2d_fwd_fp32.cpp | 6 +- .../gemm_xdl_requant_relu_requant_int8.cpp | 25 +- .../15_grouped_gemm/grouped_gemm_xdl_fp16.cpp | 25 +- .../gemm_reduce_xdl_max_fp16.cpp | 22 +- .../gemm_reduce_xdl_mean_squaremean_fp16.cpp | 25 +- .../convnd_bwd_data_xdl.cpp | 24 +- .../batched_gemm_reduce_xdl_fp16.cpp | 25 +- .../broadcast_add_2d_amn_bn.cpp | 41 +- .../broadcast_add_3d_am_bmnk.cpp | 16 +- .../elementwise_add_1d.cpp | 40 +- .../elementwise_add_4d.cpp | 41 +- .../convnd_bwd_weight_xdl.cpp | 25 +- .../convnd_bwd_weight_xdl_bf16_splitk.cpp | 27 +- .../gemm_bias_relu_add_layernorm_xdl_fp16.cpp | 23 +- .../gemm_layernorm_xdl_fp16.cpp | 23 +- example/22_cgemm/cgemm_xdl_fp16.cpp | 47 +- example/23_softmax/softmax_blockwise.cpp | 21 +- example/CMakeLists.txt | 19 +- external/include/half/half.hpp | 5670 ----------------- include/ck/{config.hpp => ck.hpp} | 9 +- .../device_prop.hpp | 1 + include/ck/device_utility/hip_check_error.hpp | 14 + include/ck/device_utility/kernel_launch.hpp | 71 + include/ck/options.hpp | 3 - .../tensor_description/cluster_descriptor.hpp | 8 +- .../multi_index_transform.hpp | 8 +- .../multi_index_transform_helper.hpp | 8 +- .../ck/tensor_description/tensor_adaptor.hpp | 10 +- .../tensor_description/tensor_descriptor.hpp | 8 +- .../tensor_descriptor_helper.hpp | 7 +- .../tensor_space_filling_curve.hpp | 16 +- .../gpu/block/blockwise_gemm_dl_v2r3.hpp | 9 +- .../gpu/block/blockwise_gemm_xdlops.hpp | 10 +- .../blockwise_tensor_slice_transfer_v5r1.hpp | 14 +- .../block/reduction_functions_blockwise.hpp | 39 +- ...hread_group_tensor_slice_transfer_v4r1.hpp | 11 +- ...hread_group_tensor_slice_transfer_v6r1.hpp | 11 +- ...hread_group_tensor_slice_transfer_v6r2.hpp | 11 +- ...hread_group_tensor_slice_transfer_v6r3.hpp | 11 +- .../thread_group_tensor_slice_transfer_v7.hpp | 10 +- .../gpu/device/device_5ary_elementwise.hpp | 18 +- .../gpu/device/device_base.hpp | 2 +- ...evice_batched_gemm_reduce_xdl_cshuffle.hpp | 19 +- .../gpu/device/device_batched_gemm_xdl.hpp | 22 +- .../gpu/device/device_binary_elementwise.hpp | 8 +- .../device_cgemm_4gemm_xdl_cshuffle.hpp | 50 +- ...rd_weight_xdl_c_shuffle_nhwc_kyxc_nhwk.hpp | 24 +- ...ice_conv2d_bwd_data_xdl_nhwc_kyxc_nhwk.hpp | 23 +- ...fle_bias_activation_add_nhwc_kyxc_nhwk.hpp | 23 +- ...shuffle_bias_activation_nhwc_kyxc_nhwk.hpp | 21 +- ...onv2d_fwd_xdl_c_shuffle_nhwc_kyxc_nhwk.hpp | 23 +- .../device_conv2d_fwd_xdl_nhwc_kyxc_nhwk.hpp | 25 +- .../device/device_conv_backward_weight.hpp | 8 +- .../gpu/device/device_conv_bwd_data.hpp | 10 +- .../gpu/device/device_conv_fwd.hpp | 8 +- .../device_conv_fwd_bias_activation.hpp | 9 +- .../device_conv_fwd_bias_activation_add.hpp | 8 +- ...rd_weight_xdl_c_shuffle_nhwc_kyxc_nhwk.hpp | 21 +- ..._convnd_bwd_data_xdl_ndhwc_kzyxc_ndhwk.hpp | 23 +- .../device_convnd_fwd_xdl_nhwc_kyxc_nhwk.hpp | 19 +- .../gpu/device/device_gemm_bias.hpp | 4 +- .../device/device_gemm_bias_activation.hpp | 7 +- ...vice_gemm_bias_add_reduce_xdl_cshuffle.hpp | 18 +- .../gpu/device/device_gemm_dl.hpp | 20 +- .../device_gemm_multiple_d_xdl_cshuffle.hpp | 18 +- .../device_gemm_reduce_xdl_cshuffle.hpp | 19 +- .../gpu/device/device_gemm_xdl.hpp | 20 +- .../device_gemm_xdl_c_shuffle_bias_2d.hpp | 18 +- ...ice_gemm_xdl_c_shuffle_bias_activation.hpp | 21 +- ...gemm_xdl_c_shuffle_bias_activation_add.hpp | 21 +- .../gpu/device/device_gemm_xdl_cshuffle.hpp | 20 +- .../gpu/device/device_gemm_xdl_splitk.hpp | 26 +- .../device_gemm_xdl_splitk_c_shuffle.hpp | 25 +- .../gpu/device/device_grouped_gemm_xdl.hpp | 23 +- .../gpu/device/device_pool2d_fwd.hpp | 9 +- .../device/device_pool2d_fwd_nhwc_nhwc.hpp | 19 +- .../gpu/device/device_reduce.hpp | 10 +- .../gpu/device/device_reduce_common.hpp | 11 +- .../gpu/device/device_reduce_multiblock.hpp | 22 +- .../gpu/device/device_reduce_threadwise.hpp | 16 +- .../gpu/device/device_softmax.hpp | 22 +- .../gpu/device/device_unary_elementwise.hpp | 8 +- .../gpu/device/gemm_specialization.hpp | 4 +- .../gpu/device/reduction_operator_mapping.hpp | 39 +- .../element/binary_element_wise_operation.hpp | 27 +- .../gpu/element/element_wise_operation.hpp | 8 +- .../element/unary_element_wise_operation.hpp | 4 +- .../gpu/grid/block_to_ctile_map.hpp | 13 +- .../grid/gridwise_2d_reduction_multiblock.hpp | 44 +- .../grid/gridwise_2d_reduction_threadwise.hpp | 43 +- .../gpu/grid/gridwise_5ary_Elementwise_1d.hpp | 8 +- .../grid/gridwise_binary_elementwise_1d.hpp | 8 +- ...e_gemm_bias_add_reduce_xdl_cshuffle_v1.hpp | 23 +- .../gpu/grid/gridwise_gemm_dl_v1r3.hpp | 21 +- .../gridwise_gemm_multiple_d_xdl_cshuffle.hpp | 21 +- .../gpu/grid/gridwise_gemm_pipeline_v1.hpp | 5 +- .../gridwise_gemm_reduce_xdl_cshuffle_v1.hpp | 24 +- .../grid/gridwise_gemm_xdl_cshuffle_v1.hpp | 22 +- .../grid/gridwise_gemm_xdlops_bwd_weight.hpp | 21 +- .../gpu/grid/gridwise_gemm_xdlops_v2r3.hpp | 19 +- .../gpu/grid/gridwise_gemm_xdlops_v2r4.hpp | 24 +- .../gpu/grid/gridwise_gemm_xdlops_v2r4r2.hpp | 26 +- .../gpu/grid/gridwise_gemm_xdlops_v3r1.hpp | 24 +- .../gpu/grid/gridwise_gemm_xdlops_v3r2.hpp | 25 +- .../gpu/grid/gridwise_gemm_xdlops_v3r3.hpp | 22 +- .../gpu/grid/gridwise_set_buffer_value.hpp | 31 +- .../gpu/grid/gridwise_softmax.hpp | 45 +- .../grid/gridwise_unary_elementwise_1d.hpp | 8 +- .../thread/reduction_functions_threadwise.hpp | 34 +- .../gpu/thread/threadwise_contraction_dl.hpp | 5 +- .../thread/threadwise_tensor_slice_set.hpp | 10 +- .../threadwise_tensor_slice_transfer.hpp | 12 +- .../threadwise_tensor_slice_transfer_v3r1.hpp | 12 +- .../threadwise_tensor_slice_transfer_v4r1.hpp | 10 +- .../threadwise_tensor_slice_transfer_v5r1.hpp | 7 +- .../threadwise_tensor_slice_transfer_v6r1.hpp | 14 +- .../threadwise_tensor_slice_transfer_v6r2.hpp | 12 +- .../threadwise_tensor_slice_transfer_v6r3.hpp | 12 +- .../threadwise_tensor_slice_transfer_v7.hpp | 8 +- .../tensor_operation/gpu/warp/xdlops_gemm.hpp | 10 +- include/ck/utility/amd_address_space.hpp | 6 +- include/ck/utility/common_header.hpp | 79 +- include/ck/utility/data_type.hpp | 2 +- include/ck/utility/dynamic_buffer.hpp | 3 +- include/ck/utility/functional2.hpp | 8 +- include/ck/utility/functional3.hpp | 13 +- include/ck/utility/get_id.hpp | 3 +- .../ck/utility/is_known_at_compile_time.hpp | 6 +- include/ck/utility/magic_division.hpp | 7 +- include/ck/utility/math.hpp | 7 +- include/ck/utility/math_v2.hpp | 10 +- include/ck/utility/multi_index.hpp | 5 +- include/ck/utility/reduction_common.hpp | 34 +- include/ck/utility/reduction_enums.hpp | 32 +- .../reduction_functions_accumulate.hpp | 43 +- include/ck/utility/reduction_operator.hpp | 41 +- include/ck/utility/synchronization.hpp | 6 +- include/ck/utility/thread_group.hpp | 1 + include/ck/utility/transpose_vectors.hpp | 6 +- include/ck/utility/type.hpp | 6 +- .../ck/library/host/host_interface.hpp | 54 - .../ck/library/host_tensor/conv_common.hpp | 20 +- .../include/ck/library/host_tensor/device.hpp | 123 - .../ck/library/host_tensor/device_memory.hpp | 37 + .../ck/library/host_tensor/device_tensor.hpp | 8 - .../library/host_tensor/host_common_util.hpp | 37 +- .../ck/library/host_tensor/host_gemm.hpp | 1 + .../ck/library/host_tensor/host_reduction.hpp | 43 +- .../ck/library/host_tensor/host_tensor.hpp | 8 +- .../host_tensor/host_tensor_generator.hpp | 2 +- .../library/obselete_driver_offline/debug.hpp | 13 - ...emm_v5r1_dlops_nc0hwc1_kc0yxc1_nk0hwk1.hpp | 220 - ...plicit_gemm_v4r1_xdlops_nhwc_kyxc_nhwk.hpp | 309 - ...icit_gemm_v4r1r2_xdlops_nhwc_kyxc_nhwk.hpp | 423 -- ..._gemm_v4r1r2_xdlops_nhwc_kyxc_nhwk_1x1.hpp | 389 -- ...mm_v4r4r2_xdlops_atomic_nchw_kcyx_nkhw.hpp | 256 - ...icit_gemm_v4r4r2_xdlops_nchw_kcyx_nkhw.hpp | 234 - ...mm_v4r4r4_xdlops_atomic_nhwc_kyxc_nhwk.hpp | 288 - ...icit_gemm_v4r4r4_xdlops_nhwc_kyxc_nhwk.hpp | 276 - ...mm_v4r4r5_xdlops_atomic_nhwc_kyxc_nhwk.hpp | 456 -- ...mplicit_gemm_v4r4_dlops_nchw_kcyx_nkhw.hpp | 201 - ...licit_gemm_v4r4r2_dlops_nhwc_kyxc_nhwk.hpp | 273 - ...icit_gemm_v4r4r2_xdlops_nchw_kcyx_nkhw.hpp | 228 - ...icit_gemm_v4r4r4_xdlops_nhwc_kyxc_nhwk.hpp | 600 -- ...emm_v5r1_dlops_nc0hwc1_kc0yxc1_nk0hwk1.hpp | 196 - ...mplicit_gemm_v6r1_dlops_nchw_kcyx_nkhw.hpp | 241 - ...emm_v5r1_dlops_nc0hwc1_kc0yxc1_nk0hwk1.hpp | 212 - .../device_gemm_xdlops_km_kn_mn.hpp | 463 -- .../device_gemm_xdlops_km_kn_nm.hpp | 263 - .../device_gemm_xdlops_km_nk_mn.hpp | 463 -- .../device_gemm_xdlops_km_nk_nm.hpp | 263 - .../device_gemm_xdlops_mk_kn_mn.hpp | 463 -- .../device_gemm_xdlops_mk_kn_nm.hpp | 291 - .../device_gemm_xdlops_mk_nk_mn.hpp | 564 -- .../device_gemm_xdlops_mk_nk_nm.hpp | 347 - .../driver_contraction_dlops_v1r2.hpp | 286 - ...emm_v5r1_dlops_nc0hwc1_kc0yxc1_nk0hwk1.hpp | 429 -- ...emm_v5r1_dlops_nc0hwc1_kc0yxc1_nk0hwk1.hpp | 386 -- ...emm_v5r1_dlops_nc0hwc1_kc0yxc1_nk0hwk1.hpp | 440 -- .../driver_gemm_dlops_v1r2.hpp | 278 - .../driver_gemm_dlops_v1r3.hpp | 275 - .../driver_gemm_xdlops_v2r3.hpp | 220 - .../driver_gemm_xdlops_v2r4.hpp | 213 - .../cpu/reference_batched_gemm.hpp | 9 +- .../cpu/reference_cgemm.hpp | 31 +- .../cpu/reference_conv_backward_weight.hpp | 5 +- .../cpu/reference_conv_bwd_data.hpp | 10 +- .../cpu/reference_conv_fwd.hpp | 5 +- .../reference_conv_fwd_bias_activation.hpp | 9 +- ...reference_conv_fwd_bias_activation_add.hpp | 9 +- .../cpu/reference_gemm.hpp | 6 +- .../cpu/reference_gemm_bias_2d.hpp | 9 +- .../cpu/reference_gemm_bias_activation.hpp | 10 +- .../reference_gemm_bias_activation_add.hpp | 10 +- .../cpu/reference_softmax.hpp | 8 +- .../gpu/reduce/device_reduce_instance.hpp | 47 +- .../device_reduce_instance_blockwise.hpp | 12 +- ..._reduce_instance_blockwise_b16_f32_b16.hpp | 11 +- ..._reduce_instance_blockwise_f16_f16_f16.hpp | 11 +- ..._reduce_instance_blockwise_f16_f32_f16.hpp | 11 +- ..._reduce_instance_blockwise_f32_f32_f32.hpp | 10 +- ..._reduce_instance_blockwise_f32_f64_f32.hpp | 10 +- ..._reduce_instance_blockwise_f64_f64_f64.hpp | 10 +- ...ce_reduce_instance_blockwise_i8_i32_i8.hpp | 10 +- ...ice_reduce_instance_blockwise_i8_i8_i8.hpp | 10 +- .../device_reduce_instance_impl_common.hpp | 6 +- ..._reduce_instance_multiblock_atomic_add.hpp | 13 +- ...ance_multiblock_atomic_add_b16_f32_f32.hpp | 11 +- ...ance_multiblock_atomic_add_f16_f32_f32.hpp | 11 +- ...ance_multiblock_atomic_add_f32_f32_f32.hpp | 10 +- ...ance_multiblock_atomic_add_f32_f64_f32.hpp | 10 +- ...ance_multiblock_atomic_add_f64_f64_f64.hpp | 10 +- .../device_reduce_instance_threadwise.hpp | 12 +- ...reduce_instance_threadwise_b16_f32_b16.hpp | 11 +- ...reduce_instance_threadwise_f16_f16_f16.hpp | 11 +- ...reduce_instance_threadwise_f16_f32_f16.hpp | 11 +- ...reduce_instance_threadwise_f32_f32_f32.hpp | 10 +- ...reduce_instance_threadwise_f32_f64_f32.hpp | 10 +- ...reduce_instance_threadwise_f64_f64_f64.hpp | 10 +- ...e_reduce_instance_threadwise_i8_i32_i8.hpp | 10 +- ...ce_reduce_instance_threadwise_i8_i8_i8.hpp | 10 +- .../include/ck/library/utility/check_err.hpp | 6 +- .../include/ck/library/utility/conv_util.hpp | 22 +- library/include/ck/library/utility/fill.hpp | 2 +- .../ck/library/utility/op_instance_engine.hpp | 9 +- library/src/host_tensor/CMakeLists.txt | 8 +- library/src/host_tensor/device.cpp | 70 - library/src/host_tensor/device_memory.cpp | 25 + library/src/host_tensor/host_tensor.cpp | 3 +- .../obselete_driver_offline/CMakeLists.txt | 37 - .../conv_add_fwd_driver_offline_nchwc.cpp | 416 -- .../conv_bwd_driver_offline.cpp | 488 -- .../conv_fwd_driver_offline.cpp | 549 -- .../conv_fwd_driver_offline_nchwc.cpp | 393 -- .../conv_maxpool_fwd_driver_offline_nchwc.cpp | 415 -- .../conv_wrw_driver_offline.cpp | 532 -- .../gemm_driver_offline.cpp | 456 -- .../gpu/CMakeLists.txt | 26 +- ...dl_bf16_bf16_bf16_gkm_gkn_gmn_instance.cpp | 12 +- ...dl_bf16_bf16_bf16_gkm_gnk_gmn_instance.cpp | 12 +- ...dl_bf16_bf16_bf16_gmk_gkn_gmn_instance.cpp | 12 +- ...dl_bf16_bf16_bf16_gmk_gnk_gmn_instance.cpp | 12 +- ...m_xdl_f16_f16_f16_gkm_gkn_gmn_instance.cpp | 12 +- ...m_xdl_f16_f16_f16_gkm_gnk_gmn_instance.cpp | 12 +- ...m_xdl_f16_f16_f16_gmk_gkn_gmn_instance.cpp | 12 +- ...m_xdl_f16_f16_f16_gmk_gnk_gmn_instance.cpp | 12 +- ...m_xdl_f32_f32_f32_gkm_gkn_gmn_instance.cpp | 12 +- ...m_xdl_f32_f32_f32_gkm_gnk_gmn_instance.cpp | 12 +- ...m_xdl_f32_f32_f32_gmk_gkn_gmn_instance.cpp | 12 +- ...m_xdl_f32_f32_f32_gmk_gnk_gmn_instance.cpp | 12 +- ...dl_int8_int8_int8_gkm_gkn_gmn_instance.cpp | 12 +- ...dl_int8_int8_int8_gkm_gnk_gmn_instance.cpp | 12 +- ...dl_int8_int8_int8_gmk_gkn_gmn_instance.cpp | 12 +- ...dl_int8_int8_int8_gmk_gnk_gmn_instance.cpp | 12 +- ...6_f16_f16_f32_f32_gkm_gkn_gmn_instance.cpp | 14 +- ...6_f16_f16_f32_f32_gkm_gnk_gmn_instance.cpp | 14 +- ...6_f16_f16_f32_f32_gmk_gkn_gmn_instance.cpp | 14 +- ...6_f16_f16_f32_f32_gmk_gnk_gmn_instance.cpp | 14 +- ...nv1d_fwd_xdl_nwc_kxc_nwk_bf16_instance.cpp | 13 +- ...onv1d_fwd_xdl_nwc_kxc_nwk_f16_instance.cpp | 13 +- ...onv1d_fwd_xdl_nwc_kxc_nwk_f32_instance.cpp | 13 +- ...nv1d_fwd_xdl_nwc_kxc_nwk_int8_instance.cpp | 13 +- ..._data_xdl_nhwc_kyxc_nhwk_bf16_instance.cpp | 12 +- ...d_data_xdl_nhwc_kyxc_nhwk_f16_instance.cpp | 12 +- ...d_data_xdl_nhwc_kyxc_nhwk_f32_instance.cpp | 12 +- ..._data_xdl_nhwc_kyxc_nhwk_int8_instance.cpp | 12 +- ...weight_xdl_nhwc_kyxc_nhwk_f16_instance.cpp | 12 +- ...weight_xdl_nhwc_kyxc_nhwk_f32_instance.cpp | 12 +- ..._c_shuffle_nhwc_kyxc_nhwk_f16_instance.cpp | 12 +- ...d_fwd_xdl_nhwc_kyxc_nhwk_bf16_instance.cpp | 12 +- ...2d_fwd_xdl_nhwc_kyxc_nhwk_f16_instance.cpp | 12 +- ...2d_fwd_xdl_nhwc_kyxc_nhwk_f32_instance.cpp | 12 +- ...d_fwd_xdl_nhwc_kyxc_nhwk_int8_instance.cpp | 12 +- ...d_fwd_xdl_nhwc_kyxc_nhwk_bf16_instance.cpp | 12 +- ...2d_fwd_xdl_nhwc_kyxc_nhwk_f16_instance.cpp | 12 +- ...2d_fwd_xdl_nhwc_kyxc_nhwk_f32_instance.cpp | 12 +- ...d_fwd_xdl_nhwc_kyxc_nhwk_int8_instance.cpp | 12 +- ..._bias_relu_nhwc_kyxc_nhwk_f16_instance.cpp | 12 +- ...s_relu_add_nhwc_kyxc_nhwk_f16_instance.cpp | 12 +- .../CMakeLists.txt | 9 - ...atomic_add_nhwc_kyxc_nhwk_f16_instance.cpp | 69 - ...wd_xdl_ndhwc_kzyxc_ndhwk_bf16_instance.cpp | 12 +- ...fwd_xdl_ndhwc_kzyxc_ndhwk_f16_instance.cpp | 12 +- ...fwd_xdl_ndhwc_kzyxc_ndhwk_f32_instance.cpp | 12 +- ...wd_xdl_ndhwc_kzyxc_ndhwk_int8_instance.cpp | 12 +- ...bwd_data_xdl_nwc_kxc_nwk_bf16_instance.cpp | 12 +- ..._bwd_data_xdl_nwc_kxc_nwk_f16_instance.cpp | 12 +- ..._bwd_data_xdl_nwc_kxc_nwk_f32_instance.cpp | 12 +- ...bwd_data_xdl_nwc_kxc_nwk_int8_instance.cpp | 12 +- ..._data_xdl_nhwc_kyxc_nhwk_bf16_instance.cpp | 12 +- ...d_data_xdl_nhwc_kyxc_nhwk_f16_instance.cpp | 14 +- ...d_data_xdl_nhwc_kyxc_nhwk_f32_instance.cpp | 12 +- ..._data_xdl_nhwc_kyxc_nhwk_int8_instance.cpp | 16 +- ...ta_xdl_ndhwc_kzyxc_ndhwk_bf16_instance.cpp | 12 +- ...ata_xdl_ndhwc_kzyxc_ndhwk_f16_instance.cpp | 14 +- ...ata_xdl_ndhwc_kzyxc_ndhwk_f32_instance.cpp | 12 +- ...ta_xdl_ndhwc_kzyxc_ndhwk_int8_instance.cpp | 14 +- .../gpu/device_conv2d.cpp | 201 - ..._gemm_dl_f16_f16_f16_km_kn_mn_instance.cpp | 12 +- ..._gemm_dl_f16_f16_f16_km_nk_mn_instance.cpp | 12 +- ..._gemm_dl_f16_f16_f16_mk_kn_mn_instance.cpp | 12 +- ..._gemm_dl_f16_f16_f16_mk_nk_mn_instance.cpp | 12 +- ..._gemm_dl_f32_f32_f32_km_kn_mn_instance.cpp | 12 +- ..._gemm_dl_f32_f32_f32_km_nk_mn_instance.cpp | 12 +- ..._gemm_dl_f32_f32_f32_mk_kn_mn_instance.cpp | 12 +- ..._gemm_dl_f32_f32_f32_mk_nk_mn_instance.cpp | 12 +- ...ice_gemm_dl_i8_i8_i8_km_kn_mn_instance.cpp | 12 +- ...ice_gemm_dl_i8_i8_i8_km_nk_mn_instance.cpp | 12 +- ...ice_gemm_dl_i8_i8_i8_mk_kn_mn_instance.cpp | 12 +- ...ice_gemm_dl_i8_i8_i8_mk_nk_mn_instance.cpp | 12 +- ..._2_stage_f16_f16_f16_mk_nk_mn_instance.cpp | 12 +- ...uffle_bf16_bf16_bf16_km_kn_mn_instance.cpp | 12 +- ...uffle_bf16_bf16_bf16_km_nk_mn_instance.cpp | 12 +- ...uffle_bf16_bf16_bf16_mk_kn_mn_instance.cpp | 12 +- ...uffle_bf16_bf16_bf16_mk_nk_mn_instance.cpp | 12 +- ..._shuffle_f16_f16_f16_km_kn_mn_instance.cpp | 12 +- ..._shuffle_f16_f16_f16_km_nk_mn_instance.cpp | 12 +- ..._shuffle_f16_f16_f16_mk_kn_mn_instance.cpp | 12 +- ..._shuffle_f16_f16_f16_mk_nk_mn_instance.cpp | 12 +- ..._shuffle_f32_f32_f32_km_kn_mn_instance.cpp | 12 +- ..._shuffle_f32_f32_f32_km_nk_mn_instance.cpp | 12 +- ..._shuffle_f32_f32_f32_mk_kn_mn_instance.cpp | 12 +- ..._shuffle_f32_f32_f32_mk_nk_mn_instance.cpp | 12 +- ...l_c_shuffle_i8_i8_i8_km_kn_mn_instance.cpp | 12 +- ...l_c_shuffle_i8_i8_i8_km_nk_mn_instance.cpp | 12 +- ...l_c_shuffle_i8_i8_i8_mk_kn_mn_instance.cpp | 12 +- ...l_c_shuffle_i8_i8_i8_mk_nk_mn_instance.cpp | 12 +- ...gemm_xdl_f16_f16_f16_km_kn_mn_instance.cpp | 12 +- ...gemm_xdl_f16_f16_f16_km_nk_mn_instance.cpp | 12 +- ...gemm_xdl_f16_f16_f16_mk_kn_mn_instance.cpp | 12 +- ...gemm_xdl_f16_f16_f16_mk_nk_mn_instance.cpp | 12 +- ...gemm_xdl_f32_f32_f32_km_kn_mn_instance.cpp | 12 +- ...gemm_xdl_f32_f32_f32_km_nk_mn_instance.cpp | 12 +- ...gemm_xdl_f32_f32_f32_mk_kn_mn_instance.cpp | 12 +- ...gemm_xdl_f32_f32_f32_mk_nk_mn_instance.cpp | 12 +- ...gemm_xdl_f64_f64_f64_km_kn_mn_instance.cpp | 12 +- ...gemm_xdl_f64_f64_f64_km_nk_mn_instance.cpp | 12 +- ...gemm_xdl_f64_f64_f64_mk_kn_mn_instance.cpp | 12 +- ...gemm_xdl_f64_f64_f64_mk_nk_mn_instance.cpp | 12 +- ...l_splitk_f16_f16_f16_km_kn_mn_instance.cpp | 12 +- ...l_splitk_f16_f16_f16_km_nk_mn_instance.cpp | 12 +- ...l_splitk_f16_f16_f16_mk_kn_mn_instance.cpp | 12 +- ...l_splitk_f16_f16_f16_mk_nk_mn_instance.cpp | 12 +- ...l_splitk_f32_f32_f32_km_kn_mn_instance.cpp | 12 +- ...l_splitk_f32_f32_f32_km_nk_mn_instance.cpp | 12 +- ...l_splitk_f32_f32_f32_mk_kn_mn_instance.cpp | 12 +- ...l_splitk_f32_f32_f32_mk_nk_mn_instance.cpp | 12 +- ..._shuffle_f16_f16_f16_km_kn_mn_instance.cpp | 14 +- ..._shuffle_f16_f16_f16_km_nk_mn_instance.cpp | 14 +- ..._shuffle_f16_f16_f16_mk_kn_mn_instance.cpp | 14 +- ..._shuffle_f16_f16_f16_mk_nk_mn_instance.cpp | 14 +- ..._bias_2d_f16_f16_f16_km_kn_mn_instance.cpp | 12 +- ..._bias_2d_f16_f16_f16_km_nk_mn_instance.cpp | 12 +- ..._bias_2d_f16_f16_f16_mk_kn_mn_instance.cpp | 12 +- ..._bias_2d_f16_f16_f16_mk_nk_mn_instance.cpp | 12 +- ..._bias_2d_f32_f32_f32_km_kn_mn_instance.cpp | 12 +- ..._bias_2d_f32_f32_f32_km_nk_mn_instance.cpp | 12 +- ..._bias_2d_f32_f32_f32_mk_kn_mn_instance.cpp | 12 +- ..._bias_2d_f32_f32_f32_mk_nk_mn_instance.cpp | 12 +- ..._f16_f16_f16_f32_f32_km_kn_mn_instance.cpp | 15 +- ..._f16_f16_f16_f32_f32_km_nk_mn_instance.cpp | 15 +- ..._f16_f16_f16_f32_f32_mk_kn_mn_instance.cpp | 15 +- ..._f16_f16_f16_f32_f32_mk_nk_mn_instance.cpp | 15 +- ...ias_relu_f16_f16_f16_km_kn_mn_instance.cpp | 12 +- ...ias_relu_f16_f16_f16_km_nk_mn_instance.cpp | 12 +- ...ias_relu_f16_f16_f16_mk_kn_mn_instance.cpp | 12 +- ...ias_relu_f16_f16_f16_mk_nk_mn_instance.cpp | 12 +- ...relu_add_f16_f16_f16_km_kn_mn_instance.cpp | 14 +- ...relu_add_f16_f16_f16_km_nk_mn_instance.cpp | 14 +- ...relu_add_f16_f16_f16_mk_kn_mn_instance.cpp | 14 +- ...relu_add_f16_f16_f16_mk_nk_mn_instance.cpp | 14 +- ..._f16_f16_f16_f32_f32_km_kn_mn_instance.cpp | 15 +- ..._f16_f16_f16_f32_f32_km_nk_mn_instance.cpp | 15 +- ..._f16_f16_f16_f32_f32_mk_kn_mn_instance.cpp | 15 +- ..._f16_f16_f16_f32_f32_mk_nk_mn_instance.cpp | 15 +- ...gemm_xdl_f16_f16_f16_km_kn_mn_instance.cpp | 12 +- ...gemm_xdl_f16_f16_f16_km_nk_mn_instance.cpp | 12 +- ...gemm_xdl_f16_f16_f16_mk_kn_mn_instance.cpp | 12 +- ...gemm_xdl_f16_f16_f16_mk_nk_mn_instance.cpp | 12 +- ..._reduce_instance_blockwise_b16_f32_b16.cpp | 2 +- ..._reduce_instance_blockwise_f16_f16_f16.cpp | 2 +- ..._reduce_instance_blockwise_f16_f32_f16.cpp | 2 +- ..._reduce_instance_blockwise_f32_f32_f32.cpp | 2 +- ..._reduce_instance_blockwise_f32_f64_f32.cpp | 3 +- ..._reduce_instance_blockwise_f64_f64_f64.cpp | 2 +- ...ce_reduce_instance_blockwise_i8_i32_i8.cpp | 2 +- ...ice_reduce_instance_blockwise_i8_i8_i8.cpp | 2 +- ...ance_multiblock_atomic_add_b16_f32_f32.cpp | 3 +- ...ance_multiblock_atomic_add_f16_f32_f32.cpp | 2 +- ...ance_multiblock_atomic_add_f32_f32_f32.cpp | 3 +- ...ance_multiblock_atomic_add_f32_f64_f32.cpp | 3 +- ...ance_multiblock_atomic_add_f64_f64_f64.cpp | 2 +- ...reduce_instance_threadwise_b16_f32_b16.cpp | 2 +- ...reduce_instance_threadwise_f16_f16_f16.cpp | 2 +- ...reduce_instance_threadwise_f16_f32_f16.cpp | 3 +- ...reduce_instance_threadwise_f32_f32_f32.cpp | 2 +- ...reduce_instance_threadwise_f32_f64_f32.cpp | 2 +- ...reduce_instance_threadwise_f64_f64_f64.cpp | 3 +- ...e_reduce_instance_threadwise_i8_i32_i8.cpp | 2 +- ...ce_reduce_instance_threadwise_i8_i8_i8.cpp | 2 +- library/src/utility/CMakeLists.txt | 10 - library/src/utility/conv_util.cpp | 2 +- profiler/CMakeLists.txt | 23 +- .../include}/data_type_enum.hpp | 4 +- .../include}/data_type_enum_helper.hpp | 8 +- .../include/profile_batched_gemm_impl.hpp | 19 +- .../profile_batched_gemm_reduce_impl.hpp | 23 +- .../include/profile_conv_bwd_weight_impl.hpp | 21 +- .../profile_conv_fwd_bias_relu_add_impl.hpp | 20 +- ...ile_conv_fwd_bias_relu_atomic_add_impl.hpp | 331 - .../profile_conv_fwd_bias_relu_impl.hpp | 21 +- .../include/profile_convnd_bwd_data_impl.hpp | 22 +- .../profile_gemm_add_add_fastgelu_impl.hpp | 21 +- .../include/profile_gemm_bias_2d_impl.hpp | 21 +- .../profile_gemm_bias_add_reduce_impl.hpp | 25 +- .../profile_gemm_bias_relu_add_impl.hpp | 22 +- .../include/profile_gemm_bias_relu_impl.hpp | 22 +- profiler/include/profile_gemm_impl.hpp | 23 +- profiler/include/profile_gemm_reduce_impl.hpp | 25 +- .../include/profile_grouped_gemm_impl.hpp | 23 +- profiler/include/profile_reduce_impl.hpp | 16 +- profiler/src/profile_batched_gemm.cpp | 14 +- profiler/src/profile_batched_gemm_reduce.cpp | 4 +- profiler/src/profile_conv_bwd_weight.cpp | 5 +- profiler/src/profile_conv_fwd_bias_relu.cpp | 5 +- .../src/profile_conv_fwd_bias_relu_add.cpp | 5 +- .../profile_conv_fwd_bias_relu_atomic_add.cpp | 116 - profiler/src/profile_convnd_bwd_data.cpp | 4 +- profiler/src/profile_convnd_fwd.cpp | 12 +- profiler/src/profile_gemm.cpp | 5 +- .../src/profile_gemm_add_add_fastgelu.cpp | 3 +- profiler/src/profile_gemm_bias_2d.cpp | 5 +- profiler/src/profile_gemm_bias_add_reduce.cpp | 5 +- profiler/src/profile_gemm_bias_relu.cpp | 5 +- profiler/src/profile_gemm_bias_relu_add.cpp | 5 +- profiler/src/profile_gemm_reduce.cpp | 5 +- profiler/src/profile_grouped_gemm.cpp | 5 +- profiler/src/profile_reduce.cpp | 9 +- profiler/src/profiler.cpp | 8 +- test/CMakeLists.txt | 22 - test/batched_gemm/batched_gemm_fp16.cpp | 2 +- test/batched_gemm_reduce/CMakeLists.txt | 6 - .../batched_gemm_reduce_fp16.cpp | 2 +- .../test_block_to_ctile_map.cpp | 7 +- test/client_app/CMakeLists.txt | 11 - test/client_app/client_app.cpp | 77 - test/client_app/client_app_impl.hpp | 214 - test/conv2d_bwd_weight/CMakeLists.txt | 5 - test/conv2d_bwd_weight/conv2d_bwd_weight.cpp | 6 +- test/conv_util/conv_util.cpp | 9 +- test/convnd_bwd_data/CMakeLists.txt | 5 - test/convnd_bwd_data/convnd_bwd_data.cpp | 4 +- test/convnd_fwd/conv1d_fwd.cpp | 10 +- test/convnd_fwd/conv2d_fwd.cpp | 10 +- test/convnd_fwd/conv3d_fwd.cpp | 13 +- test/convnd_fwd/conv_util.hpp | 12 +- test/gemm/gemm_dl_fp16.cpp | 25 +- test/gemm/gemm_dl_fp32.cpp | 25 +- test/gemm/gemm_dl_int8.cpp | 25 +- test/gemm/gemm_util.hpp | 18 +- test/gemm/gemm_xdl_bf16.cpp | 26 +- test/gemm/gemm_xdl_fp16.cpp | 24 +- test/gemm/gemm_xdl_fp32.cpp | 27 +- test/gemm/gemm_xdl_fp64.cpp | 26 +- test/gemm/gemm_xdl_int8.cpp | 27 +- test/gemm_reduce/CMakeLists.txt | 6 - test/gemm_reduce/gemm_reduce_fp16.cpp | 2 +- test/gemm_split_k/gemm_split_k.cpp | 25 +- test/grouped_gemm/grouped_gemm_fp16.cpp | 25 +- .../magic_number_division.cpp | 15 +- test/reduce/reduce_no_index.cpp | 6 +- test/reduce/reduce_with_index.cpp | 6 +- .../reference_conv_fwd/reference_conv_fwd.cpp | 20 +- test/softmax/test_softmax_util.hpp | 16 +- .../space_filling_curve.cpp | 4 +- 499 files changed, 3000 insertions(+), 24130 deletions(-) delete mode 100644 external/include/half/half.hpp rename include/ck/{config.hpp => ck.hpp} (98%) rename include/ck/{host_utility => device_utility}/device_prop.hpp (97%) create mode 100644 include/ck/device_utility/hip_check_error.hpp create mode 100644 include/ck/device_utility/kernel_launch.hpp delete mode 100644 include/ck/options.hpp rename include/ck/{utility => tensor_description}/tensor_space_filling_curve.hpp (95%) delete mode 100644 library/include/ck/library/host/host_interface.hpp delete mode 100644 library/include/ck/library/host_tensor/device.hpp create mode 100644 library/include/ck/library/host_tensor/device_memory.hpp delete mode 100644 library/include/ck/library/host_tensor/device_tensor.hpp delete mode 100644 library/include/ck/library/obselete_driver_offline/debug.hpp delete mode 100644 library/include/ck/library/obselete_driver_offline/device_convolution_add_forward_implicit_gemm_v5r1_dlops_nc0hwc1_kc0yxc1_nk0hwk1.hpp delete mode 100644 library/include/ck/library/obselete_driver_offline/device_convolution_backward_data_implicit_gemm_v4r1_xdlops_nhwc_kyxc_nhwk.hpp delete mode 100644 library/include/ck/library/obselete_driver_offline/device_convolution_backward_data_implicit_gemm_v4r1r2_xdlops_nhwc_kyxc_nhwk.hpp delete mode 100644 library/include/ck/library/obselete_driver_offline/device_convolution_backward_data_implicit_gemm_v4r1r2_xdlops_nhwc_kyxc_nhwk_1x1.hpp delete mode 100644 library/include/ck/library/obselete_driver_offline/device_convolution_backward_weight_implicit_gemm_v4r4r2_xdlops_atomic_nchw_kcyx_nkhw.hpp delete mode 100644 library/include/ck/library/obselete_driver_offline/device_convolution_backward_weight_implicit_gemm_v4r4r2_xdlops_nchw_kcyx_nkhw.hpp delete mode 100644 library/include/ck/library/obselete_driver_offline/device_convolution_backward_weight_implicit_gemm_v4r4r4_xdlops_atomic_nhwc_kyxc_nhwk.hpp delete mode 100644 library/include/ck/library/obselete_driver_offline/device_convolution_backward_weight_implicit_gemm_v4r4r4_xdlops_nhwc_kyxc_nhwk.hpp delete mode 100644 library/include/ck/library/obselete_driver_offline/device_convolution_backward_weight_implicit_gemm_v4r4r5_xdlops_atomic_nhwc_kyxc_nhwk.hpp delete mode 100644 library/include/ck/library/obselete_driver_offline/device_convolution_forward_implicit_gemm_v4r4_dlops_nchw_kcyx_nkhw.hpp delete mode 100644 library/include/ck/library/obselete_driver_offline/device_convolution_forward_implicit_gemm_v4r4r2_dlops_nhwc_kyxc_nhwk.hpp delete mode 100644 library/include/ck/library/obselete_driver_offline/device_convolution_forward_implicit_gemm_v4r4r2_xdlops_nchw_kcyx_nkhw.hpp delete mode 100644 library/include/ck/library/obselete_driver_offline/device_convolution_forward_implicit_gemm_v4r4r4_xdlops_nhwc_kyxc_nhwk.hpp delete mode 100644 library/include/ck/library/obselete_driver_offline/device_convolution_forward_implicit_gemm_v5r1_dlops_nc0hwc1_kc0yxc1_nk0hwk1.hpp delete mode 100644 library/include/ck/library/obselete_driver_offline/device_convolution_forward_implicit_gemm_v6r1_dlops_nchw_kcyx_nkhw.hpp delete mode 100644 library/include/ck/library/obselete_driver_offline/device_convolution_maxpool_forward_implicit_gemm_v5r1_dlops_nc0hwc1_kc0yxc1_nk0hwk1.hpp delete mode 100644 library/include/ck/library/obselete_driver_offline/device_gemm_xdlops_km_kn_mn.hpp delete mode 100644 library/include/ck/library/obselete_driver_offline/device_gemm_xdlops_km_kn_nm.hpp delete mode 100644 library/include/ck/library/obselete_driver_offline/device_gemm_xdlops_km_nk_mn.hpp delete mode 100644 library/include/ck/library/obselete_driver_offline/device_gemm_xdlops_km_nk_nm.hpp delete mode 100644 library/include/ck/library/obselete_driver_offline/device_gemm_xdlops_mk_kn_mn.hpp delete mode 100644 library/include/ck/library/obselete_driver_offline/device_gemm_xdlops_mk_kn_nm.hpp delete mode 100644 library/include/ck/library/obselete_driver_offline/device_gemm_xdlops_mk_nk_mn.hpp delete mode 100644 library/include/ck/library/obselete_driver_offline/device_gemm_xdlops_mk_nk_nm.hpp delete mode 100644 library/include/ck/library/obselete_driver_offline/driver_contraction_dlops_v1r2.hpp delete mode 100644 library/include/ck/library/obselete_driver_offline/driver_convolution_add_forward_implicit_gemm_v5r1_dlops_nc0hwc1_kc0yxc1_nk0hwk1.hpp delete mode 100644 library/include/ck/library/obselete_driver_offline/driver_convolution_forward_implicit_gemm_v5r1_dlops_nc0hwc1_kc0yxc1_nk0hwk1.hpp delete mode 100644 library/include/ck/library/obselete_driver_offline/driver_convolution_maxpool_forward_implicit_gemm_v5r1_dlops_nc0hwc1_kc0yxc1_nk0hwk1.hpp delete mode 100644 library/include/ck/library/obselete_driver_offline/driver_gemm_dlops_v1r2.hpp delete mode 100644 library/include/ck/library/obselete_driver_offline/driver_gemm_dlops_v1r3.hpp delete mode 100644 library/include/ck/library/obselete_driver_offline/driver_gemm_xdlops_v2r3.hpp delete mode 100644 library/include/ck/library/obselete_driver_offline/driver_gemm_xdlops_v2r4.hpp delete mode 100644 library/src/host_tensor/device.cpp create mode 100644 library/src/host_tensor/device_memory.cpp delete mode 100644 library/src/obselete_driver_offline/CMakeLists.txt delete mode 100644 library/src/obselete_driver_offline/conv_add_fwd_driver_offline_nchwc.cpp delete mode 100644 library/src/obselete_driver_offline/conv_bwd_driver_offline.cpp delete mode 100644 library/src/obselete_driver_offline/conv_fwd_driver_offline.cpp delete mode 100644 library/src/obselete_driver_offline/conv_fwd_driver_offline_nchwc.cpp delete mode 100644 library/src/obselete_driver_offline/conv_maxpool_fwd_driver_offline_nchwc.cpp delete mode 100644 library/src/obselete_driver_offline/conv_wrw_driver_offline.cpp delete mode 100644 library/src/obselete_driver_offline/gemm_driver_offline.cpp delete mode 100644 library/src/tensor_operation_instance/gpu/conv2d_fwd_bias_relu_atomic_add/CMakeLists.txt delete mode 100644 library/src/tensor_operation_instance/gpu/conv2d_fwd_bias_relu_atomic_add/device_conv2d_fwd_xdl_c_shuffle_bias_relu_atomic_add_nhwc_kyxc_nhwk_f16_instance.cpp delete mode 100644 library/src/tensor_operation_instance/gpu/device_conv2d.cpp rename {include/ck/utility => profiler/include}/data_type_enum.hpp (75%) rename {include/ck/utility => profiler/include}/data_type_enum_helper.hpp (90%) delete mode 100644 profiler/include/profile_conv_fwd_bias_relu_atomic_add_impl.hpp delete mode 100644 profiler/src/profile_conv_fwd_bias_relu_atomic_add.cpp delete mode 100644 test/client_app/CMakeLists.txt delete mode 100644 test/client_app/client_app.cpp delete mode 100644 test/client_app/client_app_impl.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index e5903f3747..39d2401fc7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -78,10 +78,6 @@ rocm_create_package( LDCONFIG ) -## half -set(HALF_INCLUDE_DIR "${PROJECT_SOURCE_DIR}/external/include/half") -message("HALF_INCLUDE_DIR: ${HALF_INCLUDE_DIR}") - ## tidy include(EnableCompilerWarnings) set(CK_TIDY_ERRORS ERRORS * -readability-inconsistent-declaration-parameter-name) @@ -229,7 +225,6 @@ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/bin) include_directories(BEFORE ${PROJECT_SOURCE_DIR}/include - ${PROJECT_BINARY_DIR}/include ${PROJECT_SOURCE_DIR}/library/include ) diff --git a/Jenkinsfile b/Jenkinsfile index 65876ea1c0..b4adc5de95 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -379,23 +379,23 @@ pipeline { } } } - stage("Client App") - { - parallel - { - stage("Run Client App") - { - agent{ label rocmnode("gfx908")} - environment{ - setup_args = """ -D -DBUILD_DEV=Off -DCMAKE_INSTALL_PREFIX=../install CMAKE_CXX_FLAGS="--offload-arch=gfx908 -O3 " """ - execute_args = """ cd ../test/client_app && rm -rf build && mkdir build && cd build && cmake -DCMAKE_PREFIX_PATH="${env.WORKSPACE}/install;/opt/rocm" .. && make """ - } - steps{ - buildHipClangJobAndReboot(setup_args: setup_args, config_targets: "install", no_reboot:true, build_type: 'Release', execute_cmd: execute_args, prefixpath: '/usr/local') - } - } - } - } + //stage("Client App") + //{ + // parallel + // { + // stage("Run Client App") + // { + // agent{ label rocmnode("gfx908")} + // environment{ + // setup_args = """ -D -DBUILD_DEV=Off -DCMAKE_INSTALL_PREFIX=../install CMAKE_CXX_FLAGS="--offload-arch=gfx908 -O3 " """ + // execute_args = """ cd ../test/client_app && rm -rf build && mkdir build && cd build && cmake -DCMAKE_PREFIX_PATH="${env.WORKSPACE}/install;/opt/rocm" .. && make """ + // } + // steps{ + // buildHipClangJobAndReboot(setup_args: setup_args, config_targets: "install", no_reboot:true, build_type: 'Release', execute_cmd: execute_args, prefixpath: '/usr/local') + // } + // } + // } + //} stage("Performance Tests") { parallel diff --git a/example/01_gemm/gemm_dl_fp16.cpp b/example/01_gemm/gemm_dl_fp16.cpp index 9a22628777..1bb6214514 100644 --- a/example/01_gemm/gemm_dl_fp16.cpp +++ b/example/01_gemm/gemm_dl_fp16.cpp @@ -2,19 +2,17 @@ #include #include #include -#include -#include -#include "check_err.hpp" -#include "config.hpp" -#include "device.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" -#include "device_tensor.hpp" -#include "device_gemm_dl.hpp" -#include "element_wise_operation.hpp" -#include "reference_gemm.hpp" -#include "gemm_specialization.hpp" +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_dl.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" + +#include "ck/library/utility/check_err.hpp" +#include "ck/library/host_tensor/device_memory.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" +#include "ck/library/host_tensor/host_tensor_generator.hpp" +#include "ck/library/reference_tensor_operation/cpu/reference_gemm.hpp" template using S = ck::Sequence; diff --git a/example/01_gemm/gemm_dl_fp32.cpp b/example/01_gemm/gemm_dl_fp32.cpp index 32b183a3a1..4b4428669d 100644 --- a/example/01_gemm/gemm_dl_fp32.cpp +++ b/example/01_gemm/gemm_dl_fp32.cpp @@ -2,19 +2,17 @@ #include #include #include -#include -#include -#include "check_err.hpp" -#include "config.hpp" -#include "device.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" -#include "device_tensor.hpp" -#include "device_gemm_dl.hpp" -#include "element_wise_operation.hpp" -#include "reference_gemm.hpp" -#include "gemm_specialization.hpp" +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_dl.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" + +#include "ck/library/utility/check_err.hpp" +#include "ck/library/host_tensor/device_memory.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" +#include "ck/library/host_tensor/host_tensor_generator.hpp" +#include "ck/library/reference_tensor_operation/cpu/reference_gemm.hpp" template using S = ck::Sequence; diff --git a/example/01_gemm/gemm_dl_int8.cpp b/example/01_gemm/gemm_dl_int8.cpp index 16c9213104..e8c827195b 100644 --- a/example/01_gemm/gemm_dl_int8.cpp +++ b/example/01_gemm/gemm_dl_int8.cpp @@ -2,19 +2,17 @@ #include #include #include -#include -#include -#include "check_err.hpp" -#include "config.hpp" -#include "device.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" -#include "device_tensor.hpp" -#include "device_gemm_dl.hpp" -#include "element_wise_operation.hpp" -#include "reference_gemm.hpp" -#include "gemm_specialization.hpp" +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_dl.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" + +#include "ck/library/utility/check_err.hpp" +#include "ck/library/host_tensor/device_memory.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" +#include "ck/library/host_tensor/host_tensor_generator.hpp" +#include "ck/library/reference_tensor_operation/cpu/reference_gemm.hpp" template using S = ck::Sequence; diff --git a/example/01_gemm/gemm_xdl_bf16.cpp b/example/01_gemm/gemm_xdl_bf16.cpp index b126736be6..8b4f5f6b68 100644 --- a/example/01_gemm/gemm_xdl_bf16.cpp +++ b/example/01_gemm/gemm_xdl_bf16.cpp @@ -2,19 +2,17 @@ #include #include #include -#include -#include -#include "check_err.hpp" -#include "config.hpp" -#include "device.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" -#include "device_tensor.hpp" -#include "device_gemm_xdl_cshuffle.hpp" -#include "element_wise_operation.hpp" -#include "reference_gemm.hpp" -#include "gemm_specialization.hpp" +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_xdl_cshuffle.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" + +#include "ck/library/host_tensor/device_memory.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" +#include "ck/library/host_tensor/host_tensor_generator.hpp" +#include "ck/library/reference_tensor_operation/cpu/reference_gemm.hpp" +#include "ck/library/utility/check_err.hpp" template using S = ck::Sequence; diff --git a/example/01_gemm/gemm_xdl_fp16.cpp b/example/01_gemm/gemm_xdl_fp16.cpp index bf7227b2b0..675ff67d18 100644 --- a/example/01_gemm/gemm_xdl_fp16.cpp +++ b/example/01_gemm/gemm_xdl_fp16.cpp @@ -2,19 +2,17 @@ #include #include #include -#include -#include -#include "check_err.hpp" -#include "config.hpp" -#include "device.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" -#include "device_tensor.hpp" -#include "device_gemm_xdl.hpp" -#include "device_gemm_xdl_cshuffle.hpp" -#include "element_wise_operation.hpp" -#include "reference_gemm.hpp" -#include "gemm_specialization.hpp" + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_xdl_cshuffle.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" + +#include "ck/library/utility/check_err.hpp" +#include "ck/library/host_tensor/device_memory.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" +#include "ck/library/host_tensor/host_tensor_generator.hpp" +#include "ck/library/reference_tensor_operation/cpu/reference_gemm.hpp" template using S = ck::Sequence; diff --git a/example/01_gemm/gemm_xdl_fp64.cpp b/example/01_gemm/gemm_xdl_fp64.cpp index 7cea68c8b0..7607668300 100644 --- a/example/01_gemm/gemm_xdl_fp64.cpp +++ b/example/01_gemm/gemm_xdl_fp64.cpp @@ -2,20 +2,18 @@ #include #include #include -#include -#include -#include "check_err.hpp" -#include "config.hpp" -#include "device.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" -#include "device_tensor.hpp" -#include "device_gemm_xdl.hpp" -#include "device_gemm_xdl_cshuffle.hpp" -#include "element_wise_operation.hpp" -#include "reference_gemm.hpp" -#include "gemm_specialization.hpp" +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_xdl.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" + +#include "ck/library/host_tensor/device_memory.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" +#include "ck/library/host_tensor/host_tensor_generator.hpp" +#include "ck/library/reference_tensor_operation/cpu/reference_gemm.hpp" +#include "ck/library/utility/check_err.hpp" template using S = ck::Sequence; diff --git a/example/01_gemm/gemm_xdl_int8.cpp b/example/01_gemm/gemm_xdl_int8.cpp index 27fcd62a2c..60309e0350 100644 --- a/example/01_gemm/gemm_xdl_int8.cpp +++ b/example/01_gemm/gemm_xdl_int8.cpp @@ -2,19 +2,18 @@ #include #include #include -#include -#include -#include "check_err.hpp" -#include "config.hpp" -#include "device.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" -#include "device_tensor.hpp" -#include "device_gemm_xdl_cshuffle.hpp" -#include "element_wise_operation.hpp" -#include "reference_gemm.hpp" -#include "gemm_specialization.hpp" +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_xdl_cshuffle.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" + +#include "ck/library/utility/check_err.hpp" +#include "ck/library/host_tensor/device_memory.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" +#include "ck/library/host_tensor/host_tensor_generator.hpp" +#include "ck/library/reference_tensor_operation/cpu/reference_gemm.hpp" template using S = ck::Sequence; diff --git a/example/02_gemm_alpha_beta/gemm_xdl_alpha_beta.cpp b/example/02_gemm_alpha_beta/gemm_xdl_alpha_beta.cpp index 1a6e1de4dc..fcd772e52c 100644 --- a/example/02_gemm_alpha_beta/gemm_xdl_alpha_beta.cpp +++ b/example/02_gemm_alpha_beta/gemm_xdl_alpha_beta.cpp @@ -2,21 +2,17 @@ #include #include #include -#include -#include -#include "check_err.hpp" -#include "config.hpp" -#include "print.hpp" -#include "device.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" -#include "host_gemm.hpp" -#include "device_tensor.hpp" -#include "device_base.hpp" -#include "device_gemm_xdl_c_shuffle_bias_2d.hpp" -#include "element_wise_operation.hpp" -#include "reference_gemm_bias_2d.hpp" +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_xdl_c_shuffle_bias_2d.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" + +#include "ck/library/utility/check_err.hpp" +#include "ck/library/host_tensor/device_memory.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" +#include "ck/library/host_tensor/host_tensor_generator.hpp" +#include "ck/library/reference_tensor_operation/cpu/reference_gemm_bias_2d.hpp" template using S = ck::Sequence; diff --git a/example/03_gemm_bias_relu/gemm_xdl_bias_relu.cpp b/example/03_gemm_bias_relu/gemm_xdl_bias_relu.cpp index f91f6ccfc7..8f6a91fc48 100644 --- a/example/03_gemm_bias_relu/gemm_xdl_bias_relu.cpp +++ b/example/03_gemm_bias_relu/gemm_xdl_bias_relu.cpp @@ -2,18 +2,18 @@ #include #include #include -#include -#include "check_err.hpp" -#include "config.hpp" -#include "device.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" -#include "device_tensor.hpp" -#include "element_wise_operation.hpp" -#include "reference_gemm.hpp" -#include "gemm_specialization.hpp" -#include "device_gemm_multiple_d_xdl_cshuffle.hpp" +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_multiple_d_xdl_cshuffle.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" + +#include "ck/library/host_tensor/device_memory.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" +#include "ck/library/host_tensor/host_tensor_generator.hpp" +#include "ck/library/reference_tensor_operation/cpu/reference_gemm.hpp" +#include "ck/library/utility/check_err.hpp" template using S = ck::Sequence; diff --git a/example/04_gemm_add_add_fastgelu/gemm_add_add_fastgelu_xdl_fp16.cpp b/example/04_gemm_add_add_fastgelu/gemm_add_add_fastgelu_xdl_fp16.cpp index 7db5be0c91..cd93e5f138 100644 --- a/example/04_gemm_add_add_fastgelu/gemm_add_add_fastgelu_xdl_fp16.cpp +++ b/example/04_gemm_add_add_fastgelu/gemm_add_add_fastgelu_xdl_fp16.cpp @@ -2,18 +2,18 @@ #include #include #include -#include -#include "check_err.hpp" -#include "config.hpp" -#include "device.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" -#include "device_tensor.hpp" -#include "element_wise_operation.hpp" -#include "reference_gemm.hpp" -#include "gemm_specialization.hpp" -#include "device_gemm_multiple_d_xdl_cshuffle.hpp" +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_multiple_d_xdl_cshuffle.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" + +#include "ck/library/host_tensor/device_memory.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" +#include "ck/library/host_tensor/host_tensor_generator.hpp" +#include "ck/library/reference_tensor_operation/cpu/reference_gemm.hpp" +#include "ck/library/utility/check_err.hpp" template using S = ck::Sequence; diff --git a/example/06_conv2d_fwd_bias_relu/conv2d_fwd_xdl_bias_relu.cpp b/example/06_conv2d_fwd_bias_relu/conv2d_fwd_xdl_bias_relu.cpp index d50afb6854..6a5f668d81 100644 --- a/example/06_conv2d_fwd_bias_relu/conv2d_fwd_xdl_bias_relu.cpp +++ b/example/06_conv2d_fwd_bias_relu/conv2d_fwd_xdl_bias_relu.cpp @@ -2,20 +2,18 @@ #include #include #include -#include -#include -#include "check_err.hpp" -#include "config.hpp" -#include "conv_util.hpp" -#include "device.hpp" -#include "device_conv2d_fwd_xdl_c_shuffle_bias_activation_nhwc_kyxc_nhwk.hpp" -#include "device_tensor.hpp" -#include "element_wise_operation.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" -#include "reference_conv_fwd_bias_activation.hpp" -#include "tensor_layout.hpp" +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_conv2d_fwd_xdl_c_shuffle_bias_activation_nhwc_kyxc_nhwk.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" + +#include "ck/library/utility/check_err.hpp" +#include "ck/library/utility/conv_util.hpp" +#include "ck/library/host_tensor/device_memory.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" +#include "ck/library/host_tensor/host_tensor_generator.hpp" +#include "ck/library/reference_tensor_operation/cpu/reference_conv_fwd_bias_activation.hpp" namespace { diff --git a/example/07_conv2d_fwd_bias_relu_add/conv2d_fwd_xdl_bias_relu_add.cpp b/example/07_conv2d_fwd_bias_relu_add/conv2d_fwd_xdl_bias_relu_add.cpp index 1a234ea851..d4b3197bfe 100644 --- a/example/07_conv2d_fwd_bias_relu_add/conv2d_fwd_xdl_bias_relu_add.cpp +++ b/example/07_conv2d_fwd_bias_relu_add/conv2d_fwd_xdl_bias_relu_add.cpp @@ -2,20 +2,18 @@ #include #include #include -#include -#include -#include "check_err.hpp" -#include "config.hpp" -#include "conv_util.hpp" -#include "device.hpp" -#include "device_conv2d_fwd_xdl_c_shuffle_bias_activation_add_nhwc_kyxc_nhwk.hpp" -#include "device_tensor.hpp" -#include "element_wise_operation.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" -#include "reference_conv_fwd_bias_activation_add.hpp" -#include "tensor_layout.hpp" +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_conv2d_fwd_xdl_c_shuffle_bias_activation_add_nhwc_kyxc_nhwk.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" + +#include "ck/library/utility/check_err.hpp" +#include "ck/library/utility/conv_util.hpp" +#include "ck/library/host_tensor/device_memory.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" +#include "ck/library/host_tensor/host_tensor_generator.hpp" +#include "ck/library/reference_tensor_operation/cpu/reference_conv_fwd_bias_activation_add.hpp" namespace { diff --git a/example/09_convnd_fwd/convnd_fwd_xdl_fp16.cpp b/example/09_convnd_fwd/convnd_fwd_xdl_fp16.cpp index d951bc4e4b..ba44113f9e 100644 --- a/example/09_convnd_fwd/convnd_fwd_xdl_fp16.cpp +++ b/example/09_convnd_fwd/convnd_fwd_xdl_fp16.cpp @@ -3,17 +3,17 @@ #include #include -#include "check_err.hpp" -#include "config.hpp" -#include "conv_util.hpp" -#include "device.hpp" -#include "device_tensor.hpp" -#include "device_convnd_fwd_xdl_nhwc_kyxc_nhwk.hpp" -#include "element_wise_operation.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" -#include "reference_conv_fwd.hpp" -#include "tensor_layout.hpp" +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_convnd_fwd_xdl_nhwc_kyxc_nhwk.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" + +#include "ck/library/utility/check_err.hpp" +#include "ck/library/utility/conv_util.hpp" +#include "ck/library/host_tensor/device_memory.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" +#include "ck/library/host_tensor/host_tensor_generator.hpp" +#include "ck/library/reference_tensor_operation/cpu/reference_conv_fwd.hpp" namespace { diff --git a/example/09_convnd_fwd/convnd_fwd_xdl_fp32.cpp b/example/09_convnd_fwd/convnd_fwd_xdl_fp32.cpp index 7fa0f0d275..a850b67bd9 100644 --- a/example/09_convnd_fwd/convnd_fwd_xdl_fp32.cpp +++ b/example/09_convnd_fwd/convnd_fwd_xdl_fp32.cpp @@ -3,17 +3,17 @@ #include #include -#include "check_err.hpp" -#include "config.hpp" -#include "conv_util.hpp" -#include "device.hpp" -#include "device_tensor.hpp" -#include "device_convnd_fwd_xdl_nhwc_kyxc_nhwk.hpp" -#include "element_wise_operation.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" -#include "reference_conv_fwd.hpp" -#include "tensor_layout.hpp" +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_convnd_fwd_xdl_nhwc_kyxc_nhwk.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" + +#include "ck/library/utility/check_err.hpp" +#include "ck/library/utility/conv_util.hpp" +#include "ck/library/host_tensor/device_memory.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" +#include "ck/library/host_tensor/host_tensor_generator.hpp" +#include "ck/library/reference_tensor_operation/cpu/reference_conv_fwd.hpp" namespace { diff --git a/example/09_convnd_fwd/convnd_fwd_xdl_fp64.cpp b/example/09_convnd_fwd/convnd_fwd_xdl_fp64.cpp index 52440e0d5f..20ffd19789 100644 --- a/example/09_convnd_fwd/convnd_fwd_xdl_fp64.cpp +++ b/example/09_convnd_fwd/convnd_fwd_xdl_fp64.cpp @@ -3,17 +3,17 @@ #include #include -#include "check_err.hpp" -#include "config.hpp" -#include "conv_util.hpp" -#include "device.hpp" -#include "device_tensor.hpp" -#include "device_convnd_fwd_xdl_nhwc_kyxc_nhwk.hpp" -#include "element_wise_operation.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" -#include "reference_conv_fwd.hpp" -#include "tensor_layout.hpp" +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_convnd_fwd_xdl_nhwc_kyxc_nhwk.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" + +#include "ck/library/utility/check_err.hpp" +#include "ck/library/utility/conv_util.hpp" +#include "ck/library/host_tensor/device_memory.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" +#include "ck/library/host_tensor/host_tensor_generator.hpp" +#include "ck/library/reference_tensor_operation/cpu/reference_conv_fwd.hpp" namespace { diff --git a/example/09_convnd_fwd/convnd_fwd_xdl_int8.cpp b/example/09_convnd_fwd/convnd_fwd_xdl_int8.cpp index 9a1028f88b..51088b6461 100644 --- a/example/09_convnd_fwd/convnd_fwd_xdl_int8.cpp +++ b/example/09_convnd_fwd/convnd_fwd_xdl_int8.cpp @@ -3,17 +3,17 @@ #include #include -#include "check_err.hpp" -#include "config.hpp" -#include "conv_util.hpp" -#include "device.hpp" -#include "device_tensor.hpp" -#include "device_convnd_fwd_xdl_nhwc_kyxc_nhwk.hpp" -#include "element_wise_operation.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" -#include "reference_conv_fwd.hpp" -#include "tensor_layout.hpp" +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_convnd_fwd_xdl_nhwc_kyxc_nhwk.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" + +#include "ck/library/utility/check_err.hpp" +#include "ck/library/utility/conv_util.hpp" +#include "ck/library/host_tensor/device_memory.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" +#include "ck/library/host_tensor/host_tensor_generator.hpp" +#include "ck/library/reference_tensor_operation/cpu/reference_conv_fwd.hpp" namespace { diff --git a/example/10_conv2d_bwd_data/conv2d_bwd_data_xdl.cpp b/example/10_conv2d_bwd_data/conv2d_bwd_data_xdl.cpp index 2d25f5ac2f..24c4424e44 100644 --- a/example/10_conv2d_bwd_data/conv2d_bwd_data_xdl.cpp +++ b/example/10_conv2d_bwd_data/conv2d_bwd_data_xdl.cpp @@ -2,20 +2,18 @@ #include #include #include -#include -#include -#include "check_err.hpp" -#include "config.hpp" -#include "print.hpp" -#include "device.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" -#include "device_tensor.hpp" -#include "tensor_layout.hpp" -#include "element_wise_operation.hpp" -#include "device_conv2d_bwd_data_xdl_nhwc_kyxc_nhwk.hpp" -#include "reference_conv_bwd_data.hpp" +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_conv2d_bwd_data_xdl_nhwc_kyxc_nhwk.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" + +#include "ck/library/utility/check_err.hpp" +#include "ck/library/utility/conv_util.hpp" +#include "ck/library/host_tensor/device_memory.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" +#include "ck/library/host_tensor/host_tensor_generator.hpp" +#include "ck/library/reference_tensor_operation/cpu/reference_conv_bwd_data.hpp" using InDataType = ck::half_t; using WeiDataType = ck::half_t; diff --git a/example/11_conv2d_bwd_weight/conv2d_bwd_weight_xdl.cpp b/example/11_conv2d_bwd_weight/conv2d_bwd_weight_xdl.cpp index 1578161116..624cf90385 100644 --- a/example/11_conv2d_bwd_weight/conv2d_bwd_weight_xdl.cpp +++ b/example/11_conv2d_bwd_weight/conv2d_bwd_weight_xdl.cpp @@ -2,20 +2,18 @@ #include #include #include -#include -#include -#include "check_err.hpp" -#include "config.hpp" -#include "print.hpp" -#include "device.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" -#include "device_tensor.hpp" -#include "tensor_layout.hpp" -#include "element_wise_operation.hpp" -#include "device_conv2d_backward_weight_xdl_c_shuffle_nhwc_kyxc_nhwk.hpp" -#include "reference_conv_backward_weight.hpp" +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_conv2d_backward_weight_xdl_c_shuffle_nhwc_kyxc_nhwk.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" + +#include "ck/library/utility/check_err.hpp" +#include "ck/library/utility/conv_util.hpp" +#include "ck/library/host_tensor/device_memory.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" +#include "ck/library/host_tensor/host_tensor_generator.hpp" +#include "ck/library/reference_tensor_operation/cpu/reference_conv_backward_weight.hpp" using InDataType = ck::half_t; using WeiDataType = ck::half_t; diff --git a/example/12_reduce/reduce_blockwise.cpp b/example/12_reduce/reduce_blockwise.cpp index 66e9762314..99633454a8 100644 --- a/example/12_reduce/reduce_blockwise.cpp +++ b/example/12_reduce/reduce_blockwise.cpp @@ -4,20 +4,17 @@ #include #include -#include "check_err.hpp" -#include "config.hpp" -#include "print.hpp" -#include "device.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" -#include "device_tensor.hpp" -#include "device_base.hpp" -#include "device_reduce_multiblock.hpp" -#include "host_common_util.hpp" -#include "host_reduction.hpp" +#include "ck/ck.hpp" +#include "ck/utility/reduction_enums.hpp" +#include "ck/tensor_operation/gpu/device/reduction_operator_mapping.hpp" +#include "ck/tensor_operation/gpu/device/device_reduce_multiblock.hpp" -#include "reduction_enums.hpp" -#include "reduction_operator_mapping.hpp" +#include "ck/library/utility/check_err.hpp" +#include "ck/library/host_tensor/device_memory.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" +#include "ck/library/host_tensor/host_tensor_generator.hpp" +#include "ck/library/host_tensor/host_common_util.hpp" +#include "ck/library/host_tensor/host_reduction.hpp" using namespace ck; using namespace ck::tensor_operation::device; diff --git a/example/12_reduce/reduce_blockwise_two_call.cpp b/example/12_reduce/reduce_blockwise_two_call.cpp index e4823667a8..3a821295f8 100644 --- a/example/12_reduce/reduce_blockwise_two_call.cpp +++ b/example/12_reduce/reduce_blockwise_two_call.cpp @@ -5,20 +5,17 @@ #include #include -#include "check_err.hpp" -#include "config.hpp" -#include "print.hpp" -#include "device.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" -#include "device_tensor.hpp" -#include "device_base.hpp" -#include "device_reduce_multiblock.hpp" -#include "host_common_util.hpp" -#include "host_reduction.hpp" +#include "ck/ck.hpp" +#include "ck/utility/reduction_enums.hpp" +#include "ck/tensor_operation/gpu/device/reduction_operator_mapping.hpp" +#include "ck/tensor_operation/gpu/device/device_reduce_multiblock.hpp" -#include "reduction_enums.hpp" -#include "reduction_operator_mapping.hpp" +#include "ck/library/utility/check_err.hpp" +#include "ck/library/host_tensor/device_memory.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" +#include "ck/library/host_tensor/host_tensor_generator.hpp" +#include "ck/library/host_tensor/host_common_util.hpp" +#include "ck/library/host_tensor/host_reduction.hpp" using namespace ck; using namespace ck::tensor_operation::device; diff --git a/example/13_pool2d_fwd/pool2d_fwd_common.hpp b/example/13_pool2d_fwd/pool2d_fwd_common.hpp index 436bbcd485..3435023dde 100644 --- a/example/13_pool2d_fwd/pool2d_fwd_common.hpp +++ b/example/13_pool2d_fwd/pool2d_fwd_common.hpp @@ -2,19 +2,17 @@ #include -#include "check_err.hpp" -#include "config.hpp" -#include "print.hpp" -#include "device.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" -#include "device_tensor.hpp" -#include "tensor_layout.hpp" -#include "reduction_enums.hpp" -#include "reduction_operator_mapping.hpp" -#include "reduction_functions_accumulate.hpp" +#include "ck/ck.hpp" +#include "ck/utility/reduction_enums.hpp" +#include "ck/utility/reduction_functions_accumulate.hpp" +#include "ck/tensor_operation/gpu/device/reduction_operator_mapping.hpp" +#include "ck/tensor_operation/gpu/device/device_pool2d_fwd_nhwc_nhwc.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "device_pool2d_fwd_nhwc_nhwc.hpp" +#include "ck/library/utility/check_err.hpp" +#include "ck/library/host_tensor/device_memory.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" +#include "ck/library/host_tensor/host_tensor_generator.hpp" template #include -#include "config.hpp" -#include "tensor_layout.hpp" -#include "reduction_enums.hpp" +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/utility/reduction_enums.hpp" #include "pool2d_fwd_common.hpp" diff --git a/example/13_pool2d_fwd/pool2d_fwd_fp32.cpp b/example/13_pool2d_fwd/pool2d_fwd_fp32.cpp index 7ca5b1aab7..5c60981f6f 100644 --- a/example/13_pool2d_fwd/pool2d_fwd_fp32.cpp +++ b/example/13_pool2d_fwd/pool2d_fwd_fp32.cpp @@ -1,9 +1,9 @@ #include #include -#include "config.hpp" -#include "tensor_layout.hpp" -#include "reduction_enums.hpp" +#include "ck/ck.hpp" +#include "ck/utility/reduction_enums.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" #include "pool2d_fwd_common.hpp" diff --git a/example/14_gemm_xdl_requant_relu_requant/gemm_xdl_requant_relu_requant_int8.cpp b/example/14_gemm_xdl_requant_relu_requant/gemm_xdl_requant_relu_requant_int8.cpp index a42df2b7f0..9e7ad05be7 100644 --- a/example/14_gemm_xdl_requant_relu_requant/gemm_xdl_requant_relu_requant_int8.cpp +++ b/example/14_gemm_xdl_requant_relu_requant/gemm_xdl_requant_relu_requant_int8.cpp @@ -2,21 +2,18 @@ #include #include #include -#include -#include -#include "check_err.hpp" -#include "config.hpp" -#include "print.hpp" -#include "device.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" -#include "host_gemm.hpp" -#include "device_tensor.hpp" -#include "device_gemm_xdl_cshuffle.hpp" -#include "element_wise_operation.hpp" -#include "reference_gemm.hpp" -#include "gemm_specialization.hpp" +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_xdl_cshuffle.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" + +#include "ck/library/host_tensor/device_memory.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" +#include "ck/library/host_tensor/host_tensor_generator.hpp" +#include "ck/library/reference_tensor_operation/cpu/reference_gemm.hpp" +#include "ck/library/utility/check_err.hpp" struct RequantReluRequant { diff --git a/example/15_grouped_gemm/grouped_gemm_xdl_fp16.cpp b/example/15_grouped_gemm/grouped_gemm_xdl_fp16.cpp index 503c87e138..751ec2c419 100644 --- a/example/15_grouped_gemm/grouped_gemm_xdl_fp16.cpp +++ b/example/15_grouped_gemm/grouped_gemm_xdl_fp16.cpp @@ -2,21 +2,18 @@ #include #include #include -#include -#include -#include "check_err.hpp" -#include "config.hpp" -#include "print.hpp" -#include "device.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" -#include "host_gemm.hpp" -#include "device_tensor.hpp" -#include "device_grouped_gemm_xdl.hpp" -#include "element_wise_operation.hpp" -#include "reference_gemm.hpp" -#include "gemm_specialization.hpp" +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_grouped_gemm_xdl.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" + +#include "ck/library/utility/check_err.hpp" +#include "ck/library/host_tensor/device_memory.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" +#include "ck/library/host_tensor/host_tensor_generator.hpp" +#include "ck/library/reference_tensor_operation/cpu/reference_gemm.hpp" template using S = ck::Sequence; diff --git a/example/16_gemm_reduce/gemm_reduce_xdl_max_fp16.cpp b/example/16_gemm_reduce/gemm_reduce_xdl_max_fp16.cpp index 92113e3c41..6d62510b33 100644 --- a/example/16_gemm_reduce/gemm_reduce_xdl_max_fp16.cpp +++ b/example/16_gemm_reduce/gemm_reduce_xdl_max_fp16.cpp @@ -2,18 +2,18 @@ #include #include #include -#include -#include "check_err.hpp" -#include "config.hpp" -#include "device.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" -#include "device_tensor.hpp" -#include "device_gemm_reduce_xdl_cshuffle.hpp" -#include "element_wise_operation.hpp" -#include "reference_gemm.hpp" -#include "gemm_specialization.hpp" +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_reduce_xdl_cshuffle.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" + +#include "ck/library/host_tensor/device_memory.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" +#include "ck/library/host_tensor/host_tensor_generator.hpp" +#include "ck/library/reference_tensor_operation/cpu/reference_gemm.hpp" +#include "ck/library/utility/check_err.hpp" template using S = ck::Sequence; diff --git a/example/16_gemm_reduce/gemm_reduce_xdl_mean_squaremean_fp16.cpp b/example/16_gemm_reduce/gemm_reduce_xdl_mean_squaremean_fp16.cpp index 018645e066..4f1f5707b3 100644 --- a/example/16_gemm_reduce/gemm_reduce_xdl_mean_squaremean_fp16.cpp +++ b/example/16_gemm_reduce/gemm_reduce_xdl_mean_squaremean_fp16.cpp @@ -2,20 +2,19 @@ #include #include #include -#include -#include "check_err.hpp" -#include "config.hpp" -#include "device.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" -#include "device_tensor.hpp" -#include "device_gemm_reduce_xdl_cshuffle.hpp" -#include "element_wise_operation.hpp" -#include "reduction_operator.hpp" -#include "reference_gemm.hpp" -#include "gemm_specialization.hpp" -#include "reduction_operator.hpp" +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_reduce_xdl_cshuffle.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" +#include "ck/utility/reduction_operator.hpp" + +#include "ck/library/utility/check_err.hpp" +#include "ck/library/host_tensor/device_memory.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" +#include "ck/library/host_tensor/host_tensor_generator.hpp" +#include "ck/library/reference_tensor_operation/cpu/reference_gemm.hpp" template using S = ck::Sequence; diff --git a/example/17_convnd_bwd_data_xdl/convnd_bwd_data_xdl.cpp b/example/17_convnd_bwd_data_xdl/convnd_bwd_data_xdl.cpp index 0383197358..2d444959ab 100644 --- a/example/17_convnd_bwd_data_xdl/convnd_bwd_data_xdl.cpp +++ b/example/17_convnd_bwd_data_xdl/convnd_bwd_data_xdl.cpp @@ -2,20 +2,18 @@ #include #include #include -#include -#include -#include "config.hpp" -#include "conv_util.hpp" -#include "print.hpp" -#include "device.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" -#include "device_tensor.hpp" -#include "tensor_layout.hpp" -#include "element_wise_operation.hpp" -#include "device_convnd_bwd_data_xdl_ndhwc_kzyxc_ndhwk.hpp" -#include "reference_conv_bwd_data.hpp" +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_convnd_bwd_data_xdl_ndhwc_kzyxc_ndhwk.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" + +#include "ck/library/utility/check_err.hpp" +#include "ck/library/utility/conv_util.hpp" +#include "ck/library/host_tensor/device_memory.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" +#include "ck/library/host_tensor/host_tensor_generator.hpp" +#include "ck/library/reference_tensor_operation/cpu/reference_conv_bwd_data.hpp" using InDataType = ck::half_t; using WeiDataType = ck::half_t; diff --git a/example/18_batched_gemm_reduce/batched_gemm_reduce_xdl_fp16.cpp b/example/18_batched_gemm_reduce/batched_gemm_reduce_xdl_fp16.cpp index de584ad7e8..c9e3ab27d2 100644 --- a/example/18_batched_gemm_reduce/batched_gemm_reduce_xdl_fp16.cpp +++ b/example/18_batched_gemm_reduce/batched_gemm_reduce_xdl_fp16.cpp @@ -2,19 +2,18 @@ #include #include #include -#include -#include -#include "check_err.hpp" -#include "config.hpp" -#include "device.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" -#include "device_tensor.hpp" -#include "device_batched_gemm_reduce_xdl_cshuffle.hpp" -#include "element_wise_operation.hpp" -#include "reduction_operator.hpp" -#include "reference_batched_gemm.hpp" -#include "gemm_specialization.hpp" + +#include "ck/ck.hpp" +#include "ck/utility/reduction_operator.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" +#include "ck/tensor_operation/gpu/device/device_batched_gemm_reduce_xdl_cshuffle.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" + +#include "ck/library/utility/check_err.hpp" +#include "ck/library/host_tensor/device_memory.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" +#include "ck/library/host_tensor/host_tensor_generator.hpp" +#include "ck/library/reference_tensor_operation/cpu/reference_batched_gemm.hpp" template using S = ck::Sequence; diff --git a/example/19_binary_elementwise/broadcast_add_2d_amn_bn.cpp b/example/19_binary_elementwise/broadcast_add_2d_amn_bn.cpp index 587882ed9c..ed855a420c 100644 --- a/example/19_binary_elementwise/broadcast_add_2d_amn_bn.cpp +++ b/example/19_binary_elementwise/broadcast_add_2d_amn_bn.cpp @@ -1,39 +1,14 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2022 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ #include #include -#include "check_err.hpp" -#include "config.hpp" -#include "device.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" -#include "device_tensor.hpp" -#include "binary_element_wise_operation.hpp" -#include "device_binary_elementwise.hpp" +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/element/binary_element_wise_operation.hpp" +#include "ck/tensor_operation/gpu/device/device_binary_elementwise.hpp" + +#include "ck/library/utility/check_err.hpp" +#include "ck/library/host_tensor/device_memory.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" +#include "ck/library/host_tensor/host_tensor_generator.hpp" using F16 = ck::half_t; using F32 = float; diff --git a/example/19_binary_elementwise/broadcast_add_3d_am_bmnk.cpp b/example/19_binary_elementwise/broadcast_add_3d_am_bmnk.cpp index e03f3fa76e..d3e9fc8a68 100644 --- a/example/19_binary_elementwise/broadcast_add_3d_am_bmnk.cpp +++ b/example/19_binary_elementwise/broadcast_add_3d_am_bmnk.cpp @@ -1,14 +1,14 @@ #include #include -#include "check_err.hpp" -#include "config.hpp" -#include "device.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" -#include "device_tensor.hpp" -#include "binary_element_wise_operation.hpp" -#include "device_binary_elementwise.hpp" +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/element/binary_element_wise_operation.hpp" +#include "ck/tensor_operation/gpu/device/device_binary_elementwise.hpp" + +#include "ck/library/utility/check_err.hpp" +#include "ck/library/host_tensor/device_memory.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" +#include "ck/library/host_tensor/host_tensor_generator.hpp" using F16 = ck::half_t; using F32 = float; diff --git a/example/19_binary_elementwise/elementwise_add_1d.cpp b/example/19_binary_elementwise/elementwise_add_1d.cpp index c96e9616d7..074f6a0475 100644 --- a/example/19_binary_elementwise/elementwise_add_1d.cpp +++ b/example/19_binary_elementwise/elementwise_add_1d.cpp @@ -1,39 +1,13 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2022 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ #include #include -#include "check_err.hpp" -#include "config.hpp" -#include "device.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" -#include "device_tensor.hpp" -#include "binary_element_wise_operation.hpp" -#include "device_binary_elementwise.hpp" +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/device_binary_elementwise.hpp" +#include "ck/tensor_operation/gpu/element/binary_element_wise_operation.hpp" +#include "ck/library/utility/check_err.hpp" +#include "ck/library/host_tensor/device_memory.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" +#include "ck/library/host_tensor/host_tensor_generator.hpp" using F16 = ck::half_t; using F32 = float; diff --git a/example/19_binary_elementwise/elementwise_add_4d.cpp b/example/19_binary_elementwise/elementwise_add_4d.cpp index 13345ec11f..f8d66dfb56 100644 --- a/example/19_binary_elementwise/elementwise_add_4d.cpp +++ b/example/19_binary_elementwise/elementwise_add_4d.cpp @@ -1,39 +1,14 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2020 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ #include #include -#include "check_err.hpp" -#include "config.hpp" -#include "device.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" -#include "device_tensor.hpp" -#include "binary_element_wise_operation.hpp" -#include "device_binary_elementwise.hpp" +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/element/binary_element_wise_operation.hpp" +#include "ck/tensor_operation/gpu/device/device_binary_elementwise.hpp" + +#include "ck/library/utility/check_err.hpp" +#include "ck/library/host_tensor/device_memory.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" +#include "ck/library/host_tensor/host_tensor_generator.hpp" using F16 = ck::half_t; using F32 = float; diff --git a/example/20_convnd_bwd_weight_xdl/convnd_bwd_weight_xdl.cpp b/example/20_convnd_bwd_weight_xdl/convnd_bwd_weight_xdl.cpp index f917c2c3ac..498438e258 100644 --- a/example/20_convnd_bwd_weight_xdl/convnd_bwd_weight_xdl.cpp +++ b/example/20_convnd_bwd_weight_xdl/convnd_bwd_weight_xdl.cpp @@ -2,21 +2,18 @@ #include #include #include -#include -#include -#include "check_err.hpp" -#include "conv_util.hpp" -#include "config.hpp" -#include "print.hpp" -#include "device.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" -#include "device_tensor.hpp" -#include "tensor_layout.hpp" -#include "element_wise_operation.hpp" -#include "device_convnd_backward_weight_xdl_c_shuffle_nhwc_kyxc_nhwk.hpp" -#include "reference_conv_backward_weight.hpp" +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_convnd_backward_weight_xdl_c_shuffle_nhwc_kyxc_nhwk.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" + +#include "ck/library/utility/check_err.hpp" +#include "ck/library/utility/conv_util.hpp" +#include "ck/library/host_tensor/device_memory.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" +#include "ck/library/host_tensor/host_tensor_generator.hpp" +#include "ck/library/reference_tensor_operation/cpu/reference_conv_backward_weight.hpp" using InDataType = ck::half_t; using WeiDataType = ck::half_t; diff --git a/example/20_convnd_bwd_weight_xdl/convnd_bwd_weight_xdl_bf16_splitk.cpp b/example/20_convnd_bwd_weight_xdl/convnd_bwd_weight_xdl_bf16_splitk.cpp index 43f0cdb7ec..a81720fd06 100644 --- a/example/20_convnd_bwd_weight_xdl/convnd_bwd_weight_xdl_bf16_splitk.cpp +++ b/example/20_convnd_bwd_weight_xdl/convnd_bwd_weight_xdl_bf16_splitk.cpp @@ -2,22 +2,19 @@ #include #include #include -#include -#include -#include "check_err.hpp" -#include "conv_util.hpp" -#include "config.hpp" -#include "print.hpp" -#include "device.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" -#include "device_tensor.hpp" -#include "tensor_layout.hpp" -#include "element_wise_operation.hpp" -#include "device_unary_elementwise.hpp" -#include "device_convnd_backward_weight_xdl_c_shuffle_nhwc_kyxc_nhwk.hpp" -#include "reference_conv_backward_weight.hpp" +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_convnd_backward_weight_xdl_c_shuffle_nhwc_kyxc_nhwk.hpp" +#include "ck/tensor_operation/gpu/device/device_unary_elementwise.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" + +#include "ck/library/utility/check_err.hpp" +#include "ck/library/utility/conv_util.hpp" +#include "ck/library/host_tensor/device_memory.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" +#include "ck/library/host_tensor/host_tensor_generator.hpp" +#include "ck/library/reference_tensor_operation/cpu/reference_conv_backward_weight.hpp" using InDataType = ck::bhalf_t; using WeiDataType = ck::bhalf_t; diff --git a/example/21_gemm_layernorm/gemm_bias_relu_add_layernorm_xdl_fp16.cpp b/example/21_gemm_layernorm/gemm_bias_relu_add_layernorm_xdl_fp16.cpp index 59cbb41005..fc8b16ae35 100644 --- a/example/21_gemm_layernorm/gemm_bias_relu_add_layernorm_xdl_fp16.cpp +++ b/example/21_gemm_layernorm/gemm_bias_relu_add_layernorm_xdl_fp16.cpp @@ -3,17 +3,18 @@ #include #include -#include "check_err.hpp" -#include "config.hpp" -#include "device.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" -#include "device_tensor.hpp" -#include "device_5ary_elementwise.hpp" -#include "device_gemm_bias_add_reduce_xdl_cshuffle.hpp" -#include "element_wise_operation.hpp" -#include "reference_gemm.hpp" -#include "gemm_specialization.hpp" +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_bias_add_reduce_xdl_cshuffle.hpp" +#include "ck/tensor_operation/gpu/device/device_5ary_elementwise.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" + +#include "ck/library/host_tensor/device_memory.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" +#include "ck/library/host_tensor/host_tensor_generator.hpp" +#include "ck/library/reference_tensor_operation/cpu/reference_gemm.hpp" +#include "ck/library/utility/check_err.hpp" template using S = ck::Sequence; diff --git a/example/21_gemm_layernorm/gemm_layernorm_xdl_fp16.cpp b/example/21_gemm_layernorm/gemm_layernorm_xdl_fp16.cpp index 05c35477aa..281512e0ff 100644 --- a/example/21_gemm_layernorm/gemm_layernorm_xdl_fp16.cpp +++ b/example/21_gemm_layernorm/gemm_layernorm_xdl_fp16.cpp @@ -3,17 +3,18 @@ #include #include -#include "check_err.hpp" -#include "config.hpp" -#include "device.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" -#include "device_tensor.hpp" -#include "device_5ary_elementwise.hpp" -#include "device_gemm_reduce_xdl_cshuffle.hpp" -#include "element_wise_operation.hpp" -#include "reference_gemm.hpp" -#include "gemm_specialization.hpp" +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_reduce_xdl_cshuffle.hpp" +#include "ck/tensor_operation/gpu/device/device_5ary_elementwise.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" + +#include "ck/library/host_tensor/device_memory.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" +#include "ck/library/host_tensor/host_tensor_generator.hpp" +#include "ck/library/reference_tensor_operation/cpu/reference_gemm.hpp" +#include "ck/library/utility/check_err.hpp" template using S = ck::Sequence; diff --git a/example/22_cgemm/cgemm_xdl_fp16.cpp b/example/22_cgemm/cgemm_xdl_fp16.cpp index 9790164e72..6857d8990e 100644 --- a/example/22_cgemm/cgemm_xdl_fp16.cpp +++ b/example/22_cgemm/cgemm_xdl_fp16.cpp @@ -1,45 +1,18 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2022 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ #include #include #include #include -#include -#include -#include "check_err.hpp" -#include "config.hpp" -#include "device.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" -#include "device_tensor.hpp" -#include "device_cgemm_4gemm_xdl_cshuffle.hpp" -#include "element_wise_operation.hpp" -#include "reference_cgemm.hpp" -#include "gemm_specialization.hpp" +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_cgemm_4gemm_xdl_cshuffle.hpp" + +#include "ck/library/utility/check_err.hpp" +#include "ck/library/host_tensor/device_memory.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" +#include "ck/library/host_tensor/host_tensor_generator.hpp" +#include "ck/library/reference_tensor_operation/cpu/reference_cgemm.hpp" template using S = ck::Sequence; diff --git a/example/23_softmax/softmax_blockwise.cpp b/example/23_softmax/softmax_blockwise.cpp index 39432ac1fe..b7addc66af 100644 --- a/example/23_softmax/softmax_blockwise.cpp +++ b/example/23_softmax/softmax_blockwise.cpp @@ -4,20 +4,15 @@ #include #include -#include "check_err.hpp" -#include "config.hpp" -#include "print.hpp" -#include "device.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" -#include "device_tensor.hpp" -#include "device_base.hpp" -#include "device_softmax.hpp" -#include "host_common_util.hpp" -#include "reference_softmax.hpp" +#include "ck/ck.hpp" +#include "ck/utility/reduction_enums.hpp" +#include "ck/tensor_operation/gpu/device/device_softmax.hpp" +#include "ck/tensor_operation/gpu/device/reduction_operator_mapping.hpp" -#include "reduction_enums.hpp" -#include "reduction_operator_mapping.hpp" +#include "ck/library/utility/check_err.hpp" +#include "ck/library/host_tensor/device_memory.hpp" +#include "ck/library/host_tensor/host_common_util.hpp" +#include "ck/library/reference_tensor_operation/cpu/reference_softmax.hpp" using namespace ck; using namespace ck::tensor_operation::device; diff --git a/example/CMakeLists.txt b/example/CMakeLists.txt index 2b80fc44a2..9bba66ad0b 100644 --- a/example/CMakeLists.txt +++ b/example/CMakeLists.txt @@ -1,21 +1,6 @@ include_directories(BEFORE - ${PROJECT_SOURCE_DIR}/include/ck - ${PROJECT_SOURCE_DIR}/include/ck/utility - ${PROJECT_SOURCE_DIR}/include/ck/host_utility - ${PROJECT_SOURCE_DIR}/include/ck/tensor_description - ${PROJECT_SOURCE_DIR}/include/ck/tensor - ${PROJECT_SOURCE_DIR}/include/ck/problem_transform - ${PROJECT_SOURCE_DIR}/include/ck/tensor_operation/gpu/device - ${PROJECT_SOURCE_DIR}/include/ck/tensor_operation/gpu/grid - ${PROJECT_SOURCE_DIR}/include/ck/tensor_operation/gpu/block - ${PROJECT_SOURCE_DIR}/include/ck/tensor_operation/gpu/warp - ${PROJECT_SOURCE_DIR}/include/ck/tensor_operation/gpu/thread - ${PROJECT_SOURCE_DIR}/include/ck/tensor_operation/gpu/element - ${PROJECT_SOURCE_DIR}/library/include/ck/library/host_tensor - ${PROJECT_SOURCE_DIR}/library/include/ck/library/reference_tensor_operation/cpu - ${PROJECT_SOURCE_DIR}/library/include/ck/library/reference_tensor_operation/gpu - ${PROJECT_SOURCE_DIR}/library/include/ck/library/utility - ${PROJECT_SOURCE_DIR}/external/include/half + ${PROJECT_SOURCE_DIR}/include + ${PROJECT_SOURCE_DIR}/library/include ) add_custom_target(examples) diff --git a/external/include/half/half.hpp b/external/include/half/half.hpp deleted file mode 100644 index 25f543881f..0000000000 --- a/external/include/half/half.hpp +++ /dev/null @@ -1,5670 +0,0 @@ -// half - IEEE 754-based half-precision floating-point library. -// -// Copyright (c) 2012-2019 Christian Rau -// -// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and -// associated documentation -// files (the "Software"), to deal in the Software without restriction, including without limitation -// the rights to use, copy, -// modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit -// persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in all copies or -// substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT -// NOT LIMITED TO THE -// WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -// SHALL THE AUTHORS OR -// COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF -// CONTRACT, TORT OR OTHERWISE, -// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. - -// Version 2.1.0 - -/// \file -/// Main header file for half-precision functionality. - -#ifndef HALF_HALF_HPP -#define HALF_HALF_HPP - -#define HALF_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) - -#if defined(__INTEL_COMPILER) -#define HALF_ICC_VERSION __INTEL_COMPILER -#elif defined(__ICC) -#define HALF_ICC_VERSION __ICC -#elif defined(__ICL) -#define HALF_ICC_VERSION __ICL -#else -#define HALF_ICC_VERSION 0 -#endif - -// check C++11 language features -#if defined(__clang__) // clang -#if __has_feature(cxx_static_assert) && !defined(HALF_ENABLE_CPP11_STATIC_ASSERT) -#define HALF_ENABLE_CPP11_STATIC_ASSERT 1 -#endif -#if __has_feature(cxx_constexpr) && !defined(HALF_ENABLE_CPP11_CONSTEXPR) -#define HALF_ENABLE_CPP11_CONSTEXPR 1 -#endif -#if __has_feature(cxx_noexcept) && !defined(HALF_ENABLE_CPP11_NOEXCEPT) -#define HALF_ENABLE_CPP11_NOEXCEPT 1 -#endif -#if __has_feature(cxx_user_literals) && !defined(HALF_ENABLE_CPP11_USER_LITERALS) -#define HALF_ENABLE_CPP11_USER_LITERALS 1 -#endif -#if __has_feature(cxx_thread_local) && !defined(HALF_ENABLE_CPP11_THREAD_LOCAL) -#define HALF_ENABLE_CPP11_THREAD_LOCAL 1 -#endif -#if(defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103L) && \ - !defined(HALF_ENABLE_CPP11_LONG_LONG) -#define HALF_ENABLE_CPP11_LONG_LONG 1 -#endif -#elif HALF_ICC_VERSION && defined(__INTEL_CXX11_MODE__) // Intel C++ -#if HALF_ICC_VERSION >= 1500 && !defined(HALF_ENABLE_CPP11_THREAD_LOCAL) -#define HALF_ENABLE_CPP11_THREAD_LOCAL 1 -#endif -#if HALF_ICC_VERSION >= 1500 && !defined(HALF_ENABLE_CPP11_USER_LITERALS) -#define HALF_ENABLE_CPP11_USER_LITERALS 1 -#endif -#if HALF_ICC_VERSION >= 1400 && !defined(HALF_ENABLE_CPP11_CONSTEXPR) -#define HALF_ENABLE_CPP11_CONSTEXPR 1 -#endif -#if HALF_ICC_VERSION >= 1400 && !defined(HALF_ENABLE_CPP11_NOEXCEPT) -#define HALF_ENABLE_CPP11_NOEXCEPT 1 -#endif -#if HALF_ICC_VERSION >= 1110 && !defined(HALF_ENABLE_CPP11_STATIC_ASSERT) -#define HALF_ENABLE_CPP11_STATIC_ASSERT 1 -#endif -#if HALF_ICC_VERSION >= 1110 && !defined(HALF_ENABLE_CPP11_LONG_LONG) -#define HALF_ENABLE_CPP11_LONG_LONG 1 -#endif -#elif defined(__GNUC__) // gcc -#if defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103L -#if HALF_GCC_VERSION >= 408 && !defined(HALF_ENABLE_CPP11_THREAD_LOCAL) -#define HALF_ENABLE_CPP11_THREAD_LOCAL 1 -#endif -#if HALF_GCC_VERSION >= 407 && !defined(HALF_ENABLE_CPP11_USER_LITERALS) -#define HALF_ENABLE_CPP11_USER_LITERALS 1 -#endif -#if HALF_GCC_VERSION >= 406 && !defined(HALF_ENABLE_CPP11_CONSTEXPR) -#define HALF_ENABLE_CPP11_CONSTEXPR 1 -#endif -#if HALF_GCC_VERSION >= 406 && !defined(HALF_ENABLE_CPP11_NOEXCEPT) -#define HALF_ENABLE_CPP11_NOEXCEPT 1 -#endif -#if HALF_GCC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_STATIC_ASSERT) -#define HALF_ENABLE_CPP11_STATIC_ASSERT 1 -#endif -#if !defined(HALF_ENABLE_CPP11_LONG_LONG) -#define HALF_ENABLE_CPP11_LONG_LONG 1 -#endif -#endif -#define HALF_TWOS_COMPLEMENT_INT 1 -#elif defined(_MSC_VER) // Visual C++ -#if _MSC_VER >= 1900 && !defined(HALF_ENABLE_CPP11_THREAD_LOCAL) -#define HALF_ENABLE_CPP11_THREAD_LOCAL 1 -#endif -#if _MSC_VER >= 1900 && !defined(HALF_ENABLE_CPP11_USER_LITERALS) -#define HALF_ENABLE_CPP11_USER_LITERALS 1 -#endif -#if _MSC_VER >= 1900 && !defined(HALF_ENABLE_CPP11_CONSTEXPR) -#define HALF_ENABLE_CPP11_CONSTEXPR 1 -#endif -#if _MSC_VER >= 1900 && !defined(HALF_ENABLE_CPP11_NOEXCEPT) -#define HALF_ENABLE_CPP11_NOEXCEPT 1 -#endif -#if _MSC_VER >= 1600 && !defined(HALF_ENABLE_CPP11_STATIC_ASSERT) -#define HALF_ENABLE_CPP11_STATIC_ASSERT 1 -#endif -#if _MSC_VER >= 1310 && !defined(HALF_ENABLE_CPP11_LONG_LONG) -#define HALF_ENABLE_CPP11_LONG_LONG 1 -#endif -#define HALF_TWOS_COMPLEMENT_INT 1 -#define HALF_POP_WARNINGS 1 -#pragma warning(push) -#pragma warning(disable : 4099 4127 4146) // struct vs class, constant in if, negative unsigned -#endif - -// check C++11 library features -#include -#if defined(_LIBCPP_VERSION) // libc++ -#if defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103 -#ifndef HALF_ENABLE_CPP11_TYPE_TRAITS -#define HALF_ENABLE_CPP11_TYPE_TRAITS 1 -#endif -#ifndef HALF_ENABLE_CPP11_CSTDINT -#define HALF_ENABLE_CPP11_CSTDINT 1 -#endif -#ifndef HALF_ENABLE_CPP11_CMATH -#define HALF_ENABLE_CPP11_CMATH 1 -#endif -#ifndef HALF_ENABLE_CPP11_HASH -#define HALF_ENABLE_CPP11_HASH 1 -#endif -#ifndef HALF_ENABLE_CPP11_CFENV -#define HALF_ENABLE_CPP11_CFENV 1 -#endif -#endif -#elif defined(__GLIBCXX__) // libstdc++ -#if defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103 -#ifdef __clang__ -#if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_TYPE_TRAITS) -#define HALF_ENABLE_CPP11_TYPE_TRAITS 1 -#endif -#if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_CSTDINT) -#define HALF_ENABLE_CPP11_CSTDINT 1 -#endif -#if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_CMATH) -#define HALF_ENABLE_CPP11_CMATH 1 -#endif -#if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_HASH) -#define HALF_ENABLE_CPP11_HASH 1 -#endif -#if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_CFENV) -#define HALF_ENABLE_CPP11_CFENV 1 -#endif -#else -#if HALF_GCC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_TYPE_TRAITS) -#define HALF_ENABLE_CPP11_TYPE_TRAITS 1 -#endif -#if HALF_GCC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_CSTDINT) -#define HALF_ENABLE_CPP11_CSTDINT 1 -#endif -#if HALF_GCC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_CMATH) -#define HALF_ENABLE_CPP11_CMATH 1 -#endif -#if HALF_GCC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_HASH) -#define HALF_ENABLE_CPP11_HASH 1 -#endif -#if HALF_GCC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_CFENV) -#define HALF_ENABLE_CPP11_CFENV 1 -#endif -#endif -#endif -#elif defined(_CPPLIB_VER) // Dinkumware/Visual C++ -#if _CPPLIB_VER >= 520 && !defined(HALF_ENABLE_CPP11_TYPE_TRAITS) -#define HALF_ENABLE_CPP11_TYPE_TRAITS 1 -#endif -#if _CPPLIB_VER >= 520 && !defined(HALF_ENABLE_CPP11_CSTDINT) -#define HALF_ENABLE_CPP11_CSTDINT 1 -#endif -#if _CPPLIB_VER >= 520 && !defined(HALF_ENABLE_CPP11_HASH) -#define HALF_ENABLE_CPP11_HASH 1 -#endif -#if _CPPLIB_VER >= 610 && !defined(HALF_ENABLE_CPP11_CMATH) -#define HALF_ENABLE_CPP11_CMATH 1 -#endif -#if _CPPLIB_VER >= 610 && !defined(HALF_ENABLE_CPP11_CFENV) -#define HALF_ENABLE_CPP11_CFENV 1 -#endif -#endif -#undef HALF_GCC_VERSION -#undef HALF_ICC_VERSION - -// any error throwing C++ exceptions? -#if defined(HALF_ERRHANDLING_THROW_INVALID) || defined(HALF_ERRHANDLING_THROW_DIVBYZERO) || \ - defined(HALF_ERRHANDLING_THROW_OVERFLOW) || defined(HALF_ERRHANDLING_THROW_UNDERFLOW) || \ - defined(HALF_ERRHANDLING_THROW_INEXACT) -#define HALF_ERRHANDLING_THROWS 1 -#endif - -// any error handling enabled? -#define HALF_ERRHANDLING \ - (HALF_ERRHANDLING_FLAGS || HALF_ERRHANDLING_ERRNO || HALF_ERRHANDLING_FENV || \ - HALF_ERRHANDLING_THROWS) - -#if HALF_ERRHANDLING -#define HALF_UNUSED_NOERR(name) name -#else -#define HALF_UNUSED_NOERR(name) -#endif - -// support constexpr -#if HALF_ENABLE_CPP11_CONSTEXPR -#define HALF_CONSTEXPR constexpr -#define HALF_CONSTEXPR_CONST constexpr -#if HALF_ERRHANDLING -#define HALF_CONSTEXPR_NOERR -#else -#define HALF_CONSTEXPR_NOERR constexpr -#endif -#else -#define HALF_CONSTEXPR -#define HALF_CONSTEXPR_CONST const -#define HALF_CONSTEXPR_NOERR -#endif - -// support noexcept -#if HALF_ENABLE_CPP11_NOEXCEPT -#define HALF_NOEXCEPT noexcept -#define HALF_NOTHROW noexcept -#else -#define HALF_NOEXCEPT -#define HALF_NOTHROW throw() -#endif - -// support thread storage -#if HALF_ENABLE_CPP11_THREAD_LOCAL -#define HALF_THREAD_LOCAL thread_local -#else -#define HALF_THREAD_LOCAL static -#endif - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#if HALF_ENABLE_CPP11_TYPE_TRAITS -#include -#endif -#if HALF_ENABLE_CPP11_CSTDINT -#include -#endif -#if HALF_ERRHANDLING_ERRNO -#include -#endif -#if HALF_ENABLE_CPP11_CFENV -#include -#endif -#if HALF_ENABLE_CPP11_HASH -#include -#endif -#if HALF_ENABLE_F16C_INTRINSICS -#include -#endif - -#ifndef HALF_ENABLE_F16C_INTRINSICS -/// Enable F16C intruction set intrinsics. -/// Defining this to 1 enables the use of [F16C compiler -/// intrinsics](https://en.wikipedia.org/wiki/F16C) for converting between -/// half-precision and single-precision values which may result in improved performance. This will -/// not perform additional checks -/// for support of the F16C instruction set, so an appropriate target platform is required when -/// enabling this feature. -/// -/// Unless predefined it will be enabled automatically when the `__F16C__` symbol is defined, which -/// some compilers do on supporting platforms. -#define HALF_ENABLE_F16C_INTRINSICS __F16C__ -#endif - -#ifdef HALF_DOXYGEN_ONLY -/// Type for internal floating-point computations. -/// This can be predefined to a built-in floating-point type (`float`, `double` or `long double`) to -/// override the internal -/// half-precision implementation to use this type for computing arithmetic operations and -/// mathematical function (if available). -/// This can result in improved performance for arithmetic operators and mathematical functions but -/// might cause results to -/// deviate from the specified half-precision rounding mode and inhibits proper detection of -/// half-precision exceptions. -#define HALF_ARITHMETIC_TYPE (undefined) - -/// Enable internal exception flags. -/// Defining this to 1 causes operations on half-precision values to raise internal floating-point -/// exception flags according to -/// the IEEE 754 standard. These can then be cleared and checked with clearexcept(), testexcept(). -#define HALF_ERRHANDLING_FLAGS 0 - -/// Enable exception propagation to `errno`. -/// Defining this to 1 causes operations on half-precision values to propagate floating-point -/// exceptions to -/// [errno](https://en.cppreference.com/w/cpp/error/errno) from ``. Specifically this will -/// propagate domain errors as -/// [EDOM](https://en.cppreference.com/w/cpp/error/errno_macros) and pole, overflow and underflow -/// errors as -/// [ERANGE](https://en.cppreference.com/w/cpp/error/errno_macros). Inexact errors won't be -/// propagated. -#define HALF_ERRHANDLING_ERRNO 0 - -/// Enable exception propagation to built-in floating-point platform. -/// Defining this to 1 causes operations on half-precision values to propagate floating-point -/// exceptions to the built-in -/// single- and double-precision implementation's exception flags using the -/// [C++11 floating-point environment control](https://en.cppreference.com/w/cpp/numeric/fenv) from -/// ``. However, this -/// does not work in reverse and single- or double-precision exceptions will not raise the -/// corresponding half-precision -/// exception flags, nor will explicitly clearing flags clear the corresponding built-in flags. -#define HALF_ERRHANDLING_FENV 0 - -/// Throw C++ exception on domain errors. -/// Defining this to a string literal causes operations on half-precision values to throw a -/// [std::domain_error](https://en.cppreference.com/w/cpp/error/domain_error) with the specified -/// message on domain errors. -#define HALF_ERRHANDLING_THROW_INVALID (undefined) - -/// Throw C++ exception on pole errors. -/// Defining this to a string literal causes operations on half-precision values to throw a -/// [std::domain_error](https://en.cppreference.com/w/cpp/error/domain_error) with the specified -/// message on pole errors. -#define HALF_ERRHANDLING_THROW_DIVBYZERO (undefined) - -/// Throw C++ exception on overflow errors. -/// Defining this to a string literal causes operations on half-precision values to throw a -/// [std::overflow_error](https://en.cppreference.com/w/cpp/error/overflow_error) with the specified -/// message on overflows. -#define HALF_ERRHANDLING_THROW_OVERFLOW (undefined) - -/// Throw C++ exception on underflow errors. -/// Defining this to a string literal causes operations on half-precision values to throw a -/// [std::underflow_error](https://en.cppreference.com/w/cpp/error/underflow_error) with the -/// specified message on underflows. -#define HALF_ERRHANDLING_THROW_UNDERFLOW (undefined) - -/// Throw C++ exception on rounding errors. -/// Defining this to 1 causes operations on half-precision values to throw a -/// [std::range_error](https://en.cppreference.com/w/cpp/error/range_error) with the specified -/// message on general rounding errors. -#define HALF_ERRHANDLING_THROW_INEXACT (undefined) -#endif - -#ifndef HALF_ERRHANDLING_OVERFLOW_TO_INEXACT -/// Raise INEXACT exception on overflow. -/// Defining this to 1 (default) causes overflow errors to automatically raise inexact exceptions in -/// addition. -/// These will be raised after any possible handling of the underflow exception. -#define HALF_ERRHANDLING_OVERFLOW_TO_INEXACT 1 -#endif - -#ifndef HALF_ERRHANDLING_UNDERFLOW_TO_INEXACT -/// Raise INEXACT exception on underflow. -/// Defining this to 1 (default) causes underflow errors to automatically raise inexact exceptions -/// in addition. -/// These will be raised after any possible handling of the underflow exception. -/// -/// **Note:** This will actually cause underflow (and the accompanying inexact) exceptions to be -/// raised *only* when the result -/// is inexact, while if disabled bare underflow errors will be raised for *any* (possibly exact) -/// subnormal result. -#define HALF_ERRHANDLING_UNDERFLOW_TO_INEXACT 1 -#endif - -/// Default rounding mode. -/// This specifies the rounding mode used for all conversions between [half](\ref half_float::half)s -/// and more precise types -/// (unless using half_cast() and specifying the rounding mode directly) as well as in arithmetic -/// operations and mathematical -/// functions. It can be redefined (before including half.hpp) to one of the standard rounding modes -/// using their respective -/// constants or the equivalent values of -/// [std::float_round_style](https://en.cppreference.com/w/cpp/types/numeric_limits/float_round_style): -/// -/// `std::float_round_style` | value | rounding -/// ---------------------------------|-------|------------------------- -/// `std::round_indeterminate` | -1 | fastest -/// `std::round_toward_zero` | 0 | toward zero -/// `std::round_to_nearest` | 1 | to nearest (default) -/// `std::round_toward_infinity` | 2 | toward positive infinity -/// `std::round_toward_neg_infinity` | 3 | toward negative infinity -/// -/// By default this is set to `1` (`std::round_to_nearest`), which rounds results to the nearest -/// representable value. It can even -/// be set to -/// [std::numeric_limits::round_style](https://en.cppreference.com/w/cpp/types/numeric_limits/round_style) -/// to synchronize -/// the rounding mode with that of the built-in single-precision implementation (which is likely -/// `std::round_to_nearest`, though). -#ifndef HALF_ROUND_STYLE -#define HALF_ROUND_STYLE 1 // = std::round_to_nearest -#endif - -/// Value signaling overflow. -/// In correspondence with `HUGE_VAL[F|L]` from `` this symbol expands to a positive value -/// signaling the overflow of an -/// operation, in particular it just evaluates to positive infinity. -/// -/// **See also:** Documentation for -/// [HUGE_VAL](https://en.cppreference.com/w/cpp/numeric/math/HUGE_VAL) -#define HUGE_VALH std::numeric_limits::infinity() - -/// Fast half-precision fma function. -/// This symbol is defined if the fma() function generally executes as fast as, or faster than, a -/// separate -/// half-precision multiplication followed by an addition, which is always the case. -/// -/// **See also:** Documentation for -/// [FP_FAST_FMA](https://en.cppreference.com/w/cpp/numeric/math/fma) -#define FP_FAST_FMAH 1 - -/// Half rounding mode. -/// In correspondence with `FLT_ROUNDS` from `` this symbol expands to the rounding mode -/// used for -/// half-precision operations. It is an alias for [HALF_ROUND_STYLE](\ref HALF_ROUND_STYLE). -/// -/// **See also:** Documentation for -/// [FLT_ROUNDS](https://en.cppreference.com/w/cpp/types/climits/FLT_ROUNDS) -#define HLF_ROUNDS HALF_ROUND_STYLE - -#ifndef FP_ILOGB0 -#define FP_ILOGB0 INT_MIN -#endif -#ifndef FP_ILOGBNAN -#define FP_ILOGBNAN INT_MAX -#endif -#ifndef FP_SUBNORMAL -#define FP_SUBNORMAL 0 -#endif -#ifndef FP_ZERO -#define FP_ZERO 1 -#endif -#ifndef FP_NAN -#define FP_NAN 2 -#endif -#ifndef FP_INFINITE -#define FP_INFINITE 3 -#endif -#ifndef FP_NORMAL -#define FP_NORMAL 4 -#endif - -#if !HALF_ENABLE_CPP11_CFENV && !defined(FE_ALL_EXCEPT) -#define FE_INVALID 0x10 -#define FE_DIVBYZERO 0x08 -#define FE_OVERFLOW 0x04 -#define FE_UNDERFLOW 0x02 -#define FE_INEXACT 0x01 -#define FE_ALL_EXCEPT (FE_INVALID | FE_DIVBYZERO | FE_OVERFLOW | FE_UNDERFLOW | FE_INEXACT) -#endif - -/// Main namespace for half-precision functionality. -/// This namespace contains all the functionality provided by the library. -namespace half_float { -class half; - -#if HALF_ENABLE_CPP11_USER_LITERALS -/// Library-defined half-precision literals. -/// Import this namespace to enable half-precision floating-point literals: -/// ~~~~{.cpp} -/// using namespace half_float::literal; -/// half_float::half = 4.2_h; -/// ~~~~ -namespace literal { -half operator"" _h(long double); -} -#endif - -/// \internal -/// \brief Implementation details. -namespace detail { -#if HALF_ENABLE_CPP11_TYPE_TRAITS -/// Conditional type. -template -struct conditional : std::conditional -{ -}; - -/// Helper for tag dispatching. -template -struct bool_type : std::integral_constant -{ -}; -using std::false_type; -using std::true_type; - -/// Type traits for floating-point types. -template -struct is_float : std::is_floating_point -{ -}; -#else -/// Conditional type. -template -struct conditional -{ - typedef T type; -}; -template -struct conditional -{ - typedef F type; -}; - -/// Helper for tag dispatching. -template -struct bool_type -{ -}; -typedef bool_type true_type; -typedef bool_type false_type; - -/// Type traits for floating-point types. -template -struct is_float : false_type -{ -}; -template -struct is_float : is_float -{ -}; -template -struct is_float : is_float -{ -}; -template -struct is_float : is_float -{ -}; -template <> -struct is_float : true_type -{ -}; -template <> -struct is_float : true_type -{ -}; -template <> -struct is_float : true_type -{ -}; -#endif - -/// Type traits for floating-point bits. -template -struct bits -{ - typedef unsigned char type; -}; -template -struct bits : bits -{ -}; -template -struct bits : bits -{ -}; -template -struct bits : bits -{ -}; - -#if HALF_ENABLE_CPP11_CSTDINT -/// Unsigned integer of (at least) 16 bits width. -typedef std::uint_least16_t uint16; - -/// Fastest unsigned integer of (at least) 32 bits width. -typedef std::uint_fast32_t uint32; - -/// Fastest signed integer of (at least) 32 bits width. -typedef std::int_fast32_t int32; - -/// Unsigned integer of (at least) 32 bits width. -template <> -struct bits -{ - typedef std::uint_least32_t type; -}; - -/// Unsigned integer of (at least) 64 bits width. -template <> -struct bits -{ - typedef std::uint_least64_t type; -}; -#else -/// Unsigned integer of (at least) 16 bits width. -typedef unsigned short uint16; - -/// Fastest unsigned integer of (at least) 32 bits width. -typedef unsigned long uint32; - -/// Fastest unsigned integer of (at least) 32 bits width. -typedef long int32; - -/// Unsigned integer of (at least) 32 bits width. -template <> -struct bits - : conditional::digits >= 32, unsigned int, unsigned long> -{ -}; - -#if HALF_ENABLE_CPP11_LONG_LONG -/// Unsigned integer of (at least) 64 bits width. -template <> -struct bits : conditional::digits >= 64, - unsigned long, - unsigned long long> -{ -}; -#else -/// Unsigned integer of (at least) 64 bits width. -template <> -struct bits -{ - typedef unsigned long type; -}; -#endif -#endif - -#ifdef HALF_ARITHMETIC_TYPE -/// Type to use for arithmetic computations and mathematic functions internally. -typedef HALF_ARITHMETIC_TYPE internal_t; -#endif - -/// Tag type for binary construction. -struct binary_t -{ -}; - -/// Tag for binary construction. -HALF_CONSTEXPR_CONST binary_t binary = binary_t(); - -/// \name Implementation defined classification and arithmetic -/// \{ - -/// Check for infinity. -/// \tparam T argument type (builtin floating-point type) -/// \param arg value to query -/// \retval true if infinity -/// \retval false else -template -bool builtin_isinf(T arg) -{ -#if HALF_ENABLE_CPP11_CMATH - return std::isinf(arg); -#elif defined(_MSC_VER) - return !::_finite(static_cast(arg)) && !::_isnan(static_cast(arg)); -#else - return arg == std::numeric_limits::infinity() || arg == -std::numeric_limits::infinity(); -#endif -} - -/// Check for NaN. -/// \tparam T argument type (builtin floating-point type) -/// \param arg value to query -/// \retval true if not a number -/// \retval false else -template -bool builtin_isnan(T arg) -{ -#if HALF_ENABLE_CPP11_CMATH - return std::isnan(arg); -#elif defined(_MSC_VER) - return ::_isnan(static_cast(arg)) != 0; -#else - return arg != arg; -#endif -} - -/// Check sign. -/// \tparam T argument type (builtin floating-point type) -/// \param arg value to query -/// \retval true if signbit set -/// \retval false else -template -bool builtin_signbit(T arg) -{ -#if HALF_ENABLE_CPP11_CMATH - return std::signbit(arg); -#else - return arg < T() || (arg == T() && T(1) / arg < T()); -#endif -} - -/// Platform-independent sign mask. -/// \param arg integer value in two's complement -/// \retval -1 if \a arg negative -/// \retval 0 if \a arg positive -inline uint32 sign_mask(uint32 arg) -{ - static const int N = std::numeric_limits::digits - 1; -#if HALF_TWOS_COMPLEMENT_INT - return static_cast(arg) >> N; -#else - return -((arg >> N) & 1); -#endif -} - -/// Platform-independent arithmetic right shift. -/// \param arg integer value in two's complement -/// \param i shift amount (at most 31) -/// \return \a arg right shifted for \a i bits with possible sign extension -inline uint32 arithmetic_shift(uint32 arg, int i) -{ -#if HALF_TWOS_COMPLEMENT_INT - return static_cast(arg) >> i; -#else - return static_cast(arg) / (static_cast(1) << i) - - ((arg >> (std::numeric_limits::digits - 1)) & 1); -#endif -} - -/// \} -/// \name Error handling -/// \{ - -/// Internal exception flags. -/// \return reference to global exception flags -inline int& errflags() -{ - HALF_THREAD_LOCAL int flags = 0; - return flags; -} - -/// Raise floating-point exception. -/// \param flags exceptions to raise -/// \param cond condition to raise exceptions for -inline void raise(int HALF_UNUSED_NOERR(flags), bool HALF_UNUSED_NOERR(cond) = true) -{ -#if HALF_ERRHANDLING - if(!cond) - return; -#if HALF_ERRHANDLING_FLAGS - errflags() |= flags; -#endif -#if HALF_ERRHANDLING_ERRNO - if(flags & FE_INVALID) - errno = EDOM; - else if(flags & (FE_DIVBYZERO | FE_OVERFLOW | FE_UNDERFLOW)) - errno = ERANGE; -#endif -#if HALF_ERRHANDLING_FENV && HALF_ENABLE_CPP11_CFENV - std::feraiseexcept(flags); -#endif -#ifdef HALF_ERRHANDLING_THROW_INVALID - if(flags & FE_INVALID) - throw std::domain_error(HALF_ERRHANDLING_THROW_INVALID); -#endif -#ifdef HALF_ERRHANDLING_THROW_DIVBYZERO - if(flags & FE_DIVBYZERO) - throw std::domain_error(HALF_ERRHANDLING_THROW_DIVBYZERO); -#endif -#ifdef HALF_ERRHANDLING_THROW_OVERFLOW - if(flags & FE_OVERFLOW) - throw std::overflow_error(HALF_ERRHANDLING_THROW_OVERFLOW); -#endif -#ifdef HALF_ERRHANDLING_THROW_UNDERFLOW - if(flags & FE_UNDERFLOW) - throw std::underflow_error(HALF_ERRHANDLING_THROW_UNDERFLOW); -#endif -#ifdef HALF_ERRHANDLING_THROW_INEXACT - if(flags & FE_INEXACT) - throw std::range_error(HALF_ERRHANDLING_THROW_INEXACT); -#endif -#if HALF_ERRHANDLING_UNDERFLOW_TO_INEXACT - if((flags & FE_UNDERFLOW) && !(flags & FE_INEXACT)) - raise(FE_INEXACT); -#endif -#if HALF_ERRHANDLING_OVERFLOW_TO_INEXACT - if((flags & FE_OVERFLOW) && !(flags & FE_INEXACT)) - raise(FE_INEXACT); -#endif -#endif -} - -/// Check and signal for any NaN. -/// \param x first half-precision value to check -/// \param y second half-precision value to check -/// \retval true if either \a x or \a y is NaN -/// \retval false else -/// \exception FE_INVALID if \a x or \a y is NaN -inline HALF_CONSTEXPR_NOERR bool compsignal(unsigned int x, unsigned int y) -{ -#if HALF_ERRHANDLING - raise(FE_INVALID, (x & 0x7FFF) > 0x7C00 || (y & 0x7FFF) > 0x7C00); -#endif - return (x & 0x7FFF) > 0x7C00 || (y & 0x7FFF) > 0x7C00; -} - -/// Signal and silence signaling NaN. -/// \param nan half-precision NaN value -/// \return quiet NaN -/// \exception FE_INVALID if \a nan is signaling NaN -inline HALF_CONSTEXPR_NOERR unsigned int signal(unsigned int nan) -{ -#if HALF_ERRHANDLING - raise(FE_INVALID, !(nan & 0x200)); -#endif - return nan | 0x200; -} - -/// Signal and silence signaling NaNs. -/// \param x first half-precision value to check -/// \param y second half-precision value to check -/// \return quiet NaN -/// \exception FE_INVALID if \a x or \a y is signaling NaN -inline HALF_CONSTEXPR_NOERR unsigned int signal(unsigned int x, unsigned int y) -{ -#if HALF_ERRHANDLING - raise(FE_INVALID, - ((x & 0x7FFF) > 0x7C00 && !(x & 0x200)) || ((y & 0x7FFF) > 0x7C00 && !(y & 0x200))); -#endif - return ((x & 0x7FFF) > 0x7C00) ? (x | 0x200) : (y | 0x200); -} - -/// Signal and silence signaling NaNs. -/// \param x first half-precision value to check -/// \param y second half-precision value to check -/// \param z third half-precision value to check -/// \return quiet NaN -/// \exception FE_INVALID if \a x, \a y or \a z is signaling NaN -inline HALF_CONSTEXPR_NOERR unsigned int signal(unsigned int x, unsigned int y, unsigned int z) -{ -#if HALF_ERRHANDLING - raise(FE_INVALID, - ((x & 0x7FFF) > 0x7C00 && !(x & 0x200)) || ((y & 0x7FFF) > 0x7C00 && !(y & 0x200)) || - ((z & 0x7FFF) > 0x7C00 && !(z & 0x200))); -#endif - return ((x & 0x7FFF) > 0x7C00) ? (x | 0x200) - : ((y & 0x7FFF) > 0x7C00) ? (y | 0x200) : (z | 0x200); -} - -/// Select value or signaling NaN. -/// \param x preferred half-precision value -/// \param y ignored half-precision value except for signaling NaN -/// \return \a y if signaling NaN, \a x otherwise -/// \exception FE_INVALID if \a y is signaling NaN -inline HALF_CONSTEXPR_NOERR unsigned int select(unsigned int x, unsigned int HALF_UNUSED_NOERR(y)) -{ -#if HALF_ERRHANDLING - return (((y & 0x7FFF) > 0x7C00) && !(y & 0x200)) ? signal(y) : x; -#else - return x; -#endif -} - -/// Raise domain error and return NaN. -/// return quiet NaN -/// \exception FE_INVALID -inline HALF_CONSTEXPR_NOERR unsigned int invalid() -{ -#if HALF_ERRHANDLING - raise(FE_INVALID); -#endif - return 0x7FFF; -} - -/// Raise pole error and return infinity. -/// \param sign half-precision value with sign bit only -/// \return half-precision infinity with sign of \a sign -/// \exception FE_DIVBYZERO -inline HALF_CONSTEXPR_NOERR unsigned int pole(unsigned int sign = 0) -{ -#if HALF_ERRHANDLING - raise(FE_DIVBYZERO); -#endif - return sign | 0x7C00; -} - -/// Check value for underflow. -/// \param arg non-zero half-precision value to check -/// \return \a arg -/// \exception FE_UNDERFLOW if arg is subnormal -inline HALF_CONSTEXPR_NOERR unsigned int check_underflow(unsigned int arg) -{ -#if HALF_ERRHANDLING && !HALF_ERRHANDLING_UNDERFLOW_TO_INEXACT - raise(FE_UNDERFLOW, !(arg & 0x7C00)); -#endif - return arg; -} - -/// \} -/// \name Conversion and rounding -/// \{ - -/// Half-precision overflow. -/// \tparam R rounding mode to use -/// \param sign half-precision value with sign bit only -/// \return rounded overflowing half-precision value -/// \exception FE_OVERFLOW -template -HALF_CONSTEXPR_NOERR unsigned int overflow(unsigned int sign = 0) -{ -#if HALF_ERRHANDLING - raise(FE_OVERFLOW); -#endif - return (R == std::round_toward_infinity) - ? (sign + 0x7C00 - (sign >> 15)) - : (R == std::round_toward_neg_infinity) - ? (sign + 0x7BFF + (sign >> 15)) - : (R == std::round_toward_zero) ? (sign | 0x7BFF) : (sign | 0x7C00); -} - -/// Half-precision underflow. -/// \tparam R rounding mode to use -/// \param sign half-precision value with sign bit only -/// \return rounded underflowing half-precision value -/// \exception FE_UNDERFLOW -template -HALF_CONSTEXPR_NOERR unsigned int underflow(unsigned int sign = 0) -{ -#if HALF_ERRHANDLING - raise(FE_UNDERFLOW); -#endif - return (R == std::round_toward_infinity) - ? (sign + 1 - (sign >> 15)) - : (R == std::round_toward_neg_infinity) ? (sign + (sign >> 15)) : sign; -} - -/// Round half-precision number. -/// \tparam R rounding mode to use -/// \tparam I `true` to always raise INEXACT exception, `false` to raise only for rounded results -/// \param value finite half-precision number to round -/// \param g guard bit (most significant discarded bit) -/// \param s sticky bit (or of all but the most significant discarded bits) -/// \return rounded half-precision value -/// \exception FE_OVERFLOW on overflows -/// \exception FE_UNDERFLOW on underflows -/// \exception FE_INEXACT if value had to be rounded or \a I is `true` -template -HALF_CONSTEXPR_NOERR unsigned int rounded(unsigned int value, int g, int s) -{ -#if HALF_ERRHANDLING - value += (R == std::round_to_nearest) - ? (g & (s | value)) - : (R == std::round_toward_infinity) - ? (~(value >> 15) & (g | s)) - : (R == std::round_toward_neg_infinity) ? ((value >> 15) & (g | s)) : 0; - if((value & 0x7C00) == 0x7C00) - raise(FE_OVERFLOW); - else if(value & 0x7C00) - raise(FE_INEXACT, I || (g | s) != 0); - else - raise(FE_UNDERFLOW, !(HALF_ERRHANDLING_UNDERFLOW_TO_INEXACT) || I || (g | s) != 0); - return value; -#else - return (R == std::round_to_nearest) - ? (value + (g & (s | value))) - : (R == std::round_toward_infinity) - ? (value + (~(value >> 15) & (g | s))) - : (R == std::round_toward_neg_infinity) ? (value + ((value >> 15) & (g | s))) - : value; -#endif -} - -/// Round half-precision number to nearest integer value. -/// \tparam R rounding mode to use -/// \tparam E `true` for round to even, `false` for round away from zero -/// \tparam I `true` to raise INEXACT exception (if inexact), `false` to never raise it -/// \param value half-precision value to round -/// \return half-precision bits for nearest integral value -/// \exception FE_INVALID for signaling NaN -/// \exception FE_INEXACT if value had to be rounded and \a I is `true` -template -unsigned int integral(unsigned int value) -{ - unsigned int abs = value & 0x7FFF; - if(abs < 0x3C00) - { - raise(FE_INEXACT, I); - return ((R == std::round_to_nearest) - ? (0x3C00 & -static_cast(abs >= (0x3800 + E))) - : (R == std::round_toward_infinity) - ? (0x3C00 & -(~(value >> 15) & (abs != 0))) - : (R == std::round_toward_neg_infinity) - ? (0x3C00 & -static_cast(value > 0x8000)) - : 0) | - (value & 0x8000); - } - if(abs >= 0x6400) - return (abs > 0x7C00) ? signal(value) : value; - unsigned int exp = 25 - (abs >> 10), mask = (1 << exp) - 1; - raise(FE_INEXACT, I && (value & mask)); - return (((R == std::round_to_nearest) - ? ((1 << (exp - 1)) - (~(value >> exp) & E)) - : (R == std::round_toward_infinity) - ? (mask & ((value >> 15) - 1)) - : (R == std::round_toward_neg_infinity) ? (mask & -(value >> 15)) : 0) + - value) & - ~mask; -} - -/// Convert fixed point to half-precision floating-point. -/// \tparam R rounding mode to use -/// \tparam F number of fractional bits (at least 11) -/// \tparam S `true` for signed, `false` for unsigned -/// \tparam N `true` for additional normalization step, `false` if already normalized to 1.F -/// \tparam I `true` to always raise INEXACT exception, `false` to raise only for rounded results -/// \param m mantissa in Q1.F fixed point format -/// \param exp exponent -/// \param sign half-precision value with sign bit only -/// \param s sticky bit (or of all but the most significant already discarded bits) -/// \return value converted to half-precision -/// \exception FE_OVERFLOW on overflows -/// \exception FE_UNDERFLOW on underflows -/// \exception FE_INEXACT if value had to be rounded or \a I is `true` -template -unsigned int fixed2half(uint32 m, int exp = 14, unsigned int sign = 0, int s = 0) -{ - if(S) - { - uint32 msign = sign_mask(m); - m = (m ^ msign) - msign; - sign = msign & 0x8000; - } - if(N) - for(; m < (static_cast(1) << F) && exp; m <<= 1, --exp) - ; - else if(exp < 0) - return rounded(sign + (m >> (F - 10 - exp)), - (m >> (F - 11 - exp)) & 1, - s | ((m & ((static_cast(1) << (F - 11 - exp)) - 1)) != 0)); - return rounded(sign + (exp << 10) + (m >> (F - 10)), - (m >> (F - 11)) & 1, - s | ((m & ((static_cast(1) << (F - 11)) - 1)) != 0)); -} - -/// Convert IEEE single-precision to half-precision. -/// Credit for this goes to [Jeroen van der -/// Zijp](ftp://ftp.fox-toolkit.org/pub/fasthalffloatconversion.pdf). -/// \tparam R rounding mode to use -/// \param value single-precision value to convert -/// \return rounded half-precision value -/// \exception FE_OVERFLOW on overflows -/// \exception FE_UNDERFLOW on underflows -/// \exception FE_INEXACT if value had to be rounded -template -unsigned int float2half_impl(float value, true_type) -{ -#if HALF_ENABLE_F16C_INTRINSICS - return _mm_cvtsi128_si32(_mm_cvtps_ph(_mm_set_ss(value), - (R == std::round_to_nearest) - ? _MM_FROUND_TO_NEAREST_INT - : (R == std::round_toward_zero) - ? _MM_FROUND_TO_ZERO - : (R == std::round_toward_infinity) - ? _MM_FROUND_TO_POS_INF - : (R == std::round_toward_neg_infinity) - ? _MM_FROUND_TO_NEG_INF - : _MM_FROUND_CUR_DIRECTION)); -#else - bits::type fbits; - std::memcpy(&fbits, &value, sizeof(float)); -#if 1 - unsigned int sign = (fbits >> 16) & 0x8000; - fbits &= 0x7FFFFFFF; - if(fbits >= 0x7F800000) - return sign | 0x7C00 | ((fbits > 0x7F800000) ? (0x200 | ((fbits >> 13) & 0x3FF)) : 0); - if(fbits >= 0x47800000) - return overflow(sign); - if(fbits >= 0x38800000) - return rounded(sign | (((fbits >> 23) - 112) << 10) | ((fbits >> 13) & 0x3FF), - (fbits >> 12) & 1, - (fbits & 0xFFF) != 0); - if(fbits >= 0x33000000) - { - int i = 125 - (fbits >> 23); - fbits = (fbits & 0x7FFFFF) | 0x800000; - return rounded(sign | (fbits >> (i + 1)), - (fbits >> i) & 1, - (fbits & ((static_cast(1) << i) - 1)) != 0); - } - if(fbits != 0) - return underflow(sign); - return sign; -#else - static const uint16 base_table[512] = { - 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, - 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, - 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, - 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, - 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, - 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, - 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, - 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, - 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, - 0x0000, 0x0000, 0x0000, 0x0000, 0x0001, 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, 0x0040, - 0x0080, 0x0100, 0x0200, 0x0400, 0x0800, 0x0C00, 0x1000, 0x1400, 0x1800, 0x1C00, 0x2000, - 0x2400, 0x2800, 0x2C00, 0x3000, 0x3400, 0x3800, 0x3C00, 0x4000, 0x4400, 0x4800, 0x4C00, - 0x5000, 0x5400, 0x5800, 0x5C00, 0x6000, 0x6400, 0x6800, 0x6C00, 0x7000, 0x7400, 0x7800, - 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, - 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, - 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, - 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, - 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, - 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, - 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, - 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, - 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, - 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, - 0x7BFF, 0x7BFF, 0x7C00, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, - 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, - 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, - 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, - 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, - 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, - 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, - 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, - 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, - 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8001, 0x8002, 0x8004, 0x8008, - 0x8010, 0x8020, 0x8040, 0x8080, 0x8100, 0x8200, 0x8400, 0x8800, 0x8C00, 0x9000, 0x9400, - 0x9800, 0x9C00, 0xA000, 0xA400, 0xA800, 0xAC00, 0xB000, 0xB400, 0xB800, 0xBC00, 0xC000, - 0xC400, 0xC800, 0xCC00, 0xD000, 0xD400, 0xD800, 0xDC00, 0xE000, 0xE400, 0xE800, 0xEC00, - 0xF000, 0xF400, 0xF800, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, - 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, - 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, - 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, - 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, - 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, - 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, - 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, - 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, - 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, - 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFC00}; - static const unsigned char shift_table[256] = { - 24, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, - 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, - 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, - 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, - 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 24, 23, 22, 21, 20, 19, 18, 17, - 16, 15, 14, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, - 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 13}; - int sexp = fbits >> 23, exp = sexp & 0xFF, i = shift_table[exp]; - fbits &= 0x7FFFFF; - uint32 m = (fbits | ((exp != 0) << 23)) & -static_cast(exp != 0xFF); - return rounded(base_table[sexp] + (fbits >> i), - (m >> (i - 1)) & 1, - (((static_cast(1) << (i - 1)) - 1) & m) != 0); -#endif -#endif -} - -/// Convert IEEE double-precision to half-precision. -/// \tparam R rounding mode to use -/// \param value double-precision value to convert -/// \return rounded half-precision value -/// \exception FE_OVERFLOW on overflows -/// \exception FE_UNDERFLOW on underflows -/// \exception FE_INEXACT if value had to be rounded -template -unsigned int float2half_impl(double value, true_type) -{ -#if HALF_ENABLE_F16C_INTRINSICS - if(R == std::round_indeterminate) - return _mm_cvtsi128_si32( - _mm_cvtps_ph(_mm_cvtpd_ps(_mm_set_sd(value)), _MM_FROUND_CUR_DIRECTION)); -#endif - bits::type dbits; - std::memcpy(&dbits, &value, sizeof(double)); - uint32 hi = dbits >> 32, lo = dbits & 0xFFFFFFFF; - unsigned int sign = (hi >> 16) & 0x8000; - hi &= 0x7FFFFFFF; - if(hi >= 0x7FF00000) - return sign | 0x7C00 | ((dbits & 0xFFFFFFFFFFFFF) ? (0x200 | ((hi >> 10) & 0x3FF)) : 0); - if(hi >= 0x40F00000) - return overflow(sign); - if(hi >= 0x3F100000) - return rounded(sign | (((hi >> 20) - 1008) << 10) | ((hi >> 10) & 0x3FF), - (hi >> 9) & 1, - ((hi & 0x1FF) | lo) != 0); - if(hi >= 0x3E600000) - { - int i = 1018 - (hi >> 20); - hi = (hi & 0xFFFFF) | 0x100000; - return rounded(sign | (hi >> (i + 1)), - (hi >> i) & 1, - ((hi & ((static_cast(1) << i) - 1)) | lo) != 0); - } - if((hi | lo) != 0) - return underflow(sign); - return sign; -} - -/// Convert non-IEEE floating-point to half-precision. -/// \tparam R rounding mode to use -/// \tparam T source type (builtin floating-point type) -/// \param value floating-point value to convert -/// \return rounded half-precision value -/// \exception FE_OVERFLOW on overflows -/// \exception FE_UNDERFLOW on underflows -/// \exception FE_INEXACT if value had to be rounded -template -unsigned int float2half_impl(T value, ...) -{ - unsigned int hbits = static_cast(builtin_signbit(value)) << 15; - if(value == T()) - return hbits; - if(builtin_isnan(value)) - return hbits | 0x7FFF; - if(builtin_isinf(value)) - return hbits | 0x7C00; - int exp; - std::frexp(value, &exp); - if(exp > 16) - return overflow(hbits); - if(exp < -13) - value = std::ldexp(value, 25); - else - { - value = std::ldexp(value, 12 - exp); - hbits |= ((exp + 13) << 10); - } - T ival, frac = std::modf(value, &ival); - int m = std::abs(static_cast(ival)); - return rounded(hbits + (m >> 1), m & 1, frac != T()); -} - -/// Convert floating-point to half-precision. -/// \tparam R rounding mode to use -/// \tparam T source type (builtin floating-point type) -/// \param value floating-point value to convert -/// \return rounded half-precision value -/// \exception FE_OVERFLOW on overflows -/// \exception FE_UNDERFLOW on underflows -/// \exception FE_INEXACT if value had to be rounded -template -unsigned int float2half(T value) -{ - return float2half_impl(value, - bool_type < std::numeric_limits::is_iec559 && - sizeof(typename bits::type) == sizeof(T) > ()); -} - -/// Convert integer to half-precision floating-point. -/// \tparam R rounding mode to use -/// \tparam T type to convert (builtin integer type) -/// \param value integral value to convert -/// \return rounded half-precision value -/// \exception FE_OVERFLOW on overflows -/// \exception FE_INEXACT if value had to be rounded -template -unsigned int int2half(T value) -{ - unsigned int bits = static_cast(value < 0) << 15; - if(!value) - return bits; - if(bits) - value = -value; - if(value > 0xFFFF) - return overflow(bits); - unsigned int m = static_cast(value), exp = 24; - for(; m < 0x400; m <<= 1, --exp) - ; - for(; m > 0x7FF; m >>= 1, ++exp) - ; - bits |= (exp << 10) + m; - return (exp > 24) ? rounded( - bits, (value >> (exp - 25)) & 1, (((1 << (exp - 25)) - 1) & value) != 0) - : bits; -} - -/// Convert half-precision to IEEE single-precision. -/// Credit for this goes to [Jeroen van der -/// Zijp](ftp://ftp.fox-toolkit.org/pub/fasthalffloatconversion.pdf). -/// \param value half-precision value to convert -/// \return single-precision value -inline float half2float_impl(unsigned int value, float, true_type) -{ -#if HALF_ENABLE_F16C_INTRINSICS - return _mm_cvtss_f32(_mm_cvtph_ps(_mm_cvtsi32_si128(value))); -#else -#if 0 - bits::type fbits = static_cast::type>(value&0x8000) << 16; - int abs = value & 0x7FFF; - if(abs) - { - fbits |= 0x38000000 << static_cast(abs>=0x7C00); - for(; abs<0x400; abs<<=1,fbits-=0x800000) ; - fbits += static_cast::type>(abs) << 13; - } -#else - static const bits::type mantissa_table[2048] = { - 0x00000000, 0x33800000, 0x34000000, 0x34400000, 0x34800000, 0x34A00000, 0x34C00000, - 0x34E00000, 0x35000000, 0x35100000, 0x35200000, 0x35300000, 0x35400000, 0x35500000, - 0x35600000, 0x35700000, 0x35800000, 0x35880000, 0x35900000, 0x35980000, 0x35A00000, - 0x35A80000, 0x35B00000, 0x35B80000, 0x35C00000, 0x35C80000, 0x35D00000, 0x35D80000, - 0x35E00000, 0x35E80000, 0x35F00000, 0x35F80000, 0x36000000, 0x36040000, 0x36080000, - 0x360C0000, 0x36100000, 0x36140000, 0x36180000, 0x361C0000, 0x36200000, 0x36240000, - 0x36280000, 0x362C0000, 0x36300000, 0x36340000, 0x36380000, 0x363C0000, 0x36400000, - 0x36440000, 0x36480000, 0x364C0000, 0x36500000, 0x36540000, 0x36580000, 0x365C0000, - 0x36600000, 0x36640000, 0x36680000, 0x366C0000, 0x36700000, 0x36740000, 0x36780000, - 0x367C0000, 0x36800000, 0x36820000, 0x36840000, 0x36860000, 0x36880000, 0x368A0000, - 0x368C0000, 0x368E0000, 0x36900000, 0x36920000, 0x36940000, 0x36960000, 0x36980000, - 0x369A0000, 0x369C0000, 0x369E0000, 0x36A00000, 0x36A20000, 0x36A40000, 0x36A60000, - 0x36A80000, 0x36AA0000, 0x36AC0000, 0x36AE0000, 0x36B00000, 0x36B20000, 0x36B40000, - 0x36B60000, 0x36B80000, 0x36BA0000, 0x36BC0000, 0x36BE0000, 0x36C00000, 0x36C20000, - 0x36C40000, 0x36C60000, 0x36C80000, 0x36CA0000, 0x36CC0000, 0x36CE0000, 0x36D00000, - 0x36D20000, 0x36D40000, 0x36D60000, 0x36D80000, 0x36DA0000, 0x36DC0000, 0x36DE0000, - 0x36E00000, 0x36E20000, 0x36E40000, 0x36E60000, 0x36E80000, 0x36EA0000, 0x36EC0000, - 0x36EE0000, 0x36F00000, 0x36F20000, 0x36F40000, 0x36F60000, 0x36F80000, 0x36FA0000, - 0x36FC0000, 0x36FE0000, 0x37000000, 0x37010000, 0x37020000, 0x37030000, 0x37040000, - 0x37050000, 0x37060000, 0x37070000, 0x37080000, 0x37090000, 0x370A0000, 0x370B0000, - 0x370C0000, 0x370D0000, 0x370E0000, 0x370F0000, 0x37100000, 0x37110000, 0x37120000, - 0x37130000, 0x37140000, 0x37150000, 0x37160000, 0x37170000, 0x37180000, 0x37190000, - 0x371A0000, 0x371B0000, 0x371C0000, 0x371D0000, 0x371E0000, 0x371F0000, 0x37200000, - 0x37210000, 0x37220000, 0x37230000, 0x37240000, 0x37250000, 0x37260000, 0x37270000, - 0x37280000, 0x37290000, 0x372A0000, 0x372B0000, 0x372C0000, 0x372D0000, 0x372E0000, - 0x372F0000, 0x37300000, 0x37310000, 0x37320000, 0x37330000, 0x37340000, 0x37350000, - 0x37360000, 0x37370000, 0x37380000, 0x37390000, 0x373A0000, 0x373B0000, 0x373C0000, - 0x373D0000, 0x373E0000, 0x373F0000, 0x37400000, 0x37410000, 0x37420000, 0x37430000, - 0x37440000, 0x37450000, 0x37460000, 0x37470000, 0x37480000, 0x37490000, 0x374A0000, - 0x374B0000, 0x374C0000, 0x374D0000, 0x374E0000, 0x374F0000, 0x37500000, 0x37510000, - 0x37520000, 0x37530000, 0x37540000, 0x37550000, 0x37560000, 0x37570000, 0x37580000, - 0x37590000, 0x375A0000, 0x375B0000, 0x375C0000, 0x375D0000, 0x375E0000, 0x375F0000, - 0x37600000, 0x37610000, 0x37620000, 0x37630000, 0x37640000, 0x37650000, 0x37660000, - 0x37670000, 0x37680000, 0x37690000, 0x376A0000, 0x376B0000, 0x376C0000, 0x376D0000, - 0x376E0000, 0x376F0000, 0x37700000, 0x37710000, 0x37720000, 0x37730000, 0x37740000, - 0x37750000, 0x37760000, 0x37770000, 0x37780000, 0x37790000, 0x377A0000, 0x377B0000, - 0x377C0000, 0x377D0000, 0x377E0000, 0x377F0000, 0x37800000, 0x37808000, 0x37810000, - 0x37818000, 0x37820000, 0x37828000, 0x37830000, 0x37838000, 0x37840000, 0x37848000, - 0x37850000, 0x37858000, 0x37860000, 0x37868000, 0x37870000, 0x37878000, 0x37880000, - 0x37888000, 0x37890000, 0x37898000, 0x378A0000, 0x378A8000, 0x378B0000, 0x378B8000, - 0x378C0000, 0x378C8000, 0x378D0000, 0x378D8000, 0x378E0000, 0x378E8000, 0x378F0000, - 0x378F8000, 0x37900000, 0x37908000, 0x37910000, 0x37918000, 0x37920000, 0x37928000, - 0x37930000, 0x37938000, 0x37940000, 0x37948000, 0x37950000, 0x37958000, 0x37960000, - 0x37968000, 0x37970000, 0x37978000, 0x37980000, 0x37988000, 0x37990000, 0x37998000, - 0x379A0000, 0x379A8000, 0x379B0000, 0x379B8000, 0x379C0000, 0x379C8000, 0x379D0000, - 0x379D8000, 0x379E0000, 0x379E8000, 0x379F0000, 0x379F8000, 0x37A00000, 0x37A08000, - 0x37A10000, 0x37A18000, 0x37A20000, 0x37A28000, 0x37A30000, 0x37A38000, 0x37A40000, - 0x37A48000, 0x37A50000, 0x37A58000, 0x37A60000, 0x37A68000, 0x37A70000, 0x37A78000, - 0x37A80000, 0x37A88000, 0x37A90000, 0x37A98000, 0x37AA0000, 0x37AA8000, 0x37AB0000, - 0x37AB8000, 0x37AC0000, 0x37AC8000, 0x37AD0000, 0x37AD8000, 0x37AE0000, 0x37AE8000, - 0x37AF0000, 0x37AF8000, 0x37B00000, 0x37B08000, 0x37B10000, 0x37B18000, 0x37B20000, - 0x37B28000, 0x37B30000, 0x37B38000, 0x37B40000, 0x37B48000, 0x37B50000, 0x37B58000, - 0x37B60000, 0x37B68000, 0x37B70000, 0x37B78000, 0x37B80000, 0x37B88000, 0x37B90000, - 0x37B98000, 0x37BA0000, 0x37BA8000, 0x37BB0000, 0x37BB8000, 0x37BC0000, 0x37BC8000, - 0x37BD0000, 0x37BD8000, 0x37BE0000, 0x37BE8000, 0x37BF0000, 0x37BF8000, 0x37C00000, - 0x37C08000, 0x37C10000, 0x37C18000, 0x37C20000, 0x37C28000, 0x37C30000, 0x37C38000, - 0x37C40000, 0x37C48000, 0x37C50000, 0x37C58000, 0x37C60000, 0x37C68000, 0x37C70000, - 0x37C78000, 0x37C80000, 0x37C88000, 0x37C90000, 0x37C98000, 0x37CA0000, 0x37CA8000, - 0x37CB0000, 0x37CB8000, 0x37CC0000, 0x37CC8000, 0x37CD0000, 0x37CD8000, 0x37CE0000, - 0x37CE8000, 0x37CF0000, 0x37CF8000, 0x37D00000, 0x37D08000, 0x37D10000, 0x37D18000, - 0x37D20000, 0x37D28000, 0x37D30000, 0x37D38000, 0x37D40000, 0x37D48000, 0x37D50000, - 0x37D58000, 0x37D60000, 0x37D68000, 0x37D70000, 0x37D78000, 0x37D80000, 0x37D88000, - 0x37D90000, 0x37D98000, 0x37DA0000, 0x37DA8000, 0x37DB0000, 0x37DB8000, 0x37DC0000, - 0x37DC8000, 0x37DD0000, 0x37DD8000, 0x37DE0000, 0x37DE8000, 0x37DF0000, 0x37DF8000, - 0x37E00000, 0x37E08000, 0x37E10000, 0x37E18000, 0x37E20000, 0x37E28000, 0x37E30000, - 0x37E38000, 0x37E40000, 0x37E48000, 0x37E50000, 0x37E58000, 0x37E60000, 0x37E68000, - 0x37E70000, 0x37E78000, 0x37E80000, 0x37E88000, 0x37E90000, 0x37E98000, 0x37EA0000, - 0x37EA8000, 0x37EB0000, 0x37EB8000, 0x37EC0000, 0x37EC8000, 0x37ED0000, 0x37ED8000, - 0x37EE0000, 0x37EE8000, 0x37EF0000, 0x37EF8000, 0x37F00000, 0x37F08000, 0x37F10000, - 0x37F18000, 0x37F20000, 0x37F28000, 0x37F30000, 0x37F38000, 0x37F40000, 0x37F48000, - 0x37F50000, 0x37F58000, 0x37F60000, 0x37F68000, 0x37F70000, 0x37F78000, 0x37F80000, - 0x37F88000, 0x37F90000, 0x37F98000, 0x37FA0000, 0x37FA8000, 0x37FB0000, 0x37FB8000, - 0x37FC0000, 0x37FC8000, 0x37FD0000, 0x37FD8000, 0x37FE0000, 0x37FE8000, 0x37FF0000, - 0x37FF8000, 0x38000000, 0x38004000, 0x38008000, 0x3800C000, 0x38010000, 0x38014000, - 0x38018000, 0x3801C000, 0x38020000, 0x38024000, 0x38028000, 0x3802C000, 0x38030000, - 0x38034000, 0x38038000, 0x3803C000, 0x38040000, 0x38044000, 0x38048000, 0x3804C000, - 0x38050000, 0x38054000, 0x38058000, 0x3805C000, 0x38060000, 0x38064000, 0x38068000, - 0x3806C000, 0x38070000, 0x38074000, 0x38078000, 0x3807C000, 0x38080000, 0x38084000, - 0x38088000, 0x3808C000, 0x38090000, 0x38094000, 0x38098000, 0x3809C000, 0x380A0000, - 0x380A4000, 0x380A8000, 0x380AC000, 0x380B0000, 0x380B4000, 0x380B8000, 0x380BC000, - 0x380C0000, 0x380C4000, 0x380C8000, 0x380CC000, 0x380D0000, 0x380D4000, 0x380D8000, - 0x380DC000, 0x380E0000, 0x380E4000, 0x380E8000, 0x380EC000, 0x380F0000, 0x380F4000, - 0x380F8000, 0x380FC000, 0x38100000, 0x38104000, 0x38108000, 0x3810C000, 0x38110000, - 0x38114000, 0x38118000, 0x3811C000, 0x38120000, 0x38124000, 0x38128000, 0x3812C000, - 0x38130000, 0x38134000, 0x38138000, 0x3813C000, 0x38140000, 0x38144000, 0x38148000, - 0x3814C000, 0x38150000, 0x38154000, 0x38158000, 0x3815C000, 0x38160000, 0x38164000, - 0x38168000, 0x3816C000, 0x38170000, 0x38174000, 0x38178000, 0x3817C000, 0x38180000, - 0x38184000, 0x38188000, 0x3818C000, 0x38190000, 0x38194000, 0x38198000, 0x3819C000, - 0x381A0000, 0x381A4000, 0x381A8000, 0x381AC000, 0x381B0000, 0x381B4000, 0x381B8000, - 0x381BC000, 0x381C0000, 0x381C4000, 0x381C8000, 0x381CC000, 0x381D0000, 0x381D4000, - 0x381D8000, 0x381DC000, 0x381E0000, 0x381E4000, 0x381E8000, 0x381EC000, 0x381F0000, - 0x381F4000, 0x381F8000, 0x381FC000, 0x38200000, 0x38204000, 0x38208000, 0x3820C000, - 0x38210000, 0x38214000, 0x38218000, 0x3821C000, 0x38220000, 0x38224000, 0x38228000, - 0x3822C000, 0x38230000, 0x38234000, 0x38238000, 0x3823C000, 0x38240000, 0x38244000, - 0x38248000, 0x3824C000, 0x38250000, 0x38254000, 0x38258000, 0x3825C000, 0x38260000, - 0x38264000, 0x38268000, 0x3826C000, 0x38270000, 0x38274000, 0x38278000, 0x3827C000, - 0x38280000, 0x38284000, 0x38288000, 0x3828C000, 0x38290000, 0x38294000, 0x38298000, - 0x3829C000, 0x382A0000, 0x382A4000, 0x382A8000, 0x382AC000, 0x382B0000, 0x382B4000, - 0x382B8000, 0x382BC000, 0x382C0000, 0x382C4000, 0x382C8000, 0x382CC000, 0x382D0000, - 0x382D4000, 0x382D8000, 0x382DC000, 0x382E0000, 0x382E4000, 0x382E8000, 0x382EC000, - 0x382F0000, 0x382F4000, 0x382F8000, 0x382FC000, 0x38300000, 0x38304000, 0x38308000, - 0x3830C000, 0x38310000, 0x38314000, 0x38318000, 0x3831C000, 0x38320000, 0x38324000, - 0x38328000, 0x3832C000, 0x38330000, 0x38334000, 0x38338000, 0x3833C000, 0x38340000, - 0x38344000, 0x38348000, 0x3834C000, 0x38350000, 0x38354000, 0x38358000, 0x3835C000, - 0x38360000, 0x38364000, 0x38368000, 0x3836C000, 0x38370000, 0x38374000, 0x38378000, - 0x3837C000, 0x38380000, 0x38384000, 0x38388000, 0x3838C000, 0x38390000, 0x38394000, - 0x38398000, 0x3839C000, 0x383A0000, 0x383A4000, 0x383A8000, 0x383AC000, 0x383B0000, - 0x383B4000, 0x383B8000, 0x383BC000, 0x383C0000, 0x383C4000, 0x383C8000, 0x383CC000, - 0x383D0000, 0x383D4000, 0x383D8000, 0x383DC000, 0x383E0000, 0x383E4000, 0x383E8000, - 0x383EC000, 0x383F0000, 0x383F4000, 0x383F8000, 0x383FC000, 0x38400000, 0x38404000, - 0x38408000, 0x3840C000, 0x38410000, 0x38414000, 0x38418000, 0x3841C000, 0x38420000, - 0x38424000, 0x38428000, 0x3842C000, 0x38430000, 0x38434000, 0x38438000, 0x3843C000, - 0x38440000, 0x38444000, 0x38448000, 0x3844C000, 0x38450000, 0x38454000, 0x38458000, - 0x3845C000, 0x38460000, 0x38464000, 0x38468000, 0x3846C000, 0x38470000, 0x38474000, - 0x38478000, 0x3847C000, 0x38480000, 0x38484000, 0x38488000, 0x3848C000, 0x38490000, - 0x38494000, 0x38498000, 0x3849C000, 0x384A0000, 0x384A4000, 0x384A8000, 0x384AC000, - 0x384B0000, 0x384B4000, 0x384B8000, 0x384BC000, 0x384C0000, 0x384C4000, 0x384C8000, - 0x384CC000, 0x384D0000, 0x384D4000, 0x384D8000, 0x384DC000, 0x384E0000, 0x384E4000, - 0x384E8000, 0x384EC000, 0x384F0000, 0x384F4000, 0x384F8000, 0x384FC000, 0x38500000, - 0x38504000, 0x38508000, 0x3850C000, 0x38510000, 0x38514000, 0x38518000, 0x3851C000, - 0x38520000, 0x38524000, 0x38528000, 0x3852C000, 0x38530000, 0x38534000, 0x38538000, - 0x3853C000, 0x38540000, 0x38544000, 0x38548000, 0x3854C000, 0x38550000, 0x38554000, - 0x38558000, 0x3855C000, 0x38560000, 0x38564000, 0x38568000, 0x3856C000, 0x38570000, - 0x38574000, 0x38578000, 0x3857C000, 0x38580000, 0x38584000, 0x38588000, 0x3858C000, - 0x38590000, 0x38594000, 0x38598000, 0x3859C000, 0x385A0000, 0x385A4000, 0x385A8000, - 0x385AC000, 0x385B0000, 0x385B4000, 0x385B8000, 0x385BC000, 0x385C0000, 0x385C4000, - 0x385C8000, 0x385CC000, 0x385D0000, 0x385D4000, 0x385D8000, 0x385DC000, 0x385E0000, - 0x385E4000, 0x385E8000, 0x385EC000, 0x385F0000, 0x385F4000, 0x385F8000, 0x385FC000, - 0x38600000, 0x38604000, 0x38608000, 0x3860C000, 0x38610000, 0x38614000, 0x38618000, - 0x3861C000, 0x38620000, 0x38624000, 0x38628000, 0x3862C000, 0x38630000, 0x38634000, - 0x38638000, 0x3863C000, 0x38640000, 0x38644000, 0x38648000, 0x3864C000, 0x38650000, - 0x38654000, 0x38658000, 0x3865C000, 0x38660000, 0x38664000, 0x38668000, 0x3866C000, - 0x38670000, 0x38674000, 0x38678000, 0x3867C000, 0x38680000, 0x38684000, 0x38688000, - 0x3868C000, 0x38690000, 0x38694000, 0x38698000, 0x3869C000, 0x386A0000, 0x386A4000, - 0x386A8000, 0x386AC000, 0x386B0000, 0x386B4000, 0x386B8000, 0x386BC000, 0x386C0000, - 0x386C4000, 0x386C8000, 0x386CC000, 0x386D0000, 0x386D4000, 0x386D8000, 0x386DC000, - 0x386E0000, 0x386E4000, 0x386E8000, 0x386EC000, 0x386F0000, 0x386F4000, 0x386F8000, - 0x386FC000, 0x38700000, 0x38704000, 0x38708000, 0x3870C000, 0x38710000, 0x38714000, - 0x38718000, 0x3871C000, 0x38720000, 0x38724000, 0x38728000, 0x3872C000, 0x38730000, - 0x38734000, 0x38738000, 0x3873C000, 0x38740000, 0x38744000, 0x38748000, 0x3874C000, - 0x38750000, 0x38754000, 0x38758000, 0x3875C000, 0x38760000, 0x38764000, 0x38768000, - 0x3876C000, 0x38770000, 0x38774000, 0x38778000, 0x3877C000, 0x38780000, 0x38784000, - 0x38788000, 0x3878C000, 0x38790000, 0x38794000, 0x38798000, 0x3879C000, 0x387A0000, - 0x387A4000, 0x387A8000, 0x387AC000, 0x387B0000, 0x387B4000, 0x387B8000, 0x387BC000, - 0x387C0000, 0x387C4000, 0x387C8000, 0x387CC000, 0x387D0000, 0x387D4000, 0x387D8000, - 0x387DC000, 0x387E0000, 0x387E4000, 0x387E8000, 0x387EC000, 0x387F0000, 0x387F4000, - 0x387F8000, 0x387FC000, 0x38000000, 0x38002000, 0x38004000, 0x38006000, 0x38008000, - 0x3800A000, 0x3800C000, 0x3800E000, 0x38010000, 0x38012000, 0x38014000, 0x38016000, - 0x38018000, 0x3801A000, 0x3801C000, 0x3801E000, 0x38020000, 0x38022000, 0x38024000, - 0x38026000, 0x38028000, 0x3802A000, 0x3802C000, 0x3802E000, 0x38030000, 0x38032000, - 0x38034000, 0x38036000, 0x38038000, 0x3803A000, 0x3803C000, 0x3803E000, 0x38040000, - 0x38042000, 0x38044000, 0x38046000, 0x38048000, 0x3804A000, 0x3804C000, 0x3804E000, - 0x38050000, 0x38052000, 0x38054000, 0x38056000, 0x38058000, 0x3805A000, 0x3805C000, - 0x3805E000, 0x38060000, 0x38062000, 0x38064000, 0x38066000, 0x38068000, 0x3806A000, - 0x3806C000, 0x3806E000, 0x38070000, 0x38072000, 0x38074000, 0x38076000, 0x38078000, - 0x3807A000, 0x3807C000, 0x3807E000, 0x38080000, 0x38082000, 0x38084000, 0x38086000, - 0x38088000, 0x3808A000, 0x3808C000, 0x3808E000, 0x38090000, 0x38092000, 0x38094000, - 0x38096000, 0x38098000, 0x3809A000, 0x3809C000, 0x3809E000, 0x380A0000, 0x380A2000, - 0x380A4000, 0x380A6000, 0x380A8000, 0x380AA000, 0x380AC000, 0x380AE000, 0x380B0000, - 0x380B2000, 0x380B4000, 0x380B6000, 0x380B8000, 0x380BA000, 0x380BC000, 0x380BE000, - 0x380C0000, 0x380C2000, 0x380C4000, 0x380C6000, 0x380C8000, 0x380CA000, 0x380CC000, - 0x380CE000, 0x380D0000, 0x380D2000, 0x380D4000, 0x380D6000, 0x380D8000, 0x380DA000, - 0x380DC000, 0x380DE000, 0x380E0000, 0x380E2000, 0x380E4000, 0x380E6000, 0x380E8000, - 0x380EA000, 0x380EC000, 0x380EE000, 0x380F0000, 0x380F2000, 0x380F4000, 0x380F6000, - 0x380F8000, 0x380FA000, 0x380FC000, 0x380FE000, 0x38100000, 0x38102000, 0x38104000, - 0x38106000, 0x38108000, 0x3810A000, 0x3810C000, 0x3810E000, 0x38110000, 0x38112000, - 0x38114000, 0x38116000, 0x38118000, 0x3811A000, 0x3811C000, 0x3811E000, 0x38120000, - 0x38122000, 0x38124000, 0x38126000, 0x38128000, 0x3812A000, 0x3812C000, 0x3812E000, - 0x38130000, 0x38132000, 0x38134000, 0x38136000, 0x38138000, 0x3813A000, 0x3813C000, - 0x3813E000, 0x38140000, 0x38142000, 0x38144000, 0x38146000, 0x38148000, 0x3814A000, - 0x3814C000, 0x3814E000, 0x38150000, 0x38152000, 0x38154000, 0x38156000, 0x38158000, - 0x3815A000, 0x3815C000, 0x3815E000, 0x38160000, 0x38162000, 0x38164000, 0x38166000, - 0x38168000, 0x3816A000, 0x3816C000, 0x3816E000, 0x38170000, 0x38172000, 0x38174000, - 0x38176000, 0x38178000, 0x3817A000, 0x3817C000, 0x3817E000, 0x38180000, 0x38182000, - 0x38184000, 0x38186000, 0x38188000, 0x3818A000, 0x3818C000, 0x3818E000, 0x38190000, - 0x38192000, 0x38194000, 0x38196000, 0x38198000, 0x3819A000, 0x3819C000, 0x3819E000, - 0x381A0000, 0x381A2000, 0x381A4000, 0x381A6000, 0x381A8000, 0x381AA000, 0x381AC000, - 0x381AE000, 0x381B0000, 0x381B2000, 0x381B4000, 0x381B6000, 0x381B8000, 0x381BA000, - 0x381BC000, 0x381BE000, 0x381C0000, 0x381C2000, 0x381C4000, 0x381C6000, 0x381C8000, - 0x381CA000, 0x381CC000, 0x381CE000, 0x381D0000, 0x381D2000, 0x381D4000, 0x381D6000, - 0x381D8000, 0x381DA000, 0x381DC000, 0x381DE000, 0x381E0000, 0x381E2000, 0x381E4000, - 0x381E6000, 0x381E8000, 0x381EA000, 0x381EC000, 0x381EE000, 0x381F0000, 0x381F2000, - 0x381F4000, 0x381F6000, 0x381F8000, 0x381FA000, 0x381FC000, 0x381FE000, 0x38200000, - 0x38202000, 0x38204000, 0x38206000, 0x38208000, 0x3820A000, 0x3820C000, 0x3820E000, - 0x38210000, 0x38212000, 0x38214000, 0x38216000, 0x38218000, 0x3821A000, 0x3821C000, - 0x3821E000, 0x38220000, 0x38222000, 0x38224000, 0x38226000, 0x38228000, 0x3822A000, - 0x3822C000, 0x3822E000, 0x38230000, 0x38232000, 0x38234000, 0x38236000, 0x38238000, - 0x3823A000, 0x3823C000, 0x3823E000, 0x38240000, 0x38242000, 0x38244000, 0x38246000, - 0x38248000, 0x3824A000, 0x3824C000, 0x3824E000, 0x38250000, 0x38252000, 0x38254000, - 0x38256000, 0x38258000, 0x3825A000, 0x3825C000, 0x3825E000, 0x38260000, 0x38262000, - 0x38264000, 0x38266000, 0x38268000, 0x3826A000, 0x3826C000, 0x3826E000, 0x38270000, - 0x38272000, 0x38274000, 0x38276000, 0x38278000, 0x3827A000, 0x3827C000, 0x3827E000, - 0x38280000, 0x38282000, 0x38284000, 0x38286000, 0x38288000, 0x3828A000, 0x3828C000, - 0x3828E000, 0x38290000, 0x38292000, 0x38294000, 0x38296000, 0x38298000, 0x3829A000, - 0x3829C000, 0x3829E000, 0x382A0000, 0x382A2000, 0x382A4000, 0x382A6000, 0x382A8000, - 0x382AA000, 0x382AC000, 0x382AE000, 0x382B0000, 0x382B2000, 0x382B4000, 0x382B6000, - 0x382B8000, 0x382BA000, 0x382BC000, 0x382BE000, 0x382C0000, 0x382C2000, 0x382C4000, - 0x382C6000, 0x382C8000, 0x382CA000, 0x382CC000, 0x382CE000, 0x382D0000, 0x382D2000, - 0x382D4000, 0x382D6000, 0x382D8000, 0x382DA000, 0x382DC000, 0x382DE000, 0x382E0000, - 0x382E2000, 0x382E4000, 0x382E6000, 0x382E8000, 0x382EA000, 0x382EC000, 0x382EE000, - 0x382F0000, 0x382F2000, 0x382F4000, 0x382F6000, 0x382F8000, 0x382FA000, 0x382FC000, - 0x382FE000, 0x38300000, 0x38302000, 0x38304000, 0x38306000, 0x38308000, 0x3830A000, - 0x3830C000, 0x3830E000, 0x38310000, 0x38312000, 0x38314000, 0x38316000, 0x38318000, - 0x3831A000, 0x3831C000, 0x3831E000, 0x38320000, 0x38322000, 0x38324000, 0x38326000, - 0x38328000, 0x3832A000, 0x3832C000, 0x3832E000, 0x38330000, 0x38332000, 0x38334000, - 0x38336000, 0x38338000, 0x3833A000, 0x3833C000, 0x3833E000, 0x38340000, 0x38342000, - 0x38344000, 0x38346000, 0x38348000, 0x3834A000, 0x3834C000, 0x3834E000, 0x38350000, - 0x38352000, 0x38354000, 0x38356000, 0x38358000, 0x3835A000, 0x3835C000, 0x3835E000, - 0x38360000, 0x38362000, 0x38364000, 0x38366000, 0x38368000, 0x3836A000, 0x3836C000, - 0x3836E000, 0x38370000, 0x38372000, 0x38374000, 0x38376000, 0x38378000, 0x3837A000, - 0x3837C000, 0x3837E000, 0x38380000, 0x38382000, 0x38384000, 0x38386000, 0x38388000, - 0x3838A000, 0x3838C000, 0x3838E000, 0x38390000, 0x38392000, 0x38394000, 0x38396000, - 0x38398000, 0x3839A000, 0x3839C000, 0x3839E000, 0x383A0000, 0x383A2000, 0x383A4000, - 0x383A6000, 0x383A8000, 0x383AA000, 0x383AC000, 0x383AE000, 0x383B0000, 0x383B2000, - 0x383B4000, 0x383B6000, 0x383B8000, 0x383BA000, 0x383BC000, 0x383BE000, 0x383C0000, - 0x383C2000, 0x383C4000, 0x383C6000, 0x383C8000, 0x383CA000, 0x383CC000, 0x383CE000, - 0x383D0000, 0x383D2000, 0x383D4000, 0x383D6000, 0x383D8000, 0x383DA000, 0x383DC000, - 0x383DE000, 0x383E0000, 0x383E2000, 0x383E4000, 0x383E6000, 0x383E8000, 0x383EA000, - 0x383EC000, 0x383EE000, 0x383F0000, 0x383F2000, 0x383F4000, 0x383F6000, 0x383F8000, - 0x383FA000, 0x383FC000, 0x383FE000, 0x38400000, 0x38402000, 0x38404000, 0x38406000, - 0x38408000, 0x3840A000, 0x3840C000, 0x3840E000, 0x38410000, 0x38412000, 0x38414000, - 0x38416000, 0x38418000, 0x3841A000, 0x3841C000, 0x3841E000, 0x38420000, 0x38422000, - 0x38424000, 0x38426000, 0x38428000, 0x3842A000, 0x3842C000, 0x3842E000, 0x38430000, - 0x38432000, 0x38434000, 0x38436000, 0x38438000, 0x3843A000, 0x3843C000, 0x3843E000, - 0x38440000, 0x38442000, 0x38444000, 0x38446000, 0x38448000, 0x3844A000, 0x3844C000, - 0x3844E000, 0x38450000, 0x38452000, 0x38454000, 0x38456000, 0x38458000, 0x3845A000, - 0x3845C000, 0x3845E000, 0x38460000, 0x38462000, 0x38464000, 0x38466000, 0x38468000, - 0x3846A000, 0x3846C000, 0x3846E000, 0x38470000, 0x38472000, 0x38474000, 0x38476000, - 0x38478000, 0x3847A000, 0x3847C000, 0x3847E000, 0x38480000, 0x38482000, 0x38484000, - 0x38486000, 0x38488000, 0x3848A000, 0x3848C000, 0x3848E000, 0x38490000, 0x38492000, - 0x38494000, 0x38496000, 0x38498000, 0x3849A000, 0x3849C000, 0x3849E000, 0x384A0000, - 0x384A2000, 0x384A4000, 0x384A6000, 0x384A8000, 0x384AA000, 0x384AC000, 0x384AE000, - 0x384B0000, 0x384B2000, 0x384B4000, 0x384B6000, 0x384B8000, 0x384BA000, 0x384BC000, - 0x384BE000, 0x384C0000, 0x384C2000, 0x384C4000, 0x384C6000, 0x384C8000, 0x384CA000, - 0x384CC000, 0x384CE000, 0x384D0000, 0x384D2000, 0x384D4000, 0x384D6000, 0x384D8000, - 0x384DA000, 0x384DC000, 0x384DE000, 0x384E0000, 0x384E2000, 0x384E4000, 0x384E6000, - 0x384E8000, 0x384EA000, 0x384EC000, 0x384EE000, 0x384F0000, 0x384F2000, 0x384F4000, - 0x384F6000, 0x384F8000, 0x384FA000, 0x384FC000, 0x384FE000, 0x38500000, 0x38502000, - 0x38504000, 0x38506000, 0x38508000, 0x3850A000, 0x3850C000, 0x3850E000, 0x38510000, - 0x38512000, 0x38514000, 0x38516000, 0x38518000, 0x3851A000, 0x3851C000, 0x3851E000, - 0x38520000, 0x38522000, 0x38524000, 0x38526000, 0x38528000, 0x3852A000, 0x3852C000, - 0x3852E000, 0x38530000, 0x38532000, 0x38534000, 0x38536000, 0x38538000, 0x3853A000, - 0x3853C000, 0x3853E000, 0x38540000, 0x38542000, 0x38544000, 0x38546000, 0x38548000, - 0x3854A000, 0x3854C000, 0x3854E000, 0x38550000, 0x38552000, 0x38554000, 0x38556000, - 0x38558000, 0x3855A000, 0x3855C000, 0x3855E000, 0x38560000, 0x38562000, 0x38564000, - 0x38566000, 0x38568000, 0x3856A000, 0x3856C000, 0x3856E000, 0x38570000, 0x38572000, - 0x38574000, 0x38576000, 0x38578000, 0x3857A000, 0x3857C000, 0x3857E000, 0x38580000, - 0x38582000, 0x38584000, 0x38586000, 0x38588000, 0x3858A000, 0x3858C000, 0x3858E000, - 0x38590000, 0x38592000, 0x38594000, 0x38596000, 0x38598000, 0x3859A000, 0x3859C000, - 0x3859E000, 0x385A0000, 0x385A2000, 0x385A4000, 0x385A6000, 0x385A8000, 0x385AA000, - 0x385AC000, 0x385AE000, 0x385B0000, 0x385B2000, 0x385B4000, 0x385B6000, 0x385B8000, - 0x385BA000, 0x385BC000, 0x385BE000, 0x385C0000, 0x385C2000, 0x385C4000, 0x385C6000, - 0x385C8000, 0x385CA000, 0x385CC000, 0x385CE000, 0x385D0000, 0x385D2000, 0x385D4000, - 0x385D6000, 0x385D8000, 0x385DA000, 0x385DC000, 0x385DE000, 0x385E0000, 0x385E2000, - 0x385E4000, 0x385E6000, 0x385E8000, 0x385EA000, 0x385EC000, 0x385EE000, 0x385F0000, - 0x385F2000, 0x385F4000, 0x385F6000, 0x385F8000, 0x385FA000, 0x385FC000, 0x385FE000, - 0x38600000, 0x38602000, 0x38604000, 0x38606000, 0x38608000, 0x3860A000, 0x3860C000, - 0x3860E000, 0x38610000, 0x38612000, 0x38614000, 0x38616000, 0x38618000, 0x3861A000, - 0x3861C000, 0x3861E000, 0x38620000, 0x38622000, 0x38624000, 0x38626000, 0x38628000, - 0x3862A000, 0x3862C000, 0x3862E000, 0x38630000, 0x38632000, 0x38634000, 0x38636000, - 0x38638000, 0x3863A000, 0x3863C000, 0x3863E000, 0x38640000, 0x38642000, 0x38644000, - 0x38646000, 0x38648000, 0x3864A000, 0x3864C000, 0x3864E000, 0x38650000, 0x38652000, - 0x38654000, 0x38656000, 0x38658000, 0x3865A000, 0x3865C000, 0x3865E000, 0x38660000, - 0x38662000, 0x38664000, 0x38666000, 0x38668000, 0x3866A000, 0x3866C000, 0x3866E000, - 0x38670000, 0x38672000, 0x38674000, 0x38676000, 0x38678000, 0x3867A000, 0x3867C000, - 0x3867E000, 0x38680000, 0x38682000, 0x38684000, 0x38686000, 0x38688000, 0x3868A000, - 0x3868C000, 0x3868E000, 0x38690000, 0x38692000, 0x38694000, 0x38696000, 0x38698000, - 0x3869A000, 0x3869C000, 0x3869E000, 0x386A0000, 0x386A2000, 0x386A4000, 0x386A6000, - 0x386A8000, 0x386AA000, 0x386AC000, 0x386AE000, 0x386B0000, 0x386B2000, 0x386B4000, - 0x386B6000, 0x386B8000, 0x386BA000, 0x386BC000, 0x386BE000, 0x386C0000, 0x386C2000, - 0x386C4000, 0x386C6000, 0x386C8000, 0x386CA000, 0x386CC000, 0x386CE000, 0x386D0000, - 0x386D2000, 0x386D4000, 0x386D6000, 0x386D8000, 0x386DA000, 0x386DC000, 0x386DE000, - 0x386E0000, 0x386E2000, 0x386E4000, 0x386E6000, 0x386E8000, 0x386EA000, 0x386EC000, - 0x386EE000, 0x386F0000, 0x386F2000, 0x386F4000, 0x386F6000, 0x386F8000, 0x386FA000, - 0x386FC000, 0x386FE000, 0x38700000, 0x38702000, 0x38704000, 0x38706000, 0x38708000, - 0x3870A000, 0x3870C000, 0x3870E000, 0x38710000, 0x38712000, 0x38714000, 0x38716000, - 0x38718000, 0x3871A000, 0x3871C000, 0x3871E000, 0x38720000, 0x38722000, 0x38724000, - 0x38726000, 0x38728000, 0x3872A000, 0x3872C000, 0x3872E000, 0x38730000, 0x38732000, - 0x38734000, 0x38736000, 0x38738000, 0x3873A000, 0x3873C000, 0x3873E000, 0x38740000, - 0x38742000, 0x38744000, 0x38746000, 0x38748000, 0x3874A000, 0x3874C000, 0x3874E000, - 0x38750000, 0x38752000, 0x38754000, 0x38756000, 0x38758000, 0x3875A000, 0x3875C000, - 0x3875E000, 0x38760000, 0x38762000, 0x38764000, 0x38766000, 0x38768000, 0x3876A000, - 0x3876C000, 0x3876E000, 0x38770000, 0x38772000, 0x38774000, 0x38776000, 0x38778000, - 0x3877A000, 0x3877C000, 0x3877E000, 0x38780000, 0x38782000, 0x38784000, 0x38786000, - 0x38788000, 0x3878A000, 0x3878C000, 0x3878E000, 0x38790000, 0x38792000, 0x38794000, - 0x38796000, 0x38798000, 0x3879A000, 0x3879C000, 0x3879E000, 0x387A0000, 0x387A2000, - 0x387A4000, 0x387A6000, 0x387A8000, 0x387AA000, 0x387AC000, 0x387AE000, 0x387B0000, - 0x387B2000, 0x387B4000, 0x387B6000, 0x387B8000, 0x387BA000, 0x387BC000, 0x387BE000, - 0x387C0000, 0x387C2000, 0x387C4000, 0x387C6000, 0x387C8000, 0x387CA000, 0x387CC000, - 0x387CE000, 0x387D0000, 0x387D2000, 0x387D4000, 0x387D6000, 0x387D8000, 0x387DA000, - 0x387DC000, 0x387DE000, 0x387E0000, 0x387E2000, 0x387E4000, 0x387E6000, 0x387E8000, - 0x387EA000, 0x387EC000, 0x387EE000, 0x387F0000, 0x387F2000, 0x387F4000, 0x387F6000, - 0x387F8000, 0x387FA000, 0x387FC000, 0x387FE000}; - static const bits::type exponent_table[64] = { - 0x00000000, 0x00800000, 0x01000000, 0x01800000, 0x02000000, 0x02800000, 0x03000000, - 0x03800000, 0x04000000, 0x04800000, 0x05000000, 0x05800000, 0x06000000, 0x06800000, - 0x07000000, 0x07800000, 0x08000000, 0x08800000, 0x09000000, 0x09800000, 0x0A000000, - 0x0A800000, 0x0B000000, 0x0B800000, 0x0C000000, 0x0C800000, 0x0D000000, 0x0D800000, - 0x0E000000, 0x0E800000, 0x0F000000, 0x47800000, 0x80000000, 0x80800000, 0x81000000, - 0x81800000, 0x82000000, 0x82800000, 0x83000000, 0x83800000, 0x84000000, 0x84800000, - 0x85000000, 0x85800000, 0x86000000, 0x86800000, 0x87000000, 0x87800000, 0x88000000, - 0x88800000, 0x89000000, 0x89800000, 0x8A000000, 0x8A800000, 0x8B000000, 0x8B800000, - 0x8C000000, 0x8C800000, 0x8D000000, 0x8D800000, 0x8E000000, 0x8E800000, 0x8F000000, - 0xC7800000}; - static const unsigned short offset_table[64] = { - 0, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, - 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, - 1024, 1024, 1024, 1024, 1024, 1024, 0, 1024, 1024, 1024, 1024, 1024, 1024, - 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, - 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024}; - bits::type fbits = - mantissa_table[offset_table[value >> 10] + (value & 0x3FF)] + exponent_table[value >> 10]; -#endif - float out; - std::memcpy(&out, &fbits, sizeof(float)); - return out; -#endif -} - -/// Convert half-precision to IEEE double-precision. -/// \param value half-precision value to convert -/// \return double-precision value -inline double half2float_impl(unsigned int value, double, true_type) -{ -#if HALF_ENABLE_F16C_INTRINSICS - return _mm_cvtsd_f64(_mm_cvtps_pd(_mm_cvtph_ps(_mm_cvtsi32_si128(value)))); -#else - uint32 hi = static_cast(value & 0x8000) << 16; - unsigned int abs = value & 0x7FFF; - if(abs) - { - hi |= 0x3F000000 << static_cast(abs >= 0x7C00); - for(; abs < 0x400; abs <<= 1, hi -= 0x100000) - ; - hi += static_cast(abs) << 10; - } - bits::type dbits = static_cast::type>(hi) << 32; - double out; - std::memcpy(&out, &dbits, sizeof(double)); - return out; -#endif -} - -/// Convert half-precision to non-IEEE floating-point. -/// \tparam T type to convert to (builtin integer type) -/// \param value half-precision value to convert -/// \return floating-point value -template -T half2float_impl(unsigned int value, T, ...) -{ - T out; - unsigned int abs = value & 0x7FFF; - if(abs > 0x7C00) - out = - (std::numeric_limits::has_signaling_NaN && !(abs & 0x200)) - ? std::numeric_limits::signaling_NaN() - : std::numeric_limits::has_quiet_NaN ? std::numeric_limits::quiet_NaN() : T(); - else if(abs == 0x7C00) - out = std::numeric_limits::has_infinity ? std::numeric_limits::infinity() - : std::numeric_limits::max(); - else if(abs > 0x3FF) - out = std::ldexp(static_cast((abs & 0x3FF) | 0x400), (abs >> 10) - 25); - else - out = std::ldexp(static_cast(abs), -24); - return (value & 0x8000) ? -out : out; -} - -/// Convert half-precision to floating-point. -/// \tparam T type to convert to (builtin integer type) -/// \param value half-precision value to convert -/// \return floating-point value -template -T half2float(unsigned int value) -{ - return half2float_impl(value, - T(), - bool_type < std::numeric_limits::is_iec559 && - sizeof(typename bits::type) == sizeof(T) > ()); -} - -/// Convert half-precision floating-point to integer. -/// \tparam R rounding mode to use -/// \tparam E `true` for round to even, `false` for round away from zero -/// \tparam I `true` to raise INEXACT exception (if inexact), `false` to never raise it -/// \tparam T type to convert to (buitlin integer type with at least 16 bits precision, excluding -/// any implicit sign bits) -/// \param value half-precision value to convert -/// \return rounded integer value -/// \exception FE_INVALID if value is not representable in type \a T -/// \exception FE_INEXACT if value had to be rounded and \a I is `true` -template -T half2int(unsigned int value) -{ - unsigned int abs = value & 0x7FFF; - if(abs >= 0x7C00) - { - raise(FE_INVALID); - return (value & 0x8000) ? std::numeric_limits::min() : std::numeric_limits::max(); - } - if(abs < 0x3800) - { - raise(FE_INEXACT, I); - return (R == std::round_toward_infinity) - ? T(~(value >> 15) & (abs != 0)) - : (R == std::round_toward_neg_infinity) ? -T(value > 0x8000) : T(); - } - int exp = 25 - (abs >> 10); - unsigned int m = (value & 0x3FF) | 0x400; - int32 i = static_cast( - (exp <= 0) - ? (m << -exp) - : ((m + ((R == std::round_to_nearest) ? ((1 << (exp - 1)) - (~(m >> exp) & E)) - : (R == std::round_toward_infinity) - ? (((1 << exp) - 1) & ((value >> 15) - 1)) - : (R == std::round_toward_neg_infinity) - ? (((1 << exp) - 1) & -(value >> 15)) - : 0)) >> - exp)); - if((!std::numeric_limits::is_signed && (value & 0x8000)) || - (std::numeric_limits::digits < 16 && - ((value & 0x8000) ? (-i < std::numeric_limits::min()) - : (i > std::numeric_limits::max())))) - raise(FE_INVALID); - else if(I && exp > 0 && (m & ((1 << exp) - 1))) - raise(FE_INEXACT); - return static_cast((value & 0x8000) ? -i : i); -} - -/// \} -/// \name Mathematics -/// \{ - -/// upper part of 64-bit multiplication. -/// \tparam R rounding mode to use -/// \param x first factor -/// \param y second factor -/// \return upper 32 bit of \a x * \a y -template -uint32 mulhi(uint32 x, uint32 y) -{ - uint32 xy = (x >> 16) * (y & 0xFFFF), yx = (x & 0xFFFF) * (y >> 16), - c = (xy & 0xFFFF) + (yx & 0xFFFF) + (((x & 0xFFFF) * (y & 0xFFFF)) >> 16); - return (x >> 16) * (y >> 16) + (xy >> 16) + (yx >> 16) + (c >> 16) + - ((R == std::round_to_nearest) - ? ((c >> 15) & 1) - : (R == std::round_toward_infinity) ? ((c & 0xFFFF) != 0) : 0); -} - -/// 64-bit multiplication. -/// \param x first factor -/// \param y second factor -/// \return upper 32 bit of \a x * \a y rounded to nearest -inline uint32 multiply64(uint32 x, uint32 y) -{ -#if HALF_ENABLE_CPP11_LONG_LONG - return static_cast( - (static_cast(x) * static_cast(y) + 0x80000000) >> - 32); -#else - return mulhi(x, y); -#endif -} - -/// 64-bit division. -/// \param x upper 32 bit of dividend -/// \param y divisor -/// \param s variable to store sticky bit for rounding -/// \return (\a x << 32) / \a y -inline uint32 divide64(uint32 x, uint32 y, int& s) -{ -#if HALF_ENABLE_CPP11_LONG_LONG - unsigned long long xx = static_cast(x) << 32; - return s = (xx % y != 0), static_cast(xx / y); -#else - y >>= 1; - uint32 rem = x, div = 0; - for(unsigned int i = 0; i < 32; ++i) - { - div <<= 1; - if(rem >= y) - { - rem -= y; - div |= 1; - } - rem <<= 1; - } - return s = rem > 1, div; -#endif -} - -/// Half precision positive modulus. -/// \tparam Q `true` to compute full quotient, `false` else -/// \tparam R `true` to compute signed remainder, `false` for positive remainder -/// \param x first operand as positive finite half-precision value -/// \param y second operand as positive finite half-precision value -/// \param quo adress to store quotient at, `nullptr` if \a Q `false` -/// \return modulus of \a x / \a y -template -unsigned int mod(unsigned int x, unsigned int y, int* quo = NULL) -{ - unsigned int q = 0; - if(x > y) - { - int absx = x, absy = y, expx = 0, expy = 0; - for(; absx < 0x400; absx <<= 1, --expx) - ; - for(; absy < 0x400; absy <<= 1, --expy) - ; - expx += absx >> 10; - expy += absy >> 10; - int mx = (absx & 0x3FF) | 0x400, my = (absy & 0x3FF) | 0x400; - for(int d = expx - expy; d; --d) - { - if(!Q && mx == my) - return 0; - if(mx >= my) - { - mx -= my; - q += Q; - } - mx <<= 1; - q <<= static_cast(Q); - } - if(!Q && mx == my) - return 0; - if(mx >= my) - { - mx -= my; - ++q; - } - if(Q) - { - q &= (1 << (std::numeric_limits::digits - 1)) - 1; - if(!mx) - return *quo = q, 0; - } - for(; mx < 0x400; mx <<= 1, --expy) - ; - x = (expy > 0) ? ((expy << 10) | (mx & 0x3FF)) : (mx >> (1 - expy)); - } - if(R) - { - unsigned int a, b; - if(y < 0x800) - { - a = (x < 0x400) ? (x << 1) : (x + 0x400); - b = y; - } - else - { - a = x; - b = y - 0x400; - } - if(a > b || (a == b && (q & 1))) - { - int exp = (y >> 10) + (y <= 0x3FF), d = exp - (x >> 10) - (x <= 0x3FF); - int m = (((y & 0x3FF) | ((y > 0x3FF) << 10)) << 1) - - (((x & 0x3FF) | ((x > 0x3FF) << 10)) << (1 - d)); - for(; m < 0x800 && exp > 1; m <<= 1, --exp) - ; - x = 0x8000 + ((exp - 1) << 10) + (m >> 1); - q += Q; - } - } - if(Q) - *quo = q; - return x; -} - -/// Fixed point square root. -/// \tparam F number of fractional bits -/// \param r radicand in Q1.F fixed point format -/// \param exp exponent -/// \return square root as Q1.F/2 -template -uint32 sqrt(uint32& r, int& exp) -{ - int i = exp & 1; - r <<= i; - exp = (exp - i) / 2; - uint32 m = 0; - for(uint32 bit = static_cast(1) << F; bit; bit >>= 2) - { - if(r < m + bit) - m >>= 1; - else - { - r -= m + bit; - m = (m >> 1) + bit; - } - } - return m; -} - -/// Fixed point binary exponential. -/// This uses the BKM algorithm in E-mode. -/// \param m exponent in [0,1) as Q0.31 -/// \param n number of iterations (at most 32) -/// \return 2 ^ \a m as Q1.31 -inline uint32 exp2(uint32 m, unsigned int n = 32) -{ - static const uint32 logs[] = { - 0x80000000, 0x4AE00D1D, 0x2934F098, 0x15C01A3A, 0x0B31FB7D, 0x05AEB4DD, 0x02DCF2D1, - 0x016FE50B, 0x00B84E23, 0x005C3E10, 0x002E24CA, 0x001713D6, 0x000B8A47, 0x0005C53B, - 0x0002E2A3, 0x00017153, 0x0000B8AA, 0x00005C55, 0x00002E2B, 0x00001715, 0x00000B8B, - 0x000005C5, 0x000002E3, 0x00000171, 0x000000B9, 0x0000005C, 0x0000002E, 0x00000017, - 0x0000000C, 0x00000006, 0x00000003, 0x00000001}; - if(!m) - return 0x80000000; - uint32 mx = 0x80000000, my = 0; - for(unsigned int i = 1; i < n; ++i) - { - uint32 mz = my + logs[i]; - if(mz <= m) - { - my = mz; - mx += mx >> i; - } - } - return mx; -} - -/// Fixed point binary logarithm. -/// This uses the BKM algorithm in L-mode. -/// \param m mantissa in [1,2) as Q1.30 -/// \param n number of iterations (at most 32) -/// \return log2(\a m) as Q0.31 -inline uint32 log2(uint32 m, unsigned int n = 32) -{ - static const uint32 logs[] = { - 0x80000000, 0x4AE00D1D, 0x2934F098, 0x15C01A3A, 0x0B31FB7D, 0x05AEB4DD, 0x02DCF2D1, - 0x016FE50B, 0x00B84E23, 0x005C3E10, 0x002E24CA, 0x001713D6, 0x000B8A47, 0x0005C53B, - 0x0002E2A3, 0x00017153, 0x0000B8AA, 0x00005C55, 0x00002E2B, 0x00001715, 0x00000B8B, - 0x000005C5, 0x000002E3, 0x00000171, 0x000000B9, 0x0000005C, 0x0000002E, 0x00000017, - 0x0000000C, 0x00000006, 0x00000003, 0x00000001}; - if(m == 0x40000000) - return 0; - uint32 mx = 0x40000000, my = 0; - for(unsigned int i = 1; i < n; ++i) - { - uint32 mz = mx + (mx >> i); - if(mz <= m) - { - mx = mz; - my += logs[i]; - } - } - return my; -} - -/// Fixed point sine and cosine. -/// This uses the CORDIC algorithm in rotation mode. -/// \param mz angle in [-pi/2,pi/2] as Q1.30 -/// \param n number of iterations (at most 31) -/// \return sine and cosine of \a mz as Q1.30 -inline std::pair sincos(uint32 mz, unsigned int n = 31) -{ - static const uint32 angles[] = { - 0x3243F6A9, 0x1DAC6705, 0x0FADBAFD, 0x07F56EA7, 0x03FEAB77, 0x01FFD55C, 0x00FFFAAB, - 0x007FFF55, 0x003FFFEB, 0x001FFFFD, 0x00100000, 0x00080000, 0x00040000, 0x00020000, - 0x00010000, 0x00008000, 0x00004000, 0x00002000, 0x00001000, 0x00000800, 0x00000400, - 0x00000200, 0x00000100, 0x00000080, 0x00000040, 0x00000020, 0x00000010, 0x00000008, - 0x00000004, 0x00000002, 0x00000001}; - uint32 mx = 0x26DD3B6A, my = 0; - for(unsigned int i = 0; i < n; ++i) - { - uint32 sign = sign_mask(mz); - uint32 tx = mx - (arithmetic_shift(my, i) ^ sign) + sign; - uint32 ty = my + (arithmetic_shift(mx, i) ^ sign) - sign; - mx = tx; - my = ty; - mz -= (angles[i] ^ sign) - sign; - } - return std::make_pair(my, mx); -} - -/// Fixed point arc tangent. -/// This uses the CORDIC algorithm in vectoring mode. -/// \param my y coordinate as Q0.30 -/// \param mx x coordinate as Q0.30 -/// \param n number of iterations (at most 31) -/// \return arc tangent of \a my / \a mx as Q1.30 -inline uint32 atan2(uint32 my, uint32 mx, unsigned int n = 31) -{ - static const uint32 angles[] = { - 0x3243F6A9, 0x1DAC6705, 0x0FADBAFD, 0x07F56EA7, 0x03FEAB77, 0x01FFD55C, 0x00FFFAAB, - 0x007FFF55, 0x003FFFEB, 0x001FFFFD, 0x00100000, 0x00080000, 0x00040000, 0x00020000, - 0x00010000, 0x00008000, 0x00004000, 0x00002000, 0x00001000, 0x00000800, 0x00000400, - 0x00000200, 0x00000100, 0x00000080, 0x00000040, 0x00000020, 0x00000010, 0x00000008, - 0x00000004, 0x00000002, 0x00000001}; - uint32 mz = 0; - for(unsigned int i = 0; i < n; ++i) - { - uint32 sign = sign_mask(my); - uint32 tx = mx + (arithmetic_shift(my, i) ^ sign) - sign; - uint32 ty = my - (arithmetic_shift(mx, i) ^ sign) + sign; - mx = tx; - my = ty; - mz += (angles[i] ^ sign) - sign; - } - return mz; -} - -/// Reduce argument for trigonometric functions. -/// \param abs half-precision floating-point value -/// \param k value to take quarter period -/// \return \a abs reduced to [-pi/4,pi/4] as Q0.30 -inline uint32 angle_arg(unsigned int abs, int& k) -{ - uint32 m = (abs & 0x3FF) | ((abs > 0x3FF) << 10); - int exp = (abs >> 10) + (abs <= 0x3FF) - 15; - if(abs < 0x3A48) - return k = 0, m << (exp + 20); -#if HALF_ENABLE_CPP11_LONG_LONG - unsigned long long y = m * 0xA2F9836E4E442, mask = (1ULL << (62 - exp)) - 1, - yi = (y + (mask >> 1)) & ~mask, f = y - yi; - uint32 sign = -static_cast(f >> 63); - k = static_cast(yi >> (62 - exp)); - return (multiply64(static_cast((sign ? -f : f) >> (31 - exp)), 0xC90FDAA2) ^ sign) - - sign; -#else - uint32 yh = m * 0xA2F98 + mulhi(m, 0x36E4E442), - yl = (m * 0x36E4E442) & 0xFFFFFFFF; - uint32 mask = (static_cast(1) << (30 - exp)) - 1, yi = (yh + (mask >> 1)) & ~mask, - sign = -static_cast(yi > yh); - k = static_cast(yi >> (30 - exp)); - uint32 fh = (yh ^ sign) + (yi ^ ~sign) - ~sign, fl = (yl ^ sign) - sign; - return (multiply64((exp > -1) - ? (((fh << (1 + exp)) & 0xFFFFFFFF) | ((fl & 0xFFFFFFFF) >> (31 - exp))) - : fh, - 0xC90FDAA2) ^ - sign) - - sign; -#endif -} - -/// Get arguments for atan2 function. -/// \param abs half-precision floating-point value -/// \return \a abs and sqrt(1 - \a abs^2) as Q0.30 -inline std::pair atan2_args(unsigned int abs) -{ - int exp = -15; - for(; abs < 0x400; abs <<= 1, --exp) - ; - exp += abs >> 10; - uint32 my = ((abs & 0x3FF) | 0x400) << 5, r = my * my; - int rexp = 2 * exp; - r = 0x40000000 - - ((rexp > -31) ? ((r >> -rexp) | ((r & ((static_cast(1) << -rexp) - 1)) != 0)) : 1); - for(rexp = 0; r < 0x40000000; r <<= 1, --rexp) - ; - uint32 mx = sqrt<30>(r, rexp); - int d = exp - rexp; - if(d < 0) - return std::make_pair((d < -14) ? ((my >> (-d - 14)) + ((my >> (-d - 15)) & 1)) - : (my << (14 + d)), - (mx << 14) + (r << 13) / mx); - if(d > 0) - return std::make_pair(my << 14, - (d > 14) - ? ((mx >> (d - 14)) + ((mx >> (d - 15)) & 1)) - : ((d == 14) ? mx : ((mx << (14 - d)) + (r << (13 - d)) / mx))); - return std::make_pair(my << 13, (mx << 13) + (r << 12) / mx); -} - -/// Get exponentials for hyperbolic computation -/// \param abs half-precision floating-point value -/// \param exp variable to take unbiased exponent of larger result -/// \param n number of BKM iterations (at most 32) -/// \return exp(abs) and exp(-\a abs) as Q1.31 with same exponent -inline std::pair hyperbolic_args(unsigned int abs, int& exp, unsigned int n = 32) -{ - uint32 mx = detail::multiply64(static_cast((abs & 0x3FF) + ((abs > 0x3FF) << 10)) << 21, - 0xB8AA3B29), - my; - int e = (abs >> 10) + (abs <= 0x3FF); - if(e < 14) - { - exp = 0; - mx >>= 14 - e; - } - else - { - exp = mx >> (45 - e); - mx = (mx << (e - 14)) & 0x7FFFFFFF; - } - mx = exp2(mx, n); - int d = exp << 1, s; - if(mx > 0x80000000) - { - my = divide64(0x80000000, mx, s); - my |= s; - ++d; - } - else - my = mx; - return std::make_pair( - mx, (d < 31) ? ((my >> d) | ((my & ((static_cast(1) << d) - 1)) != 0)) : 1); -} - -/// Postprocessing for binary exponential. -/// \tparam R rounding mode to use -/// \tparam I `true` to always raise INEXACT exception, `false` to raise only for rounded results -/// \param m mantissa as Q1.31 -/// \param exp absolute value of unbiased exponent -/// \param esign sign of actual exponent -/// \param sign sign bit of result -/// \return value converted to half-precision -/// \exception FE_OVERFLOW on overflows -/// \exception FE_UNDERFLOW on underflows -/// \exception FE_INEXACT if value had to be rounded or \a I is `true` -template -unsigned int exp2_post(uint32 m, int exp, bool esign, unsigned int sign = 0) -{ - int s = 0; - if(esign) - { - if(m > 0x80000000) - { - m = divide64(0x80000000, m, s); - ++exp; - } - if(exp > 25) - return underflow(sign); - else if(exp == 25) - return rounded(sign, 1, (m & 0x7FFFFFFF) != 0); - exp = -exp; - } - else if(exp > 15) - return overflow(sign); - return fixed2half(m, exp + 14, sign, s); -} - -/// Postprocessing for binary logarithm. -/// \tparam R rounding mode to use -/// \tparam L logarithm for base transformation as Q1.31 -/// \param m fractional part of logarithm as Q0.31 -/// \param ilog signed integer part of logarithm -/// \param exp biased exponent of result -/// \param sign sign bit of result -/// \return value base-transformed and converted to half-precision -/// \exception FE_OVERFLOW on overflows -/// \exception FE_UNDERFLOW on underflows -/// \exception FE_INEXACT if no other exception occurred -template -unsigned int log2_post(uint32 m, int ilog, int exp, unsigned int sign = 0) -{ - uint32 msign = sign_mask(ilog); - m = (((static_cast(ilog) << 27) + (m >> 4)) ^ msign) - msign; - if(!m) - return 0; - for(; m < 0x80000000; m <<= 1, --exp) - ; - int i = m >= L, s; - exp += i; - m >>= 1 + i; - sign ^= msign & 0x8000; - if(exp < -11) - return underflow(sign); - m = divide64(m, L, s); - return fixed2half(m, exp, sign, 1); -} - -/// Hypotenuse square root and postprocessing. -/// \tparam R rounding mode to use -/// \param r mantissa as Q2.30 -/// \param exp unbiased exponent -/// \return square root converted to half-precision -/// \exception FE_OVERFLOW on overflows -/// \exception FE_UNDERFLOW on underflows -/// \exception FE_INEXACT if value had to be rounded -template -unsigned int hypot_post(uint32 r, int exp) -{ - int i = r >> 31; - if((exp += i) > 46) - return overflow(); - if(exp < -34) - return underflow(); - r = (r >> i) | (r & i); - uint32 m = sqrt<30>(r, exp += 15); - return fixed2half(m, exp - 1, 0, r != 0); -} - -/// Division and postprocessing for tangents. -/// \tparam R rounding mode to use -/// \param my dividend as Q1.31 -/// \param mx divisor as Q1.31 -/// \param exp biased exponent of result -/// \param sign sign bit of result -/// \return quotient converted to half-precision -/// \exception FE_OVERFLOW on overflows -/// \exception FE_UNDERFLOW on underflows -/// \exception FE_INEXACT if no other exception occurred -template -unsigned int tangent_post(uint32 my, uint32 mx, int exp, unsigned int sign = 0) -{ - int i = my >= mx, s; - exp += i; - if(exp > 29) - return overflow(sign); - if(exp < -11) - return underflow(sign); - uint32 m = divide64(my >> (i + 1), mx, s); - return fixed2half(m, exp, sign, s); -} - -/// Area function and postprocessing. -/// This computes the value directly in Q2.30 using the representation `asinh|acosh(x) = -/// log(x+sqrt(x^2+|-1))`. -/// \tparam R rounding mode to use -/// \tparam S `true` for asinh, `false` for acosh -/// \param arg half-precision argument -/// \return asinh|acosh(\a arg) converted to half-precision -/// \exception FE_OVERFLOW on overflows -/// \exception FE_UNDERFLOW on underflows -/// \exception FE_INEXACT if no other exception occurred -template -unsigned int area(unsigned int arg) -{ - int abs = arg & 0x7FFF, expx = (abs >> 10) + (abs <= 0x3FF) - 15, expy = -15, ilog, i; - uint32 mx = static_cast((abs & 0x3FF) | ((abs > 0x3FF) << 10)) << 20, my, r; - for(; abs < 0x400; abs <<= 1, --expy) - ; - expy += abs >> 10; - r = ((abs & 0x3FF) | 0x400) << 5; - r *= r; - i = r >> 31; - expy = 2 * expy + i; - r >>= i; - if(S) - { - if(expy < 0) - { - r = 0x40000000 + ((expy > -30) ? ((r >> -expy) | - ((r & ((static_cast(1) << -expy) - 1)) != 0)) - : 1); - expy = 0; - } - else - { - r += 0x40000000 >> expy; - i = r >> 31; - r = (r >> i) | (r & i); - expy += i; - } - } - else - { - r -= 0x40000000 >> expy; - for(; r < 0x40000000; r <<= 1, --expy) - ; - } - my = sqrt<30>(r, expy); - my = (my << 15) + (r << 14) / my; - if(S) - { - mx >>= expy - expx; - ilog = expy; - } - else - { - my >>= expx - expy; - ilog = expx; - } - my += mx; - i = my >> 31; - static const int G = S && (R == std::round_to_nearest); - return log2_post( - log2(my >> i, 26 + S + G) + (G << 3), ilog + i, 17, arg & (static_cast(S) << 15)); -} - -/// Class for 1.31 unsigned floating-point computation -struct f31 -{ - /// Constructor. - /// \param mant mantissa as 1.31 - /// \param e exponent - HALF_CONSTEXPR f31(uint32 mant, int e) : m(mant), exp(e) {} - - /// Constructor. - /// \param abs unsigned half-precision value - f31(unsigned int abs) : exp(-15) - { - for(; abs < 0x400; abs <<= 1, --exp) - ; - m = static_cast((abs & 0x3FF) | 0x400) << 21; - exp += (abs >> 10); - } - - /// Addition operator. - /// \param a first operand - /// \param b second operand - /// \return \a a + \a b - friend f31 operator+(f31 a, f31 b) - { - if(b.exp > a.exp) - std::swap(a, b); - int d = a.exp - b.exp; - uint32 m = a.m + ((d < 32) ? (b.m >> d) : 0); - int i = (m & 0xFFFFFFFF) < a.m; - return f31(((m + i) >> i) | 0x80000000, a.exp + i); - } - - /// Subtraction operator. - /// \param a first operand - /// \param b second operand - /// \return \a a - \a b - friend f31 operator-(f31 a, f31 b) - { - int d = a.exp - b.exp, exp = a.exp; - uint32 m = a.m - ((d < 32) ? (b.m >> d) : 0); - if(!m) - return f31(0, -32); - for(; m < 0x80000000; m <<= 1, --exp) - ; - return f31(m, exp); - } - - /// Multiplication operator. - /// \param a first operand - /// \param b second operand - /// \return \a a * \a b - friend f31 operator*(f31 a, f31 b) - { - uint32 m = multiply64(a.m, b.m); - int i = m >> 31; - return f31(m << (1 - i), a.exp + b.exp + i); - } - - /// Division operator. - /// \param a first operand - /// \param b second operand - /// \return \a a / \a b - friend f31 operator/(f31 a, f31 b) - { - int i = a.m >= b.m, s; - uint32 m = divide64((a.m + i) >> i, b.m, s); - return f31(m, a.exp - b.exp + i - 1); - } - - uint32 m; ///< mantissa as 1.31. - int exp; ///< exponent. -}; - -/// Error function and postprocessing. -/// This computes the value directly in Q1.31 using the approximations given -/// [here](https://en.wikipedia.org/wiki/Error_function#Approximation_with_elementary_functions). -/// \tparam R rounding mode to use -/// \tparam C `true` for comlementary error function, `false` else -/// \param arg half-precision function argument -/// \return approximated value of error function in half-precision -/// \exception FE_OVERFLOW on overflows -/// \exception FE_UNDERFLOW on underflows -/// \exception FE_INEXACT if no other exception occurred -template -unsigned int erf(unsigned int arg) -{ - unsigned int abs = arg & 0x7FFF, sign = arg & 0x8000; - f31 x(abs), x2 = x * x * f31(0xB8AA3B29, 0), - t = f31(0x80000000, 0) / (f31(0x80000000, 0) + f31(0xA7BA054A, -2) * x), t2 = t * t; - f31 e = ((f31(0x87DC2213, 0) * t2 + f31(0xB5F0E2AE, 0)) * t2 + f31(0x82790637, -2) - - (f31(0xBA00E2B8, 0) * t2 + f31(0x91A98E62, -2)) * t) * - t / - ((x2.exp < 0) ? f31(exp2((x2.exp > -32) ? (x2.m >> -x2.exp) : 0, 30), 0) - : f31(exp2((x2.m << x2.exp) & 0x7FFFFFFF, 22), x2.m >> (31 - x2.exp))); - return (!C || sign) - ? fixed2half( - 0x80000000 - (e.m >> (C - e.exp)), 14 + C, sign & (C - 1U)) - : (e.exp < -25) - ? underflow() - : fixed2half(e.m >> 1, e.exp + 14, 0, e.m & 1); -} - -/// Gamma function and postprocessing. -/// This approximates the value of either the gamma function or its logarithm directly in Q1.31. -/// \tparam R rounding mode to use -/// \tparam L `true` for lograithm of gamma function, `false` for gamma function -/// \param arg half-precision floating-point value -/// \return lgamma/tgamma(\a arg) in half-precision -/// \exception FE_OVERFLOW on overflows -/// \exception FE_UNDERFLOW on underflows -/// \exception FE_INEXACT if \a arg is not a positive integer -template -unsigned int gamma(unsigned int arg) -{ - /* static const double p[] ={ 2.50662827563479526904, 225.525584619175212544, - -268.295973841304927459, 80.9030806934622512966, -5.00757863970517583837, - 0.0114684895434781459556 }; double t = arg + 4.65, s = p[0]; for(unsigned int i=0; i<5; ++i) - s += p[i+1] / (arg+i); - return std::log(s) + (arg-0.5)*std::log(t) - t; -*/ static const f31 pi(0xC90FDAA2, 1), lbe(0xB8AA3B29, 0); - unsigned int abs = arg & 0x7FFF, sign = arg & 0x8000; - bool bsign = sign != 0; - f31 z(abs), x = sign ? (z + f31(0x80000000, 0)) : z, t = x + f31(0x94CCCCCD, 2), - s = f31(0xA06C9901, 1) + f31(0xBBE654E2, -7) / (x + f31(0x80000000, 2)) + - f31(0xA1CE6098, 6) / (x + f31(0x80000000, 1)) + f31(0xE1868CB7, 7) / x - - f31(0x8625E279, 8) / (x + f31(0x80000000, 0)) - - f31(0xA03E158F, 2) / (x + f31(0xC0000000, 1)); - int i = (s.exp >= 2) + (s.exp >= 4) + (s.exp >= 8) + (s.exp >= 16); - s = f31((static_cast(s.exp) << (31 - i)) + (log2(s.m >> 1, 28) >> i), i) / lbe; - if(x.exp != -1 || x.m != 0x80000000) - { - i = (t.exp >= 2) + (t.exp >= 4) + (t.exp >= 8); - f31 l = f31((static_cast(t.exp) << (31 - i)) + (log2(t.m >> 1, 30) >> i), i) / lbe; - s = (x.exp < -1) ? (s - (f31(0x80000000, -1) - x) * l) - : (s + (x - f31(0x80000000, -1)) * l); - } - s = x.exp ? (s - t) : (t - s); - if(bsign) - { - if(z.exp >= 0) - { - sign &= (L | ((z.m >> (31 - z.exp)) & 1)) - 1; - for(z = f31((z.m << (1 + z.exp)) & 0xFFFFFFFF, -1); z.m < 0x80000000; - z.m <<= 1, --z.exp) - ; - } - if(z.exp == -1) - z = f31(0x80000000, 0) - z; - if(z.exp < -1) - { - z = z * pi; - z.m = sincos(z.m >> (1 - z.exp), 30).first; - for(z.exp = 1; z.m < 0x80000000; z.m <<= 1, --z.exp) - ; - } - else - z = f31(0x80000000, 0); - } - if(L) - { - if(bsign) - { - f31 l(0x92868247, 0); - if(z.exp < 0) - { - uint32 m = log2((z.m + 1) >> 1, 27); - z = f31(-((static_cast(z.exp) << 26) + (m >> 5)), 5); - for(; z.m < 0x80000000; z.m <<= 1, --z.exp) - ; - l = l + z / lbe; - } - sign = static_cast(x.exp && (l.exp < s.exp || (l.exp == s.exp && l.m < s.m))) - << 15; - s = sign ? (s - l) : x.exp ? (l - s) : (l + s); - } - else - { - sign = static_cast(x.exp == 0) << 15; - if(s.exp < -24) - return underflow(sign); - if(s.exp > 15) - return overflow(sign); - } - } - else - { - s = s * lbe; - uint32 m; - if(s.exp < 0) - { - m = s.m >> -s.exp; - s.exp = 0; - } - else - { - m = (s.m << s.exp) & 0x7FFFFFFF; - s.exp = (s.m >> (31 - s.exp)); - } - s.m = exp2(m, 27); - if(!x.exp) - s = f31(0x80000000, 0) / s; - if(bsign) - { - if(z.exp < 0) - s = s * z; - s = pi / s; - if(s.exp < -24) - return underflow(sign); - } - else if(z.exp > 0 && !(z.m & ((1 << (31 - z.exp)) - 1))) - return ((s.exp + 14) << 10) + (s.m >> 21); - if(s.exp > 15) - return overflow(sign); - } - return fixed2half(s.m, s.exp + 14, sign); -} -/// \} - -template -struct half_caster; -} // namespace detail - -/// Half-precision floating-point type. -/// This class implements an IEEE-conformant half-precision floating-point type with the usual -/// arithmetic -/// operators and conversions. It is implicitly convertible to single-precision floating-point, -/// which makes artihmetic -/// expressions and functions with mixed-type operands to be of the most precise operand type. -/// -/// According to the C++98/03 definition, the half type is not a POD type. But according to C++11's -/// less strict and -/// extended definitions it is both a standard layout type and a trivially copyable type (even if -/// not a POD type), which -/// means it can be standard-conformantly copied using raw binary copies. But in this context some -/// more words about the -/// actual size of the type. Although the half is representing an IEEE 16-bit type, it does not -/// neccessarily have to be of -/// exactly 16-bits size. But on any reasonable implementation the actual binary representation of -/// this type will most -/// probably not ivolve any additional "magic" or padding beyond the simple binary representation of -/// the underlying 16-bit -/// IEEE number, even if not strictly guaranteed by the standard. But even then it only has an -/// actual size of 16 bits if -/// your C++ implementation supports an unsigned integer type of exactly 16 bits width. But this -/// should be the case on -/// nearly any reasonable platform. -/// -/// So if your C++ implementation is not totally exotic or imposes special alignment requirements, -/// it is a reasonable -/// assumption that the data of a half is just comprised of the 2 bytes of the underlying IEEE -/// representation. -class half -{ - public: - /// \name Construction and assignment - /// \{ - - /// Default constructor. - /// This initializes the half to 0. Although this does not match the builtin types' - /// default-initialization semantics - /// and may be less efficient than no initialization, it is needed to provide proper - /// value-initialization semantics. - HALF_CONSTEXPR half() HALF_NOEXCEPT : data_() {} - - /// Conversion constructor. - /// \param rhs float to convert - /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding - explicit half(float rhs) - : data_(static_cast(detail::float2half(rhs))) - { - } - - /// Conversion to single-precision. - /// \return single precision value representing expression value - operator float() const { return detail::half2float(data_); } - - /// Assignment operator. - /// \param rhs single-precision value to copy from - /// \return reference to this half - /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding - half& operator=(float rhs) - { - data_ = static_cast(detail::float2half(rhs)); - return *this; - } - - /// \} - /// \name Arithmetic updates - /// \{ - - /// Arithmetic assignment. - /// \tparam T type of concrete half expression - /// \param rhs half expression to add - /// \return reference to this half - /// \exception FE_... according to operator+(half,half) - half& operator+=(half rhs) { return *this = *this + rhs; } - - /// Arithmetic assignment. - /// \tparam T type of concrete half expression - /// \param rhs half expression to subtract - /// \return reference to this half - /// \exception FE_... according to operator-(half,half) - half& operator-=(half rhs) { return *this = *this - rhs; } - - /// Arithmetic assignment. - /// \tparam T type of concrete half expression - /// \param rhs half expression to multiply with - /// \return reference to this half - /// \exception FE_... according to operator*(half,half) - half& operator*=(half rhs) { return *this = *this * rhs; } - - /// Arithmetic assignment. - /// \tparam T type of concrete half expression - /// \param rhs half expression to divide by - /// \return reference to this half - /// \exception FE_... according to operator/(half,half) - half& operator/=(half rhs) { return *this = *this / rhs; } - - /// Arithmetic assignment. - /// \param rhs single-precision value to add - /// \return reference to this half - /// \exception FE_... according to operator=() - half& operator+=(float rhs) { return *this = *this + rhs; } - - /// Arithmetic assignment. - /// \param rhs single-precision value to subtract - /// \return reference to this half - /// \exception FE_... according to operator=() - half& operator-=(float rhs) { return *this = *this - rhs; } - - /// Arithmetic assignment. - /// \param rhs single-precision value to multiply with - /// \return reference to this half - /// \exception FE_... according to operator=() - half& operator*=(float rhs) { return *this = *this * rhs; } - - /// Arithmetic assignment. - /// \param rhs single-precision value to divide by - /// \return reference to this half - /// \exception FE_... according to operator=() - half& operator/=(float rhs) { return *this = *this / rhs; } - - /// \} - /// \name Increment and decrement - /// \{ - - /// Prefix increment. - /// \return incremented half value - /// \exception FE_... according to operator+(half,half) - half& operator++() { return *this = *this + half(detail::binary, 0x3C00); } - - /// Prefix decrement. - /// \return decremented half value - /// \exception FE_... according to operator-(half,half) - half& operator--() { return *this = *this + half(detail::binary, 0xBC00); } - - /// Postfix increment. - /// \return non-incremented half value - /// \exception FE_... according to operator+(half,half) - half operator++(int) - { - half out(*this); - ++*this; - return out; - } - - /// Postfix decrement. - /// \return non-decremented half value - /// \exception FE_... according to operator-(half,half) - half operator--(int) - { - half out(*this); - --*this; - return out; - } - /// \} - - private: - /// Rounding mode to use - static const std::float_round_style round_style = (std::float_round_style)(HALF_ROUND_STYLE); - - /// Constructor. - /// \param bits binary representation to set half to - HALF_CONSTEXPR half(detail::binary_t, unsigned int bits) HALF_NOEXCEPT - : data_(static_cast(bits)) - { - } - - /// Internal binary representation - detail::uint16 data_; - -#ifndef HALF_DOXYGEN_ONLY - friend HALF_CONSTEXPR_NOERR bool operator==(half, half); - friend HALF_CONSTEXPR_NOERR bool operator!=(half, half); - friend HALF_CONSTEXPR_NOERR bool operator<(half, half); - friend HALF_CONSTEXPR_NOERR bool operator>(half, half); - friend HALF_CONSTEXPR_NOERR bool operator<=(half, half); - friend HALF_CONSTEXPR_NOERR bool operator>=(half, half); - friend HALF_CONSTEXPR half operator-(half); - friend half operator+(half, half); - friend half operator-(half, half); - friend half operator*(half, half); - friend half operator/(half, half); - template - friend std::basic_ostream& operator<<(std::basic_ostream&, half); - template - friend std::basic_istream& operator>>(std::basic_istream&, half&); - friend HALF_CONSTEXPR half fabs(half); - friend half fmod(half, half); - friend half remainder(half, half); - friend half remquo(half, half, int*); - friend half fma(half, half, half); - friend HALF_CONSTEXPR_NOERR half fmax(half, half); - friend HALF_CONSTEXPR_NOERR half fmin(half, half); - friend half fdim(half, half); - friend half nanh(const char*); - friend half exp(half); - friend half exp2(half); - friend half expm1(half); - friend half log(half); - friend half log10(half); - friend half log2(half); - friend half log1p(half); - friend half sqrt(half); - friend half cbrt(half); - friend half hypot(half, half); - friend half hypot(half, half, half); - friend half pow(half, half); - friend void sincos(half, half*, half*); - friend half sin(half); - friend half cos(half); - friend half tan(half); - friend half asin(half); - friend half acos(half); - friend half atan(half); - friend half atan2(half, half); - friend half sinh(half); - friend half cosh(half); - friend half tanh(half); - friend half asinh(half); - friend half acosh(half); - friend half atanh(half); - friend half erf(half); - friend half erfc(half); - friend half lgamma(half); - friend half tgamma(half); - friend half ceil(half); - friend half floor(half); - friend half trunc(half); - friend half round(half); - friend long lround(half); - friend half rint(half); - friend long lrint(half); - friend half nearbyint(half); -#ifdef HALF_ENABLE_CPP11_LONG_LONG - friend long long llround(half); - friend long long llrint(half); -#endif - friend half frexp(half, int*); - friend half scalbln(half, long); - friend half modf(half, half*); - friend int ilogb(half); - friend half logb(half); - friend half nextafter(half, half); - friend half nexttoward(half, long double); - friend HALF_CONSTEXPR half copysign(half, half); - friend HALF_CONSTEXPR int fpclassify(half); - friend HALF_CONSTEXPR bool isfinite(half); - friend HALF_CONSTEXPR bool isinf(half); - friend HALF_CONSTEXPR bool isnan(half); - friend HALF_CONSTEXPR bool isnormal(half); - friend HALF_CONSTEXPR bool signbit(half); - friend HALF_CONSTEXPR bool isgreater(half, half); - friend HALF_CONSTEXPR bool isgreaterequal(half, half); - friend HALF_CONSTEXPR bool isless(half, half); - friend HALF_CONSTEXPR bool islessequal(half, half); - friend HALF_CONSTEXPR bool islessgreater(half, half); - template - friend struct detail::half_caster; - friend class std::numeric_limits; -#if HALF_ENABLE_CPP11_HASH - friend struct std::hash; -#endif -#if HALF_ENABLE_CPP11_USER_LITERALS - friend half literal::operator"" _h(long double); -#endif -#endif -}; - -#if HALF_ENABLE_CPP11_USER_LITERALS -namespace literal { -/// Half literal. -/// While this returns a properly rounded half-precision value, half literals can unfortunately not -/// be constant -/// expressions due to rather involved conversions. So don't expect this to be a literal literal -/// without involving -/// conversion operations at runtime. It is a convenience feature, not a performance optimization. -/// \param value literal value -/// \return half with of given value (possibly rounded) -/// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding -inline half operator"" _h(long double value) -{ - return half(detail::binary, detail::float2half(value)); -} -} // namespace literal -#endif - -namespace detail { -/// Helper class for half casts. -/// This class template has to be specialized for all valid cast arguments to define an appropriate -/// static -/// `cast` member function and a corresponding `type` member denoting its return type. -/// \tparam T destination type -/// \tparam U source type -/// \tparam R rounding mode to use -template -struct half_caster -{ -}; -template -struct half_caster -{ -#if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS - static_assert(std::is_arithmetic::value, "half_cast from non-arithmetic type unsupported"); -#endif - - static half cast(U arg) { return cast_impl(arg, is_float()); }; - - private: - static half cast_impl(U arg, true_type) { return half(binary, float2half(arg)); } - static half cast_impl(U arg, false_type) { return half(binary, int2half(arg)); } -}; -template -struct half_caster -{ -#if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS - static_assert(std::is_arithmetic::value, "half_cast to non-arithmetic type unsupported"); -#endif - - static T cast(half arg) { return cast_impl(arg, is_float()); } - - private: - static T cast_impl(half arg, true_type) { return half2float(arg.data_); } - static T cast_impl(half arg, false_type) { return half2int(arg.data_); } -}; -template -struct half_caster -{ - static half cast(half arg) { return arg; } -}; -} // namespace detail -} // namespace half_float - -/// Extensions to the C++ standard library. -namespace std { -/// Numeric limits for half-precision floats. -/// **See also:** Documentation for -/// [std::numeric_limits](https://en.cppreference.com/w/cpp/types/numeric_limits) -template <> -class numeric_limits -{ - public: - /// Is template specialization. - static HALF_CONSTEXPR_CONST bool is_specialized = true; - - /// Supports signed values. - static HALF_CONSTEXPR_CONST bool is_signed = true; - - /// Is not an integer type. - static HALF_CONSTEXPR_CONST bool is_integer = false; - - /// Is not exact. - static HALF_CONSTEXPR_CONST bool is_exact = false; - - /// Doesn't provide modulo arithmetic. - static HALF_CONSTEXPR_CONST bool is_modulo = false; - - /// Has a finite set of values. - static HALF_CONSTEXPR_CONST bool is_bounded = true; - - /// IEEE conformant. - static HALF_CONSTEXPR_CONST bool is_iec559 = true; - - /// Supports infinity. - static HALF_CONSTEXPR_CONST bool has_infinity = true; - - /// Supports quiet NaNs. - static HALF_CONSTEXPR_CONST bool has_quiet_NaN = true; - - /// Supports signaling NaNs. - static HALF_CONSTEXPR_CONST bool has_signaling_NaN = true; - - /// Supports subnormal values. - static HALF_CONSTEXPR_CONST float_denorm_style has_denorm = denorm_present; - - /// Supports no denormalization detection. - static HALF_CONSTEXPR_CONST bool has_denorm_loss = false; - -#if HALF_ERRHANDLING_THROWS - static HALF_CONSTEXPR_CONST bool traps = true; -#else - /// Traps only if [HALF_ERRHANDLING_THROW_...](\ref HALF_ERRHANDLING_THROW_INVALID) is - /// acitvated. - static HALF_CONSTEXPR_CONST bool traps = false; -#endif - - /// Does not support no pre-rounding underflow detection. - static HALF_CONSTEXPR_CONST bool tinyness_before = false; - - /// Rounding mode. - static HALF_CONSTEXPR_CONST float_round_style round_style = half_float::half::round_style; - - /// Significant digits. - static HALF_CONSTEXPR_CONST int digits = 11; - - /// Significant decimal digits. - static HALF_CONSTEXPR_CONST int digits10 = 3; - - /// Required decimal digits to represent all possible values. - static HALF_CONSTEXPR_CONST int max_digits10 = 5; - - /// Number base. - static HALF_CONSTEXPR_CONST int radix = 2; - - /// One more than smallest exponent. - static HALF_CONSTEXPR_CONST int min_exponent = -13; - - /// Smallest normalized representable power of 10. - static HALF_CONSTEXPR_CONST int min_exponent10 = -4; - - /// One more than largest exponent - static HALF_CONSTEXPR_CONST int max_exponent = 16; - - /// Largest finitely representable power of 10. - static HALF_CONSTEXPR_CONST int max_exponent10 = 4; - - /// Smallest positive normal value. - static HALF_CONSTEXPR half_float::half min() HALF_NOTHROW - { - return half_float::half(half_float::detail::binary, 0x0400); - } - - /// Smallest finite value. - static HALF_CONSTEXPR half_float::half lowest() HALF_NOTHROW - { - return half_float::half(half_float::detail::binary, 0xFBFF); - } - - /// Largest finite value. - static HALF_CONSTEXPR half_float::half max() HALF_NOTHROW - { - return half_float::half(half_float::detail::binary, 0x7BFF); - } - - /// Difference between 1 and next representable value. - static HALF_CONSTEXPR half_float::half epsilon() HALF_NOTHROW - { - return half_float::half(half_float::detail::binary, 0x1400); - } - - /// Maximum rounding error in ULP (units in the last place). - static HALF_CONSTEXPR half_float::half round_error() HALF_NOTHROW - { - return half_float::half(half_float::detail::binary, - (round_style == std::round_to_nearest) ? 0x3800 : 0x3C00); - } - - /// Positive infinity. - static HALF_CONSTEXPR half_float::half infinity() HALF_NOTHROW - { - return half_float::half(half_float::detail::binary, 0x7C00); - } - - /// Quiet NaN. - static HALF_CONSTEXPR half_float::half quiet_NaN() HALF_NOTHROW - { - return half_float::half(half_float::detail::binary, 0x7FFF); - } - - /// Signaling NaN. - static HALF_CONSTEXPR half_float::half signaling_NaN() HALF_NOTHROW - { - return half_float::half(half_float::detail::binary, 0x7DFF); - } - - /// Smallest positive subnormal value. - static HALF_CONSTEXPR half_float::half denorm_min() HALF_NOTHROW - { - return half_float::half(half_float::detail::binary, 0x0001); - } -}; - -#if HALF_ENABLE_CPP11_HASH -/// Hash function for half-precision floats. -/// This is only defined if C++11 `std::hash` is supported and enabled. -/// -/// **See also:** Documentation for [std::hash](https://en.cppreference.com/w/cpp/utility/hash) -template <> -struct hash -{ - /// Type of function argument. - typedef half_float::half argument_type; - - /// Function return type. - typedef size_t result_type; - - /// Compute hash function. - /// \param arg half to hash - /// \return hash value - result_type operator()(argument_type arg) const - { - return hash()(arg.data_ & - -static_cast(arg.data_ != 0x8000)); - } -}; -#endif -} // namespace std - -namespace half_float { -/// \anchor compop -/// \name Comparison operators -/// \{ - -/// Comparison for equality. -/// \param x first operand -/// \param y second operand -/// \retval true if operands equal -/// \retval false else -/// \exception FE_INVALID if \a x or \a y is NaN -inline HALF_CONSTEXPR_NOERR bool operator==(half x, half y) -{ - return !detail::compsignal(x.data_, y.data_) && - (x.data_ == y.data_ || !((x.data_ | y.data_) & 0x7FFF)); -} - -/// Comparison for inequality. -/// \param x first operand -/// \param y second operand -/// \retval true if operands not equal -/// \retval false else -/// \exception FE_INVALID if \a x or \a y is NaN -inline HALF_CONSTEXPR_NOERR bool operator!=(half x, half y) -{ - return detail::compsignal(x.data_, y.data_) || - (x.data_ != y.data_ && ((x.data_ | y.data_) & 0x7FFF)); -} - -/// Comparison for less than. -/// \param x first operand -/// \param y second operand -/// \retval true if \a x less than \a y -/// \retval false else -/// \exception FE_INVALID if \a x or \a y is NaN -inline HALF_CONSTEXPR_NOERR bool operator<(half x, half y) -{ - return !detail::compsignal(x.data_, y.data_) && - ((x.data_ ^ (0x8000 | (0x8000 - (x.data_ >> 15)))) + (x.data_ >> 15)) < - ((y.data_ ^ (0x8000 | (0x8000 - (y.data_ >> 15)))) + (y.data_ >> 15)); -} - -/// Comparison for greater than. -/// \param x first operand -/// \param y second operand -/// \retval true if \a x greater than \a y -/// \retval false else -/// \exception FE_INVALID if \a x or \a y is NaN -inline HALF_CONSTEXPR_NOERR bool operator>(half x, half y) -{ - return !detail::compsignal(x.data_, y.data_) && - ((x.data_ ^ (0x8000 | (0x8000 - (x.data_ >> 15)))) + (x.data_ >> 15)) > - ((y.data_ ^ (0x8000 | (0x8000 - (y.data_ >> 15)))) + (y.data_ >> 15)); -} - -/// Comparison for less equal. -/// \param x first operand -/// \param y second operand -/// \retval true if \a x less equal \a y -/// \retval false else -/// \exception FE_INVALID if \a x or \a y is NaN -inline HALF_CONSTEXPR_NOERR bool operator<=(half x, half y) -{ - return !detail::compsignal(x.data_, y.data_) && - ((x.data_ ^ (0x8000 | (0x8000 - (x.data_ >> 15)))) + (x.data_ >> 15)) <= - ((y.data_ ^ (0x8000 | (0x8000 - (y.data_ >> 15)))) + (y.data_ >> 15)); -} - -/// Comparison for greater equal. -/// \param x first operand -/// \param y second operand -/// \retval true if \a x greater equal \a y -/// \retval false else -/// \exception FE_INVALID if \a x or \a y is NaN -inline HALF_CONSTEXPR_NOERR bool operator>=(half x, half y) -{ - return !detail::compsignal(x.data_, y.data_) && - ((x.data_ ^ (0x8000 | (0x8000 - (x.data_ >> 15)))) + (x.data_ >> 15)) >= - ((y.data_ ^ (0x8000 | (0x8000 - (y.data_ >> 15)))) + (y.data_ >> 15)); -} - -/// \} -/// \anchor arithmetics -/// \name Arithmetic operators -/// \{ - -/// Identity. -/// \param arg operand -/// \return unchanged operand -inline HALF_CONSTEXPR half operator+(half arg) { return arg; } - -/// Negation. -/// \param arg operand -/// \return negated operand -inline HALF_CONSTEXPR half operator-(half arg) { return half(detail::binary, arg.data_ ^ 0x8000); } - -/// Addition. -/// This operation is exact to rounding for all rounding modes. -/// \param x left operand -/// \param y right operand -/// \return sum of half expressions -/// \exception FE_INVALID if \a x and \a y are infinities with different signs or signaling NaNs -/// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding -inline half operator+(half x, half y) -{ -#ifdef HALF_ARITHMETIC_TYPE - return half( - detail::binary, - detail::float2half(detail::half2float(x.data_) + - detail::half2float(y.data_))); -#else - int absx = x.data_ & 0x7FFF, absy = y.data_ & 0x7FFF; - bool sub = ((x.data_ ^ y.data_) & 0x8000) != 0; - if(absx >= 0x7C00 || absy >= 0x7C00) - return half(detail::binary, - (absx > 0x7C00 || absy > 0x7C00) - ? detail::signal(x.data_, y.data_) - : (absy != 0x7C00) ? x.data_ - : (sub && absx == 0x7C00) ? detail::invalid() : y.data_); - if(!absx) - return absy ? y - : half(detail::binary, - (half::round_style == std::round_toward_neg_infinity) - ? (x.data_ | y.data_) - : (x.data_ & y.data_)); - if(!absy) - return x; - unsigned int sign = ((sub && absy > absx) ? y.data_ : x.data_) & 0x8000; - if(absy > absx) - std::swap(absx, absy); - int exp = (absx >> 10) + (absx <= 0x3FF), d = exp - (absy >> 10) - (absy <= 0x3FF), - mx = ((absx & 0x3FF) | ((absx > 0x3FF) << 10)) << 3, my; - if(d < 13) - { - my = ((absy & 0x3FF) | ((absy > 0x3FF) << 10)) << 3; - my = (my >> d) | ((my & ((1 << d) - 1)) != 0); - } - else - my = 1; - if(sub) - { - if(!(mx -= my)) - return half(detail::binary, - static_cast(half::round_style == std::round_toward_neg_infinity) - << 15); - for(; mx < 0x2000 && exp > 1; mx <<= 1, --exp) - ; - } - else - { - mx += my; - int i = mx >> 14; - if((exp += i) > 30) - return half(detail::binary, detail::overflow(sign)); - mx = (mx >> i) | (mx & i); - } - return half(detail::binary, - detail::rounded( - sign + ((exp - 1) << 10) + (mx >> 3), (mx >> 2) & 1, (mx & 0x3) != 0)); -#endif -} - -/// Subtraction. -/// This operation is exact to rounding for all rounding modes. -/// \param x left operand -/// \param y right operand -/// \return difference of half expressions -/// \exception FE_INVALID if \a x and \a y are infinities with equal signs or signaling NaNs -/// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding -inline half operator-(half x, half y) -{ -#ifdef HALF_ARITHMETIC_TYPE - return half( - detail::binary, - detail::float2half(detail::half2float(x.data_) - - detail::half2float(y.data_))); -#else - return x + -y; -#endif -} - -/// Multiplication. -/// This operation is exact to rounding for all rounding modes. -/// \param x left operand -/// \param y right operand -/// \return product of half expressions -/// \exception FE_INVALID if multiplying 0 with infinity or if \a x or \a y is signaling NaN -/// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding -inline half operator*(half x, half y) -{ -#ifdef HALF_ARITHMETIC_TYPE - return half( - detail::binary, - detail::float2half(detail::half2float(x.data_) * - detail::half2float(y.data_))); -#else - int absx = x.data_ & 0x7FFF, absy = y.data_ & 0x7FFF, exp = -16; - unsigned int sign = (x.data_ ^ y.data_) & 0x8000; - if(absx >= 0x7C00 || absy >= 0x7C00) - return half(detail::binary, - (absx > 0x7C00 || absy > 0x7C00) - ? detail::signal(x.data_, y.data_) - : ((absx == 0x7C00 && !absy) || (absy == 0x7C00 && !absx)) - ? detail::invalid() - : (sign | 0x7C00)); - if(!absx || !absy) - return half(detail::binary, sign); - for(; absx < 0x400; absx <<= 1, --exp) - ; - for(; absy < 0x400; absy <<= 1, --exp) - ; - detail::uint32 m = static_cast((absx & 0x3FF) | 0x400) * - static_cast((absy & 0x3FF) | 0x400); - int i = m >> 21, s = m & i; - exp += (absx >> 10) + (absy >> 10) + i; - if(exp > 29) - return half(detail::binary, detail::overflow(sign)); - else if(exp < -11) - return half(detail::binary, detail::underflow(sign)); - return half( - detail::binary, - detail::fixed2half(m >> i, exp, sign, s)); -#endif -} - -/// Division. -/// This operation is exact to rounding for all rounding modes. -/// \param x left operand -/// \param y right operand -/// \return quotient of half expressions -/// \exception FE_INVALID if dividing 0s or infinities with each other or if \a x or \a y is -/// signaling NaN -/// \exception FE_DIVBYZERO if dividing finite value by 0 -/// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding -inline half operator/(half x, half y) -{ -#ifdef HALF_ARITHMETIC_TYPE - return half( - detail::binary, - detail::float2half(detail::half2float(x.data_) / - detail::half2float(y.data_))); -#else - int absx = x.data_ & 0x7FFF, absy = y.data_ & 0x7FFF, exp = 14; - unsigned int sign = (x.data_ ^ y.data_) & 0x8000; - if(absx >= 0x7C00 || absy >= 0x7C00) - return half(detail::binary, - (absx > 0x7C00 || absy > 0x7C00) - ? detail::signal(x.data_, y.data_) - : (absx == absy) ? detail::invalid() - : (sign | ((absx == 0x7C00) ? 0x7C00 : 0))); - if(!absx) - return half(detail::binary, absy ? sign : detail::invalid()); - if(!absy) - return half(detail::binary, detail::pole(sign)); - for(; absx < 0x400; absx <<= 1, --exp) - ; - for(; absy < 0x400; absy <<= 1, ++exp) - ; - detail::uint32 mx = (absx & 0x3FF) | 0x400, my = (absy & 0x3FF) | 0x400; - int i = mx < my; - exp += (absx >> 10) - (absy >> 10) - i; - if(exp > 29) - return half(detail::binary, detail::overflow(sign)); - else if(exp < -11) - return half(detail::binary, detail::underflow(sign)); - mx <<= 12 + i; - my <<= 1; - return half(detail::binary, - detail::fixed2half( - mx / my, exp, sign, mx % my != 0)); -#endif -} - -/// \} -/// \anchor streaming -/// \name Input and output -/// \{ - -/// Output operator. -/// This uses the built-in functionality for streaming out floating-point numbers. -/// \param out output stream to write into -/// \param arg half expression to write -/// \return reference to output stream -template -std::basic_ostream& operator<<(std::basic_ostream& out, half arg) -{ -#ifdef HALF_ARITHMETIC_TYPE - return out << detail::half2float(arg.data_); -#else - return out << detail::half2float(arg.data_); -#endif -} - -/// Input operator. -/// This uses the built-in functionality for streaming in floating-point numbers, specifically -/// double precision floating -/// point numbers (unless overridden with [HALF_ARITHMETIC_TYPE](\ref HALF_ARITHMETIC_TYPE)). So the -/// input string is first -/// rounded to double precision using the underlying platform's current floating-point rounding mode -/// before being rounded -/// to half-precision using the library's half-precision rounding mode. -/// \param in input stream to read from -/// \param arg half to read into -/// \return reference to input stream -/// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding -template -std::basic_istream& operator>>(std::basic_istream& in, half& arg) -{ -#ifdef HALF_ARITHMETIC_TYPE - detail::internal_t f; -#else - double f; -#endif - if(in >> f) - arg.data_ = detail::float2half(f); - return in; -} - -/// \} -/// \anchor basic -/// \name Basic mathematical operations -/// \{ - -/// Absolute value. -/// **See also:** Documentation for -/// [std::fabs](https://en.cppreference.com/w/cpp/numeric/math/fabs). -/// \param arg operand -/// \return absolute value of \a arg -inline HALF_CONSTEXPR half fabs(half arg) { return half(detail::binary, arg.data_ & 0x7FFF); } - -/// Absolute value. -/// **See also:** Documentation for [std::abs](https://en.cppreference.com/w/cpp/numeric/math/fabs). -/// \param arg operand -/// \return absolute value of \a arg -inline HALF_CONSTEXPR half abs(half arg) { return fabs(arg); } - -/// Remainder of division. -/// **See also:** Documentation for -/// [std::fmod](https://en.cppreference.com/w/cpp/numeric/math/fmod). -/// \param x first operand -/// \param y second operand -/// \return remainder of floating-point division. -/// \exception FE_INVALID if \a x is infinite or \a y is 0 or if \a x or \a y is signaling NaN -inline half fmod(half x, half y) -{ - unsigned int absx = x.data_ & 0x7FFF, absy = y.data_ & 0x7FFF, sign = x.data_ & 0x8000; - if(absx >= 0x7C00 || absy >= 0x7C00) - return half(detail::binary, - (absx > 0x7C00 || absy > 0x7C00) - ? detail::signal(x.data_, y.data_) - : (absx == 0x7C00) ? detail::invalid() : x.data_); - if(!absy) - return half(detail::binary, detail::invalid()); - if(!absx) - return x; - if(absx == absy) - return half(detail::binary, sign); - return half(detail::binary, sign | detail::mod(absx, absy)); -} - -/// Remainder of division. -/// **See also:** Documentation for -/// [std::remainder](https://en.cppreference.com/w/cpp/numeric/math/remainder). -/// \param x first operand -/// \param y second operand -/// \return remainder of floating-point division. -/// \exception FE_INVALID if \a x is infinite or \a y is 0 or if \a x or \a y is signaling NaN -inline half remainder(half x, half y) -{ - unsigned int absx = x.data_ & 0x7FFF, absy = y.data_ & 0x7FFF, sign = x.data_ & 0x8000; - if(absx >= 0x7C00 || absy >= 0x7C00) - return half(detail::binary, - (absx > 0x7C00 || absy > 0x7C00) - ? detail::signal(x.data_, y.data_) - : (absx == 0x7C00) ? detail::invalid() : x.data_); - if(!absy) - return half(detail::binary, detail::invalid()); - if(absx == absy) - return half(detail::binary, sign); - return half(detail::binary, sign ^ detail::mod(absx, absy)); -} - -/// Remainder of division. -/// **See also:** Documentation for -/// [std::remquo](https://en.cppreference.com/w/cpp/numeric/math/remquo). -/// \param x first operand -/// \param y second operand -/// \param quo address to store some bits of quotient at -/// \return remainder of floating-point division. -/// \exception FE_INVALID if \a x is infinite or \a y is 0 or if \a x or \a y is signaling NaN -inline half remquo(half x, half y, int* quo) -{ - unsigned int absx = x.data_ & 0x7FFF, absy = y.data_ & 0x7FFF, value = x.data_ & 0x8000; - if(absx >= 0x7C00 || absy >= 0x7C00) - return half(detail::binary, - (absx > 0x7C00 || absy > 0x7C00) - ? detail::signal(x.data_, y.data_) - : (absx == 0x7C00) ? detail::invalid() : (*quo = 0, x.data_)); - if(!absy) - return half(detail::binary, detail::invalid()); - bool qsign = ((value ^ y.data_) & 0x8000) != 0; - int q = 1; - if(absx != absy) - value ^= detail::mod(absx, absy, &q); - return *quo = qsign ? -q : q, half(detail::binary, value); -} - -/// Fused multiply add. -/// This function is exact to rounding for all rounding modes. -/// -/// **See also:** Documentation for [std::fma](https://en.cppreference.com/w/cpp/numeric/math/fma). -/// \param x first operand -/// \param y second operand -/// \param z third operand -/// \return ( \a x * \a y ) + \a z rounded as one operation. -/// \exception FE_INVALID according to operator*() and operator+() unless any argument is a quiet -/// NaN and no argument is a signaling NaN -/// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding the final addition -inline half fma(half x, half y, half z) -{ -#ifdef HALF_ARITHMETIC_TYPE - detail::internal_t fx = detail::half2float(x.data_), - fy = detail::half2float(y.data_), - fz = detail::half2float(z.data_); -#if HALF_ENABLE_CPP11_CMATH && FP_FAST_FMA - return half(detail::binary, detail::float2half(std::fma(fx, fy, fz))); -#else - return half(detail::binary, detail::float2half(fx * fy + fz)); -#endif -#else - int absx = x.data_ & 0x7FFF, absy = y.data_ & 0x7FFF, absz = z.data_ & 0x7FFF, exp = -15; - unsigned int sign = (x.data_ ^ y.data_) & 0x8000; - bool sub = ((sign ^ z.data_) & 0x8000) != 0; - if(absx >= 0x7C00 || absy >= 0x7C00 || absz >= 0x7C00) - return (absx > 0x7C00 || absy > 0x7C00 || absz > 0x7C00) - ? half(detail::binary, detail::signal(x.data_, y.data_, z.data_)) - : (absx == 0x7C00) ? half(detail::binary, - (!absy || (sub && absz == 0x7C00)) ? detail::invalid() - : (sign | 0x7C00)) - : (absy == 0x7C00) ? half(detail::binary, - (!absx || (sub && absz == 0x7C00)) - ? detail::invalid() - : (sign | 0x7C00)) - : z; - if(!absx || !absy) - return absz - ? z - : half(detail::binary, - (half::round_style == std::round_toward_neg_infinity) ? (z.data_ | sign) - : (z.data_ & sign)); - for(; absx < 0x400; absx <<= 1, --exp) - ; - for(; absy < 0x400; absy <<= 1, --exp) - ; - detail::uint32 m = static_cast((absx & 0x3FF) | 0x400) * - static_cast((absy & 0x3FF) | 0x400); - int i = m >> 21; - exp += (absx >> 10) + (absy >> 10) + i; - m <<= 3 - i; - if(absz) - { - int expz = 0; - for(; absz < 0x400; absz <<= 1, --expz) - ; - expz += absz >> 10; - detail::uint32 mz = static_cast((absz & 0x3FF) | 0x400) << 13; - if(expz > exp || (expz == exp && mz > m)) - { - std::swap(m, mz); - std::swap(exp, expz); - if(sub) - sign = z.data_ & 0x8000; - } - int d = exp - expz; - mz = (d < 23) ? ((mz >> d) | ((mz & ((static_cast(1) << d) - 1)) != 0)) : 1; - if(sub) - { - m = m - mz; - if(!m) - return half( - detail::binary, - static_cast(half::round_style == std::round_toward_neg_infinity) - << 15); - for(; m < 0x800000; m <<= 1, --exp) - ; - } - else - { - m += mz; - i = m >> 24; - m = (m >> i) | (m & i); - exp += i; - } - } - if(exp > 30) - return half(detail::binary, detail::overflow(sign)); - else if(exp < -10) - return half(detail::binary, detail::underflow(sign)); - return half(detail::binary, - detail::fixed2half(m, exp - 1, sign)); -#endif -} - -/// Maximum of half expressions. -/// **See also:** Documentation for -/// [std::fmax](https://en.cppreference.com/w/cpp/numeric/math/fmax). -/// \param x first operand -/// \param y second operand -/// \return maximum of operands, ignoring quiet NaNs -/// \exception FE_INVALID if \a x or \a y is signaling NaN -inline HALF_CONSTEXPR_NOERR half fmax(half x, half y) -{ - return half(detail::binary, - (!isnan(y) && (isnan(x) || (x.data_ ^ (0x8000 | (0x8000 - (x.data_ >> 15)))) < - (y.data_ ^ (0x8000 | (0x8000 - (y.data_ >> 15)))))) - ? detail::select(y.data_, x.data_) - : detail::select(x.data_, y.data_)); -} - -/// Minimum of half expressions. -/// **See also:** Documentation for -/// [std::fmin](https://en.cppreference.com/w/cpp/numeric/math/fmin). -/// \param x first operand -/// \param y second operand -/// \return minimum of operands, ignoring quiet NaNs -/// \exception FE_INVALID if \a x or \a y is signaling NaN -inline HALF_CONSTEXPR_NOERR half fmin(half x, half y) -{ - return half(detail::binary, - (!isnan(y) && (isnan(x) || (x.data_ ^ (0x8000 | (0x8000 - (x.data_ >> 15)))) > - (y.data_ ^ (0x8000 | (0x8000 - (y.data_ >> 15)))))) - ? detail::select(y.data_, x.data_) - : detail::select(x.data_, y.data_)); -} - -/// Positive difference. -/// This function is exact to rounding for all rounding modes. -/// -/// **See also:** Documentation for -/// [std::fdim](https://en.cppreference.com/w/cpp/numeric/math/fdim). -/// \param x first operand -/// \param y second operand -/// \return \a x - \a y or 0 if difference negative -/// \exception FE_... according to operator-(half,half) -inline half fdim(half x, half y) -{ - if(isnan(x) || isnan(y)) - return half(detail::binary, detail::signal(x.data_, y.data_)); - return (x.data_ ^ (0x8000 | (0x8000 - (x.data_ >> 15)))) <= - (y.data_ ^ (0x8000 | (0x8000 - (y.data_ >> 15)))) - ? half(detail::binary, 0) - : (x - y); -} - -/// Get NaN value. -/// **See also:** Documentation for [std::nan](https://en.cppreference.com/w/cpp/numeric/math/nan). -/// \param arg string code -/// \return quiet NaN -inline half nanh(const char* arg) -{ - unsigned int value = 0x7FFF; - while(*arg) - value ^= static_cast(*arg++) & 0xFF; - return half(detail::binary, value); -} - -/// \} -/// \anchor exponential -/// \name Exponential functions -/// \{ - -/// Exponential function. -/// This function is exact to rounding for all rounding modes. -/// -/// **See also:** Documentation for [std::exp](https://en.cppreference.com/w/cpp/numeric/math/exp). -/// \param arg function argument -/// \return e raised to \a arg -/// \exception FE_INVALID for signaling NaN -/// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding -inline half exp(half arg) -{ -#ifdef HALF_ARITHMETIC_TYPE - return half(detail::binary, - detail::float2half( - std::exp(detail::half2float(arg.data_)))); -#else - int abs = arg.data_ & 0x7FFF; - if(!abs) - return half(detail::binary, 0x3C00); - if(abs >= 0x7C00) - return half(detail::binary, - (abs == 0x7C00) ? (0x7C00 & ((arg.data_ >> 15) - 1U)) - : detail::signal(arg.data_)); - if(abs >= 0x4C80) - return half(detail::binary, - (arg.data_ & 0x8000) ? detail::underflow() - : detail::overflow()); - detail::uint32 m = detail::multiply64( - static_cast((abs & 0x3FF) + ((abs > 0x3FF) << 10)) << 21, 0xB8AA3B29); - int e = (abs >> 10) + (abs <= 0x3FF), exp; - if(e < 14) - { - exp = 0; - m >>= 14 - e; - } - else - { - exp = m >> (45 - e); - m = (m << (e - 14)) & 0x7FFFFFFF; - } - return half(detail::binary, - detail::exp2_post( - detail::exp2(m, 26), exp, (arg.data_ & 0x8000) != 0)); -#endif -} - -/// Binary exponential. -/// This function is exact to rounding for all rounding modes. -/// -/// **See also:** Documentation for -/// [std::exp2](https://en.cppreference.com/w/cpp/numeric/math/exp2). -/// \param arg function argument -/// \return 2 raised to \a arg -/// \exception FE_INVALID for signaling NaN -/// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding -inline half exp2(half arg) -{ -#if defined(HALF_ARITHMETIC_TYPE) && HALF_ENABLE_CPP11_CMATH - return half(detail::binary, - detail::float2half( - std::exp2(detail::half2float(arg.data_)))); -#else - int abs = arg.data_ & 0x7FFF; - if(!abs) - return half(detail::binary, 0x3C00); - if(abs >= 0x7C00) - return half(detail::binary, - (abs == 0x7C00) ? (0x7C00 & ((arg.data_ >> 15) - 1U)) - : detail::signal(arg.data_)); - if(abs >= 0x4E40) - return half(detail::binary, - (arg.data_ & 0x8000) ? detail::underflow() - : detail::overflow()); - int e = (abs >> 10) + (abs <= 0x3FF), exp = (abs & 0x3FF) + ((abs > 0x3FF) << 10); - detail::uint32 m = detail::exp2((static_cast(exp) << (6 + e)) & 0x7FFFFFFF, 28); - exp >>= 25 - e; - if(m == 0x80000000) - { - if(arg.data_ & 0x8000) - exp = -exp; - else if(exp > 15) - return half(detail::binary, detail::overflow()); - return half(detail::binary, - detail::fixed2half(m, exp + 14)); - } - return half(detail::binary, - detail::exp2_post(m, exp, (arg.data_ & 0x8000) != 0)); -#endif -} - -/// Exponential minus one. -/// This function may be 1 ULP off the correctly rounded exact result in <0.05% of inputs for -/// `std::round_to_nearest` -/// and in <1% of inputs for any other rounding mode. -/// -/// **See also:** Documentation for -/// [std::expm1](https://en.cppreference.com/w/cpp/numeric/math/expm1). -/// \param arg function argument -/// \return e raised to \a arg and subtracted by 1 -/// \exception FE_INVALID for signaling NaN -/// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding -inline half expm1(half arg) -{ -#if defined(HALF_ARITHMETIC_TYPE) && HALF_ENABLE_CPP11_CMATH - return half(detail::binary, - detail::float2half( - std::expm1(detail::half2float(arg.data_)))); -#else - unsigned int abs = arg.data_ & 0x7FFF, sign = arg.data_ & 0x8000; - if(!abs) - return arg; - if(abs >= 0x7C00) - return half(detail::binary, - (abs == 0x7C00) ? (0x7C00 + (sign >> 1)) : detail::signal(arg.data_)); - if(abs >= 0x4A00) - return half(detail::binary, - (arg.data_ & 0x8000) ? detail::rounded(0xBBFF, 1, 1) - : detail::overflow()); - detail::uint32 m = detail::multiply64( - static_cast((abs & 0x3FF) + ((abs > 0x3FF) << 10)) << 21, 0xB8AA3B29); - int e = (abs >> 10) + (abs <= 0x3FF), exp; - if(e < 14) - { - exp = 0; - m >>= 14 - e; - } - else - { - exp = m >> (45 - e); - m = (m << (e - 14)) & 0x7FFFFFFF; - } - m = detail::exp2(m); - if(sign) - { - int s = 0; - if(m > 0x80000000) - { - ++exp; - m = detail::divide64(0x80000000, m, s); - } - m = 0x80000000 - - ((m >> exp) | ((m & ((static_cast(1) << exp) - 1)) != 0) | s); - exp = 0; - } - else - m -= (exp < 31) ? (0x80000000 >> exp) : 1; - for(exp += 14; m < 0x80000000 && exp; m <<= 1, --exp) - ; - if(exp > 29) - return half(detail::binary, detail::overflow()); - return half(detail::binary, - detail::rounded( - sign + (exp << 10) + (m >> 21), (m >> 20) & 1, (m & 0xFFFFF) != 0)); -#endif -} - -/// Natural logarithm. -/// This function is exact to rounding for all rounding modes. -/// -/// **See also:** Documentation for [std::log](https://en.cppreference.com/w/cpp/numeric/math/log). -/// \param arg function argument -/// \return logarithm of \a arg to base e -/// \exception FE_INVALID for signaling NaN or negative argument -/// \exception FE_DIVBYZERO for 0 -/// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding -inline half log(half arg) -{ -#ifdef HALF_ARITHMETIC_TYPE - return half(detail::binary, - detail::float2half( - std::log(detail::half2float(arg.data_)))); -#else - int abs = arg.data_ & 0x7FFF, exp = -15; - if(!abs) - return half(detail::binary, detail::pole(0x8000)); - if(arg.data_ & 0x8000) - return half(detail::binary, - (arg.data_ <= 0xFC00) ? detail::invalid() : detail::signal(arg.data_)); - if(abs >= 0x7C00) - return (abs == 0x7C00) ? arg : half(detail::binary, detail::signal(arg.data_)); - for(; abs < 0x400; abs <<= 1, --exp) - ; - exp += abs >> 10; - return half(detail::binary, - detail::log2_post( - detail::log2(static_cast((abs & 0x3FF) | 0x400) << 20, 27) + 8, - exp, - 17)); -#endif -} - -/// Common logarithm. -/// This function is exact to rounding for all rounding modes. -/// -/// **See also:** Documentation for -/// [std::log10](https://en.cppreference.com/w/cpp/numeric/math/log10). -/// \param arg function argument -/// \return logarithm of \a arg to base 10 -/// \exception FE_INVALID for signaling NaN or negative argument -/// \exception FE_DIVBYZERO for 0 -/// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding -inline half log10(half arg) -{ -#ifdef HALF_ARITHMETIC_TYPE - return half(detail::binary, - detail::float2half( - std::log10(detail::half2float(arg.data_)))); -#else - int abs = arg.data_ & 0x7FFF, exp = -15; - if(!abs) - return half(detail::binary, detail::pole(0x8000)); - if(arg.data_ & 0x8000) - return half(detail::binary, - (arg.data_ <= 0xFC00) ? detail::invalid() : detail::signal(arg.data_)); - if(abs >= 0x7C00) - return (abs == 0x7C00) ? arg : half(detail::binary, detail::signal(arg.data_)); - switch(abs) - { - case 0x4900: return half(detail::binary, 0x3C00); - case 0x5640: return half(detail::binary, 0x4000); - case 0x63D0: return half(detail::binary, 0x4200); - case 0x70E2: return half(detail::binary, 0x4400); - } - for(; abs < 0x400; abs <<= 1, --exp) - ; - exp += abs >> 10; - return half(detail::binary, - detail::log2_post( - detail::log2(static_cast((abs & 0x3FF) | 0x400) << 20, 27) + 8, - exp, - 16)); -#endif -} - -/// Binary logarithm. -/// This function is exact to rounding for all rounding modes. -/// -/// **See also:** Documentation for -/// [std::log2](https://en.cppreference.com/w/cpp/numeric/math/log2). -/// \param arg function argument -/// \return logarithm of \a arg to base 2 -/// \exception FE_INVALID for signaling NaN or negative argument -/// \exception FE_DIVBYZERO for 0 -/// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding -inline half log2(half arg) -{ -#if defined(HALF_ARITHMETIC_TYPE) && HALF_ENABLE_CPP11_CMATH - return half(detail::binary, - detail::float2half( - std::log2(detail::half2float(arg.data_)))); -#else - int abs = arg.data_ & 0x7FFF, exp = -15, s = 0; - if(!abs) - return half(detail::binary, detail::pole(0x8000)); - if(arg.data_ & 0x8000) - return half(detail::binary, - (arg.data_ <= 0xFC00) ? detail::invalid() : detail::signal(arg.data_)); - if(abs >= 0x7C00) - return (abs == 0x7C00) ? arg : half(detail::binary, detail::signal(arg.data_)); - if(abs == 0x3C00) - return half(detail::binary, 0); - for(; abs < 0x400; abs <<= 1, --exp) - ; - exp += (abs >> 10); - if(!(abs & 0x3FF)) - { - unsigned int value = static_cast(exp < 0) << 15, m = std::abs(exp) << 6; - for(exp = 18; m < 0x400; m <<= 1, --exp) - ; - return half(detail::binary, value + (exp << 10) + m); - } - detail::uint32 ilog = exp, sign = detail::sign_mask(ilog), - m = (((ilog << 27) + - (detail::log2(static_cast((abs & 0x3FF) | 0x400) << 20, - 28) >> - 4)) ^ - sign) - - sign; - if(!m) - return half(detail::binary, 0); - for(exp = 14; m < 0x8000000 && exp; m <<= 1, --exp) - ; - for(; m > 0xFFFFFFF; m >>= 1, ++exp) - s |= m & 1; - return half( - detail::binary, - detail::fixed2half(m, exp, sign & 0x8000, s)); -#endif -} - -/// Natural logarithm plus one. -/// This function may be 1 ULP off the correctly rounded exact result in <0.05% of inputs for -/// `std::round_to_nearest` -/// and in ~1% of inputs for any other rounding mode. -/// -/// **See also:** Documentation for -/// [std::log1p](https://en.cppreference.com/w/cpp/numeric/math/log1p). -/// \param arg function argument -/// \return logarithm of \a arg plus 1 to base e -/// \exception FE_INVALID for signaling NaN or argument <-1 -/// \exception FE_DIVBYZERO for -1 -/// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding -inline half log1p(half arg) -{ -#if defined(HALF_ARITHMETIC_TYPE) && HALF_ENABLE_CPP11_CMATH - return half(detail::binary, - detail::float2half( - std::log1p(detail::half2float(arg.data_)))); -#else - if(arg.data_ >= 0xBC00) - return half(detail::binary, - (arg.data_ == 0xBC00) - ? detail::pole(0x8000) - : (arg.data_ <= 0xFC00) ? detail::invalid() : detail::signal(arg.data_)); - int abs = arg.data_ & 0x7FFF, exp = -15; - if(!abs || abs >= 0x7C00) - return (abs > 0x7C00) ? half(detail::binary, detail::signal(arg.data_)) : arg; - for(; abs < 0x400; abs <<= 1, --exp) - ; - exp += abs >> 10; - detail::uint32 m = static_cast((abs & 0x3FF) | 0x400) << 20; - if(arg.data_ & 0x8000) - { - m = 0x40000000 - (m >> -exp); - for(exp = 0; m < 0x40000000; m <<= 1, --exp) - ; - } - else - { - if(exp < 0) - { - m = 0x40000000 + (m >> -exp); - exp = 0; - } - else - { - m += 0x40000000 >> exp; - int i = m >> 31; - m >>= i; - exp += i; - } - } - return half(detail::binary, - detail::log2_post(detail::log2(m), exp, 17)); -#endif -} - -/// \} -/// \anchor power -/// \name Power functions -/// \{ - -/// Square root. -/// This function is exact to rounding for all rounding modes. -/// -/// **See also:** Documentation for -/// [std::sqrt](https://en.cppreference.com/w/cpp/numeric/math/sqrt). -/// \param arg function argument -/// \return square root of \a arg -/// \exception FE_INVALID for signaling NaN and negative arguments -/// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding -inline half sqrt(half arg) -{ -#ifdef HALF_ARITHMETIC_TYPE - return half(detail::binary, - detail::float2half( - std::sqrt(detail::half2float(arg.data_)))); -#else - int abs = arg.data_ & 0x7FFF, exp = 15; - if(!abs || arg.data_ >= 0x7C00) - return half(detail::binary, - (abs > 0x7C00) ? detail::signal(arg.data_) - : (arg.data_ > 0x8000) ? detail::invalid() : arg.data_); - for(; abs < 0x400; abs <<= 1, --exp) - ; - detail::uint32 r = static_cast((abs & 0x3FF) | 0x400) << 10, - m = detail::sqrt<20>(r, exp += abs >> 10); - return half( - detail::binary, - detail::rounded((exp << 10) + (m & 0x3FF), r > m, r != 0)); -#endif -} - -/// Cubic root. -/// This function is exact to rounding for all rounding modes. -/// -/// **See also:** Documentation for -/// [std::cbrt](https://en.cppreference.com/w/cpp/numeric/math/cbrt). -/// \param arg function argument -/// \return cubic root of \a arg -/// \exception FE_INVALID for signaling NaN -/// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding -inline half cbrt(half arg) -{ -#if defined(HALF_ARITHMETIC_TYPE) && HALF_ENABLE_CPP11_CMATH - return half(detail::binary, - detail::float2half( - std::cbrt(detail::half2float(arg.data_)))); -#else - int abs = arg.data_ & 0x7FFF, exp = -15; - if(!abs || abs == 0x3C00 || abs >= 0x7C00) - return (abs > 0x7C00) ? half(detail::binary, detail::signal(arg.data_)) : arg; - for(; abs < 0x400; abs <<= 1, --exp) - ; - detail::uint32 ilog = exp + (abs >> 10), sign = detail::sign_mask(ilog), f, - m = (((ilog << 27) + - (detail::log2(static_cast((abs & 0x3FF) | 0x400) << 20, - 24) >> - 4)) ^ - sign) - - sign; - for(exp = 2; m < 0x80000000; m <<= 1, --exp) - ; - m = detail::multiply64(m, 0xAAAAAAAB); - int i = m >> 31, s; - exp += i; - m <<= 1 - i; - if(exp < 0) - { - f = m >> -exp; - exp = 0; - } - else - { - f = (m << exp) & 0x7FFFFFFF; - exp = m >> (31 - exp); - } - m = detail::exp2(f, (half::round_style == std::round_to_nearest) ? 29 : 26); - if(sign) - { - if(m > 0x80000000) - { - m = detail::divide64(0x80000000, m, s); - ++exp; - } - exp = -exp; - } - return half(detail::binary, - (half::round_style == std::round_to_nearest) - ? detail::fixed2half( - m, exp + 14, arg.data_ & 0x8000) - : detail::fixed2half( - (m + 0x80) >> 8, exp + 14, arg.data_ & 0x8000)); -#endif -} - -/// Hypotenuse function. -/// This function is exact to rounding for all rounding modes. -/// -/// **See also:** Documentation for -/// [std::hypot](https://en.cppreference.com/w/cpp/numeric/math/hypot). -/// \param x first argument -/// \param y second argument -/// \return square root of sum of squares without internal over- or underflows -/// \exception FE_INVALID if \a x or \a y is signaling NaN -/// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding of the final square root -inline half hypot(half x, half y) -{ -#ifdef HALF_ARITHMETIC_TYPE - detail::internal_t fx = detail::half2float(x.data_), - fy = detail::half2float(y.data_); -#if HALF_ENABLE_CPP11_CMATH - return half(detail::binary, detail::float2half(std::hypot(fx, fy))); -#else - return half(detail::binary, - detail::float2half(std::sqrt(fx * fx + fy * fy))); -#endif -#else - int absx = x.data_ & 0x7FFF, absy = y.data_ & 0x7FFF, expx = 0, expy = 0; - if(absx >= 0x7C00 || absy >= 0x7C00) - return half(detail::binary, - (absx == 0x7C00) ? detail::select(0x7C00, y.data_) - : (absy == 0x7C00) ? detail::select(0x7C00, x.data_) - : detail::signal(x.data_, y.data_)); - if(!absx) - return half(detail::binary, absy ? detail::check_underflow(absy) : 0); - if(!absy) - return half(detail::binary, detail::check_underflow(absx)); - if(absy > absx) - std::swap(absx, absy); - for(; absx < 0x400; absx <<= 1, --expx) - ; - for(; absy < 0x400; absy <<= 1, --expy) - ; - detail::uint32 mx = (absx & 0x3FF) | 0x400, my = (absy & 0x3FF) | 0x400; - mx *= mx; - my *= my; - int ix = mx >> 21, iy = my >> 21; - expx = 2 * (expx + (absx >> 10)) - 15 + ix; - expy = 2 * (expy + (absy >> 10)) - 15 + iy; - mx <<= 10 - ix; - my <<= 10 - iy; - int d = expx - expy; - my = (d < 30) ? ((my >> d) | ((my & ((static_cast(1) << d) - 1)) != 0)) : 1; - return half(detail::binary, detail::hypot_post(mx + my, expx)); -#endif -} - -/// Hypotenuse function. -/// This function is exact to rounding for all rounding modes. -/// -/// **See also:** Documentation for -/// [std::hypot](https://en.cppreference.com/w/cpp/numeric/math/hypot). -/// \param x first argument -/// \param y second argument -/// \param z third argument -/// \return square root of sum of squares without internal over- or underflows -/// \exception FE_INVALID if \a x, \a y or \a z is signaling NaN -/// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding of the final square root -inline half hypot(half x, half y, half z) -{ -#ifdef HALF_ARITHMETIC_TYPE - detail::internal_t fx = detail::half2float(x.data_), - fy = detail::half2float(y.data_), - fz = detail::half2float(z.data_); - return half(detail::binary, - detail::float2half(std::sqrt(fx * fx + fy * fy + fz * fz))); -#else - int absx = x.data_ & 0x7FFF, absy = y.data_ & 0x7FFF, absz = z.data_ & 0x7FFF, expx = 0, - expy = 0, expz = 0; - if(!absx) - return hypot(y, z); - if(!absy) - return hypot(x, z); - if(!absz) - return hypot(x, y); - if(absx >= 0x7C00 || absy >= 0x7C00 || absz >= 0x7C00) - return half(detail::binary, - (absx == 0x7C00) - ? detail::select(0x7C00, detail::select(y.data_, z.data_)) - : (absy == 0x7C00) - ? detail::select(0x7C00, detail::select(x.data_, z.data_)) - : (absz == 0x7C00) - ? detail::select(0x7C00, detail::select(x.data_, y.data_)) - : detail::signal(x.data_, y.data_, z.data_)); - if(absz > absy) - std::swap(absy, absz); - if(absy > absx) - std::swap(absx, absy); - if(absz > absy) - std::swap(absy, absz); - for(; absx < 0x400; absx <<= 1, --expx) - ; - for(; absy < 0x400; absy <<= 1, --expy) - ; - for(; absz < 0x400; absz <<= 1, --expz) - ; - detail::uint32 mx = (absx & 0x3FF) | 0x400, my = (absy & 0x3FF) | 0x400, - mz = (absz & 0x3FF) | 0x400; - mx *= mx; - my *= my; - mz *= mz; - int ix = mx >> 21, iy = my >> 21, iz = mz >> 21; - expx = 2 * (expx + (absx >> 10)) - 15 + ix; - expy = 2 * (expy + (absy >> 10)) - 15 + iy; - expz = 2 * (expz + (absz >> 10)) - 15 + iz; - mx <<= 10 - ix; - my <<= 10 - iy; - mz <<= 10 - iz; - int d = expy - expz; - mz = (d < 30) ? ((mz >> d) | ((mz & ((static_cast(1) << d) - 1)) != 0)) : 1; - my += mz; - if(my & 0x80000000) - { - my = (my >> 1) | (my & 1); - if(++expy > expx) - { - std::swap(mx, my); - std::swap(expx, expy); - } - } - d = expx - expy; - my = (d < 30) ? ((my >> d) | ((my & ((static_cast(1) << d) - 1)) != 0)) : 1; - return half(detail::binary, detail::hypot_post(mx + my, expx)); -#endif -} - -/// Power function. -/// This function may be 1 ULP off the correctly rounded exact result for any rounding mode in -/// ~0.00025% of inputs. -/// -/// **See also:** Documentation for [std::pow](https://en.cppreference.com/w/cpp/numeric/math/pow). -/// \param x base -/// \param y exponent -/// \return \a x raised to \a y -/// \exception FE_INVALID if \a x or \a y is signaling NaN or if \a x is finite an negative and \a y -/// is finite and not integral -/// \exception FE_DIVBYZERO if \a x is 0 and \a y is negative -/// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding -inline half pow(half x, half y) -{ -#ifdef HALF_ARITHMETIC_TYPE - return half(detail::binary, - detail::float2half( - std::pow(detail::half2float(x.data_), - detail::half2float(y.data_)))); -#else - int absx = x.data_ & 0x7FFF, absy = y.data_ & 0x7FFF, exp = -15; - if(!absy || x.data_ == 0x3C00) - return half(detail::binary, - detail::select(0x3C00, (x.data_ == 0x3C00) ? y.data_ : x.data_)); - bool is_int = absy >= 0x6400 || (absy >= 0x3C00 && !(absy & ((1 << (25 - (absy >> 10))) - 1))); - unsigned int sign = - x.data_ & - (static_cast((absy < 0x6800) && is_int && ((absy >> (25 - (absy >> 10))) & 1)) - << 15); - if(absx >= 0x7C00 || absy >= 0x7C00) - return half(detail::binary, - (absx > 0x7C00 || absy > 0x7C00) - ? detail::signal(x.data_, y.data_) - : (absy == 0x7C00) - ? ((absx == 0x3C00) - ? 0x3C00 - : (!absx && y.data_ == 0xFC00) - ? detail::pole() - : (0x7C00 & -((y.data_ >> 15) ^ (absx > 0x3C00)))) - : (sign | (0x7C00 & ((y.data_ >> 15) - 1U)))); - if(!absx) - return half(detail::binary, (y.data_ & 0x8000) ? detail::pole(sign) : sign); - if((x.data_ & 0x8000) && !is_int) - return half(detail::binary, detail::invalid()); - if(x.data_ == 0xBC00) - return half(detail::binary, sign | 0x3C00); - if(y.data_ == 0x3800) - return sqrt(x); - if(y.data_ == 0x3C00) - return half(detail::binary, detail::check_underflow(x.data_)); - if(y.data_ == 0x4000) - return x * x; - for(; absx < 0x400; absx <<= 1, --exp) - ; - detail::uint32 ilog = exp + (absx >> 10), msign = detail::sign_mask(ilog), f, - m = (((ilog << 27) + - ((detail::log2(static_cast((absx & 0x3FF) | 0x400) << 20) + - 8) >> - 4)) ^ - msign) - - msign; - for(exp = -11; m < 0x80000000; m <<= 1, --exp) - ; - for(; absy < 0x400; absy <<= 1, --exp) - ; - m = detail::multiply64(m, static_cast((absy & 0x3FF) | 0x400) << 21); - int i = m >> 31; - exp += (absy >> 10) + i; - m <<= 1 - i; - if(exp < 0) - { - f = m >> -exp; - exp = 0; - } - else - { - f = (m << exp) & 0x7FFFFFFF; - exp = m >> (31 - exp); - } - return half(detail::binary, - detail::exp2_post( - detail::exp2(f), exp, ((msign & 1) ^ (y.data_ >> 15)) != 0, sign)); -#endif -} - -/// \} -/// \anchor trigonometric -/// \name Trigonometric functions -/// \{ - -/// Compute sine and cosine simultaneously. -/// This returns the same results as sin() and cos() but is faster than calling each function -/// individually. -/// -/// This function is exact to rounding for all rounding modes. -/// \param arg function argument -/// \param sin variable to take sine of \a arg -/// \param cos variable to take cosine of \a arg -/// \exception FE_INVALID for signaling NaN or infinity -/// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding -inline void sincos(half arg, half* sin, half* cos) -{ -#ifdef HALF_ARITHMETIC_TYPE - detail::internal_t f = detail::half2float(arg.data_); - *sin = half(detail::binary, detail::float2half(std::sin(f))); - *cos = half(detail::binary, detail::float2half(std::cos(f))); -#else - int abs = arg.data_ & 0x7FFF, sign = arg.data_ >> 15, k; - if(abs >= 0x7C00) - *sin = *cos = - half(detail::binary, (abs == 0x7C00) ? detail::invalid() : detail::signal(arg.data_)); - else if(!abs) - { - *sin = arg; - *cos = half(detail::binary, 0x3C00); - } - else if(abs < 0x2500) - { - *sin = half(detail::binary, detail::rounded(arg.data_ - 1, 1, 1)); - *cos = half(detail::binary, detail::rounded(0x3BFF, 1, 1)); - } - else - { - if(half::round_style != std::round_to_nearest) - { - switch(abs) - { - case 0x48B7: - *sin = half( - detail::binary, - detail::rounded((~arg.data_ & 0x8000) | 0x1D07, 1, 1)); - *cos = half(detail::binary, detail::rounded(0xBBFF, 1, 1)); - return; - case 0x598C: - *sin = half( - detail::binary, - detail::rounded((arg.data_ & 0x8000) | 0x3BFF, 1, 1)); - *cos = half(detail::binary, detail::rounded(0x80FC, 1, 1)); - return; - case 0x6A64: - *sin = half( - detail::binary, - detail::rounded((~arg.data_ & 0x8000) | 0x3BFE, 1, 1)); - *cos = half(detail::binary, detail::rounded(0x27FF, 1, 1)); - return; - case 0x6D8C: - *sin = half( - detail::binary, - detail::rounded((arg.data_ & 0x8000) | 0x0FE6, 1, 1)); - *cos = half(detail::binary, detail::rounded(0x3BFF, 1, 1)); - return; - } - } - std::pair sc = - detail::sincos(detail::angle_arg(abs, k), 28); - switch(k & 3) - { - case 1: sc = std::make_pair(sc.second, -sc.first); break; - case 2: sc = std::make_pair(-sc.first, -sc.second); break; - case 3: sc = std::make_pair(-sc.second, sc.first); break; - } - *sin = half(detail::binary, - detail::fixed2half( - (sc.first ^ -static_cast(sign)) + sign)); - *cos = half(detail::binary, - detail::fixed2half(sc.second)); - } -#endif -} - -/// Sine function. -/// This function is exact to rounding for all rounding modes. -/// -/// **See also:** Documentation for [std::sin](https://en.cppreference.com/w/cpp/numeric/math/sin). -/// \param arg function argument -/// \return sine value of \a arg -/// \exception FE_INVALID for signaling NaN or infinity -/// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding -inline half sin(half arg) -{ -#ifdef HALF_ARITHMETIC_TYPE - return half(detail::binary, - detail::float2half( - std::sin(detail::half2float(arg.data_)))); -#else - int abs = arg.data_ & 0x7FFF, k; - if(!abs) - return arg; - if(abs >= 0x7C00) - return half(detail::binary, - (abs == 0x7C00) ? detail::invalid() : detail::signal(arg.data_)); - if(abs < 0x2900) - return half(detail::binary, detail::rounded(arg.data_ - 1, 1, 1)); - if(half::round_style != std::round_to_nearest) - switch(abs) - { - case 0x48B7: - return half( - detail::binary, - detail::rounded((~arg.data_ & 0x8000) | 0x1D07, 1, 1)); - case 0x6A64: - return half( - detail::binary, - detail::rounded((~arg.data_ & 0x8000) | 0x3BFE, 1, 1)); - case 0x6D8C: - return half( - detail::binary, - detail::rounded((arg.data_ & 0x8000) | 0x0FE6, 1, 1)); - } - std::pair sc = detail::sincos(detail::angle_arg(abs, k), 28); - detail::uint32 sign = -static_cast(((k >> 1) & 1) ^ (arg.data_ >> 15)); - return half(detail::binary, - detail::fixed2half( - (((k & 1) ? sc.second : sc.first) ^ sign) - sign)); -#endif -} - -/// Cosine function. -/// This function is exact to rounding for all rounding modes. -/// -/// **See also:** Documentation for [std::cos](https://en.cppreference.com/w/cpp/numeric/math/cos). -/// \param arg function argument -/// \return cosine value of \a arg -/// \exception FE_INVALID for signaling NaN or infinity -/// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding -inline half cos(half arg) -{ -#ifdef HALF_ARITHMETIC_TYPE - return half(detail::binary, - detail::float2half( - std::cos(detail::half2float(arg.data_)))); -#else - int abs = arg.data_ & 0x7FFF, k; - if(!abs) - return half(detail::binary, 0x3C00); - if(abs >= 0x7C00) - return half(detail::binary, - (abs == 0x7C00) ? detail::invalid() : detail::signal(arg.data_)); - if(abs < 0x2500) - return half(detail::binary, detail::rounded(0x3BFF, 1, 1)); - if(half::round_style != std::round_to_nearest && abs == 0x598C) - return half(detail::binary, detail::rounded(0x80FC, 1, 1)); - std::pair sc = detail::sincos(detail::angle_arg(abs, k), 28); - detail::uint32 sign = -static_cast(((k >> 1) ^ k) & 1); - return half(detail::binary, - detail::fixed2half( - (((k & 1) ? sc.first : sc.second) ^ sign) - sign)); -#endif -} - -/// Tangent function. -/// This function is exact to rounding for all rounding modes. -/// -/// **See also:** Documentation for [std::tan](https://en.cppreference.com/w/cpp/numeric/math/tan). -/// \param arg function argument -/// \return tangent value of \a arg -/// \exception FE_INVALID for signaling NaN or infinity -/// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding -inline half tan(half arg) -{ -#ifdef HALF_ARITHMETIC_TYPE - return half(detail::binary, - detail::float2half( - std::tan(detail::half2float(arg.data_)))); -#else - int abs = arg.data_ & 0x7FFF, exp = 13, k; - if(!abs) - return arg; - if(abs >= 0x7C00) - return half(detail::binary, - (abs == 0x7C00) ? detail::invalid() : detail::signal(arg.data_)); - if(abs < 0x2700) - return half(detail::binary, detail::rounded(arg.data_, 0, 1)); - if(half::round_style != std::round_to_nearest) - switch(abs) - { - case 0x658C: - return half( - detail::binary, - detail::rounded((arg.data_ & 0x8000) | 0x07E6, 1, 1)); - case 0x7330: - return half( - detail::binary, - detail::rounded((~arg.data_ & 0x8000) | 0x4B62, 1, 1)); - } - std::pair sc = detail::sincos(detail::angle_arg(abs, k), 30); - if(k & 1) - sc = std::make_pair(-sc.second, sc.first); - detail::uint32 signy = detail::sign_mask(sc.first), signx = detail::sign_mask(sc.second); - detail::uint32 my = (sc.first ^ signy) - signy, mx = (sc.second ^ signx) - signx; - for(; my < 0x80000000; my <<= 1, --exp) - ; - for(; mx < 0x80000000; mx <<= 1, ++exp) - ; - return half( - detail::binary, - detail::tangent_post(my, mx, exp, (signy ^ signx ^ arg.data_) & 0x8000)); -#endif -} - -/// Arc sine. -/// This function is exact to rounding for all rounding modes. -/// -/// **See also:** Documentation for -/// [std::asin](https://en.cppreference.com/w/cpp/numeric/math/asin). -/// \param arg function argument -/// \return arc sine value of \a arg -/// \exception FE_INVALID for signaling NaN or if abs(\a arg) > 1 -/// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding -inline half asin(half arg) -{ -#ifdef HALF_ARITHMETIC_TYPE - return half(detail::binary, - detail::float2half( - std::asin(detail::half2float(arg.data_)))); -#else - unsigned int abs = arg.data_ & 0x7FFF, sign = arg.data_ & 0x8000; - if(!abs) - return arg; - if(abs >= 0x3C00) - return half(detail::binary, - (abs > 0x7C00) - ? detail::signal(arg.data_) - : (abs > 0x3C00) - ? detail::invalid() - : detail::rounded(sign | 0x3E48, 0, 1)); - if(abs < 0x2900) - return half(detail::binary, detail::rounded(arg.data_, 0, 1)); - if(half::round_style != std::round_to_nearest && (abs == 0x2B44 || abs == 0x2DC3)) - return half(detail::binary, detail::rounded(arg.data_ + 1, 1, 1)); - std::pair sc = detail::atan2_args(abs); - detail::uint32 m = - detail::atan2(sc.first, sc.second, (half::round_style == std::round_to_nearest) ? 27 : 26); - return half(detail::binary, - detail::fixed2half(m, 14, sign)); -#endif -} - -/// Arc cosine function. -/// This function is exact to rounding for all rounding modes. -/// -/// **See also:** Documentation for -/// [std::acos](https://en.cppreference.com/w/cpp/numeric/math/acos). -/// \param arg function argument -/// \return arc cosine value of \a arg -/// \exception FE_INVALID for signaling NaN or if abs(\a arg) > 1 -/// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding -inline half acos(half arg) -{ -#ifdef HALF_ARITHMETIC_TYPE - return half(detail::binary, - detail::float2half( - std::acos(detail::half2float(arg.data_)))); -#else - unsigned int abs = arg.data_ & 0x7FFF, sign = arg.data_ >> 15; - if(!abs) - return half(detail::binary, detail::rounded(0x3E48, 0, 1)); - if(abs >= 0x3C00) - return half(detail::binary, - (abs > 0x7C00) - ? detail::signal(arg.data_) - : (abs > 0x3C00) - ? detail::invalid() - : sign ? detail::rounded(0x4248, 0, 1) : 0); - std::pair cs = detail::atan2_args(abs); - detail::uint32 m = detail::atan2(cs.second, cs.first, 28); - return half(detail::binary, - detail::fixed2half( - sign ? (0xC90FDAA2 - m) : m, 15, 0, sign)); -#endif -} - -/// Arc tangent function. -/// This function is exact to rounding for all rounding modes. -/// -/// **See also:** Documentation for -/// [std::atan](https://en.cppreference.com/w/cpp/numeric/math/atan). -/// \param arg function argument -/// \return arc tangent value of \a arg -/// \exception FE_INVALID for signaling NaN -/// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding -inline half atan(half arg) -{ -#ifdef HALF_ARITHMETIC_TYPE - return half(detail::binary, - detail::float2half( - std::atan(detail::half2float(arg.data_)))); -#else - unsigned int abs = arg.data_ & 0x7FFF, sign = arg.data_ & 0x8000; - if(!abs) - return arg; - if(abs >= 0x7C00) - return half(detail::binary, - (abs == 0x7C00) ? detail::rounded(sign | 0x3E48, 0, 1) - : detail::signal(arg.data_)); - if(abs <= 0x2700) - return half(detail::binary, detail::rounded(arg.data_ - 1, 1, 1)); - int exp = (abs >> 10) + (abs <= 0x3FF); - detail::uint32 my = (abs & 0x3FF) | ((abs > 0x3FF) << 10); - detail::uint32 m = (exp > 15) - ? detail::atan2(my << 19, - 0x20000000 >> (exp - 15), - (half::round_style == std::round_to_nearest) ? 26 : 24) - : detail::atan2(my << (exp + 4), - 0x20000000, - (half::round_style == std::round_to_nearest) ? 30 : 28); - return half(detail::binary, - detail::fixed2half(m, 14, sign)); -#endif -} - -/// Arc tangent function. -/// This function may be 1 ULP off the correctly rounded exact result in ~0.005% of inputs for -/// `std::round_to_nearest`, -/// in ~0.1% of inputs for `std::round_toward_zero` and in ~0.02% of inputs for any other rounding -/// mode. -/// -/// **See also:** Documentation for -/// [std::atan2](https://en.cppreference.com/w/cpp/numeric/math/atan2). -/// \param y numerator -/// \param x denominator -/// \return arc tangent value -/// \exception FE_INVALID if \a x or \a y is signaling NaN -/// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding -inline half atan2(half y, half x) -{ -#ifdef HALF_ARITHMETIC_TYPE - return half(detail::binary, - detail::float2half( - std::atan2(detail::half2float(y.data_), - detail::half2float(x.data_)))); -#else - unsigned int absx = x.data_ & 0x7FFF, absy = y.data_ & 0x7FFF, signx = x.data_ >> 15, - signy = y.data_ & 0x8000; - if(absx >= 0x7C00 || absy >= 0x7C00) - { - if(absx > 0x7C00 || absy > 0x7C00) - return half(detail::binary, detail::signal(x.data_, y.data_)); - if(absy == 0x7C00) - return half(detail::binary, - (absx < 0x7C00) - ? detail::rounded(signy | 0x3E48, 0, 1) - : signx - ? detail::rounded(signy | 0x40B6, 0, 1) - : detail::rounded(signy | 0x3A48, 0, 1)); - return (x.data_ == 0x7C00) - ? half(detail::binary, signy) - : half(detail::binary, - detail::rounded(signy | 0x4248, 0, 1)); - } - if(!absy) - return signx ? half(detail::binary, - detail::rounded(signy | 0x4248, 0, 1)) - : y; - if(!absx) - return half(detail::binary, detail::rounded(signy | 0x3E48, 0, 1)); - int d = (absy >> 10) + (absy <= 0x3FF) - (absx >> 10) - (absx <= 0x3FF); - if(d > (signx ? 18 : 12)) - return half(detail::binary, detail::rounded(signy | 0x3E48, 0, 1)); - if(signx && d < -11) - return half(detail::binary, detail::rounded(signy | 0x4248, 0, 1)); - if(!signx && d < ((half::round_style == std::round_toward_zero) ? -15 : -9)) - { - for(; absy < 0x400; absy <<= 1, --d) - ; - detail::uint32 mx = ((absx << 1) & 0x7FF) | 0x800, my = ((absy << 1) & 0x7FF) | 0x800; - int i = my < mx; - d -= i; - if(d < -25) - return half(detail::binary, detail::underflow(signy)); - my <<= 11 + i; - return half(detail::binary, - detail::fixed2half( - my / mx, d + 14, signy, my % mx != 0)); - } - detail::uint32 m = detail::atan2( - ((absy & 0x3FF) | ((absy > 0x3FF) << 10)) << (19 + ((d < 0) ? d : (d > 0) ? 0 : -1)), - ((absx & 0x3FF) | ((absx > 0x3FF) << 10)) << (19 - ((d > 0) ? d : (d < 0) ? 0 : 1))); - return half(detail::binary, - detail::fixed2half( - signx ? (0xC90FDAA2 - m) : m, 15, signy, signx)); -#endif -} - -/// \} -/// \anchor hyperbolic -/// \name Hyperbolic functions -/// \{ - -/// Hyperbolic sine. -/// This function is exact to rounding for all rounding modes. -/// -/// **See also:** Documentation for -/// [std::sinh](https://en.cppreference.com/w/cpp/numeric/math/sinh). -/// \param arg function argument -/// \return hyperbolic sine value of \a arg -/// \exception FE_INVALID for signaling NaN -/// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding -inline half sinh(half arg) -{ -#ifdef HALF_ARITHMETIC_TYPE - return half(detail::binary, - detail::float2half( - std::sinh(detail::half2float(arg.data_)))); -#else - int abs = arg.data_ & 0x7FFF, exp; - if(!abs || abs >= 0x7C00) - return (abs > 0x7C00) ? half(detail::binary, detail::signal(arg.data_)) : arg; - if(abs <= 0x2900) - return half(detail::binary, detail::rounded(arg.data_, 0, 1)); - std::pair mm = - detail::hyperbolic_args(abs, exp, (half::round_style == std::round_to_nearest) ? 29 : 27); - detail::uint32 m = mm.first - mm.second; - for(exp += 13; m < 0x80000000 && exp; m <<= 1, --exp) - ; - unsigned int sign = arg.data_ & 0x8000; - if(exp > 29) - return half(detail::binary, detail::overflow(sign)); - return half(detail::binary, - detail::fixed2half(m, exp, sign)); -#endif -} - -/// Hyperbolic cosine. -/// This function is exact to rounding for all rounding modes. -/// -/// **See also:** Documentation for -/// [std::cosh](https://en.cppreference.com/w/cpp/numeric/math/cosh). -/// \param arg function argument -/// \return hyperbolic cosine value of \a arg -/// \exception FE_INVALID for signaling NaN -/// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding -inline half cosh(half arg) -{ -#ifdef HALF_ARITHMETIC_TYPE - return half(detail::binary, - detail::float2half( - std::cosh(detail::half2float(arg.data_)))); -#else - int abs = arg.data_ & 0x7FFF, exp; - if(!abs) - return half(detail::binary, 0x3C00); - if(abs >= 0x7C00) - return half(detail::binary, (abs > 0x7C00) ? detail::signal(arg.data_) : 0x7C00); - std::pair mm = - detail::hyperbolic_args(abs, exp, (half::round_style == std::round_to_nearest) ? 23 : 26); - detail::uint32 m = mm.first + mm.second, i = (~m & 0xFFFFFFFF) >> 31; - m = (m >> i) | (m & i) | 0x80000000; - if((exp += 13 + i) > 29) - return half(detail::binary, detail::overflow()); - return half(detail::binary, - detail::fixed2half(m, exp)); -#endif -} - -/// Hyperbolic tangent. -/// This function is exact to rounding for all rounding modes. -/// -/// **See also:** Documentation for -/// [std::tanh](https://en.cppreference.com/w/cpp/numeric/math/tanh). -/// \param arg function argument -/// \return hyperbolic tangent value of \a arg -/// \exception FE_INVALID for signaling NaN -/// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding -inline half tanh(half arg) -{ -#ifdef HALF_ARITHMETIC_TYPE - return half(detail::binary, - detail::float2half( - std::tanh(detail::half2float(arg.data_)))); -#else - int abs = arg.data_ & 0x7FFF, exp; - if(!abs) - return arg; - if(abs >= 0x7C00) - return half(detail::binary, - (abs > 0x7C00) ? detail::signal(arg.data_) : (arg.data_ - 0x4000)); - if(abs >= 0x4500) - return half(detail::binary, - detail::rounded((arg.data_ & 0x8000) | 0x3BFF, 1, 1)); - if(abs < 0x2700) - return half(detail::binary, detail::rounded(arg.data_ - 1, 1, 1)); - if(half::round_style != std::round_to_nearest && abs == 0x2D3F) - return half(detail::binary, detail::rounded(arg.data_ - 3, 0, 1)); - std::pair mm = detail::hyperbolic_args(abs, exp, 27); - detail::uint32 my = mm.first - mm.second - (half::round_style != std::round_to_nearest), - mx = mm.first + mm.second, i = (~mx & 0xFFFFFFFF) >> 31; - for(exp = 13; my < 0x80000000; my <<= 1, --exp) - ; - mx = (mx >> i) | 0x80000000; - return half(detail::binary, - detail::tangent_post(my, mx, exp - i, arg.data_ & 0x8000)); -#endif -} - -/// Hyperbolic area sine. -/// This function is exact to rounding for all rounding modes. -/// -/// **See also:** Documentation for -/// [std::asinh](https://en.cppreference.com/w/cpp/numeric/math/asinh). -/// \param arg function argument -/// \return area sine value of \a arg -/// \exception FE_INVALID for signaling NaN -/// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding -inline half asinh(half arg) -{ -#if defined(HALF_ARITHMETIC_TYPE) && HALF_ENABLE_CPP11_CMATH - return half(detail::binary, - detail::float2half( - std::asinh(detail::half2float(arg.data_)))); -#else - int abs = arg.data_ & 0x7FFF; - if(!abs || abs >= 0x7C00) - return (abs > 0x7C00) ? half(detail::binary, detail::signal(arg.data_)) : arg; - if(abs <= 0x2900) - return half(detail::binary, detail::rounded(arg.data_ - 1, 1, 1)); - if(half::round_style != std::round_to_nearest) - switch(abs) - { - case 0x32D4: - return half(detail::binary, - detail::rounded(arg.data_ - 13, 1, 1)); - case 0x3B5B: - return half(detail::binary, - detail::rounded(arg.data_ - 197, 1, 1)); - } - return half(detail::binary, detail::area(arg.data_)); -#endif -} - -/// Hyperbolic area cosine. -/// This function is exact to rounding for all rounding modes. -/// -/// **See also:** Documentation for -/// [std::acosh](https://en.cppreference.com/w/cpp/numeric/math/acosh). -/// \param arg function argument -/// \return area cosine value of \a arg -/// \exception FE_INVALID for signaling NaN or arguments <1 -/// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding -inline half acosh(half arg) -{ -#if defined(HALF_ARITHMETIC_TYPE) && HALF_ENABLE_CPP11_CMATH - return half(detail::binary, - detail::float2half( - std::acosh(detail::half2float(arg.data_)))); -#else - int abs = arg.data_ & 0x7FFF; - if((arg.data_ & 0x8000) || abs < 0x3C00) - return half(detail::binary, - (abs <= 0x7C00) ? detail::invalid() : detail::signal(arg.data_)); - if(abs == 0x3C00) - return half(detail::binary, 0); - if(arg.data_ >= 0x7C00) - return (abs > 0x7C00) ? half(detail::binary, detail::signal(arg.data_)) : arg; - return half(detail::binary, detail::area(arg.data_)); -#endif -} - -/// Hyperbolic area tangent. -/// This function is exact to rounding for all rounding modes. -/// -/// **See also:** Documentation for -/// [std::atanh](https://en.cppreference.com/w/cpp/numeric/math/atanh). -/// \param arg function argument -/// \return area tangent value of \a arg -/// \exception FE_INVALID for signaling NaN or if abs(\a arg) > 1 -/// \exception FE_DIVBYZERO for +/-1 -/// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding -inline half atanh(half arg) -{ -#if defined(HALF_ARITHMETIC_TYPE) && HALF_ENABLE_CPP11_CMATH - return half(detail::binary, - detail::float2half( - std::atanh(detail::half2float(arg.data_)))); -#else - int abs = arg.data_ & 0x7FFF, exp = 0; - if(!abs) - return arg; - if(abs >= 0x3C00) - return half(detail::binary, - (abs == 0x3C00) - ? detail::pole(arg.data_ & 0x8000) - : (abs <= 0x7C00) ? detail::invalid() : detail::signal(arg.data_)); - if(abs < 0x2700) - return half(detail::binary, detail::rounded(arg.data_, 0, 1)); - detail::uint32 m = static_cast((abs & 0x3FF) | ((abs > 0x3FF) << 10)) - << ((abs >> 10) + (abs <= 0x3FF) + 6), - my = 0x80000000 + m, mx = 0x80000000 - m; - for(; mx < 0x80000000; mx <<= 1, ++exp) - ; - int i = my >= mx, s; - return half(detail::binary, - detail::log2_post( - detail::log2((detail::divide64(my >> i, mx, s) + 1) >> 1, 27) + 0x10, - exp + i - 1, - 16, - arg.data_ & 0x8000)); -#endif -} - -/// \} -/// \anchor special -/// \name Error and gamma functions -/// \{ - -/// Error function. -/// This function may be 1 ULP off the correctly rounded exact result for any rounding mode in <0.5% -/// of inputs. -/// -/// **See also:** Documentation for [std::erf](https://en.cppreference.com/w/cpp/numeric/math/erf). -/// \param arg function argument -/// \return error function value of \a arg -/// \exception FE_INVALID for signaling NaN -/// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding -inline half erf(half arg) -{ -#if defined(HALF_ARITHMETIC_TYPE) && HALF_ENABLE_CPP11_CMATH - return half(detail::binary, - detail::float2half( - std::erf(detail::half2float(arg.data_)))); -#else - unsigned int abs = arg.data_ & 0x7FFF; - if(!abs || abs >= 0x7C00) - return (abs >= 0x7C00) - ? half(detail::binary, - (abs == 0x7C00) ? (arg.data_ - 0x4000) : detail::signal(arg.data_)) - : arg; - if(abs >= 0x4200) - return half(detail::binary, - detail::rounded((arg.data_ & 0x8000) | 0x3BFF, 1, 1)); - return half(detail::binary, detail::erf(arg.data_)); -#endif -} - -/// Complementary error function. -/// This function may be 1 ULP off the correctly rounded exact result for any rounding mode in <0.5% -/// of inputs. -/// -/// **See also:** Documentation for -/// [std::erfc](https://en.cppreference.com/w/cpp/numeric/math/erfc). -/// \param arg function argument -/// \return 1 minus error function value of \a arg -/// \exception FE_INVALID for signaling NaN -/// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding -inline half erfc(half arg) -{ -#if defined(HALF_ARITHMETIC_TYPE) && HALF_ENABLE_CPP11_CMATH - return half(detail::binary, - detail::float2half( - std::erfc(detail::half2float(arg.data_)))); -#else - unsigned int abs = arg.data_ & 0x7FFF, sign = arg.data_ & 0x8000; - if(abs >= 0x7C00) - return (abs >= 0x7C00) - ? half(detail::binary, (abs == 0x7C00) ? (sign >> 1) : detail::signal(arg.data_)) - : arg; - if(!abs) - return half(detail::binary, 0x3C00); - if(abs >= 0x4400) - return half( - detail::binary, - detail::rounded((sign >> 1) - (sign >> 15), sign >> 15, 1)); - return half(detail::binary, detail::erf(arg.data_)); -#endif -} - -/// Natural logarithm of gamma function. -/// This function may be 1 ULP off the correctly rounded exact result for any rounding mode in -/// ~0.025% of inputs. -/// -/// **See also:** Documentation for -/// [std::lgamma](https://en.cppreference.com/w/cpp/numeric/math/lgamma). -/// \param arg function argument -/// \return natural logarith of gamma function for \a arg -/// \exception FE_INVALID for signaling NaN -/// \exception FE_DIVBYZERO for 0 or negative integer arguments -/// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding -inline half lgamma(half arg) -{ -#if defined(HALF_ARITHMETIC_TYPE) && HALF_ENABLE_CPP11_CMATH - return half(detail::binary, - detail::float2half( - std::lgamma(detail::half2float(arg.data_)))); -#else - int abs = arg.data_ & 0x7FFF; - if(abs >= 0x7C00) - return half(detail::binary, (abs == 0x7C00) ? 0x7C00 : detail::signal(arg.data_)); - if(!abs || arg.data_ >= 0xE400 || - (arg.data_ >= 0xBC00 && !(abs & ((1 << (25 - (abs >> 10))) - 1)))) - return half(detail::binary, detail::pole()); - if(arg.data_ == 0x3C00 || arg.data_ == 0x4000) - return half(detail::binary, 0); - return half(detail::binary, detail::gamma(arg.data_)); -#endif -} - -/// Gamma function. -/// This function may be 1 ULP off the correctly rounded exact result for any rounding mode in -/// <0.25% of inputs. -/// -/// **See also:** Documentation for -/// [std::tgamma](https://en.cppreference.com/w/cpp/numeric/math/tgamma). -/// \param arg function argument -/// \return gamma function value of \a arg -/// \exception FE_INVALID for signaling NaN, negative infinity or negative integer arguments -/// \exception FE_DIVBYZERO for 0 -/// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding -inline half tgamma(half arg) -{ -#if defined(HALF_ARITHMETIC_TYPE) && HALF_ENABLE_CPP11_CMATH - return half(detail::binary, - detail::float2half( - std::tgamma(detail::half2float(arg.data_)))); -#else - unsigned int abs = arg.data_ & 0x7FFF; - if(!abs) - return half(detail::binary, detail::pole(arg.data_)); - if(abs >= 0x7C00) - return (arg.data_ == 0x7C00) ? arg : half(detail::binary, detail::signal(arg.data_)); - if(arg.data_ >= 0xE400 || (arg.data_ >= 0xBC00 && !(abs & ((1 << (25 - (abs >> 10))) - 1)))) - return half(detail::binary, detail::invalid()); - if(arg.data_ >= 0xCA80) - return half( - detail::binary, - detail::underflow((1 - ((abs >> (25 - (abs >> 10))) & 1)) << 15)); - if(arg.data_ <= 0x100 || (arg.data_ >= 0x4900 && arg.data_ < 0x8000)) - return half(detail::binary, detail::overflow()); - if(arg.data_ == 0x3C00) - return arg; - return half(detail::binary, detail::gamma(arg.data_)); -#endif -} - -/// \} -/// \anchor rounding -/// \name Rounding -/// \{ - -/// Nearest integer not less than half value. -/// **See also:** Documentation for -/// [std::ceil](https://en.cppreference.com/w/cpp/numeric/math/ceil). -/// \param arg half to round -/// \return nearest integer not less than \a arg -/// \exception FE_INVALID for signaling NaN -/// \exception FE_INEXACT if value had to be rounded -inline half ceil(half arg) -{ - return half(detail::binary, - detail::integral(arg.data_)); -} - -/// Nearest integer not greater than half value. -/// **See also:** Documentation for -/// [std::floor](https://en.cppreference.com/w/cpp/numeric/math/floor). -/// \param arg half to round -/// \return nearest integer not greater than \a arg -/// \exception FE_INVALID for signaling NaN -/// \exception FE_INEXACT if value had to be rounded -inline half floor(half arg) -{ - return half(detail::binary, - detail::integral(arg.data_)); -} - -/// Nearest integer not greater in magnitude than half value. -/// **See also:** Documentation for -/// [std::trunc](https://en.cppreference.com/w/cpp/numeric/math/trunc). -/// \param arg half to round -/// \return nearest integer not greater in magnitude than \a arg -/// \exception FE_INVALID for signaling NaN -/// \exception FE_INEXACT if value had to be rounded -inline half trunc(half arg) -{ - return half(detail::binary, detail::integral(arg.data_)); -} - -/// Nearest integer. -/// **See also:** Documentation for -/// [std::round](https://en.cppreference.com/w/cpp/numeric/math/round). -/// \param arg half to round -/// \return nearest integer, rounded away from zero in half-way cases -/// \exception FE_INVALID for signaling NaN -/// \exception FE_INEXACT if value had to be rounded -inline half round(half arg) -{ - return half(detail::binary, detail::integral(arg.data_)); -} - -/// Nearest integer. -/// **See also:** Documentation for -/// [std::lround](https://en.cppreference.com/w/cpp/numeric/math/round). -/// \param arg half to round -/// \return nearest integer, rounded away from zero in half-way cases -/// \exception FE_INVALID if value is not representable as `long` -inline long lround(half arg) -{ - return detail::half2int(arg.data_); -} - -/// Nearest integer using half's internal rounding mode. -/// **See also:** Documentation for -/// [std::rint](https://en.cppreference.com/w/cpp/numeric/math/rint). -/// \param arg half expression to round -/// \return nearest integer using default rounding mode -/// \exception FE_INVALID for signaling NaN -/// \exception FE_INEXACT if value had to be rounded -inline half rint(half arg) -{ - return half(detail::binary, detail::integral(arg.data_)); -} - -/// Nearest integer using half's internal rounding mode. -/// **See also:** Documentation for -/// [std::lrint](https://en.cppreference.com/w/cpp/numeric/math/rint). -/// \param arg half expression to round -/// \return nearest integer using default rounding mode -/// \exception FE_INVALID if value is not representable as `long` -/// \exception FE_INEXACT if value had to be rounded -inline long lrint(half arg) -{ - return detail::half2int(arg.data_); -} - -/// Nearest integer using half's internal rounding mode. -/// **See also:** Documentation for -/// [std::nearbyint](https://en.cppreference.com/w/cpp/numeric/math/nearbyint). -/// \param arg half expression to round -/// \return nearest integer using default rounding mode -/// \exception FE_INVALID for signaling NaN -inline half nearbyint(half arg) -{ - return half(detail::binary, detail::integral(arg.data_)); -} -#if HALF_ENABLE_CPP11_LONG_LONG -/// Nearest integer. -/// **See also:** Documentation for -/// [std::llround](https://en.cppreference.com/w/cpp/numeric/math/round). -/// \param arg half to round -/// \return nearest integer, rounded away from zero in half-way cases -/// \exception FE_INVALID if value is not representable as `long long` -inline long long llround(half arg) -{ - return detail::half2int(arg.data_); -} - -/// Nearest integer using half's internal rounding mode. -/// **See also:** Documentation for -/// [std::llrint](https://en.cppreference.com/w/cpp/numeric/math/rint). -/// \param arg half expression to round -/// \return nearest integer using default rounding mode -/// \exception FE_INVALID if value is not representable as `long long` -/// \exception FE_INEXACT if value had to be rounded -inline long long llrint(half arg) -{ - return detail::half2int(arg.data_); -} -#endif - -/// \} -/// \anchor float -/// \name Floating point manipulation -/// \{ - -/// Decompress floating-point number. -/// **See also:** Documentation for -/// [std::frexp](https://en.cppreference.com/w/cpp/numeric/math/frexp). -/// \param arg number to decompress -/// \param exp address to store exponent at -/// \return significant in range [0.5, 1) -/// \exception FE_INVALID for signaling NaN -inline half frexp(half arg, int* exp) -{ - *exp = 0; - unsigned int abs = arg.data_ & 0x7FFF; - if(abs >= 0x7C00 || !abs) - return (abs > 0x7C00) ? half(detail::binary, detail::signal(arg.data_)) : arg; - for(; abs < 0x400; abs <<= 1, --*exp) - ; - *exp += (abs >> 10) - 14; - return half(detail::binary, (arg.data_ & 0x8000) | 0x3800 | (abs & 0x3FF)); -} - -/// Multiply by power of two. -/// This function is exact to rounding for all rounding modes. -/// -/// **See also:** Documentation for -/// [std::scalbln](https://en.cppreference.com/w/cpp/numeric/math/scalbn). -/// \param arg number to modify -/// \param exp power of two to multiply with -/// \return \a arg multplied by 2 raised to \a exp -/// \exception FE_INVALID for signaling NaN -/// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding -inline half scalbln(half arg, long exp) -{ - unsigned int abs = arg.data_ & 0x7FFF, sign = arg.data_ & 0x8000; - if(abs >= 0x7C00 || !abs) - return (abs > 0x7C00) ? half(detail::binary, detail::signal(arg.data_)) : arg; - for(; abs < 0x400; abs <<= 1, --exp) - ; - exp += abs >> 10; - if(exp > 30) - return half(detail::binary, detail::overflow(sign)); - else if(exp < -10) - return half(detail::binary, detail::underflow(sign)); - else if(exp > 0) - return half(detail::binary, sign | (exp << 10) | (abs & 0x3FF)); - unsigned int m = (abs & 0x3FF) | 0x400; - return half(detail::binary, - detail::rounded( - sign | (m >> (1 - exp)), (m >> -exp) & 1, (m & ((1 << -exp) - 1)) != 0)); -} - -/// Multiply by power of two. -/// This function is exact to rounding for all rounding modes. -/// -/// **See also:** Documentation for -/// [std::scalbn](https://en.cppreference.com/w/cpp/numeric/math/scalbn). -/// \param arg number to modify -/// \param exp power of two to multiply with -/// \return \a arg multplied by 2 raised to \a exp -/// \exception FE_INVALID for signaling NaN -/// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding -inline half scalbn(half arg, int exp) { return scalbln(arg, exp); } - -/// Multiply by power of two. -/// This function is exact to rounding for all rounding modes. -/// -/// **See also:** Documentation for -/// [std::ldexp](https://en.cppreference.com/w/cpp/numeric/math/ldexp). -/// \param arg number to modify -/// \param exp power of two to multiply with -/// \return \a arg multplied by 2 raised to \a exp -/// \exception FE_INVALID for signaling NaN -/// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding -inline half ldexp(half arg, int exp) { return scalbln(arg, exp); } - -/// Extract integer and fractional parts. -/// **See also:** Documentation for -/// [std::modf](https://en.cppreference.com/w/cpp/numeric/math/modf). -/// \param arg number to decompress -/// \param iptr address to store integer part at -/// \return fractional part -/// \exception FE_INVALID for signaling NaN -inline half modf(half arg, half* iptr) -{ - unsigned int abs = arg.data_ & 0x7FFF; - if(abs > 0x7C00) - { - arg = half(detail::binary, detail::signal(arg.data_)); - return *iptr = arg, arg; - } - if(abs >= 0x6400) - return *iptr = arg, half(detail::binary, arg.data_ & 0x8000); - if(abs < 0x3C00) - return iptr->data_ = arg.data_ & 0x8000, arg; - unsigned int exp = abs >> 10, mask = (1 << (25 - exp)) - 1, m = arg.data_ & mask; - iptr->data_ = arg.data_ & ~mask; - if(!m) - return half(detail::binary, arg.data_ & 0x8000); - for(; m < 0x400; m <<= 1, --exp) - ; - return half(detail::binary, (arg.data_ & 0x8000) | (exp << 10) | (m & 0x3FF)); -} - -/// Extract exponent. -/// **See also:** Documentation for -/// [std::ilogb](https://en.cppreference.com/w/cpp/numeric/math/ilogb). -/// \param arg number to query -/// \return floating-point exponent -/// \retval FP_ILOGB0 for zero -/// \retval FP_ILOGBNAN for NaN -/// \retval INT_MAX for infinity -/// \exception FE_INVALID for 0 or infinite values -inline int ilogb(half arg) -{ - int abs = arg.data_ & 0x7FFF, exp; - if(!abs || abs >= 0x7C00) - { - detail::raise(FE_INVALID); - return !abs ? FP_ILOGB0 : (abs == 0x7C00) ? INT_MAX : FP_ILOGBNAN; - } - for(exp = (abs >> 10) - 15; abs < 0x200; abs <<= 1, --exp) - ; - return exp; -} - -/// Extract exponent. -/// **See also:** Documentation for -/// [std::logb](https://en.cppreference.com/w/cpp/numeric/math/logb). -/// \param arg number to query -/// \return floating-point exponent -/// \exception FE_INVALID for signaling NaN -/// \exception FE_DIVBYZERO for 0 -inline half logb(half arg) -{ - int abs = arg.data_ & 0x7FFF, exp; - if(!abs) - return half(detail::binary, detail::pole(0x8000)); - if(abs >= 0x7C00) - return half(detail::binary, (abs == 0x7C00) ? 0x7C00 : detail::signal(arg.data_)); - for(exp = (abs >> 10) - 15; abs < 0x200; abs <<= 1, --exp) - ; - unsigned int value = static_cast(exp < 0) << 15; - if(exp) - { - unsigned int m = std::abs(exp) << 6; - for(exp = 18; m < 0x400; m <<= 1, --exp) - ; - value |= (exp << 10) + m; - } - return half(detail::binary, value); -} - -/// Next representable value. -/// **See also:** Documentation for -/// [std::nextafter](https://en.cppreference.com/w/cpp/numeric/math/nextafter). -/// \param from value to compute next representable value for -/// \param to direction towards which to compute next value -/// \return next representable value after \a from in direction towards \a to -/// \exception FE_INVALID for signaling NaN -/// \exception FE_OVERFLOW for infinite result from finite argument -/// \exception FE_UNDERFLOW for subnormal result -inline half nextafter(half from, half to) -{ - int fabs = from.data_ & 0x7FFF, tabs = to.data_ & 0x7FFF; - if(fabs > 0x7C00 || tabs > 0x7C00) - return half(detail::binary, detail::signal(from.data_, to.data_)); - if(from.data_ == to.data_ || !(fabs | tabs)) - return to; - if(!fabs) - { - detail::raise(FE_UNDERFLOW, !HALF_ERRHANDLING_UNDERFLOW_TO_INEXACT); - return half(detail::binary, (to.data_ & 0x8000) + 1); - } - unsigned int out = - from.data_ + - (((from.data_ >> 15) ^ - static_cast((from.data_ ^ (0x8000 | (0x8000 - (from.data_ >> 15)))) < - (to.data_ ^ (0x8000 | (0x8000 - (to.data_ >> 15)))))) - << 1) - - 1; - detail::raise(FE_OVERFLOW, fabs < 0x7C00 && (out & 0x7C00) == 0x7C00); - detail::raise(FE_UNDERFLOW, !HALF_ERRHANDLING_UNDERFLOW_TO_INEXACT && (out & 0x7C00) < 0x400); - return half(detail::binary, out); -} - -/// Next representable value. -/// **See also:** Documentation for -/// [std::nexttoward](https://en.cppreference.com/w/cpp/numeric/math/nexttoward). -/// \param from value to compute next representable value for -/// \param to direction towards which to compute next value -/// \return next representable value after \a from in direction towards \a to -/// \exception FE_INVALID for signaling NaN -/// \exception FE_OVERFLOW for infinite result from finite argument -/// \exception FE_UNDERFLOW for subnormal result -inline half nexttoward(half from, long double to) -{ - int fabs = from.data_ & 0x7FFF; - if(fabs > 0x7C00) - return half(detail::binary, detail::signal(from.data_)); - long double lfrom = static_cast(from); - if(detail::builtin_isnan(to) || lfrom == to) - return half(static_cast(to)); - if(!fabs) - { - detail::raise(FE_UNDERFLOW, !HALF_ERRHANDLING_UNDERFLOW_TO_INEXACT); - return half(detail::binary, (static_cast(detail::builtin_signbit(to)) << 15) + 1); - } - unsigned int out = - from.data_ + (((from.data_ >> 15) ^ static_cast(lfrom < to)) << 1) - 1; - detail::raise(FE_OVERFLOW, (out & 0x7FFF) == 0x7C00); - detail::raise(FE_UNDERFLOW, !HALF_ERRHANDLING_UNDERFLOW_TO_INEXACT && (out & 0x7FFF) < 0x400); - return half(detail::binary, out); -} - -/// Take sign. -/// **See also:** Documentation for -/// [std::copysign](https://en.cppreference.com/w/cpp/numeric/math/copysign). -/// \param x value to change sign for -/// \param y value to take sign from -/// \return value equal to \a x in magnitude and to \a y in sign -inline HALF_CONSTEXPR half copysign(half x, half y) -{ - return half(detail::binary, x.data_ ^ ((x.data_ ^ y.data_) & 0x8000)); -} - -/// \} -/// \anchor classification -/// \name Floating point classification -/// \{ - -/// Classify floating-point value. -/// **See also:** Documentation for -/// [std::fpclassify](https://en.cppreference.com/w/cpp/numeric/math/fpclassify). -/// \param arg number to classify -/// \retval FP_ZERO for positive and negative zero -/// \retval FP_SUBNORMAL for subnormal numbers -/// \retval FP_INFINITY for positive and negative infinity -/// \retval FP_NAN for NaNs -/// \retval FP_NORMAL for all other (normal) values -inline HALF_CONSTEXPR int fpclassify(half arg) -{ - return !(arg.data_ & 0x7FFF) - ? FP_ZERO - : ((arg.data_ & 0x7FFF) < 0x400) - ? FP_SUBNORMAL - : ((arg.data_ & 0x7FFF) < 0x7C00) - ? FP_NORMAL - : ((arg.data_ & 0x7FFF) == 0x7C00) ? FP_INFINITE : FP_NAN; -} - -/// Check if finite number. -/// **See also:** Documentation for -/// [std::isfinite](https://en.cppreference.com/w/cpp/numeric/math/isfinite). -/// \param arg number to check -/// \retval true if neither infinity nor NaN -/// \retval false else -inline HALF_CONSTEXPR bool isfinite(half arg) { return (arg.data_ & 0x7C00) != 0x7C00; } - -/// Check for infinity. -/// **See also:** Documentation for -/// [std::isinf](https://en.cppreference.com/w/cpp/numeric/math/isinf). -/// \param arg number to check -/// \retval true for positive or negative infinity -/// \retval false else -inline HALF_CONSTEXPR bool isinf(half arg) { return (arg.data_ & 0x7FFF) == 0x7C00; } - -/// Check for NaN. -/// **See also:** Documentation for -/// [std::isnan](https://en.cppreference.com/w/cpp/numeric/math/isnan). -/// \param arg number to check -/// \retval true for NaNs -/// \retval false else -inline HALF_CONSTEXPR bool isnan(half arg) { return (arg.data_ & 0x7FFF) > 0x7C00; } - -/// Check if normal number. -/// **See also:** Documentation for -/// [std::isnormal](https://en.cppreference.com/w/cpp/numeric/math/isnormal). -/// \param arg number to check -/// \retval true if normal number -/// \retval false if either subnormal, zero, infinity or NaN -inline HALF_CONSTEXPR bool isnormal(half arg) -{ - return ((arg.data_ & 0x7C00) != 0) & ((arg.data_ & 0x7C00) != 0x7C00); -} - -/// Check sign. -/// **See also:** Documentation for -/// [std::signbit](https://en.cppreference.com/w/cpp/numeric/math/signbit). -/// \param arg number to check -/// \retval true for negative number -/// \retval false for positive number -inline HALF_CONSTEXPR bool signbit(half arg) { return (arg.data_ & 0x8000) != 0; } - -/// \} -/// \anchor compfunc -/// \name Comparison -/// \{ - -/// Quiet comparison for greater than. -/// **See also:** Documentation for -/// [std::isgreater](https://en.cppreference.com/w/cpp/numeric/math/isgreater). -/// \param x first operand -/// \param y second operand -/// \retval true if \a x greater than \a y -/// \retval false else -inline HALF_CONSTEXPR bool isgreater(half x, half y) -{ - return ((x.data_ ^ (0x8000 | (0x8000 - (x.data_ >> 15)))) + (x.data_ >> 15)) > - ((y.data_ ^ (0x8000 | (0x8000 - (y.data_ >> 15)))) + (y.data_ >> 15)) && - !isnan(x) && !isnan(y); -} - -/// Quiet comparison for greater equal. -/// **See also:** Documentation for -/// [std::isgreaterequal](https://en.cppreference.com/w/cpp/numeric/math/isgreaterequal). -/// \param x first operand -/// \param y second operand -/// \retval true if \a x greater equal \a y -/// \retval false else -inline HALF_CONSTEXPR bool isgreaterequal(half x, half y) -{ - return ((x.data_ ^ (0x8000 | (0x8000 - (x.data_ >> 15)))) + (x.data_ >> 15)) >= - ((y.data_ ^ (0x8000 | (0x8000 - (y.data_ >> 15)))) + (y.data_ >> 15)) && - !isnan(x) && !isnan(y); -} - -/// Quiet comparison for less than. -/// **See also:** Documentation for -/// [std::isless](https://en.cppreference.com/w/cpp/numeric/math/isless). -/// \param x first operand -/// \param y second operand -/// \retval true if \a x less than \a y -/// \retval false else -inline HALF_CONSTEXPR bool isless(half x, half y) -{ - return ((x.data_ ^ (0x8000 | (0x8000 - (x.data_ >> 15)))) + (x.data_ >> 15)) < - ((y.data_ ^ (0x8000 | (0x8000 - (y.data_ >> 15)))) + (y.data_ >> 15)) && - !isnan(x) && !isnan(y); -} - -/// Quiet comparison for less equal. -/// **See also:** Documentation for -/// [std::islessequal](https://en.cppreference.com/w/cpp/numeric/math/islessequal). -/// \param x first operand -/// \param y second operand -/// \retval true if \a x less equal \a y -/// \retval false else -inline HALF_CONSTEXPR bool islessequal(half x, half y) -{ - return ((x.data_ ^ (0x8000 | (0x8000 - (x.data_ >> 15)))) + (x.data_ >> 15)) <= - ((y.data_ ^ (0x8000 | (0x8000 - (y.data_ >> 15)))) + (y.data_ >> 15)) && - !isnan(x) && !isnan(y); -} - -/// Quiet comarison for less or greater. -/// **See also:** Documentation for -/// [std::islessgreater](https://en.cppreference.com/w/cpp/numeric/math/islessgreater). -/// \param x first operand -/// \param y second operand -/// \retval true if either less or greater -/// \retval false else -inline HALF_CONSTEXPR bool islessgreater(half x, half y) -{ - return x.data_ != y.data_ && ((x.data_ | y.data_) & 0x7FFF) && !isnan(x) && !isnan(y); -} - -/// Quiet check if unordered. -/// **See also:** Documentation for -/// [std::isunordered](https://en.cppreference.com/w/cpp/numeric/math/isunordered). -/// \param x first operand -/// \param y second operand -/// \retval true if unordered (one or two NaN operands) -/// \retval false else -inline HALF_CONSTEXPR bool isunordered(half x, half y) { return isnan(x) || isnan(y); } - -/// \} -/// \anchor casting -/// \name Casting -/// \{ - -/// Cast to or from half-precision floating-point number. -/// This casts between [half](\ref half_float::half) and any built-in arithmetic type. The values -/// are converted -/// directly using the default rounding mode, without any roundtrip over `float` that a -/// `static_cast` would otherwise do. -/// -/// Using this cast with neither of the two types being a [half](\ref half_float::half) or with any -/// of the two types -/// not being a built-in arithmetic type (apart from [half](\ref half_float::half), of course) -/// results in a compiler -/// error and casting between [half](\ref half_float::half)s returns the argument unmodified. -/// \tparam T destination type (half or built-in arithmetic type) -/// \tparam U source type (half or built-in arithmetic type) -/// \param arg value to cast -/// \return \a arg converted to destination type -/// \exception FE_INVALID if \a T is integer type and result is not representable as \a T -/// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding -template -T half_cast(U arg) -{ - return detail::half_caster::cast(arg); -} - -/// Cast to or from half-precision floating-point number. -/// This casts between [half](\ref half_float::half) and any built-in arithmetic type. The values -/// are converted -/// directly using the specified rounding mode, without any roundtrip over `float` that a -/// `static_cast` would otherwise do. -/// -/// Using this cast with neither of the two types being a [half](\ref half_float::half) or with any -/// of the two types -/// not being a built-in arithmetic type (apart from [half](\ref half_float::half), of course) -/// results in a compiler -/// error and casting between [half](\ref half_float::half)s returns the argument unmodified. -/// \tparam T destination type (half or built-in arithmetic type) -/// \tparam R rounding mode to use. -/// \tparam U source type (half or built-in arithmetic type) -/// \param arg value to cast -/// \return \a arg converted to destination type -/// \exception FE_INVALID if \a T is integer type and result is not representable as \a T -/// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding -template -T half_cast(U arg) -{ - return detail::half_caster::cast(arg); -} -/// \} - -/// \} -/// \anchor errors -/// \name Error handling -/// \{ - -/// Clear exception flags. -/// This function works even if [automatic exception flag handling](\ref HALF_ERRHANDLING_FLAGS) is -/// disabled, -/// but in that case manual flag management is the only way to raise flags. -/// -/// **See also:** Documentation for -/// [std::feclearexcept](https://en.cppreference.com/w/cpp/numeric/fenv/feclearexcept). -/// \param excepts OR of exceptions to clear -/// \retval 0 all selected flags cleared successfully -inline int feclearexcept(int excepts) -{ - detail::errflags() &= ~excepts; - return 0; -} - -/// Test exception flags. -/// This function works even if [automatic exception flag handling](\ref HALF_ERRHANDLING_FLAGS) is -/// disabled, -/// but in that case manual flag management is the only way to raise flags. -/// -/// **See also:** Documentation for -/// [std::fetestexcept](https://en.cppreference.com/w/cpp/numeric/fenv/fetestexcept). -/// \param excepts OR of exceptions to test -/// \return OR of selected exceptions if raised -inline int fetestexcept(int excepts) { return detail::errflags() & excepts; } - -/// Raise exception flags. -/// This raises the specified floating point exceptions and also invokes any additional automatic -/// exception handling as -/// configured with the [HALF_ERRHANDLIG_...](\ref HALF_ERRHANDLING_ERRNO) preprocessor symbols. -/// This function works even if [automatic exception flag handling](\ref HALF_ERRHANDLING_FLAGS) is -/// disabled, -/// but in that case manual flag management is the only way to raise flags. -/// -/// **See also:** Documentation for -/// [std::feraiseexcept](https://en.cppreference.com/w/cpp/numeric/fenv/feraiseexcept). -/// \param excepts OR of exceptions to raise -/// \retval 0 all selected exceptions raised successfully -inline int feraiseexcept(int excepts) -{ - detail::errflags() |= excepts; - detail::raise(excepts); - return 0; -} - -/// Save exception flags. -/// This function works even if [automatic exception flag handling](\ref HALF_ERRHANDLING_FLAGS) is -/// disabled, -/// but in that case manual flag management is the only way to raise flags. -/// -/// **See also:** Documentation for -/// [std::fegetexceptflag](https://en.cppreference.com/w/cpp/numeric/fenv/feexceptflag). -/// \param flagp adress to store flag state at -/// \param excepts OR of flags to save -/// \retval 0 for success -inline int fegetexceptflag(int* flagp, int excepts) -{ - *flagp = detail::errflags() & excepts; - return 0; -} - -/// Restore exception flags. -/// This only copies the specified exception state (including unset flags) without incurring any -/// additional exception handling. -/// This function works even if [automatic exception flag handling](\ref HALF_ERRHANDLING_FLAGS) is -/// disabled, -/// but in that case manual flag management is the only way to raise flags. -/// -/// **See also:** Documentation for -/// [std::fesetexceptflag](https://en.cppreference.com/w/cpp/numeric/fenv/feexceptflag). -/// \param flagp adress to take flag state from -/// \param excepts OR of flags to restore -/// \retval 0 for success -inline int fesetexceptflag(const int* flagp, int excepts) -{ - detail::errflags() = (detail::errflags() | (*flagp & excepts)) & (*flagp | ~excepts); - return 0; -} - -/// Throw C++ exceptions based on set exception flags. -/// This function manually throws a corresponding C++ exception if one of the specified flags is -/// set, -/// no matter if automatic throwing (via [HALF_ERRHANDLING_THROW_...](\ref -/// HALF_ERRHANDLING_THROW_INVALID)) is enabled or not. -/// This function works even if [automatic exception flag handling](\ref HALF_ERRHANDLING_FLAGS) is -/// disabled, -/// but in that case manual flag management is the only way to raise flags. -/// \param excepts OR of exceptions to test -/// \param msg error message to use for exception description -/// \throw std::domain_error if `FE_INVALID` or `FE_DIVBYZERO` is selected and set -/// \throw std::overflow_error if `FE_OVERFLOW` is selected and set -/// \throw std::underflow_error if `FE_UNDERFLOW` is selected and set -/// \throw std::range_error if `FE_INEXACT` is selected and set -inline void fethrowexcept(int excepts, const char* msg = "") -{ - excepts &= detail::errflags(); - if(excepts & (FE_INVALID | FE_DIVBYZERO)) - throw std::domain_error(msg); - if(excepts & FE_OVERFLOW) - throw std::overflow_error(msg); - if(excepts & FE_UNDERFLOW) - throw std::underflow_error(msg); - if(excepts & FE_INEXACT) - throw std::range_error(msg); -} -/// \} -} // namespace half_float - -#undef HALF_UNUSED_NOERR -#undef HALF_CONSTEXPR -#undef HALF_CONSTEXPR_CONST -#undef HALF_CONSTEXPR_NOERR -#undef HALF_NOEXCEPT -#undef HALF_NOTHROW -#undef HALF_THREAD_LOCAL -#undef HALF_TWOS_COMPLEMENT_INT -#ifdef HALF_POP_WARNINGS -#pragma warning(pop) -#undef HALF_POP_WARNINGS -#endif - -#endif diff --git a/include/ck/config.hpp b/include/ck/ck.hpp similarity index 98% rename from include/ck/config.hpp rename to include/ck/ck.hpp index 3b4470f2cc..153fc6105a 100644 --- a/include/ck/config.hpp +++ b/include/ck/ck.hpp @@ -1,14 +1,15 @@ // SPDX-License-Identifier: MIT // Copyright (c) 2018-2022, Advanced Micro Devices, Inc. All rights reserved. -#ifndef CK_CONFIG_AMD_HPP -#define CK_CONFIG_AMD_HPP +#pragma once #ifndef CK_DONT_USE_HIP_RUNTIME_HEADERS #include "hip/hip_runtime.h" #include "hip/hip_fp16.h" #endif +#define CK_TIME_KERNEL 1 + // constant address space for kernel parameter // https://llvm.org/docs/AMDGPUUsage.html#address-spaces #define CK_CONSTANT_ADDRESS_SPACE __attribute__((address_space(4))) @@ -152,6 +153,7 @@ enum struct InMemoryDataOperationEnum Add }; +// FIXME: use regular Sequence and remove this template struct InMemoryDataOperationEnumSequence { @@ -165,6 +167,7 @@ struct InMemoryDataOperationEnumSequence } }; +#if 0 // TODO: no longer needed, remove this enum struct ActivTypeEnum { @@ -172,10 +175,10 @@ enum struct ActivTypeEnum LeakyRelu, Sigmoid }; +#endif // index type using index_t = int32_t; using long_index_t = int64_t; } // namespace ck -#endif diff --git a/include/ck/host_utility/device_prop.hpp b/include/ck/device_utility/device_prop.hpp similarity index 97% rename from include/ck/host_utility/device_prop.hpp rename to include/ck/device_utility/device_prop.hpp index 74b20acecd..8666463d98 100644 --- a/include/ck/host_utility/device_prop.hpp +++ b/include/ck/device_utility/device_prop.hpp @@ -2,6 +2,7 @@ #include #include +#include namespace ck { diff --git a/include/ck/device_utility/hip_check_error.hpp b/include/ck/device_utility/hip_check_error.hpp new file mode 100644 index 0000000000..edbf454667 --- /dev/null +++ b/include/ck/device_utility/hip_check_error.hpp @@ -0,0 +1,14 @@ +#pragma once + +#include + +inline void hip_check_error(hipError_t x) +{ + if(x != hipSuccess) + { + std::ostringstream ss; + ss << "HIP runtime error: " << hipGetErrorString(x) << ". " << __FILE__ << ": " << __LINE__ + << "in function: " << __func__; + throw std::runtime_error(ss.str()); + } +} diff --git a/include/ck/device_utility/kernel_launch.hpp b/include/ck/device_utility/kernel_launch.hpp new file mode 100644 index 0000000000..096fe9abbd --- /dev/null +++ b/include/ck/device_utility/kernel_launch.hpp @@ -0,0 +1,71 @@ +#pragma once + +#include + +#include "ck/ck.hpp" +#include "ck/stream_config.hpp" +#include "ck/device_utility/hip_check_error.hpp" + +template +float launch_and_time_kernel(const StreamConfig& stream_config, + F kernel, + dim3 grid_dim, + dim3 block_dim, + std::size_t lds_byte, + Args... args) +{ +#if CK_TIME_KERNEL + if(stream_config.time_kernel_) + { + printf("%s: grid_dim {%d, %d, %d}, block_dim {%d, %d, %d} \n", + __func__, + grid_dim.x, + grid_dim.y, + grid_dim.z, + block_dim.x, + block_dim.y, + block_dim.z); + + const int nrepeat = 10; + + printf("Warm up 1 time\n"); + + // warm up + kernel<<>>(args...); + + printf("Start running %d times...\n", nrepeat); + + hipEvent_t start, stop; + + hip_check_error(hipEventCreate(&start)); + hip_check_error(hipEventCreate(&stop)); + + hip_check_error(hipDeviceSynchronize()); + hip_check_error(hipEventRecord(start, stream_config.stream_id_)); + + for(int i = 0; i < nrepeat; ++i) + { + kernel<<>>(args...); + } + + hip_check_error(hipEventRecord(stop, stream_config.stream_id_)); + hip_check_error(hipEventSynchronize(stop)); + + float total_time = 0; + + hip_check_error(hipEventElapsedTime(&total_time, start, stop)); + + return total_time / nrepeat; + } + else + { + kernel<<>>(args...); + + return 0; + } +#else + kernel<<>>(args...); + + return 0; +#endif +} diff --git a/include/ck/options.hpp b/include/ck/options.hpp deleted file mode 100644 index 82c604f82b..0000000000 --- a/include/ck/options.hpp +++ /dev/null @@ -1,3 +0,0 @@ -#pragma once - -#define CK_TIME_KERNEL 1 diff --git a/include/ck/tensor_description/cluster_descriptor.hpp b/include/ck/tensor_description/cluster_descriptor.hpp index d69bfb70c1..c33d0588f2 100644 --- a/include/ck/tensor_description/cluster_descriptor.hpp +++ b/include/ck/tensor_description/cluster_descriptor.hpp @@ -1,8 +1,7 @@ -#ifndef CK_CLUSTER_DESCRIPTOR_HPP -#define CK_CLUSTER_DESCRIPTOR_HPP +#pragma once -#include "common_header.hpp" -#include "tensor_adaptor.hpp" +#include "ck/utility/common_header.hpp" +#include "ck/tensor_description/tensor_adaptor.hpp" namespace ck { @@ -30,4 +29,3 @@ __host__ __device__ constexpr auto make_cluster_descriptor( } } // namespace ck -#endif diff --git a/include/ck/tensor_description/multi_index_transform.hpp b/include/ck/tensor_description/multi_index_transform.hpp index fa705cc3fe..3486538cf3 100644 --- a/include/ck/tensor_description/multi_index_transform.hpp +++ b/include/ck/tensor_description/multi_index_transform.hpp @@ -1,8 +1,7 @@ -#ifndef CK_MULTI_INDEX_TRANSFORM_HPP -#define CK_MULTI_INDEX_TRANSFORM_HPP +#pragma once -#include "common_header.hpp" -#include "multi_index.hpp" +#include "ck/utility/common_header.hpp" +#include "ck/utility/multi_index.hpp" namespace ck { @@ -1950,4 +1949,3 @@ struct Modulo } }; } // namespace ck -#endif diff --git a/include/ck/tensor_description/multi_index_transform_helper.hpp b/include/ck/tensor_description/multi_index_transform_helper.hpp index bc360714b9..2558d64118 100644 --- a/include/ck/tensor_description/multi_index_transform_helper.hpp +++ b/include/ck/tensor_description/multi_index_transform_helper.hpp @@ -1,8 +1,7 @@ -#ifndef CK_MULTI_INDEX_TRANSFORM_HELPER_HPP -#define CK_MULTI_INDEX_TRANSFORM_HELPER_HPP +#pragma once -#include "common_header.hpp" -#include "multi_index_transform.hpp" +#include "ck/utility/common_header.hpp" +#include "ck/tensor_description/multi_index_transform.hpp" namespace ck { @@ -126,4 +125,3 @@ __host__ __device__ constexpr auto make_modulo_transform(const Modulus& modulus, return Modulo{modulus, up_length}; } } // namespace ck -#endif diff --git a/include/ck/tensor_description/tensor_adaptor.hpp b/include/ck/tensor_description/tensor_adaptor.hpp index e62255ff48..1ada2f35ed 100644 --- a/include/ck/tensor_description/tensor_adaptor.hpp +++ b/include/ck/tensor_description/tensor_adaptor.hpp @@ -1,9 +1,8 @@ -#ifndef CK_TENSOR_ADAPTOR_HPP -#define CK_TENSOR_ADAPTOR_HPP +#pragma once -#include "common_header.hpp" -#include "tensor_descriptor.hpp" -#include "tensor_descriptor_helper.hpp" +#include "ck/utility/common_header.hpp" +#include "ck/tensor_description/tensor_descriptor.hpp" +#include "ck/tensor_description/tensor_descriptor_helper.hpp" namespace ck { @@ -478,4 +477,3 @@ __host__ __device__ constexpr auto chain_tensor_adaptors(const X& x, const Xs&.. } } // namespace ck -#endif diff --git a/include/ck/tensor_description/tensor_descriptor.hpp b/include/ck/tensor_description/tensor_descriptor.hpp index 0ca4f6e24d..5f710b8a0b 100644 --- a/include/ck/tensor_description/tensor_descriptor.hpp +++ b/include/ck/tensor_description/tensor_descriptor.hpp @@ -1,8 +1,7 @@ -#ifndef CK_TENSOR_DESCRIPTOR_HPP -#define CK_TENSOR_DESCRIPTOR_HPP +#pragma once -#include "common_header.hpp" -#include "multi_index_transform.hpp" +#include "ck/utility/common_header.hpp" +#include "ck/tensor_description/multi_index_transform.hpp" namespace ck { @@ -604,4 +603,3 @@ using TensorCoordinateStep_t = decltype(make_tensor_coordinate_step( TensorDesc{}, MultiIndex::GetNumOfDimension()>{})); } // namespace ck -#endif diff --git a/include/ck/tensor_description/tensor_descriptor_helper.hpp b/include/ck/tensor_description/tensor_descriptor_helper.hpp index ddc0ede404..e988dcdb9c 100644 --- a/include/ck/tensor_description/tensor_descriptor_helper.hpp +++ b/include/ck/tensor_description/tensor_descriptor_helper.hpp @@ -1,7 +1,8 @@ #pragma once -#include "common_header.hpp" -#include "tensor_descriptor.hpp" -#include "multi_index_transform_helper.hpp" + +#include "ck/utility/common_header.hpp" +#include "ck/tensor_description/tensor_descriptor.hpp" +#include "ck/tensor_description/multi_index_transform_helper.hpp" namespace ck { diff --git a/include/ck/utility/tensor_space_filling_curve.hpp b/include/ck/tensor_description/tensor_space_filling_curve.hpp similarity index 95% rename from include/ck/utility/tensor_space_filling_curve.hpp rename to include/ck/tensor_description/tensor_space_filling_curve.hpp index b5f1a34d83..43b51e9295 100644 --- a/include/ck/utility/tensor_space_filling_curve.hpp +++ b/include/ck/tensor_description/tensor_space_filling_curve.hpp @@ -1,12 +1,11 @@ -#ifndef TENSOR_SPACE_FILLING_CURVE_HPP -#define TENSOR_SPACE_FILLING_CURVE_HPP +#pragma once -#include "math.hpp" -#include "sequence.hpp" -#include "sequence_helper.hpp" -#include "tensor_adaptor.hpp" -#include "statically_indexed_array_multi_index.hpp" -#include "tuple_helper.hpp" +#include "ck/utility/math.hpp" +#include "ck/utility/sequence.hpp" +#include "ck/utility/sequence_helper.hpp" +#include "ck/utility/statically_indexed_array_multi_index.hpp" +#include "ck/utility/tuple_helper.hpp" +#include "ck/tensor_description/tensor_adaptor.hpp" namespace ck { @@ -156,4 +155,3 @@ struct SpaceFillingCurve }; } // namespace ck -#endif diff --git a/include/ck/tensor_operation/gpu/block/blockwise_gemm_dl_v2r3.hpp b/include/ck/tensor_operation/gpu/block/blockwise_gemm_dl_v2r3.hpp index f7fa867e16..ebf80bb2ff 100644 --- a/include/ck/tensor_operation/gpu/block/blockwise_gemm_dl_v2r3.hpp +++ b/include/ck/tensor_operation/gpu/block/blockwise_gemm_dl_v2r3.hpp @@ -1,8 +1,9 @@ #pragma once -#include "common_header.hpp" -#include "tensor_adaptor.hpp" -#include "threadwise_tensor_slice_transfer_v4r1.hpp" -#include "threadwise_contraction_dl.hpp" + +#include "ck/utility/common_header.hpp" +#include "ck/tensor_description/tensor_adaptor.hpp" +#include "ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer_v4r1.hpp" +#include "ck/tensor_operation/gpu/thread/threadwise_contraction_dl.hpp" namespace ck { diff --git a/include/ck/tensor_operation/gpu/block/blockwise_gemm_xdlops.hpp b/include/ck/tensor_operation/gpu/block/blockwise_gemm_xdlops.hpp index b93d5ff839..23ff02cb16 100644 --- a/include/ck/tensor_operation/gpu/block/blockwise_gemm_xdlops.hpp +++ b/include/ck/tensor_operation/gpu/block/blockwise_gemm_xdlops.hpp @@ -1,9 +1,9 @@ #pragma once -#include "common_header.hpp" -#include "threadwise_tensor_slice_transfer.hpp" -#include "xdlops_gemm.hpp" -#include "tensor_adaptor.hpp" -#include "thread_group.hpp" + +#include "ck/utility/common_header.hpp" +#include "ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer.hpp" +#include "ck/tensor_operation/gpu/warp/xdlops_gemm.hpp" +#include "ck/tensor_description/tensor_adaptor.hpp" namespace ck { diff --git a/include/ck/tensor_operation/gpu/block/blockwise_tensor_slice_transfer_v5r1.hpp b/include/ck/tensor_operation/gpu/block/blockwise_tensor_slice_transfer_v5r1.hpp index e8ec164364..71dd8b1012 100644 --- a/include/ck/tensor_operation/gpu/block/blockwise_tensor_slice_transfer_v5r1.hpp +++ b/include/ck/tensor_operation/gpu/block/blockwise_tensor_slice_transfer_v5r1.hpp @@ -1,11 +1,10 @@ -#ifndef CK_BLOCKWISE_TENSOR_SLICE_TRANSFER_V5R1_HPP -#define CK_BLOCKWISE_TENSOR_SLICE_TRANSFER_V5R1_HPP +#pragma once -#include "common_header.hpp" -#include "tensor_descriptor.hpp" -#include "tensor_descriptor_helper.hpp" -#include "cluster_descriptor.hpp" -#include "threadwise_tensor_slice_transfer_v5r1.hpp" +#include "ck/utility/common_header.hpp" +#include "ck/tensor_description/tensor_descriptor.hpp" +#include "ck/tensor_description/tensor_descriptor_helper.hpp" +#include "ck/tensor_description/cluster_descriptor.hpp" +#include "ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer_v5r1.hpp" namespace ck { @@ -152,4 +151,3 @@ struct BlockwiseTensorSliceTransfer_v5r1 }; } // namespace ck -#endif diff --git a/include/ck/tensor_operation/gpu/block/reduction_functions_blockwise.hpp b/include/ck/tensor_operation/gpu/block/reduction_functions_blockwise.hpp index 8580b9ea4a..9b35dd2832 100644 --- a/include/ck/tensor_operation/gpu/block/reduction_functions_blockwise.hpp +++ b/include/ck/tensor_operation/gpu/block/reduction_functions_blockwise.hpp @@ -1,35 +1,8 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2020 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ -#ifndef CK_REDUCTION_FUNCTIONS_BLOCKWISE_HPP -#define CK_REDUCTION_FUNCTIONS_BLOCKWISE_HPP +#pragma once -#include "reduction_common.hpp" -#include "reduction_functions_accumulate.hpp" - -#include "cluster_descriptor.hpp" +#include "ck/tensor_description/cluster_descriptor.hpp" +#include "ck/utility/reduction_common.hpp" +#include "ck/utility/reduction_functions_accumulate.hpp" namespace ck { @@ -193,6 +166,4 @@ struct PartitionedBlockwiseReductionWithIndex }; }; -}; // end of namespace ck - -#endif +} // namespace ck diff --git a/include/ck/tensor_operation/gpu/block/thread_group_tensor_slice_transfer_v4r1.hpp b/include/ck/tensor_operation/gpu/block/thread_group_tensor_slice_transfer_v4r1.hpp index cbabbaf47d..807c708e74 100644 --- a/include/ck/tensor_operation/gpu/block/thread_group_tensor_slice_transfer_v4r1.hpp +++ b/include/ck/tensor_operation/gpu/block/thread_group_tensor_slice_transfer_v4r1.hpp @@ -1,9 +1,10 @@ #pragma once -#include "common_header.hpp" -#include "tensor_descriptor.hpp" -#include "tensor_descriptor_helper.hpp" -#include "cluster_descriptor.hpp" -#include "threadwise_tensor_slice_transfer_v3r1.hpp" + +#include "ck/utility/common_header.hpp" +#include "ck/tensor_description/tensor_descriptor.hpp" +#include "ck/tensor_description/tensor_descriptor_helper.hpp" +#include "ck/tensor_description/cluster_descriptor.hpp" +#include "ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer_v3r1.hpp" namespace ck { diff --git a/include/ck/tensor_operation/gpu/block/thread_group_tensor_slice_transfer_v6r1.hpp b/include/ck/tensor_operation/gpu/block/thread_group_tensor_slice_transfer_v6r1.hpp index 1f0ad3e35a..8ed9424a6b 100644 --- a/include/ck/tensor_operation/gpu/block/thread_group_tensor_slice_transfer_v6r1.hpp +++ b/include/ck/tensor_operation/gpu/block/thread_group_tensor_slice_transfer_v6r1.hpp @@ -1,9 +1,10 @@ #pragma once -#include "common_header.hpp" -#include "tensor_descriptor.hpp" -#include "tensor_descriptor_helper.hpp" -#include "cluster_descriptor.hpp" -#include "threadwise_tensor_slice_transfer_v6r1.hpp" + +#include "ck/utility/common_header.hpp" +#include "ck/tensor_description/tensor_descriptor.hpp" +#include "ck/tensor_description/tensor_descriptor_helper.hpp" +#include "ck/tensor_description/cluster_descriptor.hpp" +#include "ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer_v6r1.hpp" namespace ck { diff --git a/include/ck/tensor_operation/gpu/block/thread_group_tensor_slice_transfer_v6r2.hpp b/include/ck/tensor_operation/gpu/block/thread_group_tensor_slice_transfer_v6r2.hpp index 121ddf12ad..4b62d45f42 100644 --- a/include/ck/tensor_operation/gpu/block/thread_group_tensor_slice_transfer_v6r2.hpp +++ b/include/ck/tensor_operation/gpu/block/thread_group_tensor_slice_transfer_v6r2.hpp @@ -1,9 +1,10 @@ #pragma once -#include "common_header.hpp" -#include "tensor_descriptor.hpp" -#include "tensor_descriptor_helper.hpp" -#include "cluster_descriptor.hpp" -#include "threadwise_tensor_slice_transfer_v6r2.hpp" + +#include "ck/utility/common_header.hpp" +#include "ck/tensor_description/tensor_descriptor.hpp" +#include "ck/tensor_description/tensor_descriptor_helper.hpp" +#include "ck/tensor_description/cluster_descriptor.hpp" +#include "ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer_v6r2.hpp" namespace ck { diff --git a/include/ck/tensor_operation/gpu/block/thread_group_tensor_slice_transfer_v6r3.hpp b/include/ck/tensor_operation/gpu/block/thread_group_tensor_slice_transfer_v6r3.hpp index ca5db90f30..12d0591ada 100644 --- a/include/ck/tensor_operation/gpu/block/thread_group_tensor_slice_transfer_v6r3.hpp +++ b/include/ck/tensor_operation/gpu/block/thread_group_tensor_slice_transfer_v6r3.hpp @@ -1,9 +1,10 @@ #pragma once -#include "common_header.hpp" -#include "tensor_descriptor.hpp" -#include "tensor_descriptor_helper.hpp" -#include "cluster_descriptor.hpp" -#include "threadwise_tensor_slice_transfer_v6r3.hpp" + +#include "ck/utility/common_header.hpp" +#include "ck/tensor_description/tensor_descriptor.hpp" +#include "ck/tensor_description/tensor_descriptor_helper.hpp" +#include "ck/tensor_description/cluster_descriptor.hpp" +#include "ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer_v6r3.hpp" namespace ck { diff --git a/include/ck/tensor_operation/gpu/block/thread_group_tensor_slice_transfer_v7.hpp b/include/ck/tensor_operation/gpu/block/thread_group_tensor_slice_transfer_v7.hpp index d499eee4c5..738b85c906 100644 --- a/include/ck/tensor_operation/gpu/block/thread_group_tensor_slice_transfer_v7.hpp +++ b/include/ck/tensor_operation/gpu/block/thread_group_tensor_slice_transfer_v7.hpp @@ -1,10 +1,10 @@ #pragma once -#include "common_header.hpp" -#include "tensor_descriptor.hpp" -#include "tensor_descriptor_helper.hpp" -#include "cluster_descriptor.hpp" -#include "threadwise_tensor_slice_transfer_v7.hpp" +#include "ck/utility/common_header.hpp" +#include "ck/tensor_description/tensor_descriptor.hpp" +#include "ck/tensor_description/tensor_descriptor_helper.hpp" +#include "ck/tensor_description/cluster_descriptor.hpp" +#include "ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer_v7.hpp" namespace ck { diff --git a/include/ck/tensor_operation/gpu/device/device_5ary_elementwise.hpp b/include/ck/tensor_operation/gpu/device/device_5ary_elementwise.hpp index c093f5028c..c515f9d31c 100644 --- a/include/ck/tensor_operation/gpu/device/device_5ary_elementwise.hpp +++ b/include/ck/tensor_operation/gpu/device/device_5ary_elementwise.hpp @@ -1,12 +1,16 @@ #pragma once + #include #include -#include "device.hpp" -#include "device_base.hpp" -#include "gridwise_5ary_Elementwise_1d.hpp" -#include "tensor_layout.hpp" -#include "tensor_descriptor.hpp" -#include "tensor_descriptor_helper.hpp" +#include + +#include "ck/utility/common_header.hpp" +#include "ck/tensor_description/tensor_descriptor.hpp" +#include "ck/tensor_description/tensor_descriptor_helper.hpp" +#include "ck/tensor_operation/gpu/device/device_base.hpp" +#include "ck/tensor_operation/gpu/grid/gridwise_5ary_Elementwise_1d.hpp" +#include "ck/device_utility/device_prop.hpp" +#include "ck/device_utility/kernel_launch.hpp" namespace ck { namespace tensor_operation { @@ -325,7 +329,7 @@ struct Device5AryElementwise : public BaseOperator static auto MakeInvoker() { return Invoker{}; } std::unique_ptr MakeInvokerPointer() { return std::make_unique(); } -}; // namespace device +}; } // namespace device } // namespace tensor_operation diff --git a/include/ck/tensor_operation/gpu/device/device_base.hpp b/include/ck/tensor_operation/gpu/device/device_base.hpp index 809eba5578..31ac4a258c 100644 --- a/include/ck/tensor_operation/gpu/device/device_base.hpp +++ b/include/ck/tensor_operation/gpu/device/device_base.hpp @@ -2,7 +2,7 @@ #include -#include "stream_config.hpp" +#include "ck/stream_config.hpp" namespace ck { namespace tensor_operation { diff --git a/include/ck/tensor_operation/gpu/device/device_batched_gemm_reduce_xdl_cshuffle.hpp b/include/ck/tensor_operation/gpu/device/device_batched_gemm_reduce_xdl_cshuffle.hpp index 2379719fb9..e805e28dc3 100644 --- a/include/ck/tensor_operation/gpu/device/device_batched_gemm_reduce_xdl_cshuffle.hpp +++ b/include/ck/tensor_operation/gpu/device/device_batched_gemm_reduce_xdl_cshuffle.hpp @@ -1,14 +1,17 @@ #pragma once + #include #include -#include "device.hpp" -#include "device_gemm_reduce.hpp" -#include "common_header.hpp" -#include "tensor_layout.hpp" -#include "tensor_descriptor.hpp" -#include "tensor_descriptor_helper.hpp" -#include "gridwise_gemm_reduce_xdl_cshuffle_v1.hpp" -#include "gemm_specialization.hpp" + +#include "ck/utility/common_header.hpp" +#include "ck/tensor_description/tensor_descriptor.hpp" +#include "ck/tensor_description/tensor_descriptor_helper.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_reduce.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/grid/gridwise_gemm_reduce_xdl_cshuffle_v1.hpp" +#include "ck/device_utility/device_prop.hpp" +#include "ck/device_utility/kernel_launch.hpp" namespace ck { namespace tensor_operation { diff --git a/include/ck/tensor_operation/gpu/device/device_batched_gemm_xdl.hpp b/include/ck/tensor_operation/gpu/device/device_batched_gemm_xdl.hpp index d1ffa9df14..c716946cd1 100644 --- a/include/ck/tensor_operation/gpu/device/device_batched_gemm_xdl.hpp +++ b/include/ck/tensor_operation/gpu/device/device_batched_gemm_xdl.hpp @@ -1,16 +1,17 @@ -#ifndef DEVICE_BATCHED_GEMM_XDL_HPP -#define DEVICE_BATCHED_GEMM_XDL_HPP +#pragma once #include #include -#include "device.hpp" -#include "device_base.hpp" -#include "device_gemm.hpp" -#include "common_header.hpp" -#include "tensor_layout.hpp" -#include "tensor_descriptor.hpp" -#include "tensor_descriptor_helper.hpp" -#include "gridwise_gemm_xdlops_v2r3.hpp" + +#include "ck/utility/common_header.hpp" +#include "ck/tensor_description/tensor_descriptor.hpp" +#include "ck/tensor_description/tensor_descriptor_helper.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/grid/gridwise_gemm_xdlops_v2r3.hpp" +#include "ck/device_utility/device_prop.hpp" +#include "ck/device_utility/kernel_launch.hpp" namespace ck { namespace tensor_operation { @@ -616,4 +617,3 @@ struct DeviceBatchedGemmXdl } // namespace device } // namespace tensor_operation } // namespace ck -#endif diff --git a/include/ck/tensor_operation/gpu/device/device_binary_elementwise.hpp b/include/ck/tensor_operation/gpu/device/device_binary_elementwise.hpp index 34b3a59c74..24d75347d6 100644 --- a/include/ck/tensor_operation/gpu/device/device_binary_elementwise.hpp +++ b/include/ck/tensor_operation/gpu/device/device_binary_elementwise.hpp @@ -1,10 +1,12 @@ #pragma once + #include #include -#include "device.hpp" -#include "device_base.hpp" -#include "gridwise_binary_elementwise_1d.hpp" +#include "ck/device_utility/device_prop.hpp" +#include "ck/device_utility/kernel_launch.hpp" +#include "ck/tensor_operation/gpu/device/device_base.hpp" +#include "ck/tensor_operation/gpu/grid/gridwise_binary_elementwise_1d.hpp" namespace ck { namespace tensor_operation { diff --git a/include/ck/tensor_operation/gpu/device/device_cgemm_4gemm_xdl_cshuffle.hpp b/include/ck/tensor_operation/gpu/device/device_cgemm_4gemm_xdl_cshuffle.hpp index df2805b886..d687bef9f8 100644 --- a/include/ck/tensor_operation/gpu/device/device_cgemm_4gemm_xdl_cshuffle.hpp +++ b/include/ck/tensor_operation/gpu/device/device_cgemm_4gemm_xdl_cshuffle.hpp @@ -1,42 +1,20 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2022 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ #pragma once + #include #include -#include "device.hpp" -#include "device_gemm.hpp" -#include "device_cgemm.hpp" -#include "common_header.hpp" -#include "tensor_layout.hpp" -#include "tensor_descriptor.hpp" -#include "tensor_descriptor_helper.hpp" -#include "gridwise_gemm_xdl_cshuffle_v1.hpp" -#include "binary_element_wise_operation.hpp" -#include "gridwise_binary_elementwise_1d.hpp" -#include "tensor_operation/gpu/device/gemm_specialization.hpp" + +#include "ck/utility/common_header.hpp" +#include "ck/tensor_description/tensor_descriptor.hpp" +#include "ck/tensor_description/tensor_descriptor_helper.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm.hpp" +#include "ck/tensor_operation/gpu/device/device_cgemm.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/grid/gridwise_gemm_xdl_cshuffle_v1.hpp" +#include "ck/tensor_operation/gpu/grid/gridwise_binary_elementwise_1d.hpp" +#include "ck/tensor_operation/gpu/element/binary_element_wise_operation.hpp" +#include "ck/device_utility/device_prop.hpp" +#include "ck/device_utility/kernel_launch.hpp" namespace ck { namespace tensor_operation { diff --git a/include/ck/tensor_operation/gpu/device/device_conv2d_backward_weight_xdl_c_shuffle_nhwc_kyxc_nhwk.hpp b/include/ck/tensor_operation/gpu/device/device_conv2d_backward_weight_xdl_c_shuffle_nhwc_kyxc_nhwk.hpp index 8404f4c266..17b2ca3c52 100644 --- a/include/ck/tensor_operation/gpu/device/device_conv2d_backward_weight_xdl_c_shuffle_nhwc_kyxc_nhwk.hpp +++ b/include/ck/tensor_operation/gpu/device/device_conv2d_backward_weight_xdl_c_shuffle_nhwc_kyxc_nhwk.hpp @@ -1,17 +1,18 @@ -#ifndef DEVICE_CONV2D_WRW_XDL_C_SHUFFLE_NHWC_KYXC_NHWK_HPP -#define DEVICE_CONV2D_WRW_XDL_C_SHUFFLE_NHWC_KYXC_NHWK_HPP +#pragma once #include #include -#include "device.hpp" -#include "device_base.hpp" -#include "device_conv_backward_weight.hpp" -#include "convolution_forward_specialization.hpp" -#include "common_header.hpp" -#include "tensor_layout.hpp" -#include "tensor_descriptor.hpp" -#include "tensor_descriptor_helper.hpp" -#include "gridwise_gemm_xdlops_bwd_weight.hpp" + +#include "ck/utility/common_header.hpp" +#include "ck/tensor_description/tensor_descriptor.hpp" +#include "ck/tensor_description/tensor_descriptor_helper.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_conv_backward_weight.hpp" +#include "ck/tensor_operation/gpu/device/convolution_backward_weight_specialization.hpp" +#include "ck/tensor_operation/gpu/grid/gridwise_gemm_xdlops_bwd_weight.hpp" +#include "ck/tensor_operation/gpu/grid/gridwise_unary_elementwise_1d.hpp" +#include "ck/device_utility/device_prop.hpp" +#include "ck/device_utility/kernel_launch.hpp" namespace ck { namespace tensor_operation { @@ -773,4 +774,3 @@ struct DeviceConv2dBwdWeightXdl_C_Shuffle_Input_N_Hi_Wi_C_Weight_K_Y_X_C_Output_ } // namespace device } // namespace tensor_operation } // namespace ck -#endif diff --git a/include/ck/tensor_operation/gpu/device/device_conv2d_bwd_data_xdl_nhwc_kyxc_nhwk.hpp b/include/ck/tensor_operation/gpu/device/device_conv2d_bwd_data_xdl_nhwc_kyxc_nhwk.hpp index 83953e59bd..dfdbd39694 100644 --- a/include/ck/tensor_operation/gpu/device/device_conv2d_bwd_data_xdl_nhwc_kyxc_nhwk.hpp +++ b/include/ck/tensor_operation/gpu/device/device_conv2d_bwd_data_xdl_nhwc_kyxc_nhwk.hpp @@ -1,17 +1,17 @@ -#ifndef DEVICE_CONV2D_BWD_DATA_XDL_NHWC_KYXC_NHWK_HPP -#define DEVICE_CONV2D_BWD_DATA_XDL_NHWC_KYXC_NHWK_HPP +#pragma once #include #include -#include "device.hpp" -#include "device_base.hpp" -#include "device_conv_bwd_data.hpp" -#include "convolution_backward_data_specialization.hpp" -#include "common_header.hpp" -#include "tensor_layout.hpp" -#include "tensor_descriptor.hpp" -#include "tensor_descriptor_helper.hpp" -#include "gridwise_gemm_xdlops_v2r3.hpp" + +#include "ck/utility/common_header.hpp" +#include "ck/tensor_description/tensor_descriptor.hpp" +#include "ck/tensor_description/tensor_descriptor_helper.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_conv_bwd_data.hpp" +#include "ck/tensor_operation/gpu/device/convolution_backward_data_specialization.hpp" +#include "ck/tensor_operation/gpu/grid/gridwise_gemm_xdlops_v2r3.hpp" +#include "ck/device_utility/device_prop.hpp" +#include "ck/device_utility/kernel_launch.hpp" namespace ck { namespace tensor_operation { @@ -821,4 +821,3 @@ struct DeviceConv2dBwdDataXdl_Input_N_Hi_Wi_C_Weight_K_Y_X_C_Output_N_Ho_Wo_K } // namespace device } // namespace tensor_operation } // namespace ck -#endif diff --git a/include/ck/tensor_operation/gpu/device/device_conv2d_fwd_xdl_c_shuffle_bias_activation_add_nhwc_kyxc_nhwk.hpp b/include/ck/tensor_operation/gpu/device/device_conv2d_fwd_xdl_c_shuffle_bias_activation_add_nhwc_kyxc_nhwk.hpp index cc1c2cb2ca..ff2d04c3b1 100644 --- a/include/ck/tensor_operation/gpu/device/device_conv2d_fwd_xdl_c_shuffle_bias_activation_add_nhwc_kyxc_nhwk.hpp +++ b/include/ck/tensor_operation/gpu/device/device_conv2d_fwd_xdl_c_shuffle_bias_activation_add_nhwc_kyxc_nhwk.hpp @@ -1,17 +1,17 @@ -#ifndef DEVICE_CONV2D_FWD_XDL_C_SHUFFLE_BIAS_ACTIVATION_ADD_NHWC_KYXC_NHWK_HPP -#define DEVICE_CONV2D_FWD_XDL_C_SHUFFLE_BIAS_ACTIVATION_ADD_NHWC_KYXC_NHWK_HPP +#pragma once #include #include -#include "device.hpp" -#include "device_base.hpp" -#include "device_conv_fwd_bias_activation_add.hpp" -#include "convolution_forward_specialization.hpp" -#include "common_header.hpp" -#include "tensor_layout.hpp" -#include "tensor_descriptor.hpp" -#include "tensor_descriptor_helper.hpp" -#include "gridwise_gemm_xdlops_v3r3.hpp" + +#include "ck/utility/common_header.hpp" +#include "ck/tensor_description/tensor_descriptor.hpp" +#include "ck/tensor_description/tensor_descriptor_helper.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_conv_fwd_bias_activation_add.hpp" +#include "ck/tensor_operation/gpu/device/convolution_forward_specialization.hpp" +#include "ck/tensor_operation/gpu/grid/gridwise_gemm_xdlops_v3r3.hpp" +#include "ck/device_utility/device_prop.hpp" +#include "ck/device_utility/kernel_launch.hpp" namespace ck { namespace tensor_operation { @@ -963,4 +963,3 @@ struct } // namespace device } // namespace tensor_operation } // namespace ck -#endif diff --git a/include/ck/tensor_operation/gpu/device/device_conv2d_fwd_xdl_c_shuffle_bias_activation_nhwc_kyxc_nhwk.hpp b/include/ck/tensor_operation/gpu/device/device_conv2d_fwd_xdl_c_shuffle_bias_activation_nhwc_kyxc_nhwk.hpp index a397b5e2b1..dfdcceac42 100644 --- a/include/ck/tensor_operation/gpu/device/device_conv2d_fwd_xdl_c_shuffle_bias_activation_nhwc_kyxc_nhwk.hpp +++ b/include/ck/tensor_operation/gpu/device/device_conv2d_fwd_xdl_c_shuffle_bias_activation_nhwc_kyxc_nhwk.hpp @@ -1,15 +1,18 @@ #pragma once + #include #include -#include "device.hpp" -#include "device_base.hpp" -#include "device_conv_fwd_bias_activation.hpp" -#include "convolution_forward_specialization.hpp" -#include "common_header.hpp" -#include "tensor_layout.hpp" -#include "tensor_descriptor.hpp" -#include "tensor_descriptor_helper.hpp" -#include "gridwise_gemm_xdlops_v3r2.hpp" +#include + +#include "ck/utility/common_header.hpp" +#include "ck/tensor_description/tensor_descriptor.hpp" +#include "ck/tensor_description/tensor_descriptor_helper.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_conv_fwd_bias_activation.hpp" +#include "ck/tensor_operation/gpu/device/convolution_forward_specialization.hpp" +#include "ck/tensor_operation/gpu/grid/gridwise_gemm_xdlops_v3r2.hpp" +#include "ck/device_utility/device_prop.hpp" +#include "ck/device_utility/kernel_launch.hpp" namespace ck { namespace tensor_operation { diff --git a/include/ck/tensor_operation/gpu/device/device_conv2d_fwd_xdl_c_shuffle_nhwc_kyxc_nhwk.hpp b/include/ck/tensor_operation/gpu/device/device_conv2d_fwd_xdl_c_shuffle_nhwc_kyxc_nhwk.hpp index 707413dfd3..31e14c4f74 100644 --- a/include/ck/tensor_operation/gpu/device/device_conv2d_fwd_xdl_c_shuffle_nhwc_kyxc_nhwk.hpp +++ b/include/ck/tensor_operation/gpu/device/device_conv2d_fwd_xdl_c_shuffle_nhwc_kyxc_nhwk.hpp @@ -1,17 +1,17 @@ -#ifndef DEVICE_CONV2D_FWD_XDL_C_SHUFFLE_NHWC_KYXC_NHWK_HPP -#define DEVICE_CONV2D_FWD_XDL_C_SHUFFLE_NHWC_KYXC_NHWK_HPP +#pragma once #include #include -#include "device.hpp" -#include "device_base.hpp" -#include "device_conv_fwd.hpp" -#include "convolution_forward_specialization.hpp" -#include "common_header.hpp" -#include "tensor_layout.hpp" -#include "tensor_descriptor.hpp" -#include "tensor_descriptor_helper.hpp" -#include "gridwise_gemm_xdlops_v3r1.hpp" + +#include "ck/utility/common_header.hpp" +#include "ck/tensor_description/tensor_descriptor.hpp" +#include "ck/tensor_description/tensor_descriptor_helper.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_conv_fwd.hpp" +#include "ck/tensor_operation/gpu/device/convolution_forward_specialization.hpp" +#include "ck/tensor_operation/gpu/grid/gridwise_gemm_xdlops_v3r1.hpp" +#include "ck/device_utility/device_prop.hpp" +#include "ck/device_utility/kernel_launch.hpp" namespace ck { namespace tensor_operation { @@ -879,4 +879,3 @@ struct DeviceConv2dFwdXdl_C_Shuffle_Input_N_Hi_Wi_C_Weight_K_Y_X_C_Output_N_Ho_W } // namespace device } // namespace tensor_operation } // namespace ck -#endif diff --git a/include/ck/tensor_operation/gpu/device/device_conv2d_fwd_xdl_nhwc_kyxc_nhwk.hpp b/include/ck/tensor_operation/gpu/device/device_conv2d_fwd_xdl_nhwc_kyxc_nhwk.hpp index ece18459a0..e7b44b68c1 100644 --- a/include/ck/tensor_operation/gpu/device/device_conv2d_fwd_xdl_nhwc_kyxc_nhwk.hpp +++ b/include/ck/tensor_operation/gpu/device/device_conv2d_fwd_xdl_nhwc_kyxc_nhwk.hpp @@ -1,17 +1,17 @@ -#ifndef DEVICE_CONV2D_FWD_XDL_NHWC_KYXC_NHWK_HPP -#define DEVICE_CONV2D_FWD_XDL_NHWC_KYXC_NHWK_HPP +#pragma once #include #include -#include "device.hpp" -#include "device_base.hpp" -#include "device_conv_fwd.hpp" -#include "convolution_forward_specialization.hpp" -#include "common_header.hpp" -#include "tensor_layout.hpp" -#include "tensor_descriptor.hpp" -#include "tensor_descriptor_helper.hpp" -#include "gridwise_gemm_xdlops_v2r3.hpp" + +#include "ck/utility/common_header.hpp" +#include "ck/tensor_description/tensor_descriptor.hpp" +#include "ck/tensor_description/tensor_descriptor_helper.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_conv_fwd.hpp" +#include "ck/tensor_operation/gpu/device/convolution_forward_specialization.hpp" +#include "ck/tensor_operation/gpu/grid/gridwise_gemm_xdlops_v2r3.hpp" +#include "ck/device_utility/device_prop.hpp" +#include "ck/device_utility/kernel_launch.hpp" namespace ck { namespace tensor_operation { @@ -714,9 +714,8 @@ struct DeviceConv2dFwdXdl_Input_N_Hi_Wi_C_Weight_K_Y_X_C_Output_N_Ho_Wo_K return str.str(); } -}; // namespace device +}; } // namespace device } // namespace tensor_operation } // namespace ck -#endif diff --git a/include/ck/tensor_operation/gpu/device/device_conv_backward_weight.hpp b/include/ck/tensor_operation/gpu/device/device_conv_backward_weight.hpp index 549cfb26f3..4dd4acf9b2 100644 --- a/include/ck/tensor_operation/gpu/device/device_conv_backward_weight.hpp +++ b/include/ck/tensor_operation/gpu/device/device_conv_backward_weight.hpp @@ -1,8 +1,9 @@ -#ifndef DEVICE_CONV_WRW_HPP -#define DEVICE_CONV_WRW_HPP +#pragma once +#include #include -#include "device_base.hpp" + +#include "ck/tensor_operation/gpu/device/device_base.hpp" namespace ck { namespace tensor_operation { @@ -44,4 +45,3 @@ using DeviceConvBwdWeightPtr = std::unique_ptr< } // namespace device } // namespace tensor_operation } // namespace ck -#endif diff --git a/include/ck/tensor_operation/gpu/device/device_conv_bwd_data.hpp b/include/ck/tensor_operation/gpu/device/device_conv_bwd_data.hpp index 1d08af1a05..e66e8ec8d4 100644 --- a/include/ck/tensor_operation/gpu/device/device_conv_bwd_data.hpp +++ b/include/ck/tensor_operation/gpu/device/device_conv_bwd_data.hpp @@ -1,9 +1,10 @@ -#ifndef DEVICE_CONV_BWD_DATA_HPP -#define DEVICE_CONV_BWD_DATA_HPP +#pragma once +#include #include -#include "device_base.hpp" -#include "element_wise_operation.hpp" + +#include "ck/tensor_operation/gpu/device/device_base.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" namespace ck { namespace tensor_operation { @@ -44,4 +45,3 @@ using DeviceConvBwdDataPtr = std::unique_ptr< } // namespace device } // namespace tensor_operation } // namespace ck -#endif diff --git a/include/ck/tensor_operation/gpu/device/device_conv_fwd.hpp b/include/ck/tensor_operation/gpu/device/device_conv_fwd.hpp index d53e56f18b..979202b28d 100644 --- a/include/ck/tensor_operation/gpu/device/device_conv_fwd.hpp +++ b/include/ck/tensor_operation/gpu/device/device_conv_fwd.hpp @@ -1,8 +1,9 @@ -#ifndef DEVICE_CONV_FWD_HPP -#define DEVICE_CONV_FWD_HPP +#pragma once #include -#include "device_base.hpp" +#include + +#include "ck/tensor_operation/gpu/device/device_base.hpp" namespace ck { namespace tensor_operation { @@ -43,4 +44,3 @@ using DeviceConvFwdPtr = std::unique_ptr< } // namespace device } // namespace tensor_operation } // namespace ck -#endif diff --git a/include/ck/tensor_operation/gpu/device/device_conv_fwd_bias_activation.hpp b/include/ck/tensor_operation/gpu/device/device_conv_fwd_bias_activation.hpp index 77d4b7fb95..a3fb609d41 100644 --- a/include/ck/tensor_operation/gpu/device/device_conv_fwd_bias_activation.hpp +++ b/include/ck/tensor_operation/gpu/device/device_conv_fwd_bias_activation.hpp @@ -1,8 +1,10 @@ -#ifndef DEVICE_CONV_FWD_BIAS_ACTIVATION_HPP -#define DEVICE_CONV_FWD_BIAS_ACTIVATION_HPP +#pragma once +#include #include -#include "device_base.hpp" + +#include "ck/tensor_operation/gpu/device/device_base.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" namespace ck { namespace tensor_operation { @@ -46,4 +48,3 @@ using DeviceConvFwdBiasActivationPtr = } // namespace device } // namespace tensor_operation } // namespace ck -#endif diff --git a/include/ck/tensor_operation/gpu/device/device_conv_fwd_bias_activation_add.hpp b/include/ck/tensor_operation/gpu/device/device_conv_fwd_bias_activation_add.hpp index 2f8e780b78..e1082fca6a 100644 --- a/include/ck/tensor_operation/gpu/device/device_conv_fwd_bias_activation_add.hpp +++ b/include/ck/tensor_operation/gpu/device/device_conv_fwd_bias_activation_add.hpp @@ -1,8 +1,9 @@ -#ifndef DEVICE_CONV_FWD_BIAS_ACTIVATION_ADD_HPP -#define DEVICE_CONV_FWD_BIAS_ACTIVATION_ADD_HPP +#pragma once +#include #include -#include "device_base.hpp" + +#include "ck/tensor_operation/gpu/device/device_base.hpp" namespace ck { namespace tensor_operation { @@ -47,4 +48,3 @@ using DeviceConvFwdBiasActivationAddPtr = } // namespace device } // namespace tensor_operation } // namespace ck -#endif diff --git a/include/ck/tensor_operation/gpu/device/device_convnd_backward_weight_xdl_c_shuffle_nhwc_kyxc_nhwk.hpp b/include/ck/tensor_operation/gpu/device/device_convnd_backward_weight_xdl_c_shuffle_nhwc_kyxc_nhwk.hpp index 2991526851..7c7ba565bb 100644 --- a/include/ck/tensor_operation/gpu/device/device_convnd_backward_weight_xdl_c_shuffle_nhwc_kyxc_nhwk.hpp +++ b/include/ck/tensor_operation/gpu/device/device_convnd_backward_weight_xdl_c_shuffle_nhwc_kyxc_nhwk.hpp @@ -2,16 +2,17 @@ #include #include -#include "device.hpp" -#include "device_base.hpp" -#include "device_conv_backward_weight.hpp" -#include "convolution_backward_weight_specialization.hpp" -#include "common_header.hpp" -#include "tensor_layout.hpp" -#include "tensor_descriptor.hpp" -#include "tensor_descriptor_helper.hpp" -#include "gridwise_gemm_xdlops_bwd_weight.hpp" -#include "gridwise_unary_elementwise_1d.hpp" + +#include "ck/utility/common_header.hpp" +#include "ck/tensor_description/tensor_descriptor.hpp" +#include "ck/tensor_description/tensor_descriptor_helper.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_conv_backward_weight.hpp" +#include "ck/tensor_operation/gpu/device/convolution_backward_weight_specialization.hpp" +#include "ck/tensor_operation/gpu/grid/gridwise_gemm_xdlops_bwd_weight.hpp" +#include "ck/tensor_operation/gpu/grid/gridwise_unary_elementwise_1d.hpp" +#include "ck/device_utility/device_prop.hpp" +#include "ck/device_utility/kernel_launch.hpp" namespace ck { namespace tensor_operation { diff --git a/include/ck/tensor_operation/gpu/device/device_convnd_bwd_data_xdl_ndhwc_kzyxc_ndhwk.hpp b/include/ck/tensor_operation/gpu/device/device_convnd_bwd_data_xdl_ndhwc_kzyxc_ndhwk.hpp index 0517db4415..1388b05f61 100644 --- a/include/ck/tensor_operation/gpu/device/device_convnd_bwd_data_xdl_ndhwc_kzyxc_ndhwk.hpp +++ b/include/ck/tensor_operation/gpu/device/device_convnd_bwd_data_xdl_ndhwc_kzyxc_ndhwk.hpp @@ -1,17 +1,17 @@ -#ifndef DEVICE_CONVND_BWD_DATA_XDL_NDHWC_KZYXC_NDHWK_HPP -#define DEVICE_CONVND_BWD_DATA_XDL_NDHWC_KZYXC_NDHWK_HPP +#pragma once #include #include -#include "device.hpp" -#include "device_base.hpp" -#include "device_conv_bwd_data.hpp" -#include "convolution_backward_data_specialization.hpp" -#include "common_header.hpp" -#include "tensor_layout.hpp" -#include "tensor_descriptor.hpp" -#include "tensor_descriptor_helper.hpp" -#include "gridwise_gemm_xdlops_v2r3.hpp" + +#include "ck/utility/common_header.hpp" +#include "ck/tensor_description/tensor_descriptor.hpp" +#include "ck/tensor_description/tensor_descriptor_helper.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_conv_bwd_data.hpp" +#include "ck/tensor_operation/gpu/device/convolution_backward_data_specialization.hpp" +#include "ck/tensor_operation/gpu/grid/gridwise_gemm_xdlops_v2r3.hpp" +#include "ck/device_utility/device_prop.hpp" +#include "ck/device_utility/kernel_launch.hpp" namespace ck { namespace tensor_operation { @@ -1546,4 +1546,3 @@ struct DeviceConvndBwdDataXdl_Input_N_Di_Hi_Wi_C_Weight_K_Z_Y_X_C_Output_N_Do_Ho } // namespace device } // namespace tensor_operation } // namespace ck -#endif diff --git a/include/ck/tensor_operation/gpu/device/device_convnd_fwd_xdl_nhwc_kyxc_nhwk.hpp b/include/ck/tensor_operation/gpu/device/device_convnd_fwd_xdl_nhwc_kyxc_nhwk.hpp index c1ab44a28b..e5c3e00a47 100644 --- a/include/ck/tensor_operation/gpu/device/device_convnd_fwd_xdl_nhwc_kyxc_nhwk.hpp +++ b/include/ck/tensor_operation/gpu/device/device_convnd_fwd_xdl_nhwc_kyxc_nhwk.hpp @@ -6,16 +6,15 @@ #include #include -#include "device.hpp" -#include "device_prop.hpp" -#include "device_base.hpp" -#include "device_conv_fwd.hpp" -#include "convolution_forward_specialization.hpp" -#include "common_header.hpp" -#include "tensor_layout.hpp" -#include "tensor_descriptor.hpp" -#include "tensor_descriptor_helper.hpp" -#include "gridwise_gemm_xdlops_v2r3.hpp" +#include "ck/utility/common_header.hpp" +#include "ck/tensor_description/tensor_descriptor.hpp" +#include "ck/tensor_description/tensor_descriptor_helper.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_conv_fwd.hpp" +#include "ck/tensor_operation/gpu/device/convolution_forward_specialization.hpp" +#include "ck/tensor_operation/gpu/grid/gridwise_gemm_xdlops_v2r3.hpp" +#include "ck/device_utility/device_prop.hpp" +#include "ck/device_utility/kernel_launch.hpp" namespace ck { namespace tensor_operation { diff --git a/include/ck/tensor_operation/gpu/device/device_gemm_bias.hpp b/include/ck/tensor_operation/gpu/device/device_gemm_bias.hpp index 9f5d16a1f9..0dcfb11f33 100644 --- a/include/ck/tensor_operation/gpu/device/device_gemm_bias.hpp +++ b/include/ck/tensor_operation/gpu/device/device_gemm_bias.hpp @@ -1,6 +1,8 @@ #pragma once + #include -#include "device_base.hpp" + +#include "ck/tensor_operation/gpu/device/device_base.hpp" namespace ck { namespace tensor_operation { diff --git a/include/ck/tensor_operation/gpu/device/device_gemm_bias_activation.hpp b/include/ck/tensor_operation/gpu/device/device_gemm_bias_activation.hpp index 95736b1887..b51d502307 100644 --- a/include/ck/tensor_operation/gpu/device/device_gemm_bias_activation.hpp +++ b/include/ck/tensor_operation/gpu/device/device_gemm_bias_activation.hpp @@ -1,8 +1,8 @@ -#ifndef DEVICE_GEMM_BIAS_ACTIVATION_HPP -#define DEVICE_GEMM_BIAS_ACTIVATION_HPP +#pragma once #include -#include "device_base.hpp" + +#include "ck/tensor_operation/gpu/device/device_base.hpp" namespace ck { namespace tensor_operation { @@ -40,4 +40,3 @@ using DeviceGemmBiasActivationPtr = std::unique_ptr< } // namespace device } // namespace tensor_operation } // namespace ck -#endif diff --git a/include/ck/tensor_operation/gpu/device/device_gemm_bias_add_reduce_xdl_cshuffle.hpp b/include/ck/tensor_operation/gpu/device/device_gemm_bias_add_reduce_xdl_cshuffle.hpp index b29eb37898..023892dbdc 100644 --- a/include/ck/tensor_operation/gpu/device/device_gemm_bias_add_reduce_xdl_cshuffle.hpp +++ b/include/ck/tensor_operation/gpu/device/device_gemm_bias_add_reduce_xdl_cshuffle.hpp @@ -1,13 +1,17 @@ #pragma once + #include #include -#include "device.hpp" -#include "device_gemm_reduce.hpp" -#include "tensor_layout.hpp" -#include "tensor_descriptor.hpp" -#include "tensor_descriptor_helper.hpp" -#include "gridwise_gemm_bias_add_reduce_xdl_cshuffle_v1.hpp" -#include "gemm_specialization.hpp" + +#include "ck/utility/common_header.hpp" +#include "ck/tensor_description/tensor_descriptor.hpp" +#include "ck/tensor_description/tensor_descriptor_helper.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_reduce.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/grid/gridwise_gemm_bias_add_reduce_xdl_cshuffle_v1.hpp" +#include "ck/device_utility/device_prop.hpp" +#include "ck/device_utility/kernel_launch.hpp" namespace ck { namespace tensor_operation { diff --git a/include/ck/tensor_operation/gpu/device/device_gemm_dl.hpp b/include/ck/tensor_operation/gpu/device/device_gemm_dl.hpp index 5ccf1934fe..cf99c8c829 100644 --- a/include/ck/tensor_operation/gpu/device/device_gemm_dl.hpp +++ b/include/ck/tensor_operation/gpu/device/device_gemm_dl.hpp @@ -3,17 +3,15 @@ #include #include -#include "device.hpp" -#include "device_prop.hpp" -#include "device_base.hpp" -#include "device_gemm.hpp" -#include "common_header.hpp" -#include "tensor_layout.hpp" -#include "tensor_descriptor.hpp" -#include "tensor_descriptor_helper.hpp" -#include "gemm_specialization.hpp" -#include "element_wise_operation.hpp" -#include "gridwise_gemm_dl_v1r3.hpp" +#include "ck/utility/common_header.hpp" +#include "ck/tensor_description/tensor_descriptor.hpp" +#include "ck/tensor_description/tensor_descriptor_helper.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/grid/gridwise_gemm_dl_v1r3.hpp" +#include "ck/device_utility/device_prop.hpp" +#include "ck/device_utility/kernel_launch.hpp" namespace ck { namespace tensor_operation { diff --git a/include/ck/tensor_operation/gpu/device/device_gemm_multiple_d_xdl_cshuffle.hpp b/include/ck/tensor_operation/gpu/device/device_gemm_multiple_d_xdl_cshuffle.hpp index 2de5897311..db1fc730cb 100644 --- a/include/ck/tensor_operation/gpu/device/device_gemm_multiple_d_xdl_cshuffle.hpp +++ b/include/ck/tensor_operation/gpu/device/device_gemm_multiple_d_xdl_cshuffle.hpp @@ -3,15 +3,15 @@ #include #include -#include "device.hpp" -#include "device_gemm_multiple_d.hpp" -#include "common_header.hpp" -#include "tensor_layout.hpp" -#include "tensor_descriptor.hpp" -#include "tensor_descriptor_helper.hpp" -#include "gridwise_gemm_multiple_d_xdl_cshuffle.hpp" -#include "gemm_specialization.hpp" -#include "device_prop.hpp" +#include "ck/utility/common_header.hpp" +#include "ck/tensor_description/tensor_descriptor.hpp" +#include "ck/tensor_description/tensor_descriptor_helper.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_multiple_d.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/grid/gridwise_gemm_multiple_d_xdl_cshuffle.hpp" +#include "ck/device_utility/device_prop.hpp" +#include "ck/device_utility/kernel_launch.hpp" namespace ck { diff --git a/include/ck/tensor_operation/gpu/device/device_gemm_reduce_xdl_cshuffle.hpp b/include/ck/tensor_operation/gpu/device/device_gemm_reduce_xdl_cshuffle.hpp index 989883bd39..61e189828b 100644 --- a/include/ck/tensor_operation/gpu/device/device_gemm_reduce_xdl_cshuffle.hpp +++ b/include/ck/tensor_operation/gpu/device/device_gemm_reduce_xdl_cshuffle.hpp @@ -1,14 +1,17 @@ #pragma once + #include #include -#include "device.hpp" -#include "device_gemm_reduce.hpp" -#include "common_header.hpp" -#include "tensor_layout.hpp" -#include "tensor_descriptor.hpp" -#include "tensor_descriptor_helper.hpp" -#include "gridwise_gemm_reduce_xdl_cshuffle_v1.hpp" -#include "gemm_specialization.hpp" + +#include "ck/utility/common_header.hpp" +#include "ck/tensor_description/tensor_descriptor.hpp" +#include "ck/tensor_description/tensor_descriptor_helper.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_reduce.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/grid/gridwise_gemm_reduce_xdl_cshuffle_v1.hpp" +#include "ck/device_utility/device_prop.hpp" +#include "ck/device_utility/kernel_launch.hpp" namespace ck { namespace tensor_operation { diff --git a/include/ck/tensor_operation/gpu/device/device_gemm_xdl.hpp b/include/ck/tensor_operation/gpu/device/device_gemm_xdl.hpp index 3a8e1390e4..eb3488d784 100644 --- a/include/ck/tensor_operation/gpu/device/device_gemm_xdl.hpp +++ b/include/ck/tensor_operation/gpu/device/device_gemm_xdl.hpp @@ -2,16 +2,16 @@ #include #include -#include "device.hpp" -#include "device_prop.hpp" -#include "device_base.hpp" -#include "device_gemm.hpp" -#include "common_header.hpp" -#include "tensor_layout.hpp" -#include "tensor_descriptor.hpp" -#include "tensor_descriptor_helper.hpp" -#include "gridwise_gemm_xdlops_v2r3.hpp" -#include "gemm_specialization.hpp" + +#include "ck/utility/common_header.hpp" +#include "ck/tensor_description/tensor_descriptor.hpp" +#include "ck/tensor_description/tensor_descriptor_helper.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/grid/gridwise_gemm_xdlops_v2r3.hpp" +#include "ck/device_utility/device_prop.hpp" +#include "ck/device_utility/kernel_launch.hpp" namespace ck { namespace tensor_operation { diff --git a/include/ck/tensor_operation/gpu/device/device_gemm_xdl_c_shuffle_bias_2d.hpp b/include/ck/tensor_operation/gpu/device/device_gemm_xdl_c_shuffle_bias_2d.hpp index 1db69dd462..5f6fbc5614 100644 --- a/include/ck/tensor_operation/gpu/device/device_gemm_xdl_c_shuffle_bias_2d.hpp +++ b/include/ck/tensor_operation/gpu/device/device_gemm_xdl_c_shuffle_bias_2d.hpp @@ -1,13 +1,17 @@ #pragma once + #include #include -#include "device.hpp" -#include "device_gemm_bias.hpp" -#include "common_header.hpp" -#include "tensor_layout.hpp" -#include "tensor_descriptor.hpp" -#include "tensor_descriptor_helper.hpp" -#include "gridwise_gemm_xdlops_v3r2.hpp" + +#include "ck/utility/common_header.hpp" +#include "ck/tensor_description/tensor_descriptor.hpp" +#include "ck/tensor_description/tensor_descriptor_helper.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_bias.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/grid/gridwise_gemm_xdlops_v3r2.hpp" +#include "ck/device_utility/device_prop.hpp" +#include "ck/device_utility/kernel_launch.hpp" namespace ck { namespace tensor_operation { diff --git a/include/ck/tensor_operation/gpu/device/device_gemm_xdl_c_shuffle_bias_activation.hpp b/include/ck/tensor_operation/gpu/device/device_gemm_xdl_c_shuffle_bias_activation.hpp index b465f8e4ae..6b272bffdc 100644 --- a/include/ck/tensor_operation/gpu/device/device_gemm_xdl_c_shuffle_bias_activation.hpp +++ b/include/ck/tensor_operation/gpu/device/device_gemm_xdl_c_shuffle_bias_activation.hpp @@ -1,15 +1,17 @@ -#ifndef DEVICE_GEMM_XDL_C_SHUFFLE_BIAS_ACTIVATION_HPP -#define DEVICE_GEMM_XDL_C_SHUFFLE_BIAS_ACTIVATION_HPP +#pragma once #include #include -#include "device.hpp" -#include "device_gemm_bias_activation.hpp" -#include "common_header.hpp" -#include "tensor_layout.hpp" -#include "tensor_descriptor.hpp" -#include "tensor_descriptor_helper.hpp" -#include "gridwise_gemm_xdlops_v3r2.hpp" + +#include "ck/utility/common_header.hpp" +#include "ck/tensor_description/tensor_descriptor.hpp" +#include "ck/tensor_description/tensor_descriptor_helper.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_bias_activation.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/grid/gridwise_gemm_xdlops_v3r2.hpp" +#include "ck/device_utility/device_prop.hpp" +#include "ck/device_utility/kernel_launch.hpp" namespace ck { namespace tensor_operation { @@ -513,4 +515,3 @@ struct DeviceGemmXdl_C_Shuffle_Bias_Activation } // namespace device } // namespace tensor_operation } // namespace ck -#endif diff --git a/include/ck/tensor_operation/gpu/device/device_gemm_xdl_c_shuffle_bias_activation_add.hpp b/include/ck/tensor_operation/gpu/device/device_gemm_xdl_c_shuffle_bias_activation_add.hpp index 7a2e1886d3..eff4d21770 100644 --- a/include/ck/tensor_operation/gpu/device/device_gemm_xdl_c_shuffle_bias_activation_add.hpp +++ b/include/ck/tensor_operation/gpu/device/device_gemm_xdl_c_shuffle_bias_activation_add.hpp @@ -1,15 +1,17 @@ -#ifndef DEVICE_GEMM_XDL_C_SHUFFLE_BIAS_ACTIVATION_ADD_HPP -#define DEVICE_GEMM_XDL_C_SHUFFLE_BIAS_ACTIVATION_ADD_HPP +#pragma once #include #include -#include "device.hpp" -#include "device_gemm_bias_activation_add.hpp" -#include "common_header.hpp" -#include "tensor_layout.hpp" -#include "tensor_descriptor.hpp" -#include "tensor_descriptor_helper.hpp" -#include "gridwise_gemm_xdlops_v3r3.hpp" + +#include "ck/utility/common_header.hpp" +#include "ck/tensor_description/tensor_descriptor.hpp" +#include "ck/tensor_description/tensor_descriptor_helper.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_bias_activation_add.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/grid/gridwise_gemm_xdlops_v3r3.hpp" +#include "ck/device_utility/device_prop.hpp" +#include "ck/device_utility/kernel_launch.hpp" namespace ck { namespace tensor_operation { @@ -573,4 +575,3 @@ struct DeviceGemmXdl_C_Shuffle_Bias_Activation_Add } // namespace device } // namespace tensor_operation } // namespace ck -#endif diff --git a/include/ck/tensor_operation/gpu/device/device_gemm_xdl_cshuffle.hpp b/include/ck/tensor_operation/gpu/device/device_gemm_xdl_cshuffle.hpp index a74ee81679..130e2968c9 100644 --- a/include/ck/tensor_operation/gpu/device/device_gemm_xdl_cshuffle.hpp +++ b/include/ck/tensor_operation/gpu/device/device_gemm_xdl_cshuffle.hpp @@ -1,15 +1,17 @@ #pragma once + #include #include -#include "device.hpp" -#include "device_gemm.hpp" -#include "common_header.hpp" -#include "tensor_layout.hpp" -#include "tensor_descriptor.hpp" -#include "tensor_descriptor_helper.hpp" -#include "gridwise_gemm_xdl_cshuffle_v1.hpp" -#include "tensor_operation/gpu/device/gemm_specialization.hpp" -#include "device_prop.hpp" + +#include "ck/utility/common_header.hpp" +#include "ck/tensor_description/tensor_descriptor.hpp" +#include "ck/tensor_description/tensor_descriptor_helper.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/grid/gridwise_gemm_xdl_cshuffle_v1.hpp" +#include "ck/device_utility/device_prop.hpp" +#include "ck/device_utility/kernel_launch.hpp" namespace ck { namespace tensor_operation { diff --git a/include/ck/tensor_operation/gpu/device/device_gemm_xdl_splitk.hpp b/include/ck/tensor_operation/gpu/device/device_gemm_xdl_splitk.hpp index d9fc8f7a8a..79cbe58894 100644 --- a/include/ck/tensor_operation/gpu/device/device_gemm_xdl_splitk.hpp +++ b/include/ck/tensor_operation/gpu/device/device_gemm_xdl_splitk.hpp @@ -1,22 +1,17 @@ -#ifndef DEVICE_GEMM_SPLITK_XDL_HPP -#define DEVICE_GEMM_SPLITK_XDL_HPP +#pragma once #include #include -#include "device.hpp" -#include "device_base.hpp" -#include "device_gemm.hpp" -#include "common_header.hpp" -#include "tensor_layout.hpp" -#include "tensor_descriptor.hpp" -#include "tensor_descriptor_helper.hpp" -#include "gridwise_gemm_xdlops_v2r4.hpp" -#include "gemm_specialization.hpp" -#include "device_prop.hpp" -#ifndef CK_RUN_KERNEL_AND_TIME -#define CK_RUN_KERNEL_AND_TIME 1 -#endif +#include "ck/utility/common_header.hpp" +#include "ck/tensor_description/tensor_descriptor.hpp" +#include "ck/tensor_description/tensor_descriptor_helper.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/grid/gridwise_gemm_xdlops_v2r4.hpp" +#include "ck/device_utility/device_prop.hpp" +#include "ck/device_utility/kernel_launch.hpp" namespace ck { namespace tensor_operation { @@ -639,4 +634,3 @@ struct DeviceGemmXdlSplitK } // namespace device } // namespace tensor_operation } // namespace ck -#endif diff --git a/include/ck/tensor_operation/gpu/device/device_gemm_xdl_splitk_c_shuffle.hpp b/include/ck/tensor_operation/gpu/device/device_gemm_xdl_splitk_c_shuffle.hpp index ad424d91d9..e5cdbda4ec 100644 --- a/include/ck/tensor_operation/gpu/device/device_gemm_xdl_splitk_c_shuffle.hpp +++ b/include/ck/tensor_operation/gpu/device/device_gemm_xdl_splitk_c_shuffle.hpp @@ -1,21 +1,17 @@ -#ifndef DEVICE_GEMM_XDL_SPLITK_C_SHUFFLE_HPP -#define DEVICE_GEMM_XDL_SPLITK_C_SHUFFLE_HPP +#pragma once #include #include -#include "device.hpp" -#include "device_base.hpp" -#include "device_gemm.hpp" -#include "common_header.hpp" -#include "tensor_layout.hpp" -#include "tensor_descriptor.hpp" -#include "tensor_descriptor_helper.hpp" -#include "gridwise_gemm_xdlops_v2r4r2.hpp" -#include "gemm_specialization.hpp" -#ifndef CK_RUN_KERNEL_AND_TIME -#define CK_RUN_KERNEL_AND_TIME 1 -#endif +#include "ck/utility/common_header.hpp" +#include "ck/tensor_description/tensor_descriptor.hpp" +#include "ck/tensor_description/tensor_descriptor_helper.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/grid/gridwise_gemm_xdlops_v2r4r2.hpp" +#include "ck/device_utility/device_prop.hpp" +#include "ck/device_utility/kernel_launch.hpp" namespace ck { namespace tensor_operation { @@ -641,4 +637,3 @@ struct DeviceGemmXdlSplitKCShuffle } // namespace device } // namespace tensor_operation } // namespace ck -#endif diff --git a/include/ck/tensor_operation/gpu/device/device_grouped_gemm_xdl.hpp b/include/ck/tensor_operation/gpu/device/device_grouped_gemm_xdl.hpp index 6dfa448fa8..86b1736c4b 100644 --- a/include/ck/tensor_operation/gpu/device/device_grouped_gemm_xdl.hpp +++ b/include/ck/tensor_operation/gpu/device/device_grouped_gemm_xdl.hpp @@ -1,17 +1,17 @@ -#ifndef DEVICE_GROUPED_GEMM_XDL_HPP -#define DEVICE_GROUPED_GEMM_XDL_HPP +#pragma once #include #include -#include "device.hpp" -#include "device_base.hpp" -#include "device_gemm.hpp" -#include "common_header.hpp" -#include "tensor_layout.hpp" -#include "tensor_descriptor.hpp" -#include "tensor_descriptor_helper.hpp" -#include "gridwise_gemm_xdlops_v2r3.hpp" -#include "gemm_specialization.hpp" + +#include "ck/utility/common_header.hpp" +#include "ck/tensor_description/tensor_descriptor.hpp" +#include "ck/tensor_description/tensor_descriptor_helper.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/grid/gridwise_gemm_xdlops_v2r3.hpp" +#include "ck/device_utility/device_prop.hpp" +#include "ck/device_utility/kernel_launch.hpp" namespace ck { namespace tensor_operation { @@ -638,4 +638,3 @@ struct DeviceGroupedGemmXdl } // namespace device } // namespace tensor_operation } // namespace ck -#endif diff --git a/include/ck/tensor_operation/gpu/device/device_pool2d_fwd.hpp b/include/ck/tensor_operation/gpu/device/device_pool2d_fwd.hpp index d049f6e979..7432d8f8b0 100644 --- a/include/ck/tensor_operation/gpu/device/device_pool2d_fwd.hpp +++ b/include/ck/tensor_operation/gpu/device/device_pool2d_fwd.hpp @@ -1,10 +1,10 @@ -#ifndef DEVICE_POOL2D_FWD_HPP -#define DEVICE_POOL2D_FWD_HPP +#pragma once #include #include -#include "device_base.hpp" -#include "reduction_enums.hpp" + +#include "ck/tensor_operation/gpu/device/device_base.hpp" +#include "ck/utility/reduction_enums.hpp" namespace ck { namespace tensor_operation { @@ -35,4 +35,3 @@ using DevicePool2dFwdPtr = std::unique_ptr>; } // namespace device } // namespace tensor_operation } // namespace ck -#endif diff --git a/include/ck/tensor_operation/gpu/device/device_pool2d_fwd_nhwc_nhwc.hpp b/include/ck/tensor_operation/gpu/device/device_pool2d_fwd_nhwc_nhwc.hpp index 41fb11b7de..4c31a99189 100644 --- a/include/ck/tensor_operation/gpu/device/device_pool2d_fwd_nhwc_nhwc.hpp +++ b/include/ck/tensor_operation/gpu/device/device_pool2d_fwd_nhwc_nhwc.hpp @@ -1,13 +1,15 @@ -#ifndef DEVICE_POOL2D_FWD_NHWC_NHWC_HPP -#define DEVICE_POOL2D_FWD_NHWC_NHWC_HPP +#pragma once #include #include -#include "device_pool2d_fwd.hpp" -#include "tensor_descriptor.hpp" -#include "tensor_descriptor_helper.hpp" -#include "reduction_operator_mapping.hpp" -#include "gridwise_2d_reduction_threadwise.hpp" + +#include "ck/tensor_description/tensor_descriptor.hpp" +#include "ck/tensor_description/tensor_descriptor_helper.hpp" +#include "ck/tensor_operation/gpu/device/reduction_operator_mapping.hpp" +#include "ck/tensor_operation/gpu/device/device_pool2d_fwd.hpp" +#include "ck/tensor_operation/gpu/grid/gridwise_2d_reduction_threadwise.hpp" +#include "ck/device_utility/device_prop.hpp" +#include "ck/device_utility/kernel_launch.hpp" namespace ck { namespace tensor_operation { @@ -315,9 +317,8 @@ struct DevicePool2dFwd_Input_N_Hi_Wi_C_Output_N_Ho_Wo_C : public DevicePool2dFwd return str.str(); } -}; // namespace device +}; } // namespace device } // namespace tensor_operation } // namespace ck -#endif diff --git a/include/ck/tensor_operation/gpu/device/device_reduce.hpp b/include/ck/tensor_operation/gpu/device/device_reduce.hpp index 6f367a8747..363ae7ee52 100644 --- a/include/ck/tensor_operation/gpu/device/device_reduce.hpp +++ b/include/ck/tensor_operation/gpu/device/device_reduce.hpp @@ -1,13 +1,12 @@ -#ifndef DEVICE_REDUCE_HPP -#define DEVICE_REDUCE_HPP +#pragma once #include #include #include -#include "common_header.hpp" -#include "device_base.hpp" -#include "reduction_enums.hpp" +#include "ck/utility/common_header.hpp" +#include "ck/utility/reduction_enums.hpp" +#include "ck/tensor_operation/gpu/device/device_base.hpp" namespace ck { namespace tensor_operation { @@ -41,4 +40,3 @@ using DeviceReducePtr = } // namespace device } // namespace tensor_operation } // namespace ck -#endif diff --git a/include/ck/tensor_operation/gpu/device/device_reduce_common.hpp b/include/ck/tensor_operation/gpu/device/device_reduce_common.hpp index f68a392821..4b8a24f098 100644 --- a/include/ck/tensor_operation/gpu/device/device_reduce_common.hpp +++ b/include/ck/tensor_operation/gpu/device/device_reduce_common.hpp @@ -1,12 +1,11 @@ -#ifndef DEVICE_REDUCE_COMMON_HPP -#define DEVICE_REDUCE_COMMON_HPP +#pragma once #include #include -#include "common_header.hpp" -#include "reduction_enums.hpp" -#include "reduction_operator.hpp" +#include "ck/utility/common_header.hpp" +#include "ck/utility/reduction_enums.hpp" +#include "ck/utility/reduction_operator.hpp" namespace ck { namespace tensor_operation { @@ -85,6 +84,4 @@ std::vector shuffle_tensor_dimensions(const std::vector& origL } // namespace device } // namespace tensor_operation - } // namespace ck -#endif diff --git a/include/ck/tensor_operation/gpu/device/device_reduce_multiblock.hpp b/include/ck/tensor_operation/gpu/device/device_reduce_multiblock.hpp index 99e79e3a1a..a00e156071 100644 --- a/include/ck/tensor_operation/gpu/device/device_reduce_multiblock.hpp +++ b/include/ck/tensor_operation/gpu/device/device_reduce_multiblock.hpp @@ -1,15 +1,18 @@ -#ifndef DEVICE_REDUCE_MULTIBLOCK_HPP -#define DEVICE_REDUCE_MULTIBLOCK_HPP +#pragma once #include #include -#include "device.hpp" -#include "device_base.hpp" -#include "device_reduce.hpp" -#include "device_reduce_common.hpp" -#include "gridwise_2d_reduction_multiblock.hpp" -#include "gridwise_set_buffer_value.hpp" -#include "reduction_operator.hpp" + +#include "ck/utility/common_header.hpp" +#include "ck/utility/reduction_operator.hpp" +#include "ck/tensor_description/tensor_descriptor.hpp" +#include "ck/tensor_description/tensor_descriptor_helper.hpp" +#include "ck/tensor_operation/gpu/device/device_reduce.hpp" +#include "ck/tensor_operation/gpu/device/device_reduce_common.hpp" +#include "ck/tensor_operation/gpu/grid/gridwise_2d_reduction_multiblock.hpp" +#include "ck/tensor_operation/gpu/grid/gridwise_set_buffer_value.hpp" +#include "ck/device_utility/device_prop.hpp" +#include "ck/device_utility/kernel_launch.hpp" namespace ck { namespace tensor_operation { @@ -505,4 +508,3 @@ struct DeviceReduceMultiBlock : public DeviceReduce #include -#include "device.hpp" -#include "device_reduce.hpp" -#include "device_reduce_common.hpp" -#include "gridwise_2d_reduction_multiblock.hpp" -#include "gridwise_2d_reduction_threadwise.hpp" + +#include "ck/device_utility/device_prop.hpp" +#include "ck/device_utility/kernel_launch.hpp" +#include "ck/tensor_operation/gpu/device/device_reduce.hpp" +#include "ck/tensor_operation/gpu/device/device_reduce_common.hpp" +#include "ck/tensor_operation/gpu/grid/gridwise_2d_reduction_multiblock.hpp" +#include "ck/tensor_operation/gpu/grid/gridwise_2d_reduction_threadwise.hpp" namespace ck { namespace tensor_operation { @@ -370,4 +371,3 @@ struct DeviceReduceThreadWise : public DeviceReduce #include -#include "device.hpp" -#include "device_base.hpp" -#include "device_reduce.hpp" -#include "device_reduce_multiblock.hpp" -#include "device_reduce_common.hpp" -#include "gridwise_softmax.hpp" -#include "gridwise_set_buffer_value.hpp" -#include "reduction_operator.hpp" + +#include "ck/utility/reduction_operator.hpp" +#include "ck/tensor_operation/gpu/device/device_base.hpp" +#include "ck/tensor_operation/gpu/device/device_reduce.hpp" +#include "ck/tensor_operation/gpu/device/device_reduce_multiblock.hpp" +#include "ck/tensor_operation/gpu/device/device_reduce_common.hpp" +#include "ck/tensor_operation/gpu/grid/gridwise_softmax.hpp" +#include "ck/tensor_operation/gpu/grid/gridwise_set_buffer_value.hpp" +#include "ck/device_utility/device_prop.hpp" +#include "ck/device_utility/kernel_launch.hpp" namespace ck { namespace tensor_operation { @@ -200,4 +201,3 @@ struct DeviceSoftmax : public BaseOperator } // namespace device } // namespace tensor_operation } // namespace ck -#endif // DEVICE_SOFTMAX_HPP diff --git a/include/ck/tensor_operation/gpu/device/device_unary_elementwise.hpp b/include/ck/tensor_operation/gpu/device/device_unary_elementwise.hpp index 4fcad7004f..3bb091e277 100644 --- a/include/ck/tensor_operation/gpu/device/device_unary_elementwise.hpp +++ b/include/ck/tensor_operation/gpu/device/device_unary_elementwise.hpp @@ -1,10 +1,12 @@ #pragma once + #include #include -#include "device.hpp" -#include "device_base.hpp" -#include "gridwise_unary_elementwise_1d.hpp" +#include "ck/device_utility/device_prop.hpp" +#include "ck/device_utility/kernel_launch.hpp" +#include "ck/tensor_operation/gpu/device/device_base.hpp" +#include "ck/tensor_operation/gpu/grid/gridwise_unary_elementwise_1d.hpp" namespace ck { namespace tensor_operation { diff --git a/include/ck/tensor_operation/gpu/device/gemm_specialization.hpp b/include/ck/tensor_operation/gpu/device/gemm_specialization.hpp index d4ef61a133..3de39c5080 100644 --- a/include/ck/tensor_operation/gpu/device/gemm_specialization.hpp +++ b/include/ck/tensor_operation/gpu/device/gemm_specialization.hpp @@ -1,5 +1,4 @@ -#ifndef GEMM_SPECIALIZATION -#define GEMM_SPECIALIZATION +#pragma once namespace ck { namespace tensor_operation { @@ -20,4 +19,3 @@ enum struct GemmSpecialization } // namespace device } // namespace tensor_operation } // namespace ck -#endif diff --git a/include/ck/tensor_operation/gpu/device/reduction_operator_mapping.hpp b/include/ck/tensor_operation/gpu/device/reduction_operator_mapping.hpp index 4b3f52148d..3d355664fa 100644 --- a/include/ck/tensor_operation/gpu/device/reduction_operator_mapping.hpp +++ b/include/ck/tensor_operation/gpu/device/reduction_operator_mapping.hpp @@ -1,34 +1,9 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2020 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ -#ifndef CK_REDUCTION_OPERATOR_MAPPING_HPP -#define CK_REDUCTION_OPERATOR_MAPPING_HPP +#pragma once -#include "reduction_operator.hpp" -#include "reduction_enums.hpp" -#include "element_wise_operation.hpp" +#include "ck/utility/reduction_operator.hpp" +#include "ck/utility/reduction_enums.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" +// FIXME: can it be replaced with ck::Tuple? #include namespace ck { @@ -205,6 +180,4 @@ struct reduce_unary_operator }; }; -} // end of namespace ck - -#endif +} // namespace ck diff --git a/include/ck/tensor_operation/gpu/element/binary_element_wise_operation.hpp b/include/ck/tensor_operation/gpu/element/binary_element_wise_operation.hpp index 300ce6fc0a..fc16b2c028 100644 --- a/include/ck/tensor_operation/gpu/element/binary_element_wise_operation.hpp +++ b/include/ck/tensor_operation/gpu/element/binary_element_wise_operation.hpp @@ -1,31 +1,6 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2022 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ #pragma once -#include "data_type.hpp" +#include "ck/utility/data_type.hpp" namespace ck { namespace tensor_operation { diff --git a/include/ck/tensor_operation/gpu/element/element_wise_operation.hpp b/include/ck/tensor_operation/gpu/element/element_wise_operation.hpp index 274d398e26..3f16ddf718 100644 --- a/include/ck/tensor_operation/gpu/element/element_wise_operation.hpp +++ b/include/ck/tensor_operation/gpu/element/element_wise_operation.hpp @@ -1,9 +1,9 @@ #pragma once -#include "data_type.hpp" -#include "math_v2.hpp" -#include "unary_element_wise_operation.hpp" -#include "binary_element_wise_operation.hpp" +#include "ck/utility/data_type.hpp" +#include "ck/utility/math_v2.hpp" +#include "ck/tensor_operation/gpu/element/unary_element_wise_operation.hpp" +#include "ck/tensor_operation/gpu/element/binary_element_wise_operation.hpp" namespace ck { namespace tensor_operation { diff --git a/include/ck/tensor_operation/gpu/element/unary_element_wise_operation.hpp b/include/ck/tensor_operation/gpu/element/unary_element_wise_operation.hpp index c6142474cc..829085c329 100644 --- a/include/ck/tensor_operation/gpu/element/unary_element_wise_operation.hpp +++ b/include/ck/tensor_operation/gpu/element/unary_element_wise_operation.hpp @@ -1,7 +1,7 @@ #pragma once -#include "data_type.hpp" -#include "math_v2.hpp" +#include "ck/utility/data_type.hpp" +#include "ck/utility/math_v2.hpp" namespace ck { namespace tensor_operation { diff --git a/include/ck/tensor_operation/gpu/grid/block_to_ctile_map.hpp b/include/ck/tensor_operation/gpu/grid/block_to_ctile_map.hpp index 792060ca86..dea71e6948 100644 --- a/include/ck/tensor_operation/gpu/grid/block_to_ctile_map.hpp +++ b/include/ck/tensor_operation/gpu/grid/block_to_ctile_map.hpp @@ -1,10 +1,9 @@ -#ifndef UTILITY_BLOCK_TO_CTILE_MAP -#define UTILITY_BLOCK_TO_CTILE_MAP +#pragma once -#include "utility/math.hpp" -#include "utility/number.hpp" -#include "tensor_description/tensor_adaptor.hpp" -#include "tensor_description/multi_index_transform_helper.hpp" +#include "ck/utility/math.hpp" +#include "ck/utility/number.hpp" +#include "ck/tensor_description/tensor_adaptor.hpp" +#include "ck/tensor_description/multi_index_transform_helper.hpp" namespace ck { @@ -485,5 +484,3 @@ __host__ __device__ bool DefaultValidCTileIndex(const CTileIdx& c_tile_idx, } } // namespace ck - -#endif // UTILITY_BLOCK_TO_CTILE_MAP diff --git a/include/ck/tensor_operation/gpu/grid/gridwise_2d_reduction_multiblock.hpp b/include/ck/tensor_operation/gpu/grid/gridwise_2d_reduction_multiblock.hpp index 4206a91406..de05eee11c 100644 --- a/include/ck/tensor_operation/gpu/grid/gridwise_2d_reduction_multiblock.hpp +++ b/include/ck/tensor_operation/gpu/grid/gridwise_2d_reduction_multiblock.hpp @@ -1,39 +1,12 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2020 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ -#ifndef CK_GRIDWISE_2D_REDUCTION_MULTIBLOCK_HPP -#define CK_GRIDWISE_2D_REDUCTION_MULTIBLOCK_HPP +#pragma once -#include "reduction_common.hpp" -#include "reduction_operator.hpp" -#include "reduction_functions_accumulate.hpp" -#include "reduction_functions_blockwise.hpp" -#include "reduction_functions_threadwise.hpp" - -#include "threadwise_tensor_slice_transfer.hpp" -#include "element_wise_operation.hpp" +#include "ck/utility/reduction_common.hpp" +#include "ck/utility/reduction_operator.hpp" +#include "ck/utility/reduction_functions_accumulate.hpp" +#include "ck/tensor_operation/gpu/block/reduction_functions_blockwise.hpp" +#include "ck/tensor_operation/gpu/thread/reduction_functions_threadwise.hpp" +#include "ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" namespace ck { @@ -635,4 +608,3 @@ struct GridwiseReduction_mk_to_m_multiblock }; } // namespace ck -#endif diff --git a/include/ck/tensor_operation/gpu/grid/gridwise_2d_reduction_threadwise.hpp b/include/ck/tensor_operation/gpu/grid/gridwise_2d_reduction_threadwise.hpp index d6e4bbd4cb..44fb127a8c 100644 --- a/include/ck/tensor_operation/gpu/grid/gridwise_2d_reduction_threadwise.hpp +++ b/include/ck/tensor_operation/gpu/grid/gridwise_2d_reduction_threadwise.hpp @@ -1,38 +1,12 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2021 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ -#ifndef CK_GRIDWISE_2D_REDUCTION_THREADWISE_HPP -#define CK_GRIDWISE_2D_REDUCTION_THREADWISE_HPP +#pragma once -#include "data_type.hpp" -#include "reduction_common.hpp" -#include "reduction_operator.hpp" -#include "reduction_functions_accumulate.hpp" -#include "reduction_functions_threadwise.hpp" -#include "threadwise_tensor_slice_transfer.hpp" -#include "element_wise_operation.hpp" +#include "ck/utility/data_type.hpp" +#include "ck/utility/reduction_common.hpp" +#include "ck/utility/reduction_operator.hpp" +#include "ck/utility/reduction_functions_accumulate.hpp" +#include "ck/tensor_operation/gpu/thread/reduction_functions_threadwise.hpp" +#include "ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" namespace ck { @@ -495,4 +469,3 @@ struct GridwiseReduction_mk_to_m_threadwise }; } // namespace ck -#endif diff --git a/include/ck/tensor_operation/gpu/grid/gridwise_5ary_Elementwise_1d.hpp b/include/ck/tensor_operation/gpu/grid/gridwise_5ary_Elementwise_1d.hpp index d3342b072e..34d6a4da30 100644 --- a/include/ck/tensor_operation/gpu/grid/gridwise_5ary_Elementwise_1d.hpp +++ b/include/ck/tensor_operation/gpu/grid/gridwise_5ary_Elementwise_1d.hpp @@ -1,9 +1,9 @@ #pragma once -#include "cluster_descriptor.hpp" -#include "data_type.hpp" -#include "element_wise_operation.hpp" -#include "threadwise_tensor_slice_transfer.hpp" +#include "ck/tensor_description/cluster_descriptor.hpp" +#include "ck/utility/data_type.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" +#include "ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer.hpp" namespace ck { diff --git a/include/ck/tensor_operation/gpu/grid/gridwise_binary_elementwise_1d.hpp b/include/ck/tensor_operation/gpu/grid/gridwise_binary_elementwise_1d.hpp index 374c4fe59a..892f04d152 100644 --- a/include/ck/tensor_operation/gpu/grid/gridwise_binary_elementwise_1d.hpp +++ b/include/ck/tensor_operation/gpu/grid/gridwise_binary_elementwise_1d.hpp @@ -1,9 +1,9 @@ #pragma once -#include "cluster_descriptor.hpp" -#include "data_type.hpp" -#include "element_wise_operation.hpp" -#include "threadwise_tensor_slice_transfer.hpp" +#include "ck/utility/data_type.hpp" +#include "ck/tensor_description/cluster_descriptor.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" +#include "ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer.hpp" namespace ck { diff --git a/include/ck/tensor_operation/gpu/grid/gridwise_gemm_bias_add_reduce_xdl_cshuffle_v1.hpp b/include/ck/tensor_operation/gpu/grid/gridwise_gemm_bias_add_reduce_xdl_cshuffle_v1.hpp index 0b790d4e38..68a825f91a 100644 --- a/include/ck/tensor_operation/gpu/grid/gridwise_gemm_bias_add_reduce_xdl_cshuffle_v1.hpp +++ b/include/ck/tensor_operation/gpu/grid/gridwise_gemm_bias_add_reduce_xdl_cshuffle_v1.hpp @@ -1,14 +1,17 @@ #pragma once -#include "multi_index_transform_helper.hpp" -#include "tensor_descriptor.hpp" -#include "tensor_descriptor_helper.hpp" -#include "tensor_operation/gpu/grid/block_to_ctile_map.hpp" -#include "blockwise_gemm_xdlops.hpp" -#include "thread_group_tensor_slice_transfer_v4r1.hpp" -#include "thread_group_tensor_slice_transfer_v6r1.hpp" -#include "threadwise_tensor_slice_transfer.hpp" -#include "gridwise_gemm_pipeline_v1.hpp" -#include "reduction_functions_threadwise.hpp" + +#include "ck/utility/common_header.hpp" +#include "ck/tensor_description/multi_index_transform_helper.hpp" +#include "ck/tensor_description/tensor_descriptor.hpp" +#include "ck/tensor_description/tensor_descriptor_helper.hpp" +#include "ck/tensor_operation/gpu/grid/block_to_ctile_map.hpp" +#include "ck/tensor_operation/gpu/grid/gridwise_gemm_pipeline_v1.hpp" +#include "ck/tensor_operation/gpu/block/blockwise_gemm_xdlops.hpp" +#include "ck/tensor_operation/gpu/block/thread_group_tensor_slice_transfer_v4r1.hpp" +#include "ck/tensor_operation/gpu/block/thread_group_tensor_slice_transfer_v6r1.hpp" +#include "ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer.hpp" +#include "ck/tensor_operation/gpu/thread/reduction_functions_threadwise.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" namespace ck { diff --git a/include/ck/tensor_operation/gpu/grid/gridwise_gemm_dl_v1r3.hpp b/include/ck/tensor_operation/gpu/grid/gridwise_gemm_dl_v1r3.hpp index 3b5daf6ead..020c0a1b22 100644 --- a/include/ck/tensor_operation/gpu/grid/gridwise_gemm_dl_v1r3.hpp +++ b/include/ck/tensor_operation/gpu/grid/gridwise_gemm_dl_v1r3.hpp @@ -1,15 +1,16 @@ #pragma once -#include "common_header.hpp" -#include "multi_index_transform_helper.hpp" -#include "tensor_descriptor.hpp" -#include "tensor_descriptor_helper.hpp" -#include "tensor_operation/gpu/grid/block_to_ctile_map.hpp" -#include "blockwise_gemm_dl_v2r3.hpp" -#include "blockwise_tensor_slice_transfer_v5r1.hpp" -#include "threadwise_tensor_slice_transfer.hpp" -#include "threadwise_tensor_slice_set.hpp" -#include "element_wise_operation.hpp" +#include "ck/utility/common_header.hpp" +#include "ck/tensor_description/multi_index_transform_helper.hpp" +#include "ck/tensor_description/tensor_descriptor.hpp" +#include "ck/tensor_description/tensor_descriptor_helper.hpp" +#include "ck/tensor_operation/gpu/grid/block_to_ctile_map.hpp" +#include "ck/tensor_operation/gpu/grid/gridwise_gemm_pipeline_v1.hpp" +#include "ck/tensor_operation/gpu/block/blockwise_gemm_dl_v2r3.hpp" +#include "ck/tensor_operation/gpu/block/blockwise_tensor_slice_transfer_v5r1.hpp" +#include "ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer.hpp" +#include "ck/tensor_operation/gpu/thread/threadwise_tensor_slice_set.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" namespace ck { diff --git a/include/ck/tensor_operation/gpu/grid/gridwise_gemm_multiple_d_xdl_cshuffle.hpp b/include/ck/tensor_operation/gpu/grid/gridwise_gemm_multiple_d_xdl_cshuffle.hpp index 3ec098486b..2e1acbccd4 100644 --- a/include/ck/tensor_operation/gpu/grid/gridwise_gemm_multiple_d_xdl_cshuffle.hpp +++ b/include/ck/tensor_operation/gpu/grid/gridwise_gemm_multiple_d_xdl_cshuffle.hpp @@ -1,15 +1,16 @@ #pragma once -#include "common_header.hpp" -#include "multi_index_transform_helper.hpp" -#include "tensor_descriptor.hpp" -#include "tensor_descriptor_helper.hpp" -#include "tensor_operation/gpu/grid/block_to_ctile_map.hpp" -#include "blockwise_gemm_xdlops.hpp" -#include "thread_group_tensor_slice_transfer_v4r1.hpp" -#include "thread_group_tensor_slice_transfer_v7.hpp" -#include "threadwise_tensor_slice_transfer.hpp" -#include "gridwise_gemm_pipeline_v1.hpp" +#include "ck/utility/common_header.hpp" +#include "ck/tensor_description/multi_index_transform_helper.hpp" +#include "ck/tensor_description/tensor_descriptor.hpp" +#include "ck/tensor_description/tensor_descriptor_helper.hpp" +#include "ck/tensor_operation/gpu/grid/block_to_ctile_map.hpp" +#include "ck/tensor_operation/gpu/grid/gridwise_gemm_pipeline_v1.hpp" +#include "ck/tensor_operation/gpu/block/blockwise_gemm_xdlops.hpp" +#include "ck/tensor_operation/gpu/block/thread_group_tensor_slice_transfer_v4r1.hpp" +#include "ck/tensor_operation/gpu/block/thread_group_tensor_slice_transfer_v7.hpp" +#include "ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" namespace ck { diff --git a/include/ck/tensor_operation/gpu/grid/gridwise_gemm_pipeline_v1.hpp b/include/ck/tensor_operation/gpu/grid/gridwise_gemm_pipeline_v1.hpp index 20c3a0b618..91e8333cf7 100644 --- a/include/ck/tensor_operation/gpu/grid/gridwise_gemm_pipeline_v1.hpp +++ b/include/ck/tensor_operation/gpu/grid/gridwise_gemm_pipeline_v1.hpp @@ -1,6 +1,7 @@ #pragma once -#include "common_header.hpp" -#include "tensor_operation/gpu/block/blockwise_gemm_xdlops.hpp" + +#include "ck/utility/common_header.hpp" +#include "ck/tensor_operation/gpu/block/blockwise_gemm_xdlops.hpp" namespace ck { diff --git a/include/ck/tensor_operation/gpu/grid/gridwise_gemm_reduce_xdl_cshuffle_v1.hpp b/include/ck/tensor_operation/gpu/grid/gridwise_gemm_reduce_xdl_cshuffle_v1.hpp index 80a6eeace6..3fa55eab1c 100644 --- a/include/ck/tensor_operation/gpu/grid/gridwise_gemm_reduce_xdl_cshuffle_v1.hpp +++ b/include/ck/tensor_operation/gpu/grid/gridwise_gemm_reduce_xdl_cshuffle_v1.hpp @@ -1,15 +1,17 @@ #pragma once -#include "common_header.hpp" -#include "multi_index_transform_helper.hpp" -#include "tensor_descriptor.hpp" -#include "tensor_descriptor_helper.hpp" -#include "tensor_operation/gpu/grid/block_to_ctile_map.hpp" -#include "blockwise_gemm_xdlops.hpp" -#include "thread_group_tensor_slice_transfer_v4r1.hpp" -#include "thread_group_tensor_slice_transfer_v6r1.hpp" -#include "threadwise_tensor_slice_transfer.hpp" -#include "gridwise_gemm_pipeline_v1.hpp" -#include "reduction_functions_threadwise.hpp" + +#include "ck/utility/common_header.hpp" +#include "ck/tensor_description/multi_index_transform_helper.hpp" +#include "ck/tensor_description/tensor_descriptor.hpp" +#include "ck/tensor_description/tensor_descriptor_helper.hpp" +#include "ck/tensor_operation/gpu/grid/block_to_ctile_map.hpp" +#include "ck/tensor_operation/gpu/grid/gridwise_gemm_pipeline_v1.hpp" +#include "ck/tensor_operation/gpu/block/blockwise_gemm_xdlops.hpp" +#include "ck/tensor_operation/gpu/block/thread_group_tensor_slice_transfer_v4r1.hpp" +#include "ck/tensor_operation/gpu/block/thread_group_tensor_slice_transfer_v6r1.hpp" +#include "ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer.hpp" +#include "ck/tensor_operation/gpu/thread/reduction_functions_threadwise.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" namespace ck { diff --git a/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdl_cshuffle_v1.hpp b/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdl_cshuffle_v1.hpp index 55390dbc86..6218fc474e 100644 --- a/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdl_cshuffle_v1.hpp +++ b/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdl_cshuffle_v1.hpp @@ -1,14 +1,16 @@ #pragma once -#include "common_header.hpp" -#include "multi_index_transform_helper.hpp" -#include "tensor_descriptor.hpp" -#include "tensor_descriptor_helper.hpp" -#include "tensor_operation/gpu/grid/block_to_ctile_map.hpp" -#include "blockwise_gemm_xdlops.hpp" -#include "thread_group_tensor_slice_transfer_v4r1.hpp" -#include "thread_group_tensor_slice_transfer_v6r1.hpp" -#include "threadwise_tensor_slice_transfer.hpp" -#include "gridwise_gemm_pipeline_v1.hpp" + +#include "ck/utility/common_header.hpp" +#include "ck/tensor_description/multi_index_transform_helper.hpp" +#include "ck/tensor_description/tensor_descriptor.hpp" +#include "ck/tensor_description/tensor_descriptor_helper.hpp" +#include "ck/tensor_operation/gpu/grid/block_to_ctile_map.hpp" +#include "ck/tensor_operation/gpu/grid/gridwise_gemm_pipeline_v1.hpp" +#include "ck/tensor_operation/gpu/block/blockwise_gemm_xdlops.hpp" +#include "ck/tensor_operation/gpu/block/thread_group_tensor_slice_transfer_v4r1.hpp" +#include "ck/tensor_operation/gpu/block/thread_group_tensor_slice_transfer_v6r1.hpp" +#include "ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" namespace ck { diff --git a/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdlops_bwd_weight.hpp b/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdlops_bwd_weight.hpp index b1f3779802..2b72888d5a 100644 --- a/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdlops_bwd_weight.hpp +++ b/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdlops_bwd_weight.hpp @@ -1,15 +1,16 @@ #pragma once -#include "common_header.hpp" -#include "multi_index_transform_helper.hpp" -#include "tensor_descriptor.hpp" -#include "tensor_descriptor_helper.hpp" -#include "tensor_operation/gpu/grid/block_to_ctile_map.hpp" -#include "blockwise_gemm_xdlops.hpp" -#include "thread_group_tensor_slice_transfer_v4r1.hpp" -#include "thread_group_tensor_slice_transfer_v6r1.hpp" -#include "threadwise_tensor_slice_transfer.hpp" -#include "gridwise_gemm_pipeline_v1.hpp" +#include "ck/utility/common_header.hpp" +#include "ck/tensor_description/multi_index_transform_helper.hpp" +#include "ck/tensor_description/tensor_descriptor.hpp" +#include "ck/tensor_description/tensor_descriptor_helper.hpp" +#include "ck/tensor_operation/gpu/grid/block_to_ctile_map.hpp" +#include "ck/tensor_operation/gpu/grid/gridwise_gemm_pipeline_v1.hpp" +#include "ck/tensor_operation/gpu/block/blockwise_gemm_xdlops.hpp" +#include "ck/tensor_operation/gpu/block/thread_group_tensor_slice_transfer_v4r1.hpp" +#include "ck/tensor_operation/gpu/block/thread_group_tensor_slice_transfer_v6r1.hpp" +#include "ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" namespace ck { diff --git a/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdlops_v2r3.hpp b/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdlops_v2r3.hpp index 974455fa3b..01a1d79aed 100644 --- a/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdlops_v2r3.hpp +++ b/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdlops_v2r3.hpp @@ -1,14 +1,15 @@ #pragma once -#include "common_header.hpp" -#include "multi_index_transform_helper.hpp" -#include "tensor_descriptor.hpp" -#include "tensor_descriptor_helper.hpp" -#include "tensor_operation/gpu/grid/block_to_ctile_map.hpp" -#include "blockwise_gemm_xdlops.hpp" -#include "thread_group_tensor_slice_transfer_v4r1.hpp" -#include "threadwise_tensor_slice_transfer.hpp" -#include "gridwise_gemm_pipeline_v1.hpp" +#include "ck/utility/common_header.hpp" +#include "ck/tensor_description/multi_index_transform_helper.hpp" +#include "ck/tensor_description/tensor_descriptor.hpp" +#include "ck/tensor_description/tensor_descriptor_helper.hpp" +#include "ck/tensor_operation/gpu/grid/block_to_ctile_map.hpp" +#include "ck/tensor_operation/gpu/grid/gridwise_gemm_pipeline_v1.hpp" +#include "ck/tensor_operation/gpu/block/blockwise_gemm_xdlops.hpp" +#include "ck/tensor_operation/gpu/block/thread_group_tensor_slice_transfer_v4r1.hpp" +#include "ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" namespace ck { diff --git a/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdlops_v2r4.hpp b/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdlops_v2r4.hpp index a54906cfbc..084dd7de31 100644 --- a/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdlops_v2r4.hpp +++ b/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdlops_v2r4.hpp @@ -1,14 +1,15 @@ -#ifndef CK_GRIDWISE_GEMM_XDLOPS_V2R4_HPP -#define CK_GRIDWISE_GEMM_XDLOPS_V2R4_HPP +#pragma once -#include "common_header.hpp" -#include "multi_index_transform_helper.hpp" -#include "tensor_descriptor.hpp" -#include "tensor_descriptor_helper.hpp" -#include "tensor_operation/gpu/grid/block_to_ctile_map.hpp" -#include "blockwise_gemm_xdlops.hpp" -#include "thread_group_tensor_slice_transfer_v4r1.hpp" -#include "threadwise_tensor_slice_transfer.hpp" +#include "ck/utility/common_header.hpp" +#include "ck/tensor_description/multi_index_transform_helper.hpp" +#include "ck/tensor_description/tensor_descriptor.hpp" +#include "ck/tensor_description/tensor_descriptor_helper.hpp" +#include "ck/tensor_operation/gpu/grid/block_to_ctile_map.hpp" +#include "ck/tensor_operation/gpu/grid/gridwise_gemm_pipeline_v1.hpp" +#include "ck/tensor_operation/gpu/block/blockwise_gemm_xdlops.hpp" +#include "ck/tensor_operation/gpu/block/thread_group_tensor_slice_transfer_v4r1.hpp" +#include "ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" namespace ck { @@ -607,7 +608,6 @@ struct GridwiseGemm_bk0mk1_bk0nk1_mn_xdlops_v2r4 c_grid_buf); } } -}; // namespace ck +}; } // namespace ck -#endif diff --git a/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdlops_v2r4r2.hpp b/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdlops_v2r4r2.hpp index dbff1577e1..4de72dc0b3 100644 --- a/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdlops_v2r4r2.hpp +++ b/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdlops_v2r4r2.hpp @@ -1,15 +1,16 @@ -#ifndef CK_GRIDWISE_GEMM_XDLOPS_V2R4R2_HPP -#define CK_GRIDWISE_GEMM_XDLOPS_V2R4R2_HPP +#pragma once -#include "common_header.hpp" -#include "multi_index_transform_helper.hpp" -#include "tensor_descriptor.hpp" -#include "tensor_descriptor_helper.hpp" -#include "tensor_operation/gpu/grid/block_to_ctile_map.hpp" -#include "blockwise_gemm_xdlops.hpp" -#include "thread_group_tensor_slice_transfer_v4r1.hpp" -#include "thread_group_tensor_slice_transfer_v6r1.hpp" -#include "threadwise_tensor_slice_transfer.hpp" +#include "ck/utility/common_header.hpp" +#include "ck/tensor_description/multi_index_transform_helper.hpp" +#include "ck/tensor_description/tensor_descriptor.hpp" +#include "ck/tensor_description/tensor_descriptor_helper.hpp" +#include "ck/tensor_operation/gpu/grid/block_to_ctile_map.hpp" +#include "ck/tensor_operation/gpu/grid/gridwise_gemm_pipeline_v1.hpp" +#include "ck/tensor_operation/gpu/block/blockwise_gemm_xdlops.hpp" +#include "ck/tensor_operation/gpu/block/thread_group_tensor_slice_transfer_v4r1.hpp" +#include "ck/tensor_operation/gpu/block/thread_group_tensor_slice_transfer_v6r1.hpp" +#include "ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" namespace ck { @@ -717,7 +718,6 @@ struct GridwiseGemm_bk0mk1_bk0nk1_mn_xdlops_v2r4r2 }); } } -}; // namespace ck +}; } // namespace ck -#endif diff --git a/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdlops_v3r1.hpp b/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdlops_v3r1.hpp index 2828655f51..2fe9427808 100644 --- a/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdlops_v3r1.hpp +++ b/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdlops_v3r1.hpp @@ -1,15 +1,17 @@ #pragma once -#include "common_header.hpp" -#include "multi_index_transform_helper.hpp" -#include "tensor_descriptor.hpp" -#include "tensor_descriptor_helper.hpp" -#include "tensor_operation/gpu/grid/block_to_ctile_map.hpp" -#include "blockwise_gemm_xdlops.hpp" -#include "thread_group_tensor_slice_transfer_v4r1.hpp" -#include "thread_group_tensor_slice_transfer_v6r1.hpp" -#include "threadwise_tensor_slice_transfer.hpp" -#include "gridwise_gemm_pipeline_v1.hpp" -#include "tensor_space_filling_curve.hpp" + +#include "ck/utility/common_header.hpp" +#include "ck/tensor_description/multi_index_transform_helper.hpp" +#include "ck/tensor_description/tensor_space_filling_curve.hpp" +#include "ck/tensor_description/tensor_descriptor.hpp" +#include "ck/tensor_description/tensor_descriptor_helper.hpp" +#include "ck/tensor_operation/gpu/grid/block_to_ctile_map.hpp" +#include "ck/tensor_operation/gpu/grid/gridwise_gemm_pipeline_v1.hpp" +#include "ck/tensor_operation/gpu/block/blockwise_gemm_xdlops.hpp" +#include "ck/tensor_operation/gpu/block/thread_group_tensor_slice_transfer_v4r1.hpp" +#include "ck/tensor_operation/gpu/block/thread_group_tensor_slice_transfer_v6r1.hpp" +#include "ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" namespace ck { diff --git a/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdlops_v3r2.hpp b/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdlops_v3r2.hpp index 3a7a551181..62c6a0f18c 100644 --- a/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdlops_v3r2.hpp +++ b/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdlops_v3r2.hpp @@ -1,16 +1,16 @@ -#ifndef CK_GRIDWISE_GEMM_XDLOPS_V3R2_HPP -#define CK_GRIDWISE_GEMM_XDLOPS_V3R2_HPP +#pragma once -#include "common_header.hpp" -#include "multi_index_transform_helper.hpp" -#include "tensor_descriptor.hpp" -#include "tensor_descriptor_helper.hpp" -#include "tensor_operation/gpu/grid/block_to_ctile_map.hpp" -#include "blockwise_gemm_xdlops.hpp" -#include "thread_group_tensor_slice_transfer_v4r1.hpp" -#include "thread_group_tensor_slice_transfer_v6r2.hpp" -#include "threadwise_tensor_slice_transfer.hpp" -#include "gridwise_gemm_pipeline_v1.hpp" +#include "ck/utility/common_header.hpp" +#include "ck/tensor_description/multi_index_transform_helper.hpp" +#include "ck/tensor_description/tensor_descriptor.hpp" +#include "ck/tensor_description/tensor_descriptor_helper.hpp" +#include "ck/tensor_operation/gpu/grid/block_to_ctile_map.hpp" +#include "ck/tensor_operation/gpu/grid/gridwise_gemm_pipeline_v1.hpp" +#include "ck/tensor_operation/gpu/block/blockwise_gemm_xdlops.hpp" +#include "ck/tensor_operation/gpu/block/thread_group_tensor_slice_transfer_v4r1.hpp" +#include "ck/tensor_operation/gpu/block/thread_group_tensor_slice_transfer_v6r2.hpp" +#include "ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" namespace ck { @@ -755,4 +755,3 @@ struct GridwiseGemm_k0mk1_k0nk1_mn_xdlops_v3r2 }; } // namespace ck -#endif diff --git a/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdlops_v3r3.hpp b/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdlops_v3r3.hpp index 2e324faf13..c23bf105cb 100644 --- a/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdlops_v3r3.hpp +++ b/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdlops_v3r3.hpp @@ -1,14 +1,16 @@ #pragma once -#include "common_header.hpp" -#include "multi_index_transform_helper.hpp" -#include "tensor_descriptor.hpp" -#include "tensor_descriptor_helper.hpp" -#include "tensor_operation/gpu/grid/block_to_ctile_map.hpp" -#include "blockwise_gemm_xdlops.hpp" -#include "thread_group_tensor_slice_transfer_v4r1.hpp" -#include "thread_group_tensor_slice_transfer_v6r3.hpp" -#include "threadwise_tensor_slice_transfer.hpp" -#include "gridwise_gemm_pipeline_v1.hpp" + +#include "ck/utility/common_header.hpp" +#include "ck/tensor_description/multi_index_transform_helper.hpp" +#include "ck/tensor_description/tensor_descriptor.hpp" +#include "ck/tensor_description/tensor_descriptor_helper.hpp" +#include "ck/tensor_operation/gpu/grid/block_to_ctile_map.hpp" +#include "ck/tensor_operation/gpu/grid/gridwise_gemm_pipeline_v1.hpp" +#include "ck/tensor_operation/gpu/block/blockwise_gemm_xdlops.hpp" +#include "ck/tensor_operation/gpu/block/thread_group_tensor_slice_transfer_v4r1.hpp" +#include "ck/tensor_operation/gpu/block/thread_group_tensor_slice_transfer_v6r3.hpp" +#include "ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" namespace ck { diff --git a/include/ck/tensor_operation/gpu/grid/gridwise_set_buffer_value.hpp b/include/ck/tensor_operation/gpu/grid/gridwise_set_buffer_value.hpp index dcb45b6d5f..60a0e514c8 100644 --- a/include/ck/tensor_operation/gpu/grid/gridwise_set_buffer_value.hpp +++ b/include/ck/tensor_operation/gpu/grid/gridwise_set_buffer_value.hpp @@ -1,32 +1,6 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2020 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ -#ifndef CK_GRIDWISE_SET_BUFFER_VALUE_HPP -#define CK_GRIDWISE_SET_BUFFER_VALUE_HPP +#pragma once -#include "threadwise_tensor_slice_transfer.hpp" +#include "ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer.hpp" namespace ck { @@ -77,4 +51,3 @@ __global__ void kernel_buffer_set_value(const Grid1dBufferDescType grid_1d_buffe }; } // namespace ck -#endif diff --git a/include/ck/tensor_operation/gpu/grid/gridwise_softmax.hpp b/include/ck/tensor_operation/gpu/grid/gridwise_softmax.hpp index de293eed35..4873e8cbdc 100644 --- a/include/ck/tensor_operation/gpu/grid/gridwise_softmax.hpp +++ b/include/ck/tensor_operation/gpu/grid/gridwise_softmax.hpp @@ -1,39 +1,13 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2022 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ -#ifndef GRIDWISE_SOFTMAX_HPP -#define GRIDWISE_SOFTMAX_HPP +#pragma once -#include "reduction_common.hpp" -#include "reduction_operator.hpp" -#include "reduction_functions_accumulate.hpp" -#include "reduction_functions_blockwise.hpp" -#include "reduction_functions_threadwise.hpp" - -#include "threadwise_tensor_slice_transfer.hpp" -#include "element_wise_operation.hpp" +#include "ck/utility/data_type.hpp" +#include "ck/utility/reduction_common.hpp" +#include "ck/utility/reduction_operator.hpp" +#include "ck/utility/reduction_functions_accumulate.hpp" +#include "ck/tensor_operation/gpu/block/reduction_functions_blockwise.hpp" +#include "ck/tensor_operation/gpu/thread/reduction_functions_threadwise.hpp" +#include "ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" namespace ck { @@ -404,4 +378,3 @@ struct GridwiseSoftmax_mk_to_mk }; } // namespace ck -#endif // GRIDWISE_SOFTMAX_HPP diff --git a/include/ck/tensor_operation/gpu/grid/gridwise_unary_elementwise_1d.hpp b/include/ck/tensor_operation/gpu/grid/gridwise_unary_elementwise_1d.hpp index 5773068756..1653358beb 100644 --- a/include/ck/tensor_operation/gpu/grid/gridwise_unary_elementwise_1d.hpp +++ b/include/ck/tensor_operation/gpu/grid/gridwise_unary_elementwise_1d.hpp @@ -1,9 +1,9 @@ #pragma once -#include "cluster_descriptor.hpp" -#include "data_type.hpp" -#include "element_wise_operation.hpp" -#include "threadwise_tensor_slice_transfer.hpp" +#include "ck/utility/data_type.hpp" +#include "ck/tensor_description/cluster_descriptor.hpp" +#include "ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" namespace ck { diff --git a/include/ck/tensor_operation/gpu/thread/reduction_functions_threadwise.hpp b/include/ck/tensor_operation/gpu/thread/reduction_functions_threadwise.hpp index 35fc1b929d..45561705c5 100644 --- a/include/ck/tensor_operation/gpu/thread/reduction_functions_threadwise.hpp +++ b/include/ck/tensor_operation/gpu/thread/reduction_functions_threadwise.hpp @@ -1,32 +1,6 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2020 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ -#ifndef CK_REDUCTION_FUNCTIONS_THREADWISE_HPP -#define CK_REDUCTION_FUNCTIONS_THREADWISE_HPP +#pragma once -#include "reduction_functions_accumulate.hpp" +#include "ck/utility/reduction_functions_accumulate.hpp" namespace ck { @@ -117,6 +91,4 @@ struct ThreadwiseReductionWithIndex }; }; -}; // end of namespace ck - -#endif +} // namespace ck diff --git a/include/ck/tensor_operation/gpu/thread/threadwise_contraction_dl.hpp b/include/ck/tensor_operation/gpu/thread/threadwise_contraction_dl.hpp index 6a532c79f9..e764e88182 100644 --- a/include/ck/tensor_operation/gpu/thread/threadwise_contraction_dl.hpp +++ b/include/ck/tensor_operation/gpu/thread/threadwise_contraction_dl.hpp @@ -1,6 +1,7 @@ #pragma once -#include "common_header.hpp" -#include "math.hpp" + +#include "ck/utility/common_header.hpp" +#include "ck/utility/math.hpp" namespace ck { diff --git a/include/ck/tensor_operation/gpu/thread/threadwise_tensor_slice_set.hpp b/include/ck/tensor_operation/gpu/thread/threadwise_tensor_slice_set.hpp index 20e9a5b366..0e38cf47b3 100644 --- a/include/ck/tensor_operation/gpu/thread/threadwise_tensor_slice_set.hpp +++ b/include/ck/tensor_operation/gpu/thread/threadwise_tensor_slice_set.hpp @@ -1,9 +1,8 @@ -#ifndef CK_THREADWISE_TENSOR_SET_HPP -#define CK_THREADWISE_TENSOR_SET_HPP +#pragma once -#include "common_header.hpp" -#include "tensor_descriptor.hpp" -#include "tensor_descriptor_helper.hpp" +#include "ck/utility/common_header.hpp" +#include "ck/tensor_description/tensor_descriptor.hpp" +#include "ck/tensor_description/tensor_descriptor_helper.hpp" namespace ck { @@ -56,4 +55,3 @@ struct ThreadwiseTensorSliceSet_v1 }; } // namespace ck -#endif diff --git a/include/ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer.hpp b/include/ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer.hpp index 7a75ca5380..cadda67c42 100644 --- a/include/ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer.hpp +++ b/include/ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer.hpp @@ -1,10 +1,9 @@ -#ifndef CK_THREADWISE_TENSOR_SLICE_TRANSFER_HPP -#define CK_THREADWISE_TENSOR_SLICE_TRANSFER_HPP +#pragma once -#include "common_header.hpp" -#include "tensor_descriptor.hpp" -#include "tensor_descriptor_helper.hpp" -#include "tensor_space_filling_curve.hpp" +#include "ck/utility/common_header.hpp" +#include "ck/tensor_description/tensor_space_filling_curve.hpp" +#include "ck/tensor_description/tensor_descriptor.hpp" +#include "ck/tensor_description/tensor_descriptor_helper.hpp" namespace ck { @@ -1168,4 +1167,3 @@ struct ThreadwiseTensorSliceTransfer_v4 }; } // namespace ck -#endif diff --git a/include/ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer_v3r1.hpp b/include/ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer_v3r1.hpp index 4cd41ddb30..e3b6612437 100644 --- a/include/ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer_v3r1.hpp +++ b/include/ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer_v3r1.hpp @@ -1,10 +1,9 @@ -#ifndef CK_THREADWISE_TENSOR_SLICE_TRANSFER_V3R1_HPP -#define CK_THREADWISE_TENSOR_SLICE_TRANSFER_V3R1_HPP +#pragma once -#include "common_header.hpp" -#include "tensor_descriptor.hpp" -#include "tensor_descriptor_helper.hpp" -#include "static_tensor.hpp" +#include "ck/utility/common_header.hpp" +#include "ck/tensor_description/tensor_descriptor.hpp" +#include "ck/tensor_description/tensor_descriptor_helper.hpp" +#include "ck/tensor/static_tensor.hpp" namespace ck { @@ -789,4 +788,3 @@ struct ThreadwiseTensorSliceTransfer_v3r1 }; } // namespace ck -#endif diff --git a/include/ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer_v4r1.hpp b/include/ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer_v4r1.hpp index 2504c92856..af273ffd7f 100644 --- a/include/ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer_v4r1.hpp +++ b/include/ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer_v4r1.hpp @@ -1,9 +1,8 @@ -#ifndef CK_THREADWISE_TENSOR_SLICE_TRANSFER_V4R1_HPP -#define CK_THREADWISE_TENSOR_SLICE_TRANSFER_V4R1_HPP +#pragma once -#include "common_header.hpp" -#include "tensor_descriptor.hpp" -#include "tensor_descriptor_helper.hpp" +#include "ck/utility/common_header.hpp" +#include "ck/tensor_description/tensor_descriptor.hpp" +#include "ck/tensor_description/tensor_descriptor_helper.hpp" namespace ck { // Assume: @@ -171,4 +170,3 @@ struct ThreadwiseTensorSliceTransfer_v4r1 }; } // namespace ck -#endif diff --git a/include/ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer_v5r1.hpp b/include/ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer_v5r1.hpp index f0e9c7e761..f7704a80ce 100644 --- a/include/ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer_v5r1.hpp +++ b/include/ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer_v5r1.hpp @@ -1,8 +1,9 @@ #pragma once -#include "common_header.hpp" -#include "tensor_descriptor.hpp" -#include "tensor_descriptor_helper.hpp" +#include "ck/utility/common_header.hpp" +#include "ck/tensor_description/tensor_descriptor.hpp" +#include "ck/tensor_description/tensor_descriptor_helper.hpp" +#include "ck/tensor_description/tensor_space_filling_curve.hpp" namespace ck { diff --git a/include/ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer_v6r1.hpp b/include/ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer_v6r1.hpp index 042bc95f55..d2183179e4 100644 --- a/include/ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer_v6r1.hpp +++ b/include/ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer_v6r1.hpp @@ -1,10 +1,9 @@ -#ifndef CK_THREADWISE_TENSOR_SLICE_TRANSFER_V6R1_HPP -#define CK_THREADWISE_TENSOR_SLICE_TRANSFER_V6R1_HPP +#pragma once -#include "common_header.hpp" -#include "tensor_descriptor.hpp" -#include "tensor_descriptor_helper.hpp" -#include "tensor_space_filling_curve.hpp" +#include "ck/utility/common_header.hpp" +#include "ck/tensor_description/tensor_descriptor.hpp" +#include "ck/tensor_description/tensor_descriptor_helper.hpp" +#include "ck/tensor_description/tensor_space_filling_curve.hpp" namespace ck { @@ -206,7 +205,6 @@ struct ThreadwiseTensorSliceTransfer_v6r1 SrcCoord src_coord_; DstCoord dst_coord_; const ElementwiseOperation element_op_; -}; // namespace ck +}; } // namespace ck -#endif diff --git a/include/ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer_v6r2.hpp b/include/ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer_v6r2.hpp index ae85ba91e5..f1cb709cd4 100644 --- a/include/ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer_v6r2.hpp +++ b/include/ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer_v6r2.hpp @@ -1,10 +1,9 @@ -#ifndef CK_THREADWISE_TENSOR_SLICE_TRANSFER_V6R2_HPP -#define CK_THREADWISE_TENSOR_SLICE_TRANSFER_V6R2_HPP +#pragma once -#include "common_header.hpp" -#include "tensor_descriptor.hpp" -#include "tensor_descriptor_helper.hpp" -#include "tensor_space_filling_curve.hpp" +#include "ck/utility/common_header.hpp" +#include "ck/tensor_description/tensor_descriptor.hpp" +#include "ck/tensor_description/tensor_descriptor_helper.hpp" +#include "ck/tensor_description/tensor_space_filling_curve.hpp" namespace ck { @@ -256,4 +255,3 @@ struct ThreadwiseTensorSliceTransfer_v6r2 }; } // namespace ck -#endif diff --git a/include/ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer_v6r3.hpp b/include/ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer_v6r3.hpp index 47024d5e68..92c4fe0919 100644 --- a/include/ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer_v6r3.hpp +++ b/include/ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer_v6r3.hpp @@ -1,10 +1,9 @@ -#ifndef CK_THREADWISE_TENSOR_SLICE_TRANSFER_V6R3_HPP -#define CK_THREADWISE_TENSOR_SLICE_TRANSFER_V6R3_HPP +#pragma once -#include "common_header.hpp" -#include "tensor_descriptor.hpp" -#include "tensor_descriptor_helper.hpp" -#include "tensor_space_filling_curve.hpp" +#include "ck/utility/common_header.hpp" +#include "ck/tensor_description/tensor_descriptor.hpp" +#include "ck/tensor_description/tensor_descriptor_helper.hpp" +#include "ck/tensor_description/tensor_space_filling_curve.hpp" namespace ck { @@ -306,4 +305,3 @@ struct ThreadwiseTensorSliceTransfer_v6r3 }; } // namespace ck -#endif diff --git a/include/ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer_v7.hpp b/include/ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer_v7.hpp index 782e456f3d..694a88c1a5 100644 --- a/include/ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer_v7.hpp +++ b/include/ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer_v7.hpp @@ -1,9 +1,9 @@ #pragma once -#include "common_header.hpp" -#include "tensor_descriptor.hpp" -#include "tensor_descriptor_helper.hpp" -#include "tensor_space_filling_curve.hpp" +#include "ck/utility/common_header.hpp" +#include "ck/tensor_description/tensor_descriptor.hpp" +#include "ck/tensor_description/tensor_descriptor_helper.hpp" +#include "ck/tensor_description/tensor_space_filling_curve.hpp" namespace ck { diff --git a/include/ck/tensor_operation/gpu/warp/xdlops_gemm.hpp b/include/ck/tensor_operation/gpu/warp/xdlops_gemm.hpp index a39b795818..f0a47601bf 100644 --- a/include/ck/tensor_operation/gpu/warp/xdlops_gemm.hpp +++ b/include/ck/tensor_operation/gpu/warp/xdlops_gemm.hpp @@ -1,9 +1,8 @@ -#ifndef CK_XDLOPS_GEMM_HPP -#define CK_XDLOPS_GEMM_HPP +#pragma once -#include "common_header.hpp" -#include "math.hpp" -#include "amd_xdlops.hpp" +#include "ck/utility/common_header.hpp" +#include "ck/utility/math.hpp" +#include "ck/utility/amd_xdlops.hpp" namespace ck { @@ -786,4 +785,3 @@ struct XdlopsGemm }; } // namespace ck -#endif diff --git a/include/ck/utility/amd_address_space.hpp b/include/ck/utility/amd_address_space.hpp index 3c5939aaf3..9ca6c05dfb 100644 --- a/include/ck/utility/amd_address_space.hpp +++ b/include/ck/utility/amd_address_space.hpp @@ -1,7 +1,6 @@ -#ifndef CK_AMD_ADDRESS_SPACE_HPP -#define CK_AMD_ADDRESS_SPACE_HPP +#pragma once -#include "config.hpp" +#include "ck/ck.hpp" #include "c_style_pointer_cast.hpp" // Address Space for AMDGCN @@ -41,4 +40,3 @@ __host__ __device__ T CK_CONSTANT_ADDRESS_SPACE* cast_pointer_to_constant_addres } } // namespace ck -#endif diff --git a/include/ck/utility/common_header.hpp b/include/ck/utility/common_header.hpp index 34c0a7821b..52f1da08b8 100644 --- a/include/ck/utility/common_header.hpp +++ b/include/ck/utility/common_header.hpp @@ -1,49 +1,48 @@ #pragma once -#include "config.hpp" -#include "array.hpp" -#include "container_helper.hpp" -#include "statically_indexed_array.hpp" -#include "container_element_picker.hpp" -#include "multi_index.hpp" -#include "data_type.hpp" -#include "data_type_enum.hpp" -#include "data_type_enum_helper.hpp" -#include "functional.hpp" -#include "functional2.hpp" -#include "functional3.hpp" -#include "functional4.hpp" -#include "enable_if.hpp" -#include "ignore.hpp" -#include "integral_constant.hpp" -#include "math.hpp" -#include "number.hpp" -#include "sequence.hpp" -#include "sequence_helper.hpp" -#include "tuple.hpp" -#include "tuple_helper.hpp" -#include "type.hpp" -#include "magic_division.hpp" -#include "c_style_pointer_cast.hpp" -#include "is_known_at_compile_time.hpp" -#include "transpose_vectors.hpp" -#include "inner_product.hpp" -#include "element_wise_operation.hpp" -#include "thread_group.hpp" -#include "debug.hpp" -#include "amd_buffer_addressing.hpp" -#include "generic_memory_space_atomic.hpp" -#include "get_id.hpp" -#include "synchronization.hpp" -#include "amd_address_space.hpp" -#include "static_buffer.hpp" -#include "dynamic_buffer.hpp" +#include "ck/ck.hpp" +#include "ck/utility/array.hpp" +#include "ck/utility/container_helper.hpp" +#include "ck/utility/statically_indexed_array.hpp" +#include "ck/utility/container_element_picker.hpp" +#include "ck/utility/multi_index.hpp" +#include "ck/utility/data_type.hpp" +#include "ck/utility/functional.hpp" +#include "ck/utility/functional2.hpp" +#include "ck/utility/functional3.hpp" +#include "ck/utility/functional4.hpp" +#include "ck/utility/enable_if.hpp" +#include "ck/utility/ignore.hpp" +#include "ck/utility/integral_constant.hpp" +#include "ck/utility/math.hpp" +#include "ck/utility/number.hpp" +#include "ck/utility/sequence.hpp" +#include "ck/utility/sequence_helper.hpp" +#include "ck/utility/tuple.hpp" +#include "ck/utility/tuple_helper.hpp" +#include "ck/utility/type.hpp" +#include "ck/utility/magic_division.hpp" +#include "ck/utility/c_style_pointer_cast.hpp" +#include "ck/utility/is_known_at_compile_time.hpp" +#include "ck/utility/transpose_vectors.hpp" +#include "ck/utility/inner_product.hpp" +#include "ck/utility/thread_group.hpp" +#include "ck/utility/debug.hpp" + +#include "ck/utility/amd_buffer_addressing.hpp" +#include "ck/utility/generic_memory_space_atomic.hpp" +#include "ck/utility/get_id.hpp" +#include "ck/utility/thread_group.hpp" +#include "ck/utility/synchronization.hpp" +#include "ck/utility/amd_address_space.hpp" +#include "ck/utility/static_buffer.hpp" +#include "ck/utility/dynamic_buffer.hpp" // TODO: remove this #if CK_USE_AMD_INLINE_ASM -#include "amd_inline_asm.hpp" +#include "ck/utility/amd_inline_asm.hpp" #endif #ifdef CK_USE_AMD_MFMA -#include "amd_xdlops.hpp" +#include "ck/utility/amd_xdlops.hpp" #endif diff --git a/include/ck/utility/data_type.hpp b/include/ck/utility/data_type.hpp index a723196539..e133d0babd 100644 --- a/include/ck/utility/data_type.hpp +++ b/include/ck/utility/data_type.hpp @@ -1,6 +1,6 @@ #pragma once -#include "statically_indexed_array.hpp" +#include "ck/utility/statically_indexed_array.hpp" namespace ck { diff --git a/include/ck/utility/dynamic_buffer.hpp b/include/ck/utility/dynamic_buffer.hpp index 0ad78423fe..9b33123d5f 100644 --- a/include/ck/utility/dynamic_buffer.hpp +++ b/include/ck/utility/dynamic_buffer.hpp @@ -1,5 +1,6 @@ #pragma once -#include "config.hpp" + +#include "ck/ck.hpp" #include "enable_if.hpp" #include "c_style_pointer_cast.hpp" #include "amd_buffer_addressing.hpp" diff --git a/include/ck/utility/functional2.hpp b/include/ck/utility/functional2.hpp index 371182a05e..83e9b39c9e 100644 --- a/include/ck/utility/functional2.hpp +++ b/include/ck/utility/functional2.hpp @@ -1,8 +1,7 @@ -#ifndef CK_FUNCTIONAL2_HPP -#define CK_FUNCTIONAL2_HPP +#pragma once -#include "functional.hpp" -#include "sequence.hpp" +#include "ck/utility/functional.hpp" +#include "ck/utility/sequence.hpp" namespace ck { @@ -45,4 +44,3 @@ struct static_for }; } // namespace ck -#endif diff --git a/include/ck/utility/functional3.hpp b/include/ck/utility/functional3.hpp index 6a400f3ca6..a73adda472 100644 --- a/include/ck/utility/functional3.hpp +++ b/include/ck/utility/functional3.hpp @@ -1,10 +1,10 @@ -#ifndef CK_FUNCTIONAL3_HPP -#define CK_FUNCTIONAL3_HPP +#pragma once -#include "functional.hpp" -#include "functional2.hpp" -#include "sequence.hpp" -#include "multi_index.hpp" +#include "ck/ck.hpp" +#include "ck/utility/functional.hpp" +#include "ck/utility/functional2.hpp" +#include "ck/utility/sequence.hpp" +#include "ck/utility/multi_index.hpp" namespace ck { @@ -139,4 +139,3 @@ struct ford }; } // namespace ck -#endif diff --git a/include/ck/utility/get_id.hpp b/include/ck/utility/get_id.hpp index 7c62b890c7..1c1c284546 100644 --- a/include/ck/utility/get_id.hpp +++ b/include/ck/utility/get_id.hpp @@ -1,5 +1,6 @@ #pragma once -#include "config.hpp" + +#include "ck/ck.hpp" namespace ck { diff --git a/include/ck/utility/is_known_at_compile_time.hpp b/include/ck/utility/is_known_at_compile_time.hpp index dc44027901..4dc0418d5f 100644 --- a/include/ck/utility/is_known_at_compile_time.hpp +++ b/include/ck/utility/is_known_at_compile_time.hpp @@ -1,7 +1,6 @@ -#ifndef IS_KNOWN_AT_COMPILE_TIME_HPP -#define IS_KNOWN_AT_COMPILE_TIME_HPP +#pragma once -#include "config.hpp" +#include "ck/ck.hpp" #include "integral_constant.hpp" #include "sequence.hpp" #include "tuple.hpp" @@ -52,4 +51,3 @@ struct is_known_at_compile_time> }; } // namespace ck -#endif diff --git a/include/ck/utility/magic_division.hpp b/include/ck/utility/magic_division.hpp index 6102576717..f939ae8b66 100644 --- a/include/ck/utility/magic_division.hpp +++ b/include/ck/utility/magic_division.hpp @@ -1,7 +1,6 @@ -#ifndef CK_MAGIC_DIVISION_HPP -#define CK_MAGIC_DIVISION_HPP +#pragma once -#include "config.hpp" +#include "ck/ck.hpp" #include "integral_constant.hpp" #include "number.hpp" #include "type.hpp" @@ -156,5 +155,3 @@ struct MagicDivision }; } // namespace ck - -#endif diff --git a/include/ck/utility/math.hpp b/include/ck/utility/math.hpp index e7724a40c8..18bc5744f9 100644 --- a/include/ck/utility/math.hpp +++ b/include/ck/utility/math.hpp @@ -1,7 +1,6 @@ -#ifndef CK_MATH_HPP -#define CK_MATH_HPP +#pragma once -#include "config.hpp" +#include "ck/ck.hpp" #include "integral_constant.hpp" #include "number.hpp" #include "type.hpp" @@ -228,5 +227,3 @@ struct less } // namespace math } // namespace ck - -#endif diff --git a/include/ck/utility/math_v2.hpp b/include/ck/utility/math_v2.hpp index 438f5e12bd..66b19451ee 100644 --- a/include/ck/utility/math_v2.hpp +++ b/include/ck/utility/math_v2.hpp @@ -1,9 +1,9 @@ -#ifndef CK_MATH_V2_HPP -#define CK_MATH_V2_HPP +#pragma once #include -#include "data_type.hpp" -#include "type.hpp" + +#include "ck/utility/data_type.hpp" +#include "ck/utility/type.hpp" namespace ck { namespace math { @@ -112,5 +112,3 @@ static inline __device__ double sqrt(double x) { return ::sqrt(x); }; } // namespace math } // namespace ck - -#endif diff --git a/include/ck/utility/multi_index.hpp b/include/ck/utility/multi_index.hpp index f395b5ee71..af4658670a 100644 --- a/include/ck/utility/multi_index.hpp +++ b/include/ck/utility/multi_index.hpp @@ -1,5 +1,4 @@ -#ifndef CK_MULTI_INDEX_HPP -#define CK_MULTI_INDEX_HPP +#pragma once #include "common_header.hpp" @@ -8,5 +7,3 @@ #else #include "statically_indexed_array_multi_index.hpp" #endif - -#endif diff --git a/include/ck/utility/reduction_common.hpp b/include/ck/utility/reduction_common.hpp index a34cfce837..6534740610 100644 --- a/include/ck/utility/reduction_common.hpp +++ b/include/ck/utility/reduction_common.hpp @@ -1,32 +1,6 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2020 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ -#ifndef CK_REDUCTION_COMMON_HPP -#define CK_REDUCTION_COMMON_HPP +#pragma once -#include "reduction_enums.hpp" +#include "ck/utility/reduction_enums.hpp" namespace ck { @@ -60,6 +34,4 @@ constexpr __device__ index_t get_shift<1>() return (0); } -}; // end of namespace ck - -#endif +} // namespace ck diff --git a/include/ck/utility/reduction_enums.hpp b/include/ck/utility/reduction_enums.hpp index 9089fd6116..271743ca69 100644 --- a/include/ck/utility/reduction_enums.hpp +++ b/include/ck/utility/reduction_enums.hpp @@ -1,30 +1,4 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2020 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ -#ifndef CK_REDUCTION_ENUMS_HPP -#define CK_REDUCTION_ENUMS_HPP +#pragma once namespace ck { @@ -61,6 +35,4 @@ enum struct IndicesType INDICES_8BIT = 3, }; -}; // end of namespace ck - -#endif +} // namespace ck diff --git a/include/ck/utility/reduction_functions_accumulate.hpp b/include/ck/utility/reduction_functions_accumulate.hpp index 05ce9b16ce..7ddea554ea 100644 --- a/include/ck/utility/reduction_functions_accumulate.hpp +++ b/include/ck/utility/reduction_functions_accumulate.hpp @@ -1,36 +1,9 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2020 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ -#ifndef CK_REDUCTION_FUNCTIONS_BINOP_HPP -#define CK_REDUCTION_FUNCTIONS_BINOP_HPP +#pragma once -#include "data_type.hpp" -#include "math_v2.hpp" - -#include "reduction_common.hpp" -#include "reduction_operator.hpp" +#include "ck/utility/data_type.hpp" +#include "ck/utility/math_v2.hpp" +#include "ck/utility/reduction_common.hpp" +#include "ck/utility/reduction_operator.hpp" namespace ck { namespace detail { @@ -135,7 +108,5 @@ struct AccumulateWithIndexAndNanCheck::value; }; -}; // end of namespace reduce - -} // end of namespace ck - -#endif +} // namespace reduce +} // namespace ck diff --git a/include/ck/utility/synchronization.hpp b/include/ck/utility/synchronization.hpp index d46628d913..51fd70672f 100644 --- a/include/ck/utility/synchronization.hpp +++ b/include/ck/utility/synchronization.hpp @@ -1,7 +1,6 @@ -#ifndef CK_SYNCHRONIZATION_AMD_HPP -#define CK_SYNCHRONIZATION_AMD_HPP +#pragma once -#include "config.hpp" +#include "ck/ck.hpp" namespace ck { @@ -18,4 +17,3 @@ __device__ void block_sync_lds() } } // namespace ck -#endif diff --git a/include/ck/utility/thread_group.hpp b/include/ck/utility/thread_group.hpp index bd3563c5f1..e7a3e1c00f 100644 --- a/include/ck/utility/thread_group.hpp +++ b/include/ck/utility/thread_group.hpp @@ -1,4 +1,5 @@ #pragma once + #include "get_id.hpp" namespace ck { diff --git a/include/ck/utility/transpose_vectors.hpp b/include/ck/utility/transpose_vectors.hpp index 31f9c02c74..880464cb00 100644 --- a/include/ck/utility/transpose_vectors.hpp +++ b/include/ck/utility/transpose_vectors.hpp @@ -1,7 +1,6 @@ -#ifndef CK_TRANSPOSE_VECTORS_AMD_HPP -#define CK_TRANSPOSE_VECTORS_AMD_HPP +#pragma once -#include "config.hpp" +#include "ck/ck.hpp" #include "statically_indexed_array.hpp" #include "data_type.hpp" @@ -165,4 +164,3 @@ struct transpose_vectors }; } // namespace ck -#endif diff --git a/include/ck/utility/type.hpp b/include/ck/utility/type.hpp index ee3189ebe5..b9c97bcbf3 100644 --- a/include/ck/utility/type.hpp +++ b/include/ck/utility/type.hpp @@ -1,7 +1,6 @@ -#ifndef CK_TYPE_HPP -#define CK_TYPE_HPP +#pragma once -#include "config.hpp" +#include "ck/ck.hpp" #include "integral_constant.hpp" #include "enable_if.hpp" @@ -56,4 +55,3 @@ __host__ __device__ constexpr Y bit_cast(const X& x) } } // namespace ck -#endif diff --git a/library/include/ck/library/host/host_interface.hpp b/library/include/ck/library/host/host_interface.hpp deleted file mode 100644 index 955da0f4be..0000000000 --- a/library/include/ck/library/host/host_interface.hpp +++ /dev/null @@ -1,54 +0,0 @@ -#pragma once - -#include -#include - -#include "stream_config.hpp" -#include "config.hpp" -#include "device_base.hpp" - -struct DeviceConvFwdPtr_t -{ - using BaseArgument = ck::tensor_operation::device::BaseArgument; - using BaseInvoker = ck::tensor_operation::device::BaseInvoker; - - struct DeviceConvFwdPtrImpl; - std::unique_ptr pImpl; - DeviceConvFwdPtr_t(); - ~DeviceConvFwdPtr_t(); - DeviceConvFwdPtr_t(DeviceConvFwdPtr_t&&); - DeviceConvFwdPtr_t(DeviceConvFwdPtrImpl&); - DeviceConvFwdPtr_t& operator=(DeviceConvFwdPtr_t&) = delete; - DeviceConvFwdPtr_t& operator=(const DeviceConvFwdPtr_t&) = delete; - std::unique_ptr - MakeArgumentPointer(void* in_ptr, - void* wei_ptr, - void* out_ptr, - size_t N, - size_t K, - size_t C, - std::vector input_spatial_lengths, - std::vector filter_spatial_lengths, - std::vector output_spatial_lengths, - std::vector conv_filter_strides, - std::vector conv_filter_dilations, - std::vector input_left_pads, - std::vector input_right_pads) - const; // in,wei and out element ops are ignored for now since even if we change them, they - // cant be linked - std::unique_ptr - MakeInvokerPointer() const; // requires including BaseInvoker headers - std::string GetTypeString(); - bool IsSupportedArgument(const BaseArgument* arg_ptr); -}; - -void add_device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_f32_instances_t( - std::vector& instances); -void add_device_conv2d_fwd_xdl_c_shuffle_nhwc_kyxc_nhwk_f16_instances_t( - std::vector& instances); -void add_device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_bf16_instances_t( - std::vector& instances); -void add_device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_f16_instances_t( - std::vector& instances); -void add_device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_int8_instances_t( - std::vector& instances); diff --git a/library/include/ck/library/host_tensor/conv_common.hpp b/library/include/ck/library/host_tensor/conv_common.hpp index b60af7d664..6d389903b5 100644 --- a/library/include/ck/library/host_tensor/conv_common.hpp +++ b/library/include/ck/library/host_tensor/conv_common.hpp @@ -1,7 +1,6 @@ -#ifndef CONV_COMMON_HPP -#define CONV_COMMON_HPP +#pragma once -#include "tensor_descriptor.hpp" +#include "ck/tensor_description/tensor_descriptor.hpp" template -inline auto activ(T v, const ck::ActivTypeEnum activ_type) -{ - const T alpha = 0.3; - switch(activ_type) - { - case ck::ActivTypeEnum::None: return v; - case ck::ActivTypeEnum::LeakyRelu: return (v >= 0 ? v : alpha * v); - case ck::ActivTypeEnum::Sigmoid: return (1 / (1 + exp(-v))); - default: throw std::runtime_error("unsupported activ type"); break; - } -} - -#endif diff --git a/library/include/ck/library/host_tensor/device.hpp b/library/include/ck/library/host_tensor/device.hpp deleted file mode 100644 index 990d2f98b3..0000000000 --- a/library/include/ck/library/host_tensor/device.hpp +++ /dev/null @@ -1,123 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include -#include - -#include "stream_config.hpp" -#include "ck/options.hpp" - -template -__global__ void set_buffer_value(T* p, T x, uint64_t buffer_element_size) -{ - for(uint64_t i = threadIdx.x; i < buffer_element_size; i += blockDim.x) - { - p[i] = x; - } -} - -inline void hip_check_error(hipError_t x) -{ - if(x != hipSuccess) - { - std::ostringstream ss; - ss << "HIP runtime error: " << hipGetErrorString(x) << ". " << __FILE__ << ": " << __LINE__ - << "in function: " << __func__; - throw std::runtime_error(ss.str()); - } -} - -struct DeviceMem -{ - DeviceMem() = delete; - DeviceMem(std::size_t mem_size); - void* GetDeviceBuffer(); - std::size_t GetBufferSize(); - void ToDevice(const void* p); - void FromDevice(void* p); - void SetZero(); - template - void SetValue(T x) - { - if(mMemSize % sizeof(T) != 0) - { - throw std::runtime_error("wrong! not entire DeviceMem will be set"); - } - - set_buffer_value<<<1, 1024>>>(static_cast(mpDeviceBuf), x, mMemSize / sizeof(T)); - } - ~DeviceMem(); - - void* mpDeviceBuf; - std::size_t mMemSize; -}; - -struct KernelTimerImpl; - -struct KernelTimer -{ - KernelTimer(); - ~KernelTimer(); - void Start(); - void End(); - float GetElapsedTime() const; - - std::unique_ptr impl; -}; - -template -float launch_and_time_kernel(const StreamConfig& stream_config, - F kernel, - dim3 grid_dim, - dim3 block_dim, - std::size_t lds_byte, - Args... args) -{ -#if CK_TIME_KERNEL - if(stream_config.time_kernel_) - { - printf("%s: grid_dim {%d, %d, %d}, block_dim {%d, %d, %d} \n", - __func__, - grid_dim.x, - grid_dim.y, - grid_dim.z, - block_dim.x, - block_dim.y, - block_dim.z); - - const int nrepeat = 10; - - printf("Warm up 1 time\n"); - - // warm up - kernel<<>>(args...); - - printf("Start running %d times...\n", nrepeat); - - KernelTimer timer; - timer.Start(); - - for(int i = 0; i < nrepeat; ++i) - { - kernel<<>>(args...); - } - - timer.End(); - - return timer.GetElapsedTime() / nrepeat; - } - else - { - kernel<<>>(args...); - - return 0; - } -#else - kernel<<>>(args...); - - return 0; -#endif -} diff --git a/library/include/ck/library/host_tensor/device_memory.hpp b/library/include/ck/library/host_tensor/device_memory.hpp new file mode 100644 index 0000000000..ccf6250bc8 --- /dev/null +++ b/library/include/ck/library/host_tensor/device_memory.hpp @@ -0,0 +1,37 @@ +#pragma once + +#include + +template +__global__ void set_buffer_value(T* p, T x, uint64_t buffer_element_size) +{ + for(uint64_t i = threadIdx.x; i < buffer_element_size; i += blockDim.x) + { + p[i] = x; + } +} + +struct DeviceMem +{ + DeviceMem() = delete; + DeviceMem(std::size_t mem_size); + void* GetDeviceBuffer(); + std::size_t GetBufferSize(); + void ToDevice(const void* p); + void FromDevice(void* p); + void SetZero(); + template + void SetValue(T x) + { + if(mMemSize % sizeof(T) != 0) + { + throw std::runtime_error("wrong! not entire DeviceMem will be set"); + } + + set_buffer_value<<<1, 1024>>>(static_cast(mpDeviceBuf), x, mMemSize / sizeof(T)); + } + ~DeviceMem(); + + void* mpDeviceBuf; + std::size_t mMemSize; +}; diff --git a/library/include/ck/library/host_tensor/device_tensor.hpp b/library/include/ck/library/host_tensor/device_tensor.hpp deleted file mode 100644 index b8d3ccc8a0..0000000000 --- a/library/include/ck/library/host_tensor/device_tensor.hpp +++ /dev/null @@ -1,8 +0,0 @@ -#pragma once -#include "host_tensor.hpp" - -template -void ostream_tensor_descriptor(TensorDesc, std::ostream& os = std::cout) -{ - ostream_HostTensorDescriptor(make_HostTensorDescriptor(TensorDesc{}), os); -} diff --git a/library/include/ck/library/host_tensor/host_common_util.hpp b/library/include/ck/library/host_tensor/host_common_util.hpp index 8fc1d36430..a227d4b456 100644 --- a/library/include/ck/library/host_tensor/host_common_util.hpp +++ b/library/include/ck/library/host_tensor/host_common_util.hpp @@ -1,37 +1,11 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2020 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ -#ifndef GUARD_HOST_COMMON_UTIL_HPP -#define GUARD_HOST_COMMON_UTIL_HPP +#pragma once #include #include #include #include -#include "config.hpp" +#include "ck/ck.hpp" namespace ck { @@ -95,8 +69,5 @@ static inline std::vector getTypeValuesFromString(const char* cstr_values) return (values); } -}; // namespace host_common - -}; // namespace ck - -#endif +} // namespace host_common +} // namespace ck diff --git a/library/include/ck/library/host_tensor/host_gemm.hpp b/library/include/ck/library/host_tensor/host_gemm.hpp index 211c01c01a..14233e9058 100644 --- a/library/include/ck/library/host_tensor/host_gemm.hpp +++ b/library/include/ck/library/host_tensor/host_gemm.hpp @@ -1,4 +1,5 @@ #pragma once + #include "host_tensor.hpp" template #include #include -#include "reduction_enums.hpp" -#include "reduction_common.hpp" -#include "host_common_util.hpp" -#include "host_tensor.hpp" -#include "data_type.hpp" -#include "reduction_functions_accumulate.hpp" +#include "ck/utility/data_type.hpp" +#include "ck/utility/reduction_enums.hpp" +#include "ck/utility/reduction_common.hpp" +#include "ck/utility/reduction_functions_accumulate.hpp" +#include "ck/library/host_tensor/host_common_util.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" template static void get_all_indexes(const std::array& dimLengths, @@ -400,5 +373,3 @@ struct ReductionHost }; }; }; - -#endif diff --git a/library/include/ck/library/host_tensor/host_tensor.hpp b/library/include/ck/library/host_tensor/host_tensor.hpp index 6cbc15c2cd..a6a2a53ee3 100644 --- a/library/include/ck/library/host_tensor/host_tensor.hpp +++ b/library/include/ck/library/host_tensor/host_tensor.hpp @@ -1,5 +1,4 @@ -#ifndef HOST_TENSOR_HPP -#define HOST_TENSOR_HPP +#pragma once #include #include @@ -8,7 +7,8 @@ #include #include #include -#include "data_type.hpp" + +#include "ck/utility/data_type.hpp" template std::ostream& LogRange(std::ostream& os, Range&& range, std::string delim) @@ -413,5 +413,3 @@ float check_error(const Tensor& ref, const Tensor& result) return linf_error; } - -#endif diff --git a/library/include/ck/library/host_tensor/host_tensor_generator.hpp b/library/include/ck/library/host_tensor/host_tensor_generator.hpp index 2813d6a9ae..ce7921531f 100644 --- a/library/include/ck/library/host_tensor/host_tensor_generator.hpp +++ b/library/include/ck/library/host_tensor/host_tensor_generator.hpp @@ -3,7 +3,7 @@ #include #include -#include "config.hpp" +#include "ck/ck.hpp" template struct GeneratorTensor_0 diff --git a/library/include/ck/library/obselete_driver_offline/debug.hpp b/library/include/ck/library/obselete_driver_offline/debug.hpp deleted file mode 100644 index 72fd0763ba..0000000000 --- a/library/include/ck/library/obselete_driver_offline/debug.hpp +++ /dev/null @@ -1,13 +0,0 @@ -#ifndef DEBUG_HPP -#define DEBUG_HPP - -namespace debug { -namespace debug_driver_gemm_xdlops_v2r3 { - -// these vars are on host, they control block_id to C matrix tile idx (m0, n0) mapping -static ck::index_t M01 = 1; -static ck::index_t N01 = 1; - -} // namespace debug_driver_gemm_xdlops_v2r3 -} // namespace debug -#endif diff --git a/library/include/ck/library/obselete_driver_offline/device_convolution_add_forward_implicit_gemm_v5r1_dlops_nc0hwc1_kc0yxc1_nk0hwk1.hpp b/library/include/ck/library/obselete_driver_offline/device_convolution_add_forward_implicit_gemm_v5r1_dlops_nc0hwc1_kc0yxc1_nk0hwk1.hpp deleted file mode 100644 index debb5058e7..0000000000 --- a/library/include/ck/library/obselete_driver_offline/device_convolution_add_forward_implicit_gemm_v5r1_dlops_nc0hwc1_kc0yxc1_nk0hwk1.hpp +++ /dev/null @@ -1,220 +0,0 @@ -#include -#include "device.hpp" -#include "host_tensor.hpp" -#include "driver_convolution_add_forward_implicit_gemm_v5r1_dlops_nc0hwc1_kc0yxc1_nk0hwk1.hpp" - -template -void device_convolution_add_forward_implicit_gemm_v5r1_dlops_nc0hwc1_kc0yxc1_nk0hwk1( - const InLengths& in_n_c0_hi_wi_c1_lengths, - const WeiLengths& wei_k_c0_y_x_c1_lengths, - const AddLengths& add_n_k0_hox2_wox2_k1_lengths, - const OutLengths& out_n_k0_ho_wo_k1_lengths, - const ConvStrides& conv_strides, - const ConvDilations& conv_dilations, - const InLeftPads& in_left_pads, - const InRightPads& in_right_pads, - const Tensor& in_n_c0_hi_wi_c1, - const Tensor& wei_k_c0_y_x_c1, - const Tensor& bias_k0_k1, - const Tensor& add_n_k0_hox2_wox2_k1, - Tensor& add_n_k0_hox2_wox2_k1_out, - ck::index_t nrepeat) -{ - using namespace ck; - - std::cout << __func__ << std::endl; - - constexpr auto I0 = Number<0>{}; - constexpr auto I1 = Number<1>{}; - constexpr auto I2 = Number<2>{}; - constexpr auto I3 = Number<3>{}; - constexpr auto I4 = Number<4>{}; - - const auto N = out_n_k0_ho_wo_k1_lengths[I0]; - const auto K0 = out_n_k0_ho_wo_k1_lengths[I1]; - const auto Ho = out_n_k0_ho_wo_k1_lengths[I2]; - const auto Wo = out_n_k0_ho_wo_k1_lengths[I3]; - const auto K1 = out_n_k0_ho_wo_k1_lengths[I4]; - - const auto C0 = in_n_c0_hi_wi_c1_lengths[I1]; - const auto Hi = in_n_c0_hi_wi_c1_lengths[I2]; - const auto Wi = in_n_c0_hi_wi_c1_lengths[I3]; - const auto C1 = in_n_c0_hi_wi_c1_lengths[I4]; - - const auto K = wei_k_c0_y_x_c1_lengths[I0]; - const auto Y = wei_k_c0_y_x_c1_lengths[I2]; - const auto X = wei_k_c0_y_x_c1_lengths[I3]; - - const auto Hox2 = add_n_k0_hox2_wox2_k1_lengths[I2]; - const auto Wox2 = add_n_k0_hox2_wox2_k1_lengths[I3]; - - DeviceMem in_n_c0_hi_wi_c1_device_buf(sizeof(TInWei) * - in_n_c0_hi_wi_c1.mDesc.GetElementSpace()); - DeviceMem wei_k_c0_y_x_c1_device_buf(sizeof(TInWei) * wei_k_c0_y_x_c1.mDesc.GetElementSpace()); - DeviceMem bias_k0_k1_device_buf(sizeof(TOut) * bias_k0_k1.mDesc.GetElementSpace()); - DeviceMem add_n_k0_hox2_wox2_k1_device_buf(sizeof(TOut) * - add_n_k0_hox2_wox2_k1.mDesc.GetElementSpace()); - - in_n_c0_hi_wi_c1_device_buf.ToDevice(in_n_c0_hi_wi_c1.mData.data()); - wei_k_c0_y_x_c1_device_buf.ToDevice(wei_k_c0_y_x_c1.mData.data()); - bias_k0_k1_device_buf.ToDevice(bias_k0_k1.mData.data()); - add_n_k0_hox2_wox2_k1_device_buf.ToDevice(add_n_k0_hox2_wox2_k1.mData.data()); - - constexpr index_t InWeiVectorSize = 8; - - if(C1 % InWeiVectorSize != 0) - { - throw std::runtime_error("wrong! C1 cannot be divided by InWeiVectorSize"); - } - -#if 0 - constexpr index_t BlockSize = 256; - - constexpr index_t KPerBlock = 32; - constexpr index_t HoPerBlock = 8; - constexpr index_t WoPerBlock = 64; - - constexpr index_t E1 = C0 * 9; - constexpr index_t E2 = 1; - constexpr index_t E1PerBlock = C0; - - constexpr index_t KPerThread = 16; - constexpr index_t HoPerThread = 2; - constexpr index_t WoPerThread = 2; - constexpr index_t EPerThread = 1; - - using ABlockTransferThreadSliceLengths_E0_E1_K0_K1_E2 = Sequence<1, 9, 1, E2>; - using ABlockTransferThreadClusterLengths_E0_E1_K0_K1_E2 = Sequence<1, E1PerBlock, KPerBlock, 1>; - - constexpr index_t ABlockTransferSrcScalarPerVector_E2 = E2; - constexpr index_t ABlockTransferDstScalarPerVector_E2 = E2; - - constexpr index_t BThreadTransferSrcScalarPerVector_E2 = E2; - - constexpr index_t CThreadTransferDstScalarPerVector_K = K1; -#elif 1 - constexpr auto BlockSize = 64; - - constexpr auto KPerBlock = 8; - constexpr auto HoPerBlock = 8; - constexpr auto WoPerBlock = 32; - - constexpr auto E1 = 2 * 9; - constexpr auto E2 = 1; - constexpr auto K2 = 2; - constexpr auto E1PerBlock = 2; - - constexpr auto KPerThread = KPerBlock; - constexpr auto HoPerThread = 2; - constexpr auto WoPerThread = 2; - constexpr auto EPerThread = 1; - - using ABlockTransferThreadSliceLengths_E0_E1_K0_K1_E2 = Sequence<1, 9, 1, 1, E2>; - using ABlockTransferThreadClusterLengths_E0_E1_K0_K1_E2 = - Sequence<1, E1PerBlock, 1, KPerBlock, 1>; - - constexpr auto ABlockTransferSrcScalarPerVector_E2 = E2; - constexpr auto ABlockTransferDstScalarPerVector_E2 = E2; - constexpr auto BThreadTransferSrcScalarPerVector_E2 = E2; - constexpr auto CThreadTransferDstScalarPerVector_K = InWeiVectorSize; -#endif - - const auto in_n_c0_hi_wi_c1_desc = - make_naive_tensor_descriptor_packed(make_tuple(N, C0, Hi, Wi, E2)); - const auto wei_k_c0_y_x_c1_desc = - make_naive_tensor_descriptor_packed(make_tuple(K, C0, Y, X, E2)); - const auto add_n_k0_hox2_wox2_k1_desc = - make_naive_tensor_descriptor_packed(make_tuple(N, K0, Hox2, Wox2, K1)); - const auto out_n_k0_ho_wo_k1_desc = - make_naive_tensor_descriptor_packed(make_tuple(N, K0, Ho, Wo, K1)); - - constexpr auto conv_driver = - DriverDynamicConvolutionForwardImplicitGemmDlops_v5r1_nc0hwc1_kc0yxc1_nk0hwk1_add< - BlockSize, - typename vector_type::type, - TAcc, - TOut, - E1, - E2, - K2, - KPerBlock, - HoPerBlock, - WoPerBlock, - E1PerBlock, - KPerThread, - HoPerThread, - WoPerThread, - EPerThread, - ABlockTransferThreadSliceLengths_E0_E1_K0_K1_E2, - ABlockTransferThreadClusterLengths_E0_E1_K0_K1_E2, - ABlockTransferSrcScalarPerVector_E2, - ABlockTransferDstScalarPerVector_E2, - BThreadTransferSrcScalarPerVector_E2, - CThreadTransferDstScalarPerVector_K, - activ_type>{}; - - std::cerr << "conv_bias_activ_resize_add_input_" - << "n" << N << "c" << C0 << "h" << Hi << "w" << Wi << "c" << C1 << "_filter_k" << K - << "c" << C0 << "y" << Y << "x" << X << "c" << C1 << "_addout_n" << N << "k" << K0 - << "h" << Ho * 2 << "w" << Wo * 2 << "k" << K1 << std::endl; - - for(int i = 0; i < 5; i++) - { - - const auto ave_time = - conv_driver.Run(wei_k_c0_y_x_c1_desc, - in_n_c0_hi_wi_c1_desc, - out_n_k0_ho_wo_k1_desc, - add_n_k0_hox2_wox2_k1_desc, - conv_strides, - conv_dilations, - in_left_pads, - in_right_pads, - static_cast::type*>( - wei_k_c0_y_x_c1_device_buf.GetDeviceBuffer()), - static_cast::type*>( - in_n_c0_hi_wi_c1_device_buf.GetDeviceBuffer()), - static_cast(bias_k0_k1_device_buf.GetDeviceBuffer()), - static_cast(add_n_k0_hox2_wox2_k1_device_buf.GetDeviceBuffer()), - nrepeat); - - { - float perf = static_cast(std::size_t(2) * N * K * Ho * Wo * C0 * C1 * Y * X) / - (std::size_t(1000) * 1000 * 1000) / ave_time; - - std::cout << "Average time : " << ave_time << " ms, " << perf << " TFlop/s" - << std::endl; - } - } - - add_n_k0_hox2_wox2_k1_device_buf.ToDevice(add_n_k0_hox2_wox2_k1.mData.data()); - - conv_driver.Run(wei_k_c0_y_x_c1_desc, - in_n_c0_hi_wi_c1_desc, - out_n_k0_ho_wo_k1_desc, - add_n_k0_hox2_wox2_k1_desc, - conv_strides, - conv_dilations, - in_left_pads, - in_right_pads, - static_cast::type*>( - wei_k_c0_y_x_c1_device_buf.GetDeviceBuffer()), - static_cast::type*>( - in_n_c0_hi_wi_c1_device_buf.GetDeviceBuffer()), - static_cast(bias_k0_k1_device_buf.GetDeviceBuffer()), - static_cast(add_n_k0_hox2_wox2_k1_device_buf.GetDeviceBuffer()), - 0); - - add_n_k0_hox2_wox2_k1_device_buf.FromDevice(add_n_k0_hox2_wox2_k1_out.mData.data()); -} diff --git a/library/include/ck/library/obselete_driver_offline/device_convolution_backward_data_implicit_gemm_v4r1_xdlops_nhwc_kyxc_nhwk.hpp b/library/include/ck/library/obselete_driver_offline/device_convolution_backward_data_implicit_gemm_v4r1_xdlops_nhwc_kyxc_nhwk.hpp deleted file mode 100644 index 79d31ba246..0000000000 --- a/library/include/ck/library/obselete_driver_offline/device_convolution_backward_data_implicit_gemm_v4r1_xdlops_nhwc_kyxc_nhwk.hpp +++ /dev/null @@ -1,309 +0,0 @@ -#include -#include "device.hpp" -#include "host_tensor.hpp" -#include "transform_backward_data_convolution_into_gemm_v4r1_nhwc_kyxc_nhwk.hpp" -#include "driver_gemm_xdlops_v2r3.hpp" -#include "debug.hpp" - -template -void device_convolution_backward_data_implicit_gemm_v4r1_xdlops_nhwc_kyxc_nhwk( - const InLengths& in_n_hi_wi_c_lengths, - const WeiLengths& wei_k_y_x_c_lengths, - const OutLengths& out_n_ho_wo_k_lengths, - const ConvStrides& conv_strides, - const ConvDilations& conv_dilations, - const InLeftPads& in_left_pads, - const InRightPads& in_right_pads, - Tensor& in_n_hi_wi_c, - const Tensor& wei_k_y_x_c, - const Tensor& out_n_ho_wo_k, - ck::index_t nrepeat) -{ - using namespace ck; - - std::cout << __func__ << std::endl; - - constexpr auto I0 = Number<0>{}; - constexpr auto I1 = Number<1>{}; - constexpr auto I2 = Number<2>{}; - constexpr auto I3 = Number<3>{}; - - DeviceMem in_n_hi_wi_c_device_buf(sizeof(TInWei) * in_n_hi_wi_c.mDesc.GetElementSpace()); - DeviceMem wei_k_y_x_c_device_buf(sizeof(TInWei) * wei_k_y_x_c.mDesc.GetElementSpace()); - DeviceMem out_n_ho_wo_k_device_buf(sizeof(TOut) * out_n_ho_wo_k.mDesc.GetElementSpace()); - - in_n_hi_wi_c_device_buf.ToDevice(in_n_hi_wi_c.mData.data()); - wei_k_y_x_c_device_buf.ToDevice(wei_k_y_x_c.mData.data()); - out_n_ho_wo_k_device_buf.ToDevice(out_n_ho_wo_k.mData.data()); - - const auto in_n_hi_wi_c_desc = make_naive_tensor_descriptor_packed(in_n_hi_wi_c_lengths); - const auto wei_k_y_x_c_desc = make_naive_tensor_descriptor_packed(wei_k_y_x_c_lengths); - const auto out_n_ho_wo_k_desc = make_naive_tensor_descriptor_packed(out_n_ho_wo_k_lengths); - -#if 0 - // [M, N, K0, K1] = [128, 128, 4, 4], C = 64, for fp32 - constexpr index_t BlockSize = 256; - - constexpr index_t GemmMPerBlock = 128; - constexpr index_t GemmNPerBlock = 128; - constexpr index_t GemmKPerBlock = 4; - - constexpr index_t GemmMPerXDL = 32; - constexpr index_t GemmNPerXDL = 32; - constexpr index_t GemmK1 = 4; - - constexpr index_t MRepeat = 2; - constexpr index_t NRepeat = 2; - - using GemmABlockTransferThreadSliceLengths_GemmK0_GemmM_GemmK1 = Sequence<1, 2, 4>; - using GemmABlockTransferThreadClusterLengths_GemmK0_GemmM_GemmK1 = Sequence<4, 64, 1>; - - constexpr index_t GemmABlockTransferSrcScalarPerVector_GemmM = 2; - constexpr index_t GemmABlockTransferDstScalarPerVector_GemmK1 = 4; - - using GemmBBlockTransferThreadSliceLengths_GemmK0_GemmN_GemmK1 = Sequence<1, 2, 4>; - using GemmBBlockTransferThreadClusterLengths_GemmK0_GemmN_GemmK1 = Sequence<4, 64, 1>; - - constexpr index_t GemmBBlockTransferSrcScalarPerVector_GemmK1 = 4; - constexpr index_t GemmBBlockTransferDstScalarPerVector_GemmK1 = 4; - - constexpr index_t GemmCThreadTransferDstScalarPerVector = 4; -#elif 0 - // [M, N, K0, K1] = [128, 128, 4, 8] for fp16 - constexpr index_t BlockSize = 256; - - constexpr index_t GemmMPerBlock = 128; - constexpr index_t GemmNPerBlock = 128; - constexpr index_t GemmKPerBlock = 4; - - constexpr index_t GemmMPerXDL = 32; - constexpr index_t GemmNPerXDL = 32; - constexpr index_t GemmK1 = 8; - - constexpr index_t MRepeat = 2; - constexpr index_t NRepeat = 2; - - using GemmABlockTransferThreadSliceLengths_GemmK0_GemmM_GemmK1 = Sequence<1, 2, 8>; - using GemmABlockTransferThreadClusterLengths_GemmK0_GemmM_GemmK1 = Sequence<4, 64, 1>; - - constexpr index_t GemmABlockTransferSrcScalarPerVector_GemmM = 2; - constexpr index_t GemmABlockTransferDstScalarPerVector_GemmK1 = 8; - - using GemmBBlockTransferThreadSliceLengths_GemmK0_GemmN_GemmK1 = Sequence<1, 2, 8>; - using GemmBBlockTransferThreadClusterLengths_GemmK0_GemmN_GemmK1 = Sequence<4, 64, 1>; - - constexpr index_t GemmBBlockTransferSrcScalarPerVector_GemmK1 = 8; - constexpr index_t GemmBBlockTransferDstScalarPerVector_GemmK1 = 8; - - constexpr index_t GemmCThreadTransferDstScalarPerVector = 4; -#elif 1 - // [M, N, K0, K1] = [256, 128, 4, 8], C = 128, for fp16 - constexpr index_t BlockSize = 256; - - constexpr index_t GemmMPerBlock = 256; - constexpr index_t GemmNPerBlock = 128; - constexpr index_t GemmKPerBlock = 4; - - constexpr index_t GemmMPerXDL = 32; - constexpr index_t GemmNPerXDL = 32; - constexpr index_t GemmK1 = 8; - - constexpr index_t MRepeat = 4; - constexpr index_t NRepeat = 2; - - using GemmABlockTransferThreadSliceLengths_GemmK0_GemmM_GemmK1 = Sequence<1, 4, 8>; - using GemmABlockTransferThreadClusterLengths_GemmK0_GemmM_GemmK1 = Sequence<4, 64, 1>; - - constexpr index_t GemmABlockTransferSrcScalarPerVector_GemmM = 4; - constexpr index_t GemmABlockTransferDstScalarPerVector_GemmK1 = 8; - - using GemmBBlockTransferThreadSliceLengths_GemmK0_GemmN_GemmK1 = Sequence<1, 2, 8>; - using GemmBBlockTransferThreadClusterLengths_GemmK0_GemmN_GemmK1 = Sequence<4, 64, 1>; - - constexpr index_t GemmBBlockTransferSrcScalarPerVector_GemmK1 = 8; - constexpr index_t GemmBBlockTransferDstScalarPerVector_GemmK1 = 8; - - constexpr index_t GemmCThreadTransferDstScalarPerVector = 4; -#elif 1 - // [M, N, K0, K1] = [128, 256, 4, 8], C = 128, for fp16 - constexpr index_t BlockSize = 256; - - constexpr index_t GemmMPerBlock = 128; - constexpr index_t GemmNPerBlock = 256; - constexpr index_t GemmKPerBlock = 4; - - constexpr index_t GemmMPerXDL = 32; - constexpr index_t GemmNPerXDL = 32; - constexpr index_t GemmK1 = 8; - - constexpr index_t MRepeat = 2; - constexpr index_t NRepeat = 4; - - using GemmABlockTransferThreadSliceLengths_GemmK0_GemmM_GemmK1 = Sequence<1, 2, 8>; - using GemmABlockTransferThreadClusterLengths_GemmK0_GemmM_GemmK1 = Sequence<4, 64, 1>; - - constexpr index_t GemmABlockTransferSrcScalarPerVector_GemmM = 2; - constexpr index_t GemmABlockTransferDstScalarPerVector_GemmK1 = 8; - - using GemmBBlockTransferThreadSliceLengths_GemmK0_GemmN_GemmK1 = Sequence<1, 4, 8>; - using GemmBBlockTransferThreadClusterLengths_GemmK0_GemmN_GemmK1 = Sequence<4, 64, 1>; - - constexpr index_t GemmBBlockTransferSrcScalarPerVector_GemmK1 = 8; - constexpr index_t GemmBBlockTransferDstScalarPerVector_GemmK1 = 8; - - constexpr index_t GemmCThreadTransferDstScalarPerVector = 4; -#endif - - const auto descs = - transform_backward_data_convolution_into_gemm_v4r1_nhwc_kyxc_nhwk(wei_k_y_x_c_desc, - out_n_ho_wo_k_desc, - in_n_hi_wi_c_desc, - conv_strides, - conv_dilations, - in_left_pads, - in_right_pads, - I0, - I0, - Number{}); - - const auto wei_gemmk0_gemmm_gemmk1_grid_desc = descs[I0]; - const auto out_gemmk0_gemmn_gemmk1_grid_desc = descs[I1]; - const auto in_gemmm_gemmn_grid_desc = descs[I2]; - - // HACK: hacks that control index calculation when iterating over A, B, C matrix - constexpr auto wei_gemmk0_gemmm_gemmk1_grid_step_hacks = - make_tuple(make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0>{}, // 0+: GemmK0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 1+: GemmM - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}), // 2+: GemmK1 - make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0>{}, // 0-: GemmK0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 1-: GemmM - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{})); // 2-: GemmK1 - - constexpr auto out_gemmk0_gemmn_gemmk1_grid_step_hacks = make_tuple( - make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0>{}, // 0+: GemmK0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0>{}, // 1+: GemmN - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}), // 2+: GemmK1 - make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0>{}, // 0-: GemmK0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0>{}, // 1-: GemmN - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{})); // 2-: GemmK1 - - // clang-format off - constexpr auto in_m0_n0_m1_n1_m2_m3_m4_n2_grid_step_hacks = make_tuple( - make_tuple( - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 0+: M0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 1+: N0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 2+: M1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 3+: N1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 4+: M2 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 5+: M3 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 6+: M4 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0>{}), // 7+: N2 - make_tuple( - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 0-: M0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 1-: N0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 2-: M1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 3-: N1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 4-: M2 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 5-: M3 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 6-: M4 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0>{})); // 7-: N2 - //clang-format on - - constexpr auto wei_gemmk0_gemmm_gemmk1_grid_move_slice_window_step_hacks = - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0>{}; - - constexpr auto out_gemmk0_gemmn_gemmk1_grid_move_slice_window_step_hacks = - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0>{}; - - for(index_t i = 0; i < 5; ++i) - { - float ave_time = driver_gemm_xdlops_v2r3< - BlockSize, - TInWei, - TAcc, - TOut, - InMemoryDataOperationEnum::Set, - decltype(wei_gemmk0_gemmm_gemmk1_grid_desc), - decltype(out_gemmk0_gemmn_gemmk1_grid_desc), - decltype(in_gemmm_gemmn_grid_desc), - GemmMPerBlock, - GemmNPerBlock, - GemmKPerBlock, - GemmMPerXDL, - GemmNPerXDL, - GemmK1, - MRepeat, - NRepeat, - GemmABlockTransferThreadSliceLengths_GemmK0_GemmM_GemmK1, - GemmABlockTransferThreadClusterLengths_GemmK0_GemmM_GemmK1, - Sequence<2, 0, 1>, - Sequence<0, 2, 1>, - 1, - GemmABlockTransferSrcScalarPerVector_GemmM, - GemmABlockTransferDstScalarPerVector_GemmK1, - false, // don't move back src coordinate after threadwise copy - GemmBBlockTransferThreadSliceLengths_GemmK0_GemmN_GemmK1, - GemmBBlockTransferThreadClusterLengths_GemmK0_GemmN_GemmK1, - Sequence<1, 0, 2>, - Sequence<1, 0, 2>, - 2, - GemmBBlockTransferSrcScalarPerVector_GemmK1, - GemmBBlockTransferDstScalarPerVector_GemmK1, - false, // don't move back src coordinate after threadwise copy - Sequence<1, 3, 7, 0, 2, 4, 5, 6>, - 6, - GemmCThreadTransferDstScalarPerVector, - decltype(wei_gemmk0_gemmm_gemmk1_grid_step_hacks), - decltype(out_gemmk0_gemmn_gemmk1_grid_step_hacks), - decltype(in_m0_n0_m1_n1_m2_m3_m4_n2_grid_step_hacks), - decltype(wei_gemmk0_gemmm_gemmk1_grid_move_slice_window_step_hacks), - decltype(out_gemmk0_gemmn_gemmk1_grid_move_slice_window_step_hacks), - false, // CAccessOrderMRepeatNRepeat - false, // ABlockLdsExtraM - false // BBlockLdsExtraN - >(static_cast(wei_k_y_x_c_device_buf.GetDeviceBuffer()), - static_cast(out_n_ho_wo_k_device_buf.GetDeviceBuffer()), - static_cast(in_n_hi_wi_c_device_buf.GetDeviceBuffer()), - wei_gemmk0_gemmm_gemmk1_grid_desc, - out_gemmk0_gemmn_gemmk1_grid_desc, - in_gemmm_gemmn_grid_desc, - debug::debug_driver_gemm_xdlops_v2r3::M01, - debug::debug_driver_gemm_xdlops_v2r3::N01, - wei_gemmk0_gemmm_gemmk1_grid_step_hacks, - out_gemmk0_gemmn_gemmk1_grid_step_hacks, - in_m0_n0_m1_n1_m2_m3_m4_n2_grid_step_hacks, - wei_gemmk0_gemmm_gemmk1_grid_move_slice_window_step_hacks, - out_gemmk0_gemmn_gemmk1_grid_move_slice_window_step_hacks, - nrepeat); - - { - const auto N = out_n_ho_wo_k_lengths[I0]; - const auto K = out_n_ho_wo_k_lengths[I3]; - const auto C = wei_k_y_x_c_lengths[I3]; - - const auto Ho = out_n_ho_wo_k_lengths[I1]; - const auto Wo = out_n_ho_wo_k_lengths[I2]; - - const auto Y = wei_k_y_x_c_lengths[I1]; - const auto X = wei_k_y_x_c_lengths[I2]; - - float perf = static_cast((std::size_t(2) * N * K * Ho * Wo * C * Y * X)) / - (std::size_t(1000) * 1000 * 1000) / ave_time; - - std::cout << "Average time : " << ave_time << " ms, " << perf << " TFlop/s" - << std::endl; - } - } - - // copy result back to host - in_n_hi_wi_c_device_buf.FromDevice(in_n_hi_wi_c.mData.data()); -} diff --git a/library/include/ck/library/obselete_driver_offline/device_convolution_backward_data_implicit_gemm_v4r1r2_xdlops_nhwc_kyxc_nhwk.hpp b/library/include/ck/library/obselete_driver_offline/device_convolution_backward_data_implicit_gemm_v4r1r2_xdlops_nhwc_kyxc_nhwk.hpp deleted file mode 100644 index e3b6a6c8c2..0000000000 --- a/library/include/ck/library/obselete_driver_offline/device_convolution_backward_data_implicit_gemm_v4r1r2_xdlops_nhwc_kyxc_nhwk.hpp +++ /dev/null @@ -1,423 +0,0 @@ -#include -#include "device.hpp" -#include "host_tensor.hpp" -#include "transform_backward_data_convolution_into_gemm_v4r1r2_nhwc_kyxc_nhwk.hpp" -#include "driver_gemm_xdlops_v2r3.hpp" - -template -void device_convolution_backward_data_implicit_gemm_v4r1r2_xdlops_nhwc_kyxc_nhwk( - const InLengths& in_n_hi_wi_c_lengths, - const WeiLengths& wei_k_y_x_c_lengths, - const OutLengths& out_n_ho_wo_k_lengths, - const ConvStrides& conv_strides, - const ConvDilations& conv_dilations, - const InLeftPads& in_left_pads, - const InRightPads& in_right_pads, - Tensor& in_n_hi_wi_c, - const Tensor& wei_k_y_x_c, - const Tensor& out_n_ho_wo_k, - ck::index_t nrepeat) -{ - using namespace ck; - - std::cout << __func__ << std::endl; - - constexpr auto I0 = Number<0>{}; - constexpr auto I1 = Number<1>{}; - constexpr auto I2 = Number<2>{}; - constexpr auto I3 = Number<3>{}; - - DeviceMem in_n_hi_wi_c_device_buf(sizeof(TInWei) * in_n_hi_wi_c.mDesc.GetElementSpace()); - DeviceMem wei_k_y_x_c_device_buf(sizeof(TInWei) * wei_k_y_x_c.mDesc.GetElementSpace()); - DeviceMem out_n_ho_wo_k_device_buf(sizeof(TOut) * out_n_ho_wo_k.mDesc.GetElementSpace()); - - in_n_hi_wi_c_device_buf.ToDevice(in_n_hi_wi_c.mData.data()); - wei_k_y_x_c_device_buf.ToDevice(wei_k_y_x_c.mData.data()); - out_n_ho_wo_k_device_buf.ToDevice(out_n_ho_wo_k.mData.data()); - - const auto in_n_hi_wi_c_desc = make_naive_tensor_descriptor_packed(in_n_hi_wi_c_lengths); - const auto wei_k_y_x_c_desc = make_naive_tensor_descriptor_packed(wei_k_y_x_c_lengths); - const auto out_n_ho_wo_k_desc = make_naive_tensor_descriptor_packed(out_n_ho_wo_k_lengths); - -#if 0 - // [M, N, K0, K1] = [256, 128, 4, 4], C = 128, for fp32 - constexpr index_t BlockSize = 256; - - constexpr index_t GemmMPerBlock = 256; - constexpr index_t GemmNPerBlock = 128; - constexpr index_t GemmKPerBlock = 4; - - constexpr index_t GemmMPerWave = 32; - constexpr index_t GemmNPerWave = 32; - constexpr index_t GemmK1 = 4; - - constexpr index_t MRepeat = 4; - constexpr index_t NRepeat = 2; - - using GemmABlockTransferThreadSliceLengths_GemmK0_GemmM_GemmK1 = Sequence<1, 4, 4>; - using GemmABlockTransferThreadClusterLengths_GemmK0_GemmM_GemmK1 = Sequence<4, 64, 1>; - - constexpr index_t GemmABlockTransferSrcScalarPerVector_GemmK1 = 4; - constexpr index_t GemmABlockTransferDstScalarPerVector_GemmK1 = 4; - - using GemmBBlockTransferThreadSliceLengths_GemmK0_GemmN_GemmK1 = Sequence<1, 2, 4>; - using GemmBBlockTransferThreadClusterLengths_GemmK0_GemmN_GemmK1 = Sequence<4, 64, 1>; - - constexpr index_t GemmBBlockTransferSrcScalarPerVector_GemmN = 2; - constexpr index_t GemmBBlockTransferDstScalarPerVector_GemmK1 = 4; - - constexpr index_t GemmCThreadTransferDstScalarPerVector = 1; -#elif 0 - // [M, N, K0, K1] = [128, 128, 4, 4], C = 64, for fp32 - constexpr index_t BlockSize = 256; - - constexpr index_t GemmMPerBlock = 128; - constexpr index_t GemmNPerBlock = 128; - constexpr index_t GemmKPerBlock = 4; - - constexpr index_t GemmMPerWave = 32; - constexpr index_t GemmNPerWave = 32; - constexpr index_t GemmK1 = 4; - - constexpr index_t MRepeat = 2; - constexpr index_t NRepeat = 2; - - using GemmABlockTransferThreadSliceLengths_GemmK0_GemmM_GemmK1 = Sequence<1, 2, 4>; - using GemmABlockTransferThreadClusterLengths_GemmK0_GemmM_GemmK1 = Sequence<4, 64, 1>; - - constexpr index_t GemmABlockTransferSrcScalarPerVector_GemmK1 = 4; - constexpr index_t GemmABlockTransferDstScalarPerVector_GemmK1 = 4; - - using GemmBBlockTransferThreadSliceLengths_GemmK0_GemmN_GemmK1 = Sequence<1, 2, 4>; - using GemmBBlockTransferThreadClusterLengths_GemmK0_GemmN_GemmK1 = Sequence<4, 64, 1>; - - constexpr index_t GemmBBlockTransferSrcScalarPerVector_GemmN = 2; - constexpr index_t GemmBBlockTransferDstScalarPerVector_GemmK1 = 4; - - constexpr index_t GemmCThreadTransferDstScalarPerVector = 1; -#elif 0 - // [M, N, K0, K1] = [256, 128, 4, 8], C = 128, for fp16 - constexpr index_t BlockSize = 256; - - constexpr index_t GemmMPerBlock = 256; - constexpr index_t GemmNPerBlock = 128; - constexpr index_t GemmKPerBlock = 4; - - constexpr index_t GemmMPerWave = 32; - constexpr index_t GemmNPerWave = 32; - constexpr index_t GemmK1 = 8; - - constexpr index_t MRepeat = 4; - constexpr index_t NRepeat = 2; - - using GemmABlockTransferThreadSliceLengths_GemmK0_GemmM_GemmK1 = Sequence<1, 4, 8>; - using GemmABlockTransferThreadClusterLengths_GemmK0_GemmM_GemmK1 = Sequence<4, 64, 1>; - - constexpr index_t GemmABlockTransferSrcScalarPerVector_GemmK1 = 8; - constexpr index_t GemmABlockTransferDstScalarPerVector_GemmK1 = 8; - - using GemmBBlockTransferThreadSliceLengths_GemmK0_GemmN_GemmK1 = Sequence<1, 2, 8>; - using GemmBBlockTransferThreadClusterLengths_GemmK0_GemmN_GemmK1 = Sequence<4, 64, 1>; - - constexpr index_t GemmBBlockTransferSrcScalarPerVector_GemmN = 2; - constexpr index_t GemmBBlockTransferDstScalarPerVector_GemmK1 = 8; - - constexpr index_t GemmCThreadTransferDstScalarPerVector = 1; -#elif 1 - // [M, N, K0, K1] = [128, 256, 4, 8], C = 128, for fp16 - constexpr index_t BlockSize = 256; - - constexpr index_t GemmMPerBlock = 128; - constexpr index_t GemmNPerBlock = 256; - constexpr index_t GemmKPerBlock = 4; - - constexpr index_t GemmMPerWave = 32; - constexpr index_t GemmNPerWave = 32; - constexpr index_t GemmK1 = 8; - - constexpr index_t MRepeat = 2; - constexpr index_t NRepeat = 4; - - using GemmABlockTransferThreadSliceLengths_GemmK0_GemmM_GemmK1 = Sequence<1, 2, 8>; - using GemmABlockTransferThreadClusterLengths_GemmK0_GemmM_GemmK1 = Sequence<4, 64, 1>; - - constexpr index_t GemmABlockTransferSrcScalarPerVector_GemmK1 = 8; - constexpr index_t GemmABlockTransferDstScalarPerVector_GemmK1 = 8; - - using GemmBBlockTransferThreadSliceLengths_GemmK0_GemmN_GemmK1 = Sequence<1, 4, 8>; - using GemmBBlockTransferThreadClusterLengths_GemmK0_GemmN_GemmK1 = Sequence<4, 64, 1>; - - constexpr index_t GemmBBlockTransferSrcScalarPerVector_GemmN = 4; - constexpr index_t GemmBBlockTransferDstScalarPerVector_GemmK1 = 8; - - constexpr index_t GemmCThreadTransferDstScalarPerVector = 1; -#elif 1 - // [M, N, K0, K1] = [128, 128, 4, 8], C = 64, for fp16 - constexpr index_t BlockSize = 256; - - constexpr index_t GemmMPerBlock = 128; - constexpr index_t GemmNPerBlock = 128; - constexpr index_t GemmKPerBlock = 4; - - constexpr index_t GemmMPerWave = 32; - constexpr index_t GemmNPerWave = 32; - constexpr index_t GemmK1 = 8; - - constexpr index_t MRepeat = 2; - constexpr index_t NRepeat = 2; - - using GemmABlockTransferThreadSliceLengths_GemmK0_GemmM_GemmK1 = Sequence<1, 2, 8>; - using GemmABlockTransferThreadClusterLengths_GemmK0_GemmM_GemmK1 = Sequence<4, 64, 1>; - - constexpr index_t GemmABlockTransferSrcScalarPerVector_GemmK1 = 8; - constexpr index_t GemmABlockTransferDstScalarPerVector_GemmK1 = 8; - - using GemmBBlockTransferThreadSliceLengths_GemmK0_GemmN_GemmK1 = Sequence<1, 2, 8>; - using GemmBBlockTransferThreadClusterLengths_GemmK0_GemmN_GemmK1 = Sequence<4, 64, 1>; - - constexpr index_t GemmBBlockTransferSrcScalarPerVector_GemmN = 2; - constexpr index_t GemmBBlockTransferDstScalarPerVector_GemmK1 = 8; - - constexpr index_t GemmCThreadTransferDstScalarPerVector = 1; -#elif 0 - // [M, N, K0, K1] = [128, 64, 4, 8], C = 64, for fp16 - constexpr index_t BlockSize = 128; - - constexpr index_t GemmMPerBlock = 128; - constexpr index_t GemmNPerBlock = 64; - constexpr index_t GemmKPerBlock = 4; - - constexpr index_t GemmMPerWave = 32; - constexpr index_t GemmNPerWave = 32; - constexpr index_t GemmK1 = 8; - - constexpr index_t MRepeat = 2; - constexpr index_t NRepeat = 2; - - using GemmABlockTransferThreadSliceLengths_GemmK0_GemmM_GemmK1 = Sequence<1, 4, 8>; - using GemmABlockTransferThreadClusterLengths_GemmK0_GemmM_GemmK1 = Sequence<4, 32, 1>; - - constexpr index_t GemmABlockTransferSrcScalarPerVector_GemmK1 = 8; - constexpr index_t GemmABlockTransferDstScalarPerVector_GemmK1 = 8; - - using GemmBBlockTransferThreadSliceLengths_GemmK0_GemmN_GemmK1 = Sequence<1, 2, 8>; - using GemmBBlockTransferThreadClusterLengths_GemmK0_GemmN_GemmK1 = Sequence<4, 32, 1>; - - constexpr index_t GemmBBlockTransferSrcScalarPerVector_GemmN = 2; - constexpr index_t GemmBBlockTransferDstScalarPerVector_GemmK1 = 8; - - constexpr index_t GemmCThreadTransferDstScalarPerVector = 1; -#elif 0 - // [M, N, K0, K1] = [128, 64, 4, 8], C = 32, for fp16 - constexpr index_t BlockSize = 256; - - constexpr index_t GemmMPerBlock = 128; - constexpr index_t GemmNPerBlock = 64; - constexpr index_t GemmKPerBlock = 4; - - constexpr index_t GemmMPerWave = 32; - constexpr index_t GemmNPerWave = 32; - constexpr index_t GemmK1 = 8; - - constexpr index_t MRepeat = 2; - constexpr index_t NRepeat = 1; - - using GemmABlockTransferThreadSliceLengths_GemmK0_GemmM_GemmK1 = Sequence<1, 2, 8>; - using GemmABlockTransferThreadClusterLengths_GemmK0_GemmM_GemmK1 = Sequence<4, 64, 1>; - - constexpr index_t GemmABlockTransferSrcScalarPerVector_GemmK1 = 8; - constexpr index_t GemmABlockTransferDstScalarPerVector_GemmK1 = 8; - - using GemmBBlockTransferThreadSliceLengths_GemmK0_GemmN_GemmK1 = Sequence<1, 1, 8>; - using GemmBBlockTransferThreadClusterLengths_GemmK0_GemmN_GemmK1 = Sequence<4, 64, 1>; - - constexpr index_t GemmBBlockTransferSrcScalarPerVector_GemmN = 1; - constexpr index_t GemmBBlockTransferDstScalarPerVector_GemmK1 = 8; - - constexpr index_t GemmCThreadTransferDstScalarPerVector = 1; -#endif - - // HACK: hacks that control index calculation when iterating over A, B, C matrix - constexpr auto out_gemmk0_gemmm_gemmk1_grid_step_hacks = make_tuple( - make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0>{}, // 0+: gemmk0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0>{}, // 1+: gemmm - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}), // 2+: gemmk1 - make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0>{}, // 0-: gemmk0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0>{}, // 1-: gemmm - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{})); // 2-: - // gemmk1 - - constexpr auto wei_gemmk0_gemmn_gemmk1_grid_step_hacks = - make_tuple(make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0>{}, // 0+: gemmk0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 1+: gemmn - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}), // 2+: gemmk1 - make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0>{}, // 0-: Gemmk0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 1-: Gemmn - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{})); // 2-: Gemmk1 - - // clang-format off - constexpr auto in_m0_n0_m1_n1_m2_m3_m4_n2_grid_step_hacks = make_tuple( - make_tuple( - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 0+: M0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 1+: N0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 2+: M1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 3+: N1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 4+: M2 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 5+: M3 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 6+: M4 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}), // 7+: N2 - make_tuple( - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 0-: M0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 1-: N0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 2-: M1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 3-: N1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 4-: M2 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 5-: M3 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 6-: M4 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{})); // 7-: N2 - // clang-format on - - constexpr auto out_gemmk0_gemmm_gemmk1_grid_move_slice_window_step_hacks = - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0>{}; - - constexpr auto wei_gemmk0_gemmn_gemmk1_grid_move_slice_window_step_hacks = - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0>{}; - - for(index_t i = 0; i < 5; ++i) - { - const auto ConvStrideH = conv_strides[I0]; - const auto ConvStrideW = conv_strides[I1]; - - const auto ConvDilationH = conv_dilations[I0]; - const auto ConvDilationW = conv_dilations[I1]; - - const auto GcdStrideDilationH = math::gcd(ConvStrideH, ConvDilationH); - const auto GcdStrideDilationW = math::gcd(ConvStrideW, ConvDilationW); - - const auto YTilde = ConvStrideH / GcdStrideDilationH; - const auto XTilde = ConvStrideW / GcdStrideDilationW; - - float ave_time = 0; - - for(index_t i_ytilde = 0; i_ytilde < YTilde; ++i_ytilde) - { - for(index_t i_xtilde = 0; i_xtilde < XTilde; ++i_xtilde) - { - const auto descs = - transform_backward_data_convolution_into_gemm_v4r1r2_nhwc_kyxc_nhwk( - out_n_ho_wo_k_desc, - wei_k_y_x_c_desc, - in_n_hi_wi_c_desc, - conv_strides, - conv_dilations, - in_left_pads, - in_right_pads, - i_ytilde, - i_xtilde, - Number{}); - - const auto out_gemmk0_gemmm_gemmk1_grid_desc = descs[I0]; - const auto wei_gemmk0_gemmn_gemmk1_grid_desc = descs[I1]; - const auto in_gemmm_gemmn_grid_desc = descs[I2]; - - const auto GemmK0 = out_gemmk0_gemmm_gemmk1_grid_desc.GetLength(I0); - - if(GemmK0 != 0) - { - ave_time += driver_gemm_xdlops_v2r3< - BlockSize, - TInWei, - TAcc, - TOut, - InMemoryDataOperationEnum::Set, - decltype(out_gemmk0_gemmm_gemmk1_grid_desc), - decltype(wei_gemmk0_gemmn_gemmk1_grid_desc), - decltype(in_gemmm_gemmn_grid_desc), - GemmMPerBlock, - GemmNPerBlock, - GemmKPerBlock, - GemmMPerWave, - GemmNPerWave, - GemmK1, - MRepeat, - NRepeat, - GemmABlockTransferThreadSliceLengths_GemmK0_GemmM_GemmK1, - GemmABlockTransferThreadClusterLengths_GemmK0_GemmM_GemmK1, - Sequence<1, 0, 2>, - Sequence<1, 0, 2>, - 2, - GemmABlockTransferSrcScalarPerVector_GemmK1, - GemmABlockTransferDstScalarPerVector_GemmK1, - false, // don't move back src coordinate after threadwise copy - GemmBBlockTransferThreadSliceLengths_GemmK0_GemmN_GemmK1, - GemmBBlockTransferThreadClusterLengths_GemmK0_GemmN_GemmK1, - Sequence<2, 0, 1>, - Sequence<0, 2, 1>, - 1, - GemmBBlockTransferSrcScalarPerVector_GemmN, - GemmBBlockTransferDstScalarPerVector_GemmK1, - false, // don't move back src coordinate after threadwise copy -#if 0 - Sequence<0, 2, 4, 5, 6, 1, 3, 7>, -#else - Sequence<0, 1, 2, 3, 4, 5, 6, 7>, -#endif - 7, - GemmCThreadTransferDstScalarPerVector, - decltype(out_gemmk0_gemmm_gemmk1_grid_step_hacks), - decltype(wei_gemmk0_gemmn_gemmk1_grid_step_hacks), - decltype(in_m0_n0_m1_n1_m2_m3_m4_n2_grid_step_hacks), - decltype(out_gemmk0_gemmm_gemmk1_grid_move_slice_window_step_hacks), - decltype(wei_gemmk0_gemmn_gemmk1_grid_move_slice_window_step_hacks), - true, // CAccessOrderMRepeatNRepeat - false, // ABlockLdsExtraM - false // BBlockLdsExtraN - >(static_cast(out_n_ho_wo_k_device_buf.GetDeviceBuffer()), - static_cast(wei_k_y_x_c_device_buf.GetDeviceBuffer()), - static_cast(in_n_hi_wi_c_device_buf.GetDeviceBuffer()), - out_gemmk0_gemmm_gemmk1_grid_desc, - wei_gemmk0_gemmn_gemmk1_grid_desc, - in_gemmm_gemmn_grid_desc, - debug::debug_driver_gemm_xdlops_v2r3::M01, - debug::debug_driver_gemm_xdlops_v2r3::N01, - out_gemmk0_gemmm_gemmk1_grid_step_hacks, - wei_gemmk0_gemmn_gemmk1_grid_step_hacks, - in_m0_n0_m1_n1_m2_m3_m4_n2_grid_step_hacks, - out_gemmk0_gemmm_gemmk1_grid_move_slice_window_step_hacks, - wei_gemmk0_gemmn_gemmk1_grid_move_slice_window_step_hacks, - nrepeat); - } - } - } - - { - const auto N = out_n_ho_wo_k_lengths[I0]; - const auto K = out_n_ho_wo_k_lengths[I3]; - const auto C = wei_k_y_x_c_lengths[I3]; - - const auto Ho = out_n_ho_wo_k_lengths[I1]; - const auto Wo = out_n_ho_wo_k_lengths[I2]; - - const auto Y = wei_k_y_x_c_lengths[I1]; - const auto X = wei_k_y_x_c_lengths[I2]; - - float perf = static_cast((std::size_t(2) * N * K * Ho * Wo * C * Y * X)) / - (std::size_t(1000) * 1000 * 1000) / ave_time; - - std::cout << "Average time : " << ave_time << " ms, " << perf << " TFlop/s" - << std::endl; - } - } - - // copy result back to host - in_n_hi_wi_c_device_buf.FromDevice(in_n_hi_wi_c.mData.data()); -} diff --git a/library/include/ck/library/obselete_driver_offline/device_convolution_backward_data_implicit_gemm_v4r1r2_xdlops_nhwc_kyxc_nhwk_1x1.hpp b/library/include/ck/library/obselete_driver_offline/device_convolution_backward_data_implicit_gemm_v4r1r2_xdlops_nhwc_kyxc_nhwk_1x1.hpp deleted file mode 100644 index 9cc4052f77..0000000000 --- a/library/include/ck/library/obselete_driver_offline/device_convolution_backward_data_implicit_gemm_v4r1r2_xdlops_nhwc_kyxc_nhwk_1x1.hpp +++ /dev/null @@ -1,389 +0,0 @@ -#include -#include "device.hpp" -#include "host_tensor.hpp" -#include "transform_backward_data_convolution_into_gemm_v4r1r2_nhwc_kyxc_nhwk.hpp" -#include "driver_gemm_xdlops_v2r3.hpp" - -template -void device_convolution_backward_data_implicit_gemm_v4r1r2_xdlops_nhwc_kyxc_nhwk_1x1( - const InLengths& in_n_hi_wi_c_lengths, - const WeiLengths& wei_k_y_x_c_lengths, - const OutLengths& out_n_ho_wo_k_lengths, - const ConvStrides& conv_strides, - const ConvDilations&, - const InLeftPads&, - const InRightPads&, - Tensor& in_n_hi_wi_c, - const Tensor& wei_k_y_x_c, - const Tensor& out_n_ho_wo_k, - ck::index_t nrepeat) -{ - using namespace ck; - - std::cout << __func__ << std::endl; - - constexpr auto I0 = Number<0>{}; - constexpr auto I1 = Number<1>{}; - constexpr auto I2 = Number<2>{}; - constexpr auto I3 = Number<3>{}; - - DeviceMem in_n_hi_wi_c_device_buf(sizeof(TInWei) * in_n_hi_wi_c.mDesc.GetElementSpace()); - DeviceMem wei_k_y_x_c_device_buf(sizeof(TInWei) * wei_k_y_x_c.mDesc.GetElementSpace()); - DeviceMem out_n_ho_wo_k_device_buf(sizeof(TOut) * out_n_ho_wo_k.mDesc.GetElementSpace()); - - in_n_hi_wi_c_device_buf.ToDevice(in_n_hi_wi_c.mData.data()); - wei_k_y_x_c_device_buf.ToDevice(wei_k_y_x_c.mData.data()); - out_n_ho_wo_k_device_buf.ToDevice(out_n_ho_wo_k.mData.data()); - - const auto in_n_hi_wi_c_desc = make_naive_tensor_descriptor_packed(in_n_hi_wi_c_lengths); - const auto wei_k_y_x_c_desc = make_naive_tensor_descriptor_packed(wei_k_y_x_c_lengths); - const auto out_n_ho_wo_k_desc = make_naive_tensor_descriptor_packed(out_n_ho_wo_k_lengths); - -#if 0 - // [M, N, K0, K1] = [256, 128, 4, 4], C = 128, for fp32 - constexpr index_t BlockSize = 256; - - constexpr index_t GemmMPerBlock = 256; - constexpr index_t GemmNPerBlock = 128; - constexpr index_t GemmKPerBlock = 4; - - constexpr index_t GemmMPerWave = 32; - constexpr index_t GemmNPerWave = 32; - constexpr index_t GemmK1 = 4; - - constexpr index_t MRepeat = 4; - constexpr index_t NRepeat = 2; - - using GemmABlockTransferThreadSliceLengths_GemmK0_GemmM_GemmK1 = Sequence<1, 4, 4>; - using GemmABlockTransferThreadClusterLengths_GemmK0_GemmM_GemmK1 = Sequence<4, 64, 1>; - - constexpr index_t GemmABlockTransferSrcScalarPerVector_GemmK1 = 4; - constexpr index_t GemmABlockTransferDstScalarPerVector_GemmK1 = 4; - - using GemmBBlockTransferThreadSliceLengths_GemmK0_GemmN_GemmK1 = Sequence<1, 2, 4>; - using GemmBBlockTransferThreadClusterLengths_GemmK0_GemmN_GemmK1 = Sequence<4, 64, 1>; - - constexpr index_t GemmBBlockTransferSrcScalarPerVector_GemmN = 2; - constexpr index_t GemmBBlockTransferDstScalarPerVector_GemmK1 = 4; - - constexpr index_t GemmCThreadTransferDstScalarPerVector = 1; -#elif 0 - // [M, N, K0, K1] = [128, 128, 4, 4], C = 64, for fp32 - constexpr index_t BlockSize = 256; - - constexpr index_t GemmMPerBlock = 128; - constexpr index_t GemmNPerBlock = 128; - constexpr index_t GemmKPerBlock = 4; - - constexpr index_t GemmMPerWave = 32; - constexpr index_t GemmNPerWave = 32; - constexpr index_t GemmK1 = 4; - - constexpr index_t MRepeat = 2; - constexpr index_t NRepeat = 2; - - using GemmABlockTransferThreadSliceLengths_GemmK0_GemmM_GemmK1 = Sequence<1, 2, 4>; - using GemmABlockTransferThreadClusterLengths_GemmK0_GemmM_GemmK1 = Sequence<4, 64, 1>; - - constexpr index_t GemmABlockTransferSrcScalarPerVector_GemmK1 = 4; - constexpr index_t GemmABlockTransferDstScalarPerVector_GemmK1 = 4; - - using GemmBBlockTransferThreadSliceLengths_GemmK0_GemmN_GemmK1 = Sequence<1, 2, 4>; - using GemmBBlockTransferThreadClusterLengths_GemmK0_GemmN_GemmK1 = Sequence<4, 64, 1>; - - constexpr index_t GemmBBlockTransferSrcScalarPerVector_GemmN = 2; - constexpr index_t GemmBBlockTransferDstScalarPerVector_GemmK1 = 4; - - constexpr index_t GemmCThreadTransferDstScalarPerVector = 1; -#elif 0 - // [M, N, K0, K1] = [256, 128, 4, 8], C = 128, for fp16 - constexpr index_t BlockSize = 256; - - constexpr index_t GemmMPerBlock = 256; - constexpr index_t GemmNPerBlock = 128; - constexpr index_t GemmKPerBlock = 4; - - constexpr index_t GemmMPerWave = 32; - constexpr index_t GemmNPerWave = 32; - constexpr index_t GemmK1 = 8; - - constexpr index_t MRepeat = 4; - constexpr index_t NRepeat = 2; - - using GemmABlockTransferThreadSliceLengths_GemmK0_GemmM_GemmK1 = Sequence<1, 4, 8>; - using GemmABlockTransferThreadClusterLengths_GemmK0_GemmM_GemmK1 = Sequence<4, 64, 1>; - - constexpr index_t GemmABlockTransferSrcScalarPerVector_GemmK1 = 8; - constexpr index_t GemmABlockTransferDstScalarPerVector_GemmK1 = 8; - - using GemmBBlockTransferThreadSliceLengths_GemmK0_GemmN_GemmK1 = Sequence<1, 2, 8>; - using GemmBBlockTransferThreadClusterLengths_GemmK0_GemmN_GemmK1 = Sequence<4, 64, 1>; - - constexpr index_t GemmBBlockTransferSrcScalarPerVector_GemmN = 2; - constexpr index_t GemmBBlockTransferDstScalarPerVector_GemmK1 = 8; - - constexpr index_t GemmCThreadTransferDstScalarPerVector = 1; -#elif 1 - // [M, N, K0, K1] = [128, 256, 4, 8], C = 128, for fp16 - constexpr index_t BlockSize = 256; - - constexpr index_t GemmMPerBlock = 128; - constexpr index_t GemmNPerBlock = 256; - constexpr index_t GemmKPerBlock = 4; - - constexpr index_t GemmMPerWave = 32; - constexpr index_t GemmNPerWave = 32; - constexpr index_t GemmK1 = 8; - - constexpr index_t MRepeat = 2; - constexpr index_t NRepeat = 4; - - using GemmABlockTransferThreadSliceLengths_GemmK0_GemmM_GemmK1 = Sequence<1, 2, 8>; - using GemmABlockTransferThreadClusterLengths_GemmK0_GemmM_GemmK1 = Sequence<4, 64, 1>; - - constexpr index_t GemmABlockTransferSrcScalarPerVector_GemmK1 = 8; - constexpr index_t GemmABlockTransferDstScalarPerVector_GemmK1 = 8; - - using GemmBBlockTransferThreadSliceLengths_GemmK0_GemmN_GemmK1 = Sequence<1, 4, 8>; - using GemmBBlockTransferThreadClusterLengths_GemmK0_GemmN_GemmK1 = Sequence<4, 64, 1>; - - constexpr index_t GemmBBlockTransferSrcScalarPerVector_GemmN = 4; - constexpr index_t GemmBBlockTransferDstScalarPerVector_GemmK1 = 8; - - constexpr index_t GemmCThreadTransferDstScalarPerVector = 1; -#elif 0 - // [M, N, K0, K1] = [128, 128, 4, 8], C = 64, for fp16 - constexpr index_t BlockSize = 256; - - constexpr index_t GemmMPerBlock = 128; - constexpr index_t GemmNPerBlock = 128; - constexpr index_t GemmKPerBlock = 4; - - constexpr index_t GemmMPerWave = 32; - constexpr index_t GemmNPerWave = 32; - constexpr index_t GemmK1 = 8; - - constexpr index_t MRepeat = 2; - constexpr index_t NRepeat = 2; - - using GemmABlockTransferThreadSliceLengths_GemmK0_GemmM_GemmK1 = Sequence<1, 2, 8>; - using GemmABlockTransferThreadClusterLengths_GemmK0_GemmM_GemmK1 = Sequence<4, 64, 1>; - - constexpr index_t GemmABlockTransferSrcScalarPerVector_GemmK1 = 8; - constexpr index_t GemmABlockTransferDstScalarPerVector_GemmK1 = 8; - - using GemmBBlockTransferThreadSliceLengths_GemmK0_GemmN_GemmK1 = Sequence<1, 2, 8>; - using GemmBBlockTransferThreadClusterLengths_GemmK0_GemmN_GemmK1 = Sequence<4, 64, 1>; - - constexpr index_t GemmBBlockTransferSrcScalarPerVector_GemmN = 2; - constexpr index_t GemmBBlockTransferDstScalarPerVector_GemmK1 = 8; - - constexpr index_t GemmCThreadTransferDstScalarPerVector = 1; -#elif 0 - // [M, N, K0, K1] = [128, 64, 4, 8], C = 64, for fp16 - constexpr index_t BlockSize = 128; - - constexpr index_t GemmMPerBlock = 128; - constexpr index_t GemmNPerBlock = 64; - constexpr index_t GemmKPerBlock = 4; - - constexpr index_t GemmMPerWave = 32; - constexpr index_t GemmNPerWave = 32; - constexpr index_t GemmK1 = 8; - - constexpr index_t MRepeat = 2; - constexpr index_t NRepeat = 2; - - using GemmABlockTransferThreadSliceLengths_GemmK0_GemmM_GemmK1 = Sequence<1, 4, 8>; - using GemmABlockTransferThreadClusterLengths_GemmK0_GemmM_GemmK1 = Sequence<4, 32, 1>; - - constexpr index_t GemmABlockTransferSrcScalarPerVector_GemmK1 = 8; - constexpr index_t GemmABlockTransferDstScalarPerVector_GemmK1 = 8; - - using GemmBBlockTransferThreadSliceLengths_GemmK0_GemmN_GemmK1 = Sequence<1, 2, 8>; - using GemmBBlockTransferThreadClusterLengths_GemmK0_GemmN_GemmK1 = Sequence<4, 32, 1>; - - constexpr index_t GemmBBlockTransferSrcScalarPerVector_GemmN = 2; - constexpr index_t GemmBBlockTransferDstScalarPerVector_GemmK1 = 8; - - constexpr index_t GemmCThreadTransferDstScalarPerVector = 1; -#elif 0 - // [M, N, K0, K1] = [128, 64, 4, 8], C = 32, for fp16 - constexpr index_t BlockSize = 256; - - constexpr index_t GemmMPerBlock = 128; - constexpr index_t GemmNPerBlock = 64; - constexpr index_t GemmKPerBlock = 4; - - constexpr index_t GemmMPerWave = 32; - constexpr index_t GemmNPerWave = 32; - constexpr index_t GemmK1 = 8; - - constexpr index_t MRepeat = 2; - constexpr index_t NRepeat = 1; - - using GemmABlockTransferThreadSliceLengths_GemmK0_GemmM_GemmK1 = Sequence<1, 2, 8>; - using GemmABlockTransferThreadClusterLengths_GemmK0_GemmM_GemmK1 = Sequence<4, 64, 1>; - - constexpr index_t GemmABlockTransferSrcScalarPerVector_GemmK1 = 8; - constexpr index_t GemmABlockTransferDstScalarPerVector_GemmK1 = 8; - - using GemmBBlockTransferThreadSliceLengths_GemmK0_GemmN_GemmK1 = Sequence<1, 1, 8>; - using GemmBBlockTransferThreadClusterLengths_GemmK0_GemmN_GemmK1 = Sequence<4, 64, 1>; - - constexpr index_t GemmBBlockTransferSrcScalarPerVector_GemmN = 1; - constexpr index_t GemmBBlockTransferDstScalarPerVector_GemmK1 = 8; - - constexpr index_t GemmCThreadTransferDstScalarPerVector = 1; -#endif - - // HACK: hacks that control index calculation when iterating over A, B, C matrix - constexpr auto out_gemmk0_gemmm_gemmk1_grid_step_hacks = - make_tuple(make_tuple(Sequence<0, 0, 0>{}, // 0+: gemmk0 - Sequence<0, 0, 0>{}, // 1+: gemmm - Sequence<0, 0, 0>{}), // 2+: gemmk1 - make_tuple(Sequence<0, 0, 0>{}, // 0-: gemmk0 - Sequence<0, 0, 0>{}, // 1-: gemmm - Sequence<0, 0, 0>{})); // 2-: gemmk1 - - constexpr auto wei_gemmk0_gemmn_gemmk1_grid_step_hacks = - make_tuple(make_tuple(Sequence<0, 0, 0>{}, // 0+: gemmk0 - Sequence<0, 0, 0>{}, // 1+: gemmn - Sequence<0, 0, 0>{}), // 2+: gemmk1 - make_tuple(Sequence<0, 0, 0>{}, // 0-: Gemmk0 - Sequence<0, 0, 0>{}, // 1-: Gemmn - Sequence<0, 0, 0>{})); // 2-: Gemmk1 - - // clang-format off - constexpr auto in_m0_n0_m1_n1_m2_m3_m4_n2_grid_step_hacks = make_tuple( - make_tuple( - Sequence<0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 0+: M0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 1+: N0 - Sequence<0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 2+: M1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 3+: N1 - Sequence<0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 4+: M2 - Sequence<0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 5+: M3 - Sequence<0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 6+: M4 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}), // 7+: N2 - make_tuple( - Sequence<0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 0-: M0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 1-: N0 - Sequence<0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 2-: M1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 3-: N1 - Sequence<0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 4-: M2 - Sequence<0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 5-: M3 - Sequence<0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 6-: M4 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{})); // 7-: N2 - // clang-format on - - constexpr auto out_gemmk0_gemmm_gemmk1_grid_move_slice_window_step_hacks = Sequence<0, 0, 0>{}; - - constexpr auto wei_gemmk0_gemmn_gemmk1_grid_move_slice_window_step_hacks = Sequence<0, 0, 0>{}; - - for(index_t i = 0; i < 5; ++i) - { - const auto descs = transform_backward_data_convolution_into_gemm_v4r1r2_nhwc_kyxc_nhwk_1x1( - out_n_ho_wo_k_desc, - wei_k_y_x_c_desc, - in_n_hi_wi_c_desc, - conv_strides, - Number{}); - - const auto out_gemmk0_gemmm_gemmk1_grid_desc = descs[I0]; - const auto wei_gemmk0_gemmn_gemmk1_grid_desc = descs[I1]; - const auto in_gemmm_gemmn_grid_desc = descs[I2]; - - float ave_time = driver_gemm_xdlops_v2r3< - BlockSize, - TInWei, - TAcc, - TOut, - InMemoryDataOperationEnum::Set, - decltype(out_gemmk0_gemmm_gemmk1_grid_desc), - decltype(wei_gemmk0_gemmn_gemmk1_grid_desc), - decltype(in_gemmm_gemmn_grid_desc), - GemmMPerBlock, - GemmNPerBlock, - GemmKPerBlock, - GemmMPerWave, - GemmNPerWave, - GemmK1, - MRepeat, - NRepeat, - GemmABlockTransferThreadSliceLengths_GemmK0_GemmM_GemmK1, - GemmABlockTransferThreadClusterLengths_GemmK0_GemmM_GemmK1, - Sequence<1, 0, 2>, - Sequence<1, 0, 2>, - 2, - GemmABlockTransferSrcScalarPerVector_GemmK1, - GemmABlockTransferDstScalarPerVector_GemmK1, - false, // don't move back src coordinate after threadwise copy - GemmBBlockTransferThreadSliceLengths_GemmK0_GemmN_GemmK1, - GemmBBlockTransferThreadClusterLengths_GemmK0_GemmN_GemmK1, - Sequence<2, 0, 1>, - Sequence<0, 2, 1>, - 1, - GemmBBlockTransferSrcScalarPerVector_GemmN, - GemmBBlockTransferDstScalarPerVector_GemmK1, - false, // don't move back src coordinate after threadwise copy -#if 0 - Sequence<0, 2, 4, 5, 6, 1, 3, 7>, -#else - Sequence<0, 1, 2, 3, 4, 5, 6, 7>, -#endif - 7, - GemmCThreadTransferDstScalarPerVector, - decltype(out_gemmk0_gemmm_gemmk1_grid_step_hacks), - decltype(wei_gemmk0_gemmn_gemmk1_grid_step_hacks), - decltype(in_m0_n0_m1_n1_m2_m3_m4_n2_grid_step_hacks), - decltype(out_gemmk0_gemmm_gemmk1_grid_move_slice_window_step_hacks), - decltype(wei_gemmk0_gemmn_gemmk1_grid_move_slice_window_step_hacks), - true, // CAccessOrderMRepeatNRepeat - false, // ABlockLdsExtraM - false // BBlockLdsExtraN - >(static_cast(out_n_ho_wo_k_device_buf.GetDeviceBuffer()), - static_cast(wei_k_y_x_c_device_buf.GetDeviceBuffer()), - static_cast(in_n_hi_wi_c_device_buf.GetDeviceBuffer()), - out_gemmk0_gemmm_gemmk1_grid_desc, - wei_gemmk0_gemmn_gemmk1_grid_desc, - in_gemmm_gemmn_grid_desc, - debug::debug_driver_gemm_xdlops_v2r3::M01, - debug::debug_driver_gemm_xdlops_v2r3::N01, - out_gemmk0_gemmm_gemmk1_grid_step_hacks, - wei_gemmk0_gemmn_gemmk1_grid_step_hacks, - in_m0_n0_m1_n1_m2_m3_m4_n2_grid_step_hacks, - out_gemmk0_gemmm_gemmk1_grid_move_slice_window_step_hacks, - wei_gemmk0_gemmn_gemmk1_grid_move_slice_window_step_hacks, - nrepeat); - - { - const auto N = out_n_ho_wo_k_lengths[I0]; - const auto K = out_n_ho_wo_k_lengths[I3]; - const auto C = wei_k_y_x_c_lengths[I3]; - - const auto Ho = out_n_ho_wo_k_lengths[I1]; - const auto Wo = out_n_ho_wo_k_lengths[I2]; - - const auto Y = wei_k_y_x_c_lengths[I1]; - const auto X = wei_k_y_x_c_lengths[I2]; - - float perf = static_cast((std::size_t(2) * N * K * Ho * Wo * C * Y * X)) / - (std::size_t(1000) * 1000 * 1000) / ave_time; - - std::cout << "Average time : " << ave_time << " ms, " << perf << " TFlop/s" - << std::endl; - } - } - - // copy result back to host - in_n_hi_wi_c_device_buf.FromDevice(in_n_hi_wi_c.mData.data()); -} diff --git a/library/include/ck/library/obselete_driver_offline/device_convolution_backward_weight_implicit_gemm_v4r4r2_xdlops_atomic_nchw_kcyx_nkhw.hpp b/library/include/ck/library/obselete_driver_offline/device_convolution_backward_weight_implicit_gemm_v4r4r2_xdlops_atomic_nchw_kcyx_nkhw.hpp deleted file mode 100644 index 993630f3f8..0000000000 --- a/library/include/ck/library/obselete_driver_offline/device_convolution_backward_weight_implicit_gemm_v4r4r2_xdlops_atomic_nchw_kcyx_nkhw.hpp +++ /dev/null @@ -1,256 +0,0 @@ -#include -#include "device.hpp" -#include "host_tensor.hpp" -#include "transform_backward_weight_convolution_into_gemm_v4r4r2_atomic_nchw_kcyx_nkhw.hpp" -#include "driver_gemm_xdlops_v2r4.hpp" - -template -void device_convolution_backward_weight_implicit_gemm_v4r4r2_xdlops_atomic_nchw_kcyx_nkhw( - const InLengths& in_n_c_hi_wi_lengths, - const WeiLengths& wei_k_c_y_x_lengths, - const OutLengths& out_n_k_ho_wo_lengths, - const ConvStrides& conv_strides, - const ConvDilations& conv_dilations, - const InLeftPads& in_left_pads, - const InRightPads& in_right_pads, - const Tensor& in_n_c_hi_wi, - Tensor& wei_k_c_y_x, - const Tensor& out_n_k_ho_wo, - GridSizeType desired_grid_size, - ck::index_t nrepeat) -{ - using namespace ck; - - std::cout << __func__ << std::endl; - - constexpr auto I0 = Number<0>{}; - constexpr auto I1 = Number<1>{}; - constexpr auto I2 = Number<2>{}; - constexpr auto I3 = Number<3>{}; - - DeviceMem in_n_c_hi_wi_device_buf(sizeof(TIn) * in_n_c_hi_wi.mDesc.GetElementSpace()); - DeviceMem wei_k_c_y_x_device_buf(sizeof(TWei) * wei_k_c_y_x.mDesc.GetElementSpace()); - DeviceMem out_n_k_ho_wo_device_buf(sizeof(TOut) * out_n_k_ho_wo.mDesc.GetElementSpace()); - - in_n_c_hi_wi_device_buf.ToDevice(in_n_c_hi_wi.mData.data()); - wei_k_c_y_x_device_buf.ToDevice(wei_k_c_y_x.mData.data()); - out_n_k_ho_wo_device_buf.ToDevice(out_n_k_ho_wo.mData.data()); - - const auto in_n_c_hi_wi_desc = make_naive_tensor_descriptor_packed(in_n_c_hi_wi_lengths); - const auto wei_k_c_y_x_desc = make_naive_tensor_descriptor_packed(wei_k_c_y_x_lengths); - const auto out_n_k_ho_wo_desc = make_naive_tensor_descriptor_packed(out_n_k_ho_wo_lengths); - -#if 1 - // [M, N, K0, K1] = [128, 128, 4, 8] for fp32 - constexpr index_t BlockSize = 256; - - constexpr index_t GemmMPerBlock = 128; - constexpr index_t GemmNPerBlock = 128; - constexpr index_t GemmKPerBlock = 4; - - constexpr index_t GemmMPerWave = 32; - constexpr index_t GemmNPerWave = 32; - constexpr index_t GemmK1 = 8; - - constexpr index_t MRepeat = 2; - constexpr index_t NRepeat = 2; - - using GemmABlockTransferThreadSliceLengths_GemmB_GemmK0_GemmM_GemmK1 = Sequence<1, 1, 2, 8>; - using GemmABlockTransferThreadClusterLengths_GemmB_GemmK0_GemmM_GemmK1 = Sequence<1, 4, 64, 1>; - // using vector load 4, so config's wo*ho must be a multiple of 4 - constexpr index_t GemmABlockTransferSrcScalarPerVector_GemmK1 = 4; - constexpr index_t GemmABlockTransferDstScalarPerVector_GemmK1 = 4; - - using GemmBBlockTransferThreadSliceLengths_GemmB_GemmK0_GemmN_GemmK1 = Sequence<1, 1, 2, 8>; - using GemmBBlockTransferThreadClusterLengths_GemmB_GemmK0_GemmN_GemmK1 = Sequence<1, 4, 64, 1>; - - constexpr index_t GemmBBlockTransferSrcScalarPerVector_GemmN = 1; - constexpr index_t GemmBBlockTransferDstScalarPerVector_GemmK1 = 8; - - constexpr index_t GemmCThreadTransferDstScalarPerVector = 1; -#endif - - const auto N = in_n_c_hi_wi_desc.GetLength(I0); - const auto C = in_n_c_hi_wi_desc.GetLength(I1); - const auto K = out_n_k_ho_wo_desc.GetLength(I1); - - const auto Ho = out_n_k_ho_wo_desc.GetLength(I2); - const auto Wo = out_n_k_ho_wo_desc.GetLength(I3); - - const auto Y = wei_k_c_y_x_desc.GetLength(I2); - const auto X = wei_k_c_y_x_desc.GetLength(I3); - - const auto GemmM = K; - const auto GemmN = Y * X * C; - const auto GemmKTotal = N * Ho * Wo; - - const auto GridMN = GemmM * GemmN / (GemmMPerBlock * GemmNPerBlock); - const index_t GemmKBatch = std::max(desired_grid_size / GridMN, 1); - const index_t GemmK0 = - math::integer_divide_ceil(GemmKTotal, GemmK1 * GemmKPerBlock * GemmKBatch) * GemmKPerBlock; - const index_t GemmKPad = GemmKBatch * GemmK0 * GemmK1; - - std::cout << "GemmKTotal: " << GemmKTotal << " GrideSizeMN: " << GridMN - << " GemmKBatch: " << GemmKBatch << " GemmK0: " << GemmK0 << " gemmKPad: " << GemmKPad - << std::endl; - const auto descs = - transform_backward_weight_convolution_into_gemm_v4r4r2_atomic_nchw_kcyx_nkhw_pad( - wei_k_c_y_x_desc, - in_n_c_hi_wi_desc, - out_n_k_ho_wo_desc, - conv_strides, - conv_dilations, - in_left_pads, - in_right_pads, - Number{}, - GemmKBatch, - GemmKPad); - - const auto out_gemmk0_gemmm_gemmk1_grid_desc = descs[I0]; - const auto in_gemmk0_gemmn_gemmk1_grid_desc = descs[I1]; - const auto wei_gemmm_gemmn_grid_desc = descs[I2]; - - // HACK: hacks that control index calculation when iterating over A, B, C matrix - constexpr auto out_gemmk0_gemmm_gemmk1_grid_step_hacks = - make_tuple(make_tuple(Sequence<0, 0, 1, 0, 0, 0, 0>{}, // 0+: GemmB - Sequence<0, 0, 1, 0, 0, 0, 0>{}, // 1+: GemmK0 - Sequence<0, 0, 0, 0, 0, 0, 0>{}, // 2+: GemmM - Sequence<0, 0, 1, 0, 0, 0, 0>{}), // 3+: GemmK1 - make_tuple(Sequence<0, 0, 2, 0, 0, 0, 0>{}, // 0-: GemB - Sequence<0, 0, 2, 0, 0, 0, 0>{}, // 1-: GemmK0 - Sequence<0, 0, 0, 0, 0, 0, 0>{}, // 2-: GemmM - Sequence<0, 0, 2, 0, 0, 0, 0>{})); // 3-: GemmK1 - - constexpr auto in_gemmk0_gemmn_gemmk1_grid_step_hacks = make_tuple( - make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0>{}, // 0+: GemmB - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0>{}, // 1+: GemmK0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0>{}, // 2+: GemmN - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0>{}), // 3+: GemmK1 - make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0>{}, // 0-: GemmB - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0>{}, // 1-: GemmK0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0>{}, // 2-: GemmN - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0>{})); // 3-: GemmK1 - - constexpr auto wei_m0_n0_m1_n1_m2_m3_m4_n2_grid_step_hacks = - make_tuple(make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 0+: M0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 1+: N0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 2+: M1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 3+: N1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 4+: M2 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 5+: M3 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 6+: M4 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}), // 7+: N2 - make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 0-: M0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 1-: N0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 2-: M1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 3-: N1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 4-: M2 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 5-: M3 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 6-: M4 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{})); // 7-: N2 - - constexpr auto out_gemmk0_gemmm_gemmk1_grid_move_slice_window_step_hacks = - Sequence<0, 0, 1, 0, 0, 0, 0>{}; - - constexpr auto in_gemmk0_gemmn_gemmk1_grid_move_slice_window_step_hacks = - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0>{}; - - const auto driver_gemm_xdlops = - driver_gemm_xdlops_v2r4, - Sequence<0, 2, 1, 3>, - 3, - GemmABlockTransferSrcScalarPerVector_GemmK1, - GemmABlockTransferDstScalarPerVector_GemmK1, - false, // don't move back src coordinate after threadwise copy - GemmBBlockTransferThreadSliceLengths_GemmB_GemmK0_GemmN_GemmK1, - GemmBBlockTransferThreadClusterLengths_GemmB_GemmK0_GemmN_GemmK1, - Sequence<0, 2, 1, 3>, - Sequence<0, 2, 1, 3>, - 3, - GemmBBlockTransferSrcScalarPerVector_GemmN, - GemmBBlockTransferDstScalarPerVector_GemmK1, - false, // don't move back src coordinate after threadwise copy - Sequence<3, 0, 1, 2, 7, 5, 4, 6>, - 7, - GemmCThreadTransferDstScalarPerVector, - decltype(out_gemmk0_gemmm_gemmk1_grid_step_hacks), - decltype(in_gemmk0_gemmn_gemmk1_grid_step_hacks), - decltype(wei_m0_n0_m1_n1_m2_m3_m4_n2_grid_step_hacks), - decltype(out_gemmk0_gemmm_gemmk1_grid_move_slice_window_step_hacks), - decltype(in_gemmk0_gemmn_gemmk1_grid_move_slice_window_step_hacks), - false, - true, - true>; - - for(index_t i = 0; i < 5; ++i) - { - float ave_time = - driver_gemm_xdlops(static_cast(out_n_k_ho_wo_device_buf.GetDeviceBuffer()), - static_cast(in_n_c_hi_wi_device_buf.GetDeviceBuffer()), - static_cast(wei_k_c_y_x_device_buf.GetDeviceBuffer()), - out_gemmk0_gemmm_gemmk1_grid_desc, - in_gemmk0_gemmn_gemmk1_grid_desc, - wei_gemmm_gemmn_grid_desc, - debug::debug_driver_gemm_xdlops_v2r3::M01, - debug::debug_driver_gemm_xdlops_v2r3::N01, - out_gemmk0_gemmm_gemmk1_grid_step_hacks, - in_gemmk0_gemmn_gemmk1_grid_step_hacks, - wei_m0_n0_m1_n1_m2_m3_m4_n2_grid_step_hacks, - out_gemmk0_gemmm_gemmk1_grid_move_slice_window_step_hacks, - in_gemmk0_gemmn_gemmk1_grid_move_slice_window_step_hacks, - nrepeat); - - float perf = static_cast(calculate_convolution_flops( - in_n_c_hi_wi_desc, wei_k_c_y_x_desc, out_n_k_ho_wo_desc)) / - (std::size_t(1000) * 1000 * 1000) / ave_time; - - std::cout << "Average time : " << ave_time << " ms, " << perf << " TFlop/s" << std::endl; - } - - wei_k_c_y_x_device_buf.ToDevice(wei_k_c_y_x.mData.data()); - driver_gemm_xdlops(static_cast(out_n_k_ho_wo_device_buf.GetDeviceBuffer()), - static_cast(in_n_c_hi_wi_device_buf.GetDeviceBuffer()), - static_cast(wei_k_c_y_x_device_buf.GetDeviceBuffer()), - out_gemmk0_gemmm_gemmk1_grid_desc, - in_gemmk0_gemmn_gemmk1_grid_desc, - wei_gemmm_gemmn_grid_desc, - debug::debug_driver_gemm_xdlops_v2r3::M01, - debug::debug_driver_gemm_xdlops_v2r3::N01, - out_gemmk0_gemmm_gemmk1_grid_step_hacks, - in_gemmk0_gemmn_gemmk1_grid_step_hacks, - wei_m0_n0_m1_n1_m2_m3_m4_n2_grid_step_hacks, - out_gemmk0_gemmm_gemmk1_grid_move_slice_window_step_hacks, - in_gemmk0_gemmn_gemmk1_grid_move_slice_window_step_hacks, - 0); - // copy result back to host - wei_k_c_y_x_device_buf.FromDevice(wei_k_c_y_x.mData.data()); -} diff --git a/library/include/ck/library/obselete_driver_offline/device_convolution_backward_weight_implicit_gemm_v4r4r2_xdlops_nchw_kcyx_nkhw.hpp b/library/include/ck/library/obselete_driver_offline/device_convolution_backward_weight_implicit_gemm_v4r4r2_xdlops_nchw_kcyx_nkhw.hpp deleted file mode 100644 index dfb612f690..0000000000 --- a/library/include/ck/library/obselete_driver_offline/device_convolution_backward_weight_implicit_gemm_v4r4r2_xdlops_nchw_kcyx_nkhw.hpp +++ /dev/null @@ -1,234 +0,0 @@ -#include -#include "device.hpp" -#include "host_tensor.hpp" -#include "transform_backward_weight_convolution_into_gemm_v4r4r2_nchw_kcyx_nkhw.hpp" -#include "driver_gemm_xdlops_v2r3.hpp" - -template -void device_convolution_backward_weight_implicit_gemm_v4r4r2_xdlops_nchw_kcyx_nkhw( - const InLengths& in_n_c_hi_wi_lengths, - const WeiLengths& wei_k_c_y_x_lengths, - const OutLengths& out_n_k_ho_wo_lengths, - const ConvStrides& conv_strides, - const ConvDilations& conv_dilations, - const InLeftPads& in_left_pads, - const InRightPads& in_right_pads, - const Tensor& in_n_c_hi_wi, - Tensor& wei_k_c_y_x, - const Tensor& out_n_k_ho_wo, - ck::index_t nrepeat) -{ - using namespace ck; - - std::cout << __func__ << std::endl; - - constexpr auto I0 = Number<0>{}; - constexpr auto I1 = Number<1>{}; - constexpr auto I2 = Number<2>{}; - - DeviceMem in_n_c_hi_wi_device_buf(sizeof(TIn) * in_n_c_hi_wi.mDesc.GetElementSpace()); - DeviceMem wei_k_c_y_x_device_buf(sizeof(TWei) * wei_k_c_y_x.mDesc.GetElementSpace()); - DeviceMem out_n_k_ho_wo_device_buf(sizeof(TOut) * out_n_k_ho_wo.mDesc.GetElementSpace()); - - in_n_c_hi_wi_device_buf.ToDevice(in_n_c_hi_wi.mData.data()); - wei_k_c_y_x_device_buf.ToDevice(wei_k_c_y_x.mData.data()); - out_n_k_ho_wo_device_buf.ToDevice(out_n_k_ho_wo.mData.data()); - - const auto in_n_c_hi_wi_desc = make_naive_tensor_descriptor_packed(in_n_c_hi_wi_lengths); - const auto wei_k_c_y_x_desc = make_naive_tensor_descriptor_packed(wei_k_c_y_x_lengths); - const auto out_n_k_ho_wo_desc = make_naive_tensor_descriptor_packed(out_n_k_ho_wo_lengths); - -#if 0 - // [M, N, K0, K1] = [128, 128, 4, 8] for fp16 - constexpr index_t BlockSize = 256; - - constexpr index_t GemmMPerBlock = 128; - constexpr index_t GemmNPerBlock = 128; - constexpr index_t GemmKPerBlock = 4; - - constexpr index_t GemmMPerWave = 32; - constexpr index_t GemmNPerWave = 32; - constexpr index_t GemmK1 = 8; - - constexpr index_t MRepeat = 2; - constexpr index_t NRepeat = 2; - - using GemmABlockTransferThreadSliceLengths_GemmK0_GemmM_GemmK1 = Sequence<1, 2, 8>; - using GemmABlockTransferThreadClusterLengths_GemmK0_GemmM_GemmK1 = Sequence<4, 64, 1>; - // using vector load 4, so config's wo*ho must be a multiple of 4 - constexpr index_t GemmABlockTransferSrcScalarPerVector_GemmK1 = 4; - constexpr index_t GemmABlockTransferDstScalarPerVector_GemmK1 = 4; - - using GemmBBlockTransferThreadSliceLengths_GemmK0_GemmN_GemmK1 = Sequence<1, 2, 8>; - using GemmBBlockTransferThreadClusterLengths_GemmK0_GemmN_GemmK1 = Sequence<4, 64, 1>; - - constexpr index_t GemmBBlockTransferSrcScalarPerVector_GemmN = 1; - constexpr index_t GemmBBlockTransferDstScalarPerVector_GemmK1 = 8; - - constexpr index_t GemmCThreadTransferDstScalarPerVector = 1; -#elif 1 - // [M, N, K0, K1] = [128, 128, 4, 8] for fp16 - constexpr index_t BlockSize = 256; - - constexpr index_t GemmMPerBlock = 256; - constexpr index_t GemmNPerBlock = 128; - constexpr index_t GemmKPerBlock = 4; - - constexpr index_t GemmMPerWave = 32; - constexpr index_t GemmNPerWave = 32; - constexpr index_t GemmK1 = 8; - - constexpr index_t MRepeat = 4; - constexpr index_t NRepeat = 2; - - using GemmABlockTransferThreadSliceLengths_GemmK0_GemmM_GemmK1 = Sequence<1, 4, 8>; - using GemmABlockTransferThreadClusterLengths_GemmK0_GemmM_GemmK1 = Sequence<4, 64, 1>; - // using vector load 4, so config's wo*ho must be a multiple of 4 - constexpr index_t GemmABlockTransferSrcScalarPerVector_GemmK1 = 4; - constexpr index_t GemmABlockTransferDstScalarPerVector_GemmK1 = 4; - - using GemmBBlockTransferThreadSliceLengths_GemmK0_GemmN_GemmK1 = Sequence<1, 2, 8>; - using GemmBBlockTransferThreadClusterLengths_GemmK0_GemmN_GemmK1 = Sequence<4, 64, 1>; - - constexpr index_t GemmBBlockTransferSrcScalarPerVector_GemmN = 1; - constexpr index_t GemmBBlockTransferDstScalarPerVector_GemmK1 = 8; - - constexpr index_t GemmCThreadTransferDstScalarPerVector = 1; -#endif - - const auto descs = transform_backward_weight_convolution_into_gemm_v4r4r2_nchw_kcyx_nkhw_pad( - wei_k_c_y_x_desc, - in_n_c_hi_wi_desc, - out_n_k_ho_wo_desc, - conv_strides, - conv_dilations, - in_left_pads, - in_right_pads, - Number{}); - - const auto out_gemmk0_gemmm_gemmk1_grid_desc = descs[I0]; - const auto in_gemmk0_gemmn_gemmk1_grid_desc = descs[I1]; - const auto wei_gemmm_gemmn_grid_desc = descs[I2]; - - // HACK: hacks that control index calculation when iterating over A, B, C matrix - constexpr auto out_gemmk0_gemmm_gemmk1_grid_step_hacks = - make_tuple(make_tuple(Sequence<0, 0, 1, 0, 0>{}, // 0+: GemmK0 - Sequence<0, 0, 0, 0, 0>{}, // 1+: GemmM - Sequence<0, 0, 1, 0, 0>{}), // 2+: GemmK1 - make_tuple(Sequence<0, 0, 2, 0, 0>{}, // 0-: GemmK0 - Sequence<0, 0, 0, 0, 0>{}, // 1-: GemmM - Sequence<0, 0, 2, 0, 0>{})); // 2-: GemmK1 - - constexpr auto in_gemmk0_gemmn_gemmk1_grid_step_hacks = - make_tuple(make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0>{}, // 0+: GemmK0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0>{}, // 1+: GemmN - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0>{}), // 2+: GemmK1 - make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0>{}, // 0-: GemmK0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0>{}, // 1-: GemmN - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0>{})); // 2-: GemmK1 - - constexpr auto wei_m0_n0_m1_n1_m2_m3_m4_n2_grid_step_hacks = - make_tuple(make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 0+: M0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 1+: N0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 2+: M1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 3+: N1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 4+: M2 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 5+: M3 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 6+: M4 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}), // 7+: N2 - make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 0-: M0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 1-: N0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 2-: M1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 3-: N1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 4-: M2 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 5-: M3 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 6-: M4 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{})); // 7-: N2 - - constexpr auto out_gemmk0_gemmm_gemmk1_grid_move_slice_window_step_hacks = - Sequence<0, 0, 1, 0, 0>{}; - - constexpr auto in_gemmk0_gemmn_gemmk1_grid_move_slice_window_step_hacks = - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0>{}; - - for(index_t i = 0; i < 5; ++i) - { - float ave_time = driver_gemm_xdlops_v2r3< - BlockSize, - TIn, - TAcc, - TWei, - InMemoryDataOperationEnum::Set, - decltype(out_gemmk0_gemmm_gemmk1_grid_desc), - decltype(in_gemmk0_gemmn_gemmk1_grid_desc), - decltype(wei_gemmm_gemmn_grid_desc), - GemmMPerBlock, - GemmNPerBlock, - GemmKPerBlock, - GemmMPerWave, - GemmNPerWave, - GemmK1, - MRepeat, - NRepeat, - GemmABlockTransferThreadSliceLengths_GemmK0_GemmM_GemmK1, - GemmABlockTransferThreadClusterLengths_GemmK0_GemmM_GemmK1, - Sequence<1, 0, 2>, - Sequence<1, 0, 2>, - 2, - GemmABlockTransferSrcScalarPerVector_GemmK1, - GemmABlockTransferDstScalarPerVector_GemmK1, - false, // don't move back src coordinate after threadwise copy - GemmBBlockTransferThreadSliceLengths_GemmK0_GemmN_GemmK1, - GemmBBlockTransferThreadClusterLengths_GemmK0_GemmN_GemmK1, - Sequence<1, 0, 2>, - Sequence<1, 0, 2>, - 2, - GemmBBlockTransferSrcScalarPerVector_GemmN, - GemmBBlockTransferDstScalarPerVector_GemmK1, - false, // don't move back src coordinate after threadwise copy - Sequence<3, 0, 1, 2, 7, 5, 4, 6>, - 7, - GemmCThreadTransferDstScalarPerVector, - decltype(out_gemmk0_gemmm_gemmk1_grid_step_hacks), - decltype(in_gemmk0_gemmn_gemmk1_grid_step_hacks), - decltype(wei_m0_n0_m1_n1_m2_m3_m4_n2_grid_step_hacks), - decltype(out_gemmk0_gemmm_gemmk1_grid_move_slice_window_step_hacks), - decltype(in_gemmk0_gemmn_gemmk1_grid_move_slice_window_step_hacks), - false, // CAccessOrderMRepeatNRepeat - true, // ABlockLdsExtraM - true // BBlockLdsExtraN - >(static_cast(out_n_k_ho_wo_device_buf.GetDeviceBuffer()), - static_cast(in_n_c_hi_wi_device_buf.GetDeviceBuffer()), - static_cast(wei_k_c_y_x_device_buf.GetDeviceBuffer()), - out_gemmk0_gemmm_gemmk1_grid_desc, - in_gemmk0_gemmn_gemmk1_grid_desc, - wei_gemmm_gemmn_grid_desc, - debug::debug_driver_gemm_xdlops_v2r3::M01, - debug::debug_driver_gemm_xdlops_v2r3::N01, - out_gemmk0_gemmm_gemmk1_grid_step_hacks, - in_gemmk0_gemmn_gemmk1_grid_step_hacks, - wei_m0_n0_m1_n1_m2_m3_m4_n2_grid_step_hacks, - out_gemmk0_gemmm_gemmk1_grid_move_slice_window_step_hacks, - in_gemmk0_gemmn_gemmk1_grid_move_slice_window_step_hacks, - nrepeat); - - float perf = static_cast(calculate_convolution_flops( - in_n_c_hi_wi_desc, wei_k_c_y_x_desc, out_n_k_ho_wo_desc)) / - (std::size_t(1000) * 1000 * 1000) / ave_time; - - std::cout << "Average time : " << ave_time << " ms, " << perf << " TFlop/s" << std::endl; - } - - // copy result back to host - wei_k_c_y_x_device_buf.FromDevice(wei_k_c_y_x.mData.data()); -} diff --git a/library/include/ck/library/obselete_driver_offline/device_convolution_backward_weight_implicit_gemm_v4r4r4_xdlops_atomic_nhwc_kyxc_nhwk.hpp b/library/include/ck/library/obselete_driver_offline/device_convolution_backward_weight_implicit_gemm_v4r4r4_xdlops_atomic_nhwc_kyxc_nhwk.hpp deleted file mode 100644 index 06d0ea684f..0000000000 --- a/library/include/ck/library/obselete_driver_offline/device_convolution_backward_weight_implicit_gemm_v4r4r4_xdlops_atomic_nhwc_kyxc_nhwk.hpp +++ /dev/null @@ -1,288 +0,0 @@ -#include -#include "device.hpp" -#include "host_tensor.hpp" -#include "transform_backward_weight_convolution_into_gemm_v4r4r4_atomic_nhwc_kyxc_nhwk.hpp" -#include "driver_gemm_xdlops_v2r4.hpp" - -template -void device_convolution_backward_weight_implicit_gemm_v4r4r4_xdlops_atomic_nhwc_kyxc_nhwk( - const InLengths& in_n_hi_wi_c_lengths, - const WeiLengths& wei_k_y_x_c_lengths, - const OutLengths& out_n_ho_wo_k_lengths, - const ConvStrides& conv_strides, - const ConvDilations& conv_dilations, - const InLeftPads& in_left_pads, - const InRightPads& in_right_pads, - const Tensor& in_n_hi_wi_c, - Tensor& wei_k_y_x_c, - const Tensor& out_n_ho_wo_k, - GridSizeType desired_grid_size, - ck::index_t nrepeat) -{ - using namespace ck; - - std::cout << __func__ << std::endl; - - constexpr auto I0 = Number<0>{}; - constexpr auto I1 = Number<1>{}; - constexpr auto I2 = Number<2>{}; - constexpr auto I3 = Number<3>{}; - - DeviceMem in_n_hi_wi_c_device_buf(sizeof(TIn) * in_n_hi_wi_c.mDesc.GetElementSpace()); - DeviceMem wei_k_y_x_c_device_buf(sizeof(TWei) * wei_k_y_x_c.mDesc.GetElementSpace()); - DeviceMem out_n_ho_wo_k_device_buf(sizeof(TOut) * out_n_ho_wo_k.mDesc.GetElementSpace()); - - in_n_hi_wi_c_device_buf.ToDevice(in_n_hi_wi_c.mData.data()); - wei_k_y_x_c_device_buf.ToDevice(wei_k_y_x_c.mData.data()); - out_n_ho_wo_k_device_buf.ToDevice(out_n_ho_wo_k.mData.data()); - - const auto in_n_hi_wi_c_desc = make_naive_tensor_descriptor_packed(in_n_hi_wi_c_lengths); - const auto wei_k_y_x_c_desc = make_naive_tensor_descriptor_packed(wei_k_y_x_c_lengths); - const auto out_n_ho_wo_k_desc = make_naive_tensor_descriptor_packed(out_n_ho_wo_k_lengths); - -#if 0 - // [M, N, K0, K1] = [128, 256, 4, 4] for fp32 - constexpr index_t BlockSize = 256; - - constexpr index_t GemmMPerBlock = 128; - constexpr index_t GemmNPerBlock = 256; - constexpr index_t GemmKPerBlock = 4; - - constexpr index_t GemmMPerXDL = 32; - constexpr index_t GemmNPerXDL = 32; - constexpr index_t GemmK1 = 4; - - constexpr index_t MRepeat = 2; - constexpr index_t NRepeat = 4; - - using GemmABlockTransferThreadSliceLengths_GemmK0_GemmM_GemmK1 = Sequence<1, 1, 4, 2>; - using GemmABlockTransferThreadClusterLengths_GemmK0_GemmM_GemmK1 = Sequence<1, 4, 32, 2>; - - constexpr index_t GemmABlockTransferSrcScalarPerVector_GemmM = 4; - constexpr index_t GemmABlockTransferDstScalarPerVector_GemmK1 = 2; - - using GemmBBlockTransferThreadSliceLengths_GemmK0_GemmN_GemmK1 = Sequence<1, 1, 8, 2>; - using GemmBBlockTransferThreadClusterLengths_GemmK0_GemmN_GemmK1 = Sequence<1, 4, 32, 2>; - - constexpr index_t GemmBBlockTransferSrcScalarPerVector_GemmN = 8; - constexpr index_t GemmBBlockTransferDstScalarPerVector_GemmK1 = 2; - - constexpr index_t GemmCThreadTransferDstScalarPerVector = 4; -#elif 1 - // [M, N, K0, K1] = [128, 128, 4, 4] for fp32 - constexpr index_t BlockSize = 256; - - constexpr index_t GemmMPerBlock = 128; - constexpr index_t GemmNPerBlock = 128; - constexpr index_t GemmKPerBlock = 4; - - constexpr index_t GemmMPerXDL = 32; - constexpr index_t GemmNPerXDL = 32; - constexpr index_t GemmK1 = 4; - - constexpr index_t MRepeat = 2; - constexpr index_t NRepeat = 2; - - using GemmABlockTransferThreadSliceLengths_GemmK0_GemmM_GemmK1 = Sequence<1, 1, 4, 2>; - using GemmABlockTransferThreadClusterLengths_GemmK0_GemmM_GemmK1 = Sequence<1, 4, 32, 2>; - - constexpr index_t GemmABlockTransferSrcScalarPerVector_GemmM = 4; - constexpr index_t GemmABlockTransferDstScalarPerVector_GemmK1 = 2; - - using GemmBBlockTransferThreadSliceLengths_GemmK0_GemmN_GemmK1 = Sequence<1, 1, 4, 2>; - using GemmBBlockTransferThreadClusterLengths_GemmK0_GemmN_GemmK1 = Sequence<1, 4, 32, 2>; - - constexpr index_t GemmBBlockTransferSrcScalarPerVector_GemmN = 4; - constexpr index_t GemmBBlockTransferDstScalarPerVector_GemmK1 = 2; - - constexpr index_t GemmCThreadTransferDstScalarPerVector = 1; -#endif - - const auto N = in_n_hi_wi_c_desc.GetLength(I0); - const auto C = in_n_hi_wi_c_desc.GetLength(I3); - const auto K = out_n_ho_wo_k_desc.GetLength(I3); - - const auto Ho = out_n_ho_wo_k_desc.GetLength(I1); - const auto Wo = out_n_ho_wo_k_desc.GetLength(I2); - - const auto Y = wei_k_y_x_c_desc.GetLength(I1); - const auto X = wei_k_y_x_c_desc.GetLength(I2); - - const auto GemmM = Y * X * C; - const auto GemmN = K; - const auto GemmKTotal = N * Ho * Wo; - - const auto GridMN = GemmM * GemmN / (GemmMPerBlock * GemmNPerBlock); - const index_t GemmKBatch = std::max(desired_grid_size / GridMN, 1); - const index_t GemmK0 = - math::integer_divide_ceil(GemmKTotal, GemmK1 * GemmKPerBlock * GemmKBatch) * GemmKPerBlock; - const index_t GemmKPad = GemmKBatch * GemmK0 * GemmK1; - - std::cout << "GemmKTotal: " << GemmKTotal << " GrideSizeMN: " << GridMN - << " GemmKBatch: " << GemmKBatch << " GemmK0: " << GemmK0 << " gemmKPad: " << GemmKPad - << std::endl; - - const auto descs = - transform_backward_weight_convolution_into_gemm_v4r4r4_atomic_nhwc_kyxc_nhwk_pad( - in_n_hi_wi_c_desc, - wei_k_y_x_c_desc, - out_n_ho_wo_k_desc, - conv_strides, - conv_dilations, - in_left_pads, - in_right_pads, - Number{}, - GemmKBatch, - GemmKPad); - - const auto in_gemmkbatch_gemmk0_gemmm_gemmk1_grid_desc = descs[I0]; - const auto out_gemmkbatch_gemmk0_gemmn_gemmk1_grid_desc = descs[I1]; - const auto wei_gemmm_gemmn_grid_desc = descs[I2]; - - // HACK: hacks that control index calculation when iterating over A, B, C matrix - constexpr auto in_gemmkbatch_gemmk0_gemmm_gemmk1_grid_step_hacks = make_tuple( - make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0>{}, // 0+: GemmKBatch - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0>{}, // 1+: GemmK0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0>{}, // 2+: GemmM - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0>{}), // 3+: GemmK1 - make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0>{}, // 0-: GemmKBatch - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0>{}, // 1-: GemmK0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0>{}, // 2-: GemmM - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0>{})); // 3-: GemmK1 - - constexpr auto out_gemmkbatch_gemmk0_gemmn_gemmk1_grid_step_hacks = - make_tuple(make_tuple(Sequence<0, 0, 0, 0, 0>{}, // 0+: GemmK0 - Sequence<0, 0, 0, 0, 0>{}, // 0+: GemmK0 - Sequence<0, 0, 0, 0, 0>{}, // 1+: GemmN - Sequence<0, 0, 0, 0, 0>{}), // 2+: GemmK1 - make_tuple(Sequence<0, 0, 0, 0, 0>{}, // 0+: GemmK0 - Sequence<0, 0, 0, 0, 0>{}, // 0-: GemmK0 - Sequence<0, 0, 0, 0, 0>{}, // 1-: GemmN - Sequence<0, 0, 0, 0, 0>{})); // 2-: GemmK1 - - constexpr auto wei_m0_n0_m1_n1_m2_m3_m4_n2_grid_step_hacks = - make_tuple(make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 0+: M0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 1+: N0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 2+: M1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 3+: N1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 4+: M2 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 5+: M3 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 6+: M4 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}), // 7+: N2 - make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 0-: M0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 1-: N0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 2-: M1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 3-: N1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 4-: M2 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 5-: M3 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 6-: M4 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{})); // 7-: N2 - - constexpr auto in_gemmkbatch_gemmk0_gemmm_gemmk1_grid_move_slice_window_step_hacks = - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0>{}; - - constexpr auto out_gemmkbatch_gemmk0_gemmn_gemmk1_grid_move_slice_window_step_hacks = - Sequence<0, 0, 0, 0, 0>{}; - - const auto driver_gemm_xdlops = driver_gemm_xdlops_v2r4< - BlockSize, - TIn, - TAcc, - TWei, - InMemoryDataOperationEnum::AtomicAdd, - decltype(in_gemmkbatch_gemmk0_gemmm_gemmk1_grid_desc), - decltype(out_gemmkbatch_gemmk0_gemmn_gemmk1_grid_desc), - decltype(wei_gemmm_gemmn_grid_desc), - GemmMPerBlock, - GemmNPerBlock, - GemmKPerBlock, - GemmMPerXDL, - GemmNPerXDL, - GemmK1, - MRepeat, - NRepeat, - GemmABlockTransferThreadSliceLengths_GemmK0_GemmM_GemmK1, - GemmABlockTransferThreadClusterLengths_GemmK0_GemmM_GemmK1, - Sequence<0, 1, 2, 3>, - Sequence<0, 1, 2, 3>, - 2, - GemmABlockTransferSrcScalarPerVector_GemmM, - GemmABlockTransferDstScalarPerVector_GemmK1, - false, // don't move back src coordinate after threadwise copy - GemmBBlockTransferThreadSliceLengths_GemmK0_GemmN_GemmK1, - GemmBBlockTransferThreadClusterLengths_GemmK0_GemmN_GemmK1, - Sequence<0, 1, 2, 3>, - Sequence<0, 1, 2, 3>, - 2, - GemmBBlockTransferSrcScalarPerVector_GemmN, - GemmBBlockTransferDstScalarPerVector_GemmK1, - false, // don't move back src coordinate after threadwise copy - Sequence<2, 3, 0, 1, 7, 5, 4, 6>, - 6, - GemmCThreadTransferDstScalarPerVector, - decltype(in_gemmkbatch_gemmk0_gemmm_gemmk1_grid_step_hacks), - decltype(out_gemmkbatch_gemmk0_gemmn_gemmk1_grid_step_hacks), - decltype(wei_m0_n0_m1_n1_m2_m3_m4_n2_grid_step_hacks), - decltype(in_gemmkbatch_gemmk0_gemmm_gemmk1_grid_move_slice_window_step_hacks), - decltype(out_gemmkbatch_gemmk0_gemmn_gemmk1_grid_move_slice_window_step_hacks), - false, // CAccessOrderMRepeatNRepeat - true, - true>; - - for(index_t i = 0; i < 5; ++i) - { - float ave_time = - driver_gemm_xdlops(static_cast(in_n_hi_wi_c_device_buf.GetDeviceBuffer()), - static_cast(out_n_ho_wo_k_device_buf.GetDeviceBuffer()), - static_cast(wei_k_y_x_c_device_buf.GetDeviceBuffer()), - in_gemmkbatch_gemmk0_gemmm_gemmk1_grid_desc, - out_gemmkbatch_gemmk0_gemmn_gemmk1_grid_desc, - wei_gemmm_gemmn_grid_desc, - debug::debug_driver_gemm_xdlops_v2r3::M01, - debug::debug_driver_gemm_xdlops_v2r3::N01, - in_gemmkbatch_gemmk0_gemmm_gemmk1_grid_step_hacks, - out_gemmkbatch_gemmk0_gemmn_gemmk1_grid_step_hacks, - wei_m0_n0_m1_n1_m2_m3_m4_n2_grid_step_hacks, - in_gemmkbatch_gemmk0_gemmm_gemmk1_grid_move_slice_window_step_hacks, - out_gemmkbatch_gemmk0_gemmn_gemmk1_grid_move_slice_window_step_hacks, - nrepeat); - - { - - float perf = static_cast((std::size_t(2) * N * K * Ho * Wo * C * Y * X)) / - (std::size_t(1000) * 1000 * 1000) / ave_time; - - std::cout << "Average time : " << ave_time << " ms, " << perf << " TFlop/s" - << std::endl; - } - } - - wei_k_y_x_c_device_buf.ToDevice(wei_k_y_x_c.mData.data()); - driver_gemm_xdlops(static_cast(in_n_hi_wi_c_device_buf.GetDeviceBuffer()), - static_cast(out_n_ho_wo_k_device_buf.GetDeviceBuffer()), - static_cast(wei_k_y_x_c_device_buf.GetDeviceBuffer()), - in_gemmkbatch_gemmk0_gemmm_gemmk1_grid_desc, - out_gemmkbatch_gemmk0_gemmn_gemmk1_grid_desc, - wei_gemmm_gemmn_grid_desc, - debug::debug_driver_gemm_xdlops_v2r3::M01, - debug::debug_driver_gemm_xdlops_v2r3::N01, - in_gemmkbatch_gemmk0_gemmm_gemmk1_grid_step_hacks, - out_gemmkbatch_gemmk0_gemmn_gemmk1_grid_step_hacks, - wei_m0_n0_m1_n1_m2_m3_m4_n2_grid_step_hacks, - in_gemmkbatch_gemmk0_gemmm_gemmk1_grid_move_slice_window_step_hacks, - out_gemmkbatch_gemmk0_gemmn_gemmk1_grid_move_slice_window_step_hacks, - 0); - // copy result back to host - wei_k_y_x_c_device_buf.FromDevice(wei_k_y_x_c.mData.data()); -} diff --git a/library/include/ck/library/obselete_driver_offline/device_convolution_backward_weight_implicit_gemm_v4r4r4_xdlops_nhwc_kyxc_nhwk.hpp b/library/include/ck/library/obselete_driver_offline/device_convolution_backward_weight_implicit_gemm_v4r4r4_xdlops_nhwc_kyxc_nhwk.hpp deleted file mode 100644 index 5221ec582d..0000000000 --- a/library/include/ck/library/obselete_driver_offline/device_convolution_backward_weight_implicit_gemm_v4r4r4_xdlops_nhwc_kyxc_nhwk.hpp +++ /dev/null @@ -1,276 +0,0 @@ -#include -#include "device.hpp" -#include "host_tensor.hpp" -#include "transform_backward_weight_convolution_into_gemm_v4r4r4_nhwc_kyxc_nhwk.hpp" -#include "driver_gemm_xdlops_v2r3.hpp" -#include "debug.hpp" - -template -void device_convolution_backward_weight_implicit_gemm_v4r4r4_xdlops_nhwc_kyxc_nhwk( - const InLengths& in_n_hi_wi_c_lengths, - const WeiLengths& wei_k_y_x_c_lengths, - const OutLengths& out_n_ho_wo_k_lengths, - const ConvStrides& conv_strides, - const ConvDilations& conv_dilations, - const InLeftPads& in_left_pads, - const InRightPads& in_right_pads, - const Tensor& in_n_hi_wi_c, - Tensor& wei_k_y_x_c, - const Tensor& out_n_ho_wo_k, - ck::index_t nrepeat) -{ - using namespace ck; - - std::cout << __func__ << std::endl; - - constexpr auto I0 = Number<0>{}; - constexpr auto I1 = Number<1>{}; - constexpr auto I2 = Number<2>{}; - constexpr auto I3 = Number<3>{}; - - DeviceMem in_n_hi_wi_c_device_buf(sizeof(TIn) * in_n_hi_wi_c.mDesc.GetElementSpace()); - DeviceMem wei_k_y_x_c_device_buf(sizeof(TWei) * wei_k_y_x_c.mDesc.GetElementSpace()); - DeviceMem out_n_ho_wo_k_device_buf(sizeof(TOut) * out_n_ho_wo_k.mDesc.GetElementSpace()); - - in_n_hi_wi_c_device_buf.ToDevice(in_n_hi_wi_c.mData.data()); - wei_k_y_x_c_device_buf.ToDevice(wei_k_y_x_c.mData.data()); - out_n_ho_wo_k_device_buf.ToDevice(out_n_ho_wo_k.mData.data()); - - const auto in_n_hi_wi_c_desc = make_naive_tensor_descriptor_packed(in_n_hi_wi_c_lengths); - const auto wei_k_y_x_c_desc = make_naive_tensor_descriptor_packed(wei_k_y_x_c_lengths); - const auto out_n_ho_wo_k_desc = make_naive_tensor_descriptor_packed(out_n_ho_wo_k_lengths); - -#if 0 - // [M, N, K0, K1] = [256, 128, 4, 4] for fp32 - constexpr index_t BlockSize = 256; - - constexpr index_t GemmMPerBlock = 256; - constexpr index_t GemmNPerBlock = 128; - constexpr index_t GemmKPerBlock = 4; - - constexpr index_t GemmMPerXDL = 32; - constexpr index_t GemmNPerXDL = 32; - constexpr index_t GemmK1 = 4; - - constexpr index_t MRepeat = 4; - constexpr index_t NRepeat = 2; - - using GemmABlockTransferThreadSliceLengths_GemmK0_GemmM_GemmK1 = Sequence<1, 4, 4>; - using GemmABlockTransferThreadClusterLengths_GemmK0_GemmM_GemmK1 = Sequence<4, 64, 1>; - - constexpr index_t GemmABlockTransferSrcScalarPerVector_GemmM = 2; - constexpr index_t GemmABlockTransferDstScalarPerVector_GemmK1 = 4; - - using GemmBBlockTransferThreadSliceLengths_GemmK0_GemmN_GemmK1 = Sequence<1, 2, 4>; - using GemmBBlockTransferThreadClusterLengths_GemmK0_GemmN_GemmK1 = Sequence<4, 64, 1>; - - constexpr index_t GemmBBlockTransferSrcScalarPerVector_GemmN = 2; - constexpr index_t GemmBBlockTransferDstScalarPerVector_GemmK1 = 4; - - constexpr index_t GemmCThreadTransferDstScalarPerVector = 1; -#elif 1 - // [M, N, K0, K1] = [128, 128, 4, 4] for fp32 - constexpr index_t BlockSize = 256; - - constexpr index_t GemmMPerBlock = 128; - constexpr index_t GemmNPerBlock = 128; - constexpr index_t GemmKPerBlock = 4; - - constexpr index_t GemmMPerXDL = 32; - constexpr index_t GemmNPerXDL = 32; - constexpr index_t GemmK1 = 4; - - constexpr index_t MRepeat = 2; - constexpr index_t NRepeat = 2; - - using GemmABlockTransferThreadSliceLengths_GemmK0_GemmM_GemmK1 = Sequence<1, 4, 2>; - using GemmABlockTransferThreadClusterLengths_GemmK0_GemmM_GemmK1 = Sequence<4, 32, 2>; - - constexpr index_t GemmABlockTransferSrcScalarPerVector_GemmM = 4; - constexpr index_t GemmABlockTransferDstScalarPerVector_GemmK1 = 2; - - using GemmBBlockTransferThreadSliceLengths_GemmK0_GemmN_GemmK1 = Sequence<1, 4, 2>; - using GemmBBlockTransferThreadClusterLengths_GemmK0_GemmN_GemmK1 = Sequence<4, 32, 2>; - - constexpr index_t GemmBBlockTransferSrcScalarPerVector_GemmN = 4; - constexpr index_t GemmBBlockTransferDstScalarPerVector_GemmK1 = 2; - - constexpr index_t GemmCThreadTransferDstScalarPerVector = 1; - -#elif 0 - // [M, N, K0, K1] = [128, 128, 4, 8] for fp16 - constexpr index_t BlockSize = 256; - - constexpr index_t GemmMPerBlock = 128; - constexpr index_t GemmNPerBlock = 128; - constexpr index_t GemmKPerBlock = 4; - - constexpr index_t GemmMPerXDL = 32; - constexpr index_t GemmNPerXDL = 32; - constexpr index_t GemmK1 = 8; - - constexpr index_t MRepeat = 2; - constexpr index_t NRepeat = 2; - - using GemmABlockTransferThreadSliceLengths_GemmK0_GemmM_GemmK1 = Sequence<1, 4, 4>; - using GemmABlockTransferThreadClusterLengths_GemmK0_GemmM_GemmK1 = Sequence<4, 32, 2>; - - constexpr index_t GemmABlockTransferSrcScalarPerVector_GemmM = 4; - constexpr index_t GemmABlockTransferDstScalarPerVector_GemmK1 = 4; - - using GemmBBlockTransferThreadSliceLengths_GemmK0_GemmN_GemmK1 = Sequence<1, 4, 4>; - using GemmBBlockTransferThreadClusterLengths_GemmK0_GemmN_GemmK1 = Sequence<4, 32, 2>; - - constexpr index_t GemmBBlockTransferSrcScalarPerVector_GemmN = 4; - constexpr index_t GemmBBlockTransferDstScalarPerVector_GemmK1 = 4; - - constexpr index_t GemmCThreadTransferDstScalarPerVector = 1; -#endif - - const auto descs = transform_backward_weight_convolution_into_gemm_v4r4r4_nhwc_kyxc_nhwk_pad( - in_n_hi_wi_c_desc, - wei_k_y_x_c_desc, - out_n_ho_wo_k_desc, - conv_strides, - conv_dilations, - in_left_pads, - in_right_pads, - Number{}); - - const auto in_gemmk0_gemmm_gemmk1_grid_desc = descs[I0]; - const auto out_gemmk0_gemmn_gemmk1_grid_desc = descs[I1]; - const auto wei_gemmm_gemmn_grid_desc = descs[I2]; - - // HACK: hacks that control index calculation when iterating over A, B, C matrix - constexpr auto in_gemmk0_gemmm_gemmk1_grid_step_hacks = - make_tuple(make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0>{}, // 0+: GemmK0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0>{}, // 1+: GemmM - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0>{}), // 2+: GemmK1 - make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0>{}, // 0-: GemmK0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0>{}, // 1-: GemmM - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0>{})); // 2-: GemmK1 - - constexpr auto out_gemmk0_gemmn_gemmk1_grid_step_hacks = - make_tuple(make_tuple(Sequence<0, 0, 0, 0, 0>{}, // 0+: GemmK0 - Sequence<0, 0, 0, 0, 0>{}, // 1+: GemmN - Sequence<0, 0, 0, 0, 0>{}), // 2+: GemmK1 - make_tuple(Sequence<0, 0, 0, 0, 0>{}, // 0-: GemmK0 - Sequence<0, 0, 0, 0, 0>{}, // 1-: GemmN - Sequence<0, 0, 0, 0, 0>{})); // 2-: GemmK1 - - constexpr auto wei_m0_n0_m1_n1_m2_m3_m4_n2_grid_step_hacks = - make_tuple(make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 0+: M0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 1+: N0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 2+: M1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 3+: N1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 4+: M2 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 5+: M3 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 6+: M4 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}), // 7+: N2 - make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 0-: M0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 1-: N0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 2-: M1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 3-: N1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 4-: M2 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 5-: M3 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 6-: M4 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{})); // 7-: N2 - - constexpr auto in_gemmk0_gemmm_gemmk1_grid_move_slice_window_step_hacks = - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0>{}; - - constexpr auto out_gemmk0_gemmn_gemmk1_grid_move_slice_window_step_hacks = - Sequence<0, 0, 0, 0, 0>{}; - - for(index_t i = 0; i < 5; ++i) - { - float ave_time = driver_gemm_xdlops_v2r3< - BlockSize, - TIn, - TAcc, - TWei, - InMemoryDataOperationEnum::Set, - decltype(in_gemmk0_gemmm_gemmk1_grid_desc), - decltype(out_gemmk0_gemmn_gemmk1_grid_desc), - decltype(wei_gemmm_gemmn_grid_desc), - GemmMPerBlock, - GemmNPerBlock, - GemmKPerBlock, - GemmMPerXDL, - GemmNPerXDL, - GemmK1, - MRepeat, - NRepeat, - GemmABlockTransferThreadSliceLengths_GemmK0_GemmM_GemmK1, - GemmABlockTransferThreadClusterLengths_GemmK0_GemmM_GemmK1, - Sequence<0, 2, 1>, - Sequence<0, 2, 1>, - 1, - GemmABlockTransferSrcScalarPerVector_GemmM, - GemmABlockTransferDstScalarPerVector_GemmK1, - false, // don't move back src coordinate after threadwise copy - GemmBBlockTransferThreadSliceLengths_GemmK0_GemmN_GemmK1, - GemmBBlockTransferThreadClusterLengths_GemmK0_GemmN_GemmK1, - Sequence<0, 2, 1>, - Sequence<0, 2, 1>, - 1, - GemmBBlockTransferSrcScalarPerVector_GemmN, - GemmBBlockTransferDstScalarPerVector_GemmK1, - false, // don't move back src coordinate after threadwise copy - Sequence<2, 3, 0, 1, 7, 5, 4, 6>, - 7, - GemmCThreadTransferDstScalarPerVector, - decltype(in_gemmk0_gemmm_gemmk1_grid_step_hacks), - decltype(out_gemmk0_gemmn_gemmk1_grid_step_hacks), - decltype(wei_m0_n0_m1_n1_m2_m3_m4_n2_grid_step_hacks), - decltype(in_gemmk0_gemmm_gemmk1_grid_move_slice_window_step_hacks), - decltype(out_gemmk0_gemmn_gemmk1_grid_move_slice_window_step_hacks), - false, // CAccessOrderMRepeatNRepeat - true, - true>(static_cast(in_n_hi_wi_c_device_buf.GetDeviceBuffer()), - static_cast(out_n_ho_wo_k_device_buf.GetDeviceBuffer()), - static_cast(wei_k_y_x_c_device_buf.GetDeviceBuffer()), - in_gemmk0_gemmm_gemmk1_grid_desc, - out_gemmk0_gemmn_gemmk1_grid_desc, - wei_gemmm_gemmn_grid_desc, - debug::debug_driver_gemm_xdlops_v2r3::M01, - debug::debug_driver_gemm_xdlops_v2r3::N01, - in_gemmk0_gemmm_gemmk1_grid_step_hacks, - out_gemmk0_gemmn_gemmk1_grid_step_hacks, - wei_m0_n0_m1_n1_m2_m3_m4_n2_grid_step_hacks, - in_gemmk0_gemmm_gemmk1_grid_move_slice_window_step_hacks, - out_gemmk0_gemmn_gemmk1_grid_move_slice_window_step_hacks, - nrepeat); - - { - const auto N = out_n_ho_wo_k_lengths[I0]; - const auto K = out_n_ho_wo_k_lengths[I3]; - const auto C = wei_k_y_x_c_lengths[I3]; - - const auto Ho = out_n_ho_wo_k_lengths[I1]; - const auto Wo = out_n_ho_wo_k_lengths[I2]; - - const auto Y = wei_k_y_x_c_lengths[I1]; - const auto X = wei_k_y_x_c_lengths[I2]; - - float perf = static_cast((std::size_t(2) * N * K * Ho * Wo * C * Y * X)) / - (std::size_t(1000) * 1000 * 1000) / ave_time; - - std::cout << "Average time : " << ave_time << " ms, " << perf << " TFlop/s" - << std::endl; - } - } - - // copy result back to host - wei_k_y_x_c_device_buf.FromDevice(wei_k_y_x_c.mData.data()); -} diff --git a/library/include/ck/library/obselete_driver_offline/device_convolution_backward_weight_implicit_gemm_v4r4r5_xdlops_atomic_nhwc_kyxc_nhwk.hpp b/library/include/ck/library/obselete_driver_offline/device_convolution_backward_weight_implicit_gemm_v4r4r5_xdlops_atomic_nhwc_kyxc_nhwk.hpp deleted file mode 100644 index 1bdad6e97b..0000000000 --- a/library/include/ck/library/obselete_driver_offline/device_convolution_backward_weight_implicit_gemm_v4r4r5_xdlops_atomic_nhwc_kyxc_nhwk.hpp +++ /dev/null @@ -1,456 +0,0 @@ -#include -#include "device.hpp" -#include "host_tensor.hpp" -#include "transform_backward_weight_convolution_into_gemm_v4r4r5_nhwc_kyxc_nhwk.hpp" -#include "driver_gemm_xdlops_v2r4.hpp" - -template -void device_convolution_backward_weight_implicit_gemm_v4r4r5_xdlops_atomic_nhwc_kyxc_nhwk( - const InLengths& in_n_hi_wi_c_lengths, - const WeiLengths& wei_k_y_x_c_lengths, - const OutLengths& out_n_ho_wo_k_lengths, - const ConvStrides& conv_strides, - const ConvDilations& conv_dilations, - const InLeftPads& in_left_pads, - const InRightPads& in_right_pads, - const Tensor& in_n_hi_wi_c, - Tensor& wei_k_y_x_c, - const Tensor& out_n_ho_wo_k, - GridSizeType desired_grid_size, - ck::index_t nrepeat) -{ - using namespace ck; - - std::cout << __func__ << std::endl; - - constexpr auto I0 = Number<0>{}; - constexpr auto I1 = Number<1>{}; - constexpr auto I2 = Number<2>{}; - constexpr auto I3 = Number<3>{}; - - DeviceMem in_n_hi_wi_c_device_buf(sizeof(TIn) * in_n_hi_wi_c.mDesc.GetElementSpace()); - DeviceMem wei_k_y_x_c_device_buf(sizeof(TWei) * wei_k_y_x_c.mDesc.GetElementSpace()); - DeviceMem out_n_ho_wo_k_device_buf(sizeof(TOut) * out_n_ho_wo_k.mDesc.GetElementSpace()); - - in_n_hi_wi_c_device_buf.ToDevice(in_n_hi_wi_c.mData.data()); - wei_k_y_x_c_device_buf.ToDevice(wei_k_y_x_c.mData.data()); - out_n_ho_wo_k_device_buf.ToDevice(out_n_ho_wo_k.mData.data()); - - const auto in_n_hi_wi_c_desc = make_naive_tensor_descriptor_packed(in_n_hi_wi_c_lengths); - const auto wei_k_y_x_c_desc = make_naive_tensor_descriptor_packed(wei_k_y_x_c_lengths); - const auto out_n_ho_wo_k_desc = make_naive_tensor_descriptor_packed(out_n_ho_wo_k_lengths); - -#if 0 - // [M, N, K0, K1] = [256, 128, 4, 4], C 128, for fp32 - constexpr index_t BlockSize = 256; - - constexpr index_t GemmMPerBlock = 256; - constexpr index_t GemmNPerBlock = 128; - constexpr index_t GemmKPerBlock = 4; - - constexpr index_t GemmMPerXDL = 32; - constexpr index_t GemmNPerXDL = 32; - constexpr index_t GemmK1 = 4; - - constexpr index_t MRepeat = 4; - constexpr index_t NRepeat = 2; - - using GemmABlockTransferThreadSliceLengths_GemmK0_GemmM_GemmK1 = Sequence<1, 1, 8, 2>; - using GemmABlockTransferThreadClusterLengths_GemmK0_GemmM_GemmK1 = Sequence<1, 4, 32, 2>; - - constexpr index_t GemmABlockTransferSrcScalarPerVector_GemmM = 8; - constexpr index_t GemmABlockTransferDstScalarPerVector_GemmK1 = 2; - - using GemmBBlockTransferThreadSliceLengths_GemmK0_GemmN_GemmK1 = Sequence<1, 1, 4, 2>; - using GemmBBlockTransferThreadClusterLengths_GemmK0_GemmN_GemmK1 = Sequence<1, 4, 32, 2>; - - constexpr index_t GemmBBlockTransferSrcScalarPerVector_GemmN = 4; - constexpr index_t GemmBBlockTransferDstScalarPerVector_GemmK1 = 2; - - constexpr index_t GemmCThreadTransferDstScalarPerVector = 1; -#elif 0 - // [M, N, K0, K1] = [128, 256, 4, 4], C 128, for fp32 - constexpr index_t BlockSize = 256; - - constexpr index_t GemmMPerBlock = 128; - constexpr index_t GemmNPerBlock = 256; - constexpr index_t GemmKPerBlock = 4; - - constexpr index_t GemmMPerXDL = 32; - constexpr index_t GemmNPerXDL = 32; - constexpr index_t GemmK1 = 4; - - constexpr index_t MRepeat = 2; - constexpr index_t NRepeat = 4; - - using GemmABlockTransferThreadSliceLengths_GemmK0_GemmM_GemmK1 = Sequence<1, 1, 4, 2>; - using GemmABlockTransferThreadClusterLengths_GemmK0_GemmM_GemmK1 = Sequence<1, 4, 32, 2>; - - constexpr index_t GemmABlockTransferSrcScalarPerVector_GemmM = 4; - constexpr index_t GemmABlockTransferDstScalarPerVector_GemmK1 = 2; - - using GemmBBlockTransferThreadSliceLengths_GemmK0_GemmN_GemmK1 = Sequence<1, 1, 8, 2>; - using GemmBBlockTransferThreadClusterLengths_GemmK0_GemmN_GemmK1 = Sequence<1, 4, 32, 2>; - - constexpr index_t GemmBBlockTransferSrcScalarPerVector_GemmN = 8; - constexpr index_t GemmBBlockTransferDstScalarPerVector_GemmK1 = 2; - - constexpr index_t GemmCThreadTransferDstScalarPerVector = 1; -#elif 1 - // [M, N, K0, K1] = [128, 128, 4, 4], C 64, for fp32 and fp16 - constexpr index_t BlockSize = 256; - - constexpr index_t GemmMPerBlock = 128; - constexpr index_t GemmNPerBlock = 128; - constexpr index_t GemmKPerBlock = 4; - - constexpr index_t GemmMPerXDL = 32; - constexpr index_t GemmNPerXDL = 32; - constexpr index_t GemmK1 = 4; - - constexpr index_t MRepeat = 2; - constexpr index_t NRepeat = 2; - - using GemmABlockTransferThreadSliceLengths_GemmK0_GemmM_GemmK1 = Sequence<1, 1, 4, 2>; - using GemmABlockTransferThreadClusterLengths_GemmK0_GemmM_GemmK1 = Sequence<1, 4, 32, 2>; - - constexpr index_t GemmABlockTransferSrcScalarPerVector_GemmM = 4; - constexpr index_t GemmABlockTransferDstScalarPerVector_GemmK1 = 2; - - using GemmBBlockTransferThreadSliceLengths_GemmK0_GemmN_GemmK1 = Sequence<1, 1, 4, 2>; - using GemmBBlockTransferThreadClusterLengths_GemmK0_GemmN_GemmK1 = Sequence<1, 4, 32, 2>; - - constexpr index_t GemmBBlockTransferSrcScalarPerVector_GemmN = 4; - constexpr index_t GemmBBlockTransferDstScalarPerVector_GemmK1 = 2; - - constexpr index_t GemmCThreadTransferDstScalarPerVector = 1; -#elif 1 - // [M, N, K0, K1] = [256, 128, 4, 8], C 128, for fp16 - constexpr index_t BlockSize = 256; - - constexpr index_t GemmMPerBlock = 256; - constexpr index_t GemmNPerBlock = 128; - constexpr index_t GemmKPerBlock = 4; - - constexpr index_t GemmMPerXDL = 32; - constexpr index_t GemmNPerXDL = 32; - constexpr index_t GemmK1 = 8; - - constexpr index_t MRepeat = 4; - constexpr index_t NRepeat = 2; - - using GemmABlockTransferThreadSliceLengths_GemmK0_GemmM_GemmK1 = Sequence<1, 1, 16, 2>; - using GemmABlockTransferThreadClusterLengths_GemmK0_GemmM_GemmK1 = Sequence<1, 4, 16, 4>; - - constexpr index_t GemmABlockTransferSrcScalarPerVector_GemmM = 8; - constexpr index_t GemmABlockTransferDstScalarPerVector_GemmK1 = 2; - - using GemmBBlockTransferThreadSliceLengths_GemmK0_GemmN_GemmK1 = Sequence<1, 1, 8, 2>; - using GemmBBlockTransferThreadClusterLengths_GemmK0_GemmN_GemmK1 = Sequence<1, 4, 16, 4>; - - constexpr index_t GemmBBlockTransferSrcScalarPerVector_GemmN = 8; - constexpr index_t GemmBBlockTransferDstScalarPerVector_GemmK1 = 2; - - constexpr index_t GemmCThreadTransferDstScalarPerVector = 1; -#elif 1 - // [M, N, K0, K1] = [128, 128, 4, 8], C 64, for fp16 - constexpr index_t BlockSize = 256; - - constexpr index_t GemmMPerBlock = 128; - constexpr index_t GemmNPerBlock = 128; - constexpr index_t GemmKPerBlock = 4; - - constexpr index_t GemmMPerXDL = 32; - constexpr index_t GemmNPerXDL = 32; - constexpr index_t GemmK1 = 8; - - constexpr index_t MRepeat = 2; - constexpr index_t NRepeat = 2; - - using GemmABlockTransferThreadSliceLengths_GemmK0_GemmM_GemmK1 = Sequence<1, 1, 8, 2>; - using GemmABlockTransferThreadClusterLengths_GemmK0_GemmM_GemmK1 = Sequence<1, 4, 16, 4>; - - constexpr index_t GemmABlockTransferSrcScalarPerVector_GemmM = 8; - constexpr index_t GemmABlockTransferDstScalarPerVector_GemmK1 = 2; - - using GemmBBlockTransferThreadSliceLengths_GemmK0_GemmN_GemmK1 = Sequence<1, 1, 8, 2>; - using GemmBBlockTransferThreadClusterLengths_GemmK0_GemmN_GemmK1 = Sequence<1, 4, 16, 4>; - - constexpr index_t GemmBBlockTransferSrcScalarPerVector_GemmN = 8; - constexpr index_t GemmBBlockTransferDstScalarPerVector_GemmK1 = 2; - - constexpr index_t GemmCThreadTransferDstScalarPerVector = 1; -#elif 0 - // [M, N, K0, K1] = [128, 64, 4, 8], C 64, for fp16 - constexpr index_t BlockSize = 128; - - constexpr index_t GemmMPerBlock = 128; - constexpr index_t GemmNPerBlock = 64; - constexpr index_t GemmKPerBlock = 4; - - constexpr index_t GemmMPerXDL = 32; - constexpr index_t GemmNPerXDL = 32; - constexpr index_t GemmK1 = 8; - - constexpr index_t MRepeat = 2; - constexpr index_t NRepeat = 2; - - using GemmABlockTransferThreadSliceLengths_GemmK0_GemmM_GemmK1 = Sequence<1, 1, 16, 2>; - using GemmABlockTransferThreadClusterLengths_GemmK0_GemmM_GemmK1 = Sequence<1, 4, 8, 4>; - - constexpr index_t GemmABlockTransferSrcScalarPerVector_GemmM = 8; - constexpr index_t GemmABlockTransferDstScalarPerVector_GemmK1 = 2; - - using GemmBBlockTransferThreadSliceLengths_GemmK0_GemmN_GemmK1 = Sequence<1, 1, 8, 2>; - using GemmBBlockTransferThreadClusterLengths_GemmK0_GemmN_GemmK1 = Sequence<1, 4, 8, 4>; - - constexpr index_t GemmBBlockTransferSrcScalarPerVector_GemmN = 8; - constexpr index_t GemmBBlockTransferDstScalarPerVector_GemmK1 = 2; - - constexpr index_t GemmCThreadTransferDstScalarPerVector = 1; -#elif 1 - // [M, N, K0, K1] = [64, 128, 4, 8], C 64, for fp16 - constexpr index_t BlockSize = 128; - - constexpr index_t GemmMPerBlock = 64; - constexpr index_t GemmNPerBlock = 128; - constexpr index_t GemmKPerBlock = 4; - - constexpr index_t GemmMPerXDL = 32; - constexpr index_t GemmNPerXDL = 32; - constexpr index_t GemmK1 = 8; - - constexpr index_t MRepeat = 2; - constexpr index_t NRepeat = 2; - - using GemmABlockTransferThreadSliceLengths_GemmK0_GemmM_GemmK1 = Sequence<1, 1, 8, 2>; - using GemmABlockTransferThreadClusterLengths_GemmK0_GemmM_GemmK1 = Sequence<1, 4, 8, 4>; - - constexpr index_t GemmABlockTransferSrcScalarPerVector_GemmM = 8; - constexpr index_t GemmABlockTransferDstScalarPerVector_GemmK1 = 2; - - using GemmBBlockTransferThreadSliceLengths_GemmK0_GemmN_GemmK1 = Sequence<1, 1, 16, 2>; - using GemmBBlockTransferThreadClusterLengths_GemmK0_GemmN_GemmK1 = Sequence<1, 4, 8, 4>; - - constexpr index_t GemmBBlockTransferSrcScalarPerVector_GemmN = 8; - constexpr index_t GemmBBlockTransferDstScalarPerVector_GemmK1 = 2; - - constexpr index_t GemmCThreadTransferDstScalarPerVector = 1; -#elif 1 - // [M, N, K0, K1] = [64, 64, 4, 8], C 32, for fp16 - constexpr index_t BlockSize = 128; - - constexpr index_t GemmMPerBlock = 64; - constexpr index_t GemmNPerBlock = 64; - constexpr index_t GemmKPerBlock = 4; - - constexpr index_t GemmMPerXDL = 32; - constexpr index_t GemmNPerXDL = 32; - constexpr index_t GemmK1 = 8; - - constexpr index_t MRepeat = 2; - constexpr index_t NRepeat = 1; - - using GemmABlockTransferThreadSliceLengths_GemmK0_GemmM_GemmK1 = Sequence<1, 1, 8, 2>; - using GemmABlockTransferThreadClusterLengths_GemmK0_GemmM_GemmK1 = Sequence<1, 4, 8, 4>; - - constexpr index_t GemmABlockTransferSrcScalarPerVector_GemmM = 8; - constexpr index_t GemmABlockTransferDstScalarPerVector_GemmK1 = 2; - - using GemmBBlockTransferThreadSliceLengths_GemmK0_GemmN_GemmK1 = Sequence<1, 1, 8, 2>; - using GemmBBlockTransferThreadClusterLengths_GemmK0_GemmN_GemmK1 = Sequence<1, 4, 8, 4>; - - constexpr index_t GemmBBlockTransferSrcScalarPerVector_GemmN = 8; - constexpr index_t GemmBBlockTransferDstScalarPerVector_GemmK1 = 2; - - constexpr index_t GemmCThreadTransferDstScalarPerVector = 1; -#endif - - const auto N = in_n_hi_wi_c_desc.GetLength(I0); - const auto C = in_n_hi_wi_c_desc.GetLength(I3); - const auto K = out_n_ho_wo_k_desc.GetLength(I3); - - const auto Ho = out_n_ho_wo_k_desc.GetLength(I1); - const auto Wo = out_n_ho_wo_k_desc.GetLength(I2); - - const auto Y = wei_k_y_x_c_desc.GetLength(I1); - const auto X = wei_k_y_x_c_desc.GetLength(I2); - - const auto GemmM = K; - const auto GemmN = Y * X * C; - const auto GemmKTotal = N * Ho * Wo; - - const auto GridMN = GemmM * GemmN / (GemmMPerBlock * GemmNPerBlock); - const index_t GemmKBatch = std::max(desired_grid_size / GridMN, 1); - const index_t GemmK0 = - math::integer_divide_ceil(GemmKTotal, GemmK1 * GemmKPerBlock * GemmKBatch) * GemmKPerBlock; - const index_t GemmKPad = GemmKBatch * GemmK0 * GemmK1; - - std::cout << "GemmKTotal: " << GemmKTotal << " GrideSizeMN: " << GridMN - << " GemmKBatch: " << GemmKBatch << " GemmK0: " << GemmK0 << " gemmKPad: " << GemmKPad - << std::endl; - - const auto descs = transform_backward_weight_convolution_into_gemm_v4r4r5_nhwc_kyxc_nhwk_pad( - in_n_hi_wi_c_desc, - wei_k_y_x_c_desc, - out_n_ho_wo_k_desc, - conv_strides, - conv_dilations, - in_left_pads, - in_right_pads, - Number{}, - GemmKBatch, - GemmKPad); - - const auto out_gemmkbatch_gemmk0_gemmm_gemmk1_grid_desc = descs[I0]; - const auto in_gemmkbatch_gemmk0_gemmn_gemmk1_grid_desc = descs[I1]; - const auto wei_gemmm_gemmn_grid_desc = descs[I2]; - - // HACK: hacks that control index calculation when iterating over A, B, C matrix - constexpr auto out_gemmkbatch_gemmk0_gemmm_gemmk1_grid_step_hacks = - make_tuple(make_tuple(Sequence<0, 0, 0, 0, 0>{}, // 0+: GemmK0 - Sequence<0, 0, 0, 0, 0>{}, // 0+: GemmK0 - Sequence<0, 0, 0, 0, 0>{}, // 1+: GemmN - Sequence<0, 0, 0, 0, 0>{}), // 2+: GemmK1 - make_tuple(Sequence<0, 0, 0, 0, 0>{}, // 0+: GemmK0 - Sequence<0, 0, 0, 0, 0>{}, // 0-: GemmK0 - Sequence<0, 0, 0, 0, 0>{}, // 1-: GemmN - Sequence<0, 0, 0, 0, 0>{})); // 2-: GemmK1 - - constexpr auto in_gemmkbatch_gemmk0_gemmn_gemmk1_grid_step_hacks = make_tuple( - make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0>{}, // 0+: GemmK0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0>{}, // 0+: GemmK0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0>{}, // 1+: GemmM - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0>{}), // 2+: GemmK1 - make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0>{}, // 0-: GemmK0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0>{}, // 0-: GemmK0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0>{}, // 1-: GemmM - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0>{})); // 2-: GemmK1 - - constexpr auto wei_m0_n0_m1_n1_m2_m3_m4_n2_grid_step_hacks = - make_tuple(make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 0+: M0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 1+: N0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 2+: M1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 3+: N1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 4+: M2 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 5+: M3 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 6+: M4 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}), // 7+: N2 - make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 0-: M0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 1-: N0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 2-: M1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 3-: N1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 4-: M2 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 5-: M3 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 6-: M4 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{})); // 7-: N2 - - constexpr auto out_gemmkbatch_gemmk0_gemmm_gemmk1_grid_move_slice_window_step_hacks = - Sequence<0, 0, 0, 0, 0>{}; - - constexpr auto in_gemmkbatch_gemmk0_gemmn_gemmk1_grid_move_slice_window_step_hacks = - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0>{}; - - const auto driver_gemm_xdlops = driver_gemm_xdlops_v2r4< - BlockSize, - TIn, - TAcc, - TWei, - InMemoryDataOperationEnum::AtomicAdd, - decltype(out_gemmkbatch_gemmk0_gemmm_gemmk1_grid_desc), - decltype(in_gemmkbatch_gemmk0_gemmn_gemmk1_grid_desc), - decltype(wei_gemmm_gemmn_grid_desc), - GemmMPerBlock, - GemmNPerBlock, - GemmKPerBlock, - GemmMPerXDL, - GemmNPerXDL, - GemmK1, - MRepeat, - NRepeat, - GemmABlockTransferThreadSliceLengths_GemmK0_GemmM_GemmK1, - GemmABlockTransferThreadClusterLengths_GemmK0_GemmM_GemmK1, - Sequence<0, 1, 2, 3>, - Sequence<0, 1, 2, 3>, - 2, - GemmABlockTransferSrcScalarPerVector_GemmM, - GemmABlockTransferDstScalarPerVector_GemmK1, - false, // don't move back src coordinate after threadwise copy - GemmBBlockTransferThreadSliceLengths_GemmK0_GemmN_GemmK1, - GemmBBlockTransferThreadClusterLengths_GemmK0_GemmN_GemmK1, - Sequence<0, 1, 2, 3>, - Sequence<0, 1, 3, 2>, - 2, - GemmBBlockTransferSrcScalarPerVector_GemmN, - GemmBBlockTransferDstScalarPerVector_GemmK1, - false, // don't move back src coordinate after threadwise copy - Sequence<2, 3, 0, 1, 7, 5, 4, 6>, - 7, - GemmCThreadTransferDstScalarPerVector, - decltype(out_gemmkbatch_gemmk0_gemmm_gemmk1_grid_step_hacks), - decltype(in_gemmkbatch_gemmk0_gemmn_gemmk1_grid_step_hacks), - decltype(wei_m0_n0_m1_n1_m2_m3_m4_n2_grid_step_hacks), - decltype(out_gemmkbatch_gemmk0_gemmm_gemmk1_grid_move_slice_window_step_hacks), - decltype(in_gemmkbatch_gemmk0_gemmn_gemmk1_grid_move_slice_window_step_hacks), - false, // CAccessOrderMRepeatNRepeat - true, - true>; - - // timing - for(index_t i = 0; i < 5; ++i) - { - float ave_time = - driver_gemm_xdlops(static_cast(out_n_ho_wo_k_device_buf.GetDeviceBuffer()), - static_cast(in_n_hi_wi_c_device_buf.GetDeviceBuffer()), - static_cast(wei_k_y_x_c_device_buf.GetDeviceBuffer()), - out_gemmkbatch_gemmk0_gemmm_gemmk1_grid_desc, - in_gemmkbatch_gemmk0_gemmn_gemmk1_grid_desc, - wei_gemmm_gemmn_grid_desc, - debug::debug_driver_gemm_xdlops_v2r3::M01, - debug::debug_driver_gemm_xdlops_v2r3::N01, - out_gemmkbatch_gemmk0_gemmm_gemmk1_grid_step_hacks, - in_gemmkbatch_gemmk0_gemmn_gemmk1_grid_step_hacks, - wei_m0_n0_m1_n1_m2_m3_m4_n2_grid_step_hacks, - out_gemmkbatch_gemmk0_gemmm_gemmk1_grid_move_slice_window_step_hacks, - in_gemmkbatch_gemmk0_gemmn_gemmk1_grid_move_slice_window_step_hacks, - nrepeat); - - { - float perf = static_cast((std::size_t(2) * N * K * Ho * Wo * C * Y * X)) / - (std::size_t(1000) * 1000 * 1000) / ave_time; - - std::cout << "Average time : " << ave_time << " ms, " << perf << " TFlop/s" - << std::endl; - } - } - - // verification - wei_k_y_x_c_device_buf.ToDevice(wei_k_y_x_c.mData.data()); - driver_gemm_xdlops(static_cast(out_n_ho_wo_k_device_buf.GetDeviceBuffer()), - static_cast(in_n_hi_wi_c_device_buf.GetDeviceBuffer()), - static_cast(wei_k_y_x_c_device_buf.GetDeviceBuffer()), - out_gemmkbatch_gemmk0_gemmm_gemmk1_grid_desc, - in_gemmkbatch_gemmk0_gemmn_gemmk1_grid_desc, - wei_gemmm_gemmn_grid_desc, - debug::debug_driver_gemm_xdlops_v2r3::M01, - debug::debug_driver_gemm_xdlops_v2r3::N01, - out_gemmkbatch_gemmk0_gemmm_gemmk1_grid_step_hacks, - in_gemmkbatch_gemmk0_gemmn_gemmk1_grid_step_hacks, - wei_m0_n0_m1_n1_m2_m3_m4_n2_grid_step_hacks, - out_gemmkbatch_gemmk0_gemmm_gemmk1_grid_move_slice_window_step_hacks, - in_gemmkbatch_gemmk0_gemmn_gemmk1_grid_move_slice_window_step_hacks, - 0); - // copy result back to host - wei_k_y_x_c_device_buf.FromDevice(wei_k_y_x_c.mData.data()); -} diff --git a/library/include/ck/library/obselete_driver_offline/device_convolution_forward_implicit_gemm_v4r4_dlops_nchw_kcyx_nkhw.hpp b/library/include/ck/library/obselete_driver_offline/device_convolution_forward_implicit_gemm_v4r4_dlops_nchw_kcyx_nkhw.hpp deleted file mode 100644 index a9df58bedd..0000000000 --- a/library/include/ck/library/obselete_driver_offline/device_convolution_forward_implicit_gemm_v4r4_dlops_nchw_kcyx_nkhw.hpp +++ /dev/null @@ -1,201 +0,0 @@ -#include -#include "device.hpp" -#include "host_tensor.hpp" -#include "transform_forward_convolution_into_gemm_v4r4_nchw_kcyx_nkhw.hpp" -#include "driver_gemm_dlops_v1r2.hpp" - -template -void device_convolution_forward_implicit_gemm_v4r4_dlops_nchw_kcyx_nkhw( - const InLengths& in_n_c_hi_wi_lengths, - const WeiLengths& wei_k_c_y_x_lengths, - const OutLengths& out_n_k_ho_wo_lengths, - const ConvStrides& conv_strides, - const ConvDilations& conv_dilations, - const InLeftPads& in_left_pads, - const InRightPads& in_right_pads, - const Tensor& in_n_c_hi_wi, - const Tensor& wei_k_c_y_x, - Tensor& out_n_k_ho_wo, - ck::index_t nrepeat) -{ - using namespace ck; - - std::cout << __func__ << std::endl; - - constexpr auto I0 = Number<0>{}; - constexpr auto I1 = Number<1>{}; - constexpr auto I2 = Number<2>{}; - - DeviceMem in_n_c_hi_wi_device_buf(sizeof(TInWei) * in_n_c_hi_wi.mDesc.GetElementSpace()); - DeviceMem wei_k_c_y_x_device_buf(sizeof(TInWei) * wei_k_c_y_x.mDesc.GetElementSpace()); - DeviceMem out_n_k_ho_wo_device_buf(sizeof(TOut) * out_n_k_ho_wo.mDesc.GetElementSpace()); - - in_n_c_hi_wi_device_buf.ToDevice(in_n_c_hi_wi.mData.data()); - wei_k_c_y_x_device_buf.ToDevice(wei_k_c_y_x.mData.data()); - out_n_k_ho_wo_device_buf.ToDevice(out_n_k_ho_wo.mData.data()); - - const auto in_n_c_hi_wi_desc = make_naive_tensor_descriptor_packed(in_n_c_hi_wi_lengths); - const auto wei_k_c_y_x_desc = make_naive_tensor_descriptor_packed(wei_k_c_y_x_lengths); - const auto out_n_k_ho_wo_desc = make_naive_tensor_descriptor_packed(out_n_k_ho_wo_lengths); - -#if 1 - // cdata = 64, BlockSize = 256, 128x128x8 - constexpr index_t BlockSize = 256; - - constexpr index_t GemmMPerBlockM1 = 128; - constexpr index_t GemmNPerBlockN1 = 128; - constexpr index_t GemmKPerBlock = 8; - - constexpr index_t GemmM1PerThreadM111 = 4; - constexpr index_t GemmN1PerThreadN111 = 4; - constexpr index_t GemmKPerThread = 1; - - constexpr index_t GemmM11N11ThreadClusterM1100 = 8; - constexpr index_t GemmM11N11ThreadClusterN1100 = 8; - constexpr index_t GemmM11N11ThreadClusterM1101 = 2; - constexpr index_t GemmM11N11ThreadClusterN1101 = 2; - - using GemmABlockTransferThreadSliceLengths_K_M0_M1 = Sequence<4, 1, 1>; - using GemmABlockTransferThreadClusterLengths_K_M0_M1 = Sequence<2, 1, 128>; - - constexpr index_t GemmABlockTransferSrcScalarPerVector_K = 4; - constexpr index_t GemmABlockTransferDstScalarPerVector_M1 = 1; - - using GemmBBlockTransferThreadSliceLengths_K_N0_N1 = Sequence<4, 1, 1>; - using GemmBBlockTransferThreadClusterLengths_K_N0_N1 = Sequence<2, 1, 128>; - - constexpr index_t GemmBBlockTransferSrcScalarPerVector_N1 = 1; - constexpr index_t GemmBBlockTransferDstScalarPerVector_N1 = 1; - - constexpr index_t GemmCThreadTransferDstScalarPerVector_N11 = 1; -#endif - - const auto descs = - transform_forward_convolution_into_gemm_v4r4_nchw_kcyx_nkhw_pad(wei_k_c_y_x_desc, - in_n_c_hi_wi_desc, - out_n_k_ho_wo_desc, - conv_strides, - conv_dilations, - in_left_pads, - in_right_pads); - - // HACK: hacks that control index calculation when iterating over A, B, C matrix - constexpr auto wei_gemmk_gemmm0_gemmn1_grid_step_hacks = - make_tuple(make_tuple(Sequence<0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0>{}), - make_tuple(Sequence<0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0>{})); - - constexpr auto in_gemmk_gemmn0_gemmn1_grid_step_hacks = - make_tuple(make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0>{}), - make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0>{})); - - constexpr auto out_gemmm0_gemmm10_gemmm11_gemmn0_gemmn10_gemmn11_grid_step_hacks = - make_tuple(make_tuple(Sequence<0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0>{}, - Sequence<0, 0, 1, 0, 0>{}, - Sequence<0, 0, 1, 0, 0>{}, - Sequence<0, 0, 1, 0, 0>{}), - make_tuple(Sequence<0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0>{}, - Sequence<0, 0, 2, 0, 0>{}, - Sequence<0, 0, 2, 0, 0>{}, - Sequence<0, 0, 2, 0, 0>{})); - - constexpr auto wei_gemmk_gemmm0_gemmm1_grid_move_slice_window_step_hacks = - Sequence<0, 0, 0, 0, 0>{}; - - constexpr auto in_gemmk_gemmn0_gemmn1_grid_move_slice_window_step_hacks = - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0>{}; - - const auto wei_gemmk_gemmm_grid_desc = descs[I0]; - const auto in_gemmk_gemmn_grid_desc = descs[I1]; - const auto out_gemmm_gemmn_grid_desc = descs[I2]; - - for(index_t i = 0; i < 5; ++i) - { - float ave_time = driver_gemm_dlops_v1r2< - BlockSize, - TInWei, - TAcc, - TOut, - InMemoryDataOperationEnum::Set, - decltype(wei_gemmk_gemmm_grid_desc), - decltype(in_gemmk_gemmn_grid_desc), - decltype(out_gemmm_gemmn_grid_desc), - GemmMPerBlockM1, - GemmNPerBlockN1, - GemmKPerBlock, - GemmM1PerThreadM111, - GemmN1PerThreadN111, - GemmKPerThread, - GemmM11N11ThreadClusterM1100, - GemmM11N11ThreadClusterN1100, - GemmM11N11ThreadClusterM1101, - GemmM11N11ThreadClusterN1101, - GemmABlockTransferThreadSliceLengths_K_M0_M1, - GemmABlockTransferThreadClusterLengths_K_M0_M1, - Sequence<2, 1, 0>, // ABlockTransferThreadClusterArrangeOrder - Sequence<2, 1, 0>, // ABlockTransferSrcAccessOrder - 0, // ABlockTransferSrcVectorDim - GemmABlockTransferSrcScalarPerVector_K, - GemmABlockTransferDstScalarPerVector_M1, - false, // don't move back src coordinate after threadwise copy - GemmBBlockTransferThreadSliceLengths_K_N0_N1, - GemmBBlockTransferThreadClusterLengths_K_N0_N1, - Sequence<0, 1, 2>, // BBlockTransferThreadClusterArrangeOrder - Sequence<0, 1, 2>, // BBlockTransferSrcAccessOrder - 2, // BBlockTransferSrcVectorDim - GemmBBlockTransferSrcScalarPerVector_N1, - GemmBBlockTransferDstScalarPerVector_N1, - false, // don't move back src coordinate after threadwise copy - Sequence<3, 4, 5, 0, 1, 2>, // CThreadTransferSrcDstAccessOrder - 5, // CThreadTransferSrcDstVectorDim - GemmCThreadTransferDstScalarPerVector_N11, - decltype(wei_gemmk_gemmm0_gemmn1_grid_step_hacks), - decltype(in_gemmk_gemmn0_gemmn1_grid_step_hacks), - decltype(out_gemmm0_gemmm10_gemmm11_gemmn0_gemmn10_gemmn11_grid_step_hacks), - decltype(wei_gemmk_gemmm0_gemmm1_grid_move_slice_window_step_hacks), - decltype(in_gemmk_gemmn0_gemmn1_grid_move_slice_window_step_hacks)>( - static_cast(wei_k_c_y_x_device_buf.GetDeviceBuffer()), - static_cast(in_n_c_hi_wi_device_buf.GetDeviceBuffer()), - static_cast(out_n_k_ho_wo_device_buf.GetDeviceBuffer()), - wei_gemmk_gemmm_grid_desc, - in_gemmk_gemmn_grid_desc, - out_gemmm_gemmn_grid_desc, - wei_gemmk_gemmm0_gemmn1_grid_step_hacks, - in_gemmk_gemmn0_gemmn1_grid_step_hacks, - out_gemmm0_gemmm10_gemmm11_gemmn0_gemmn10_gemmn11_grid_step_hacks, - wei_gemmk_gemmm0_gemmm1_grid_move_slice_window_step_hacks, - in_gemmk_gemmn0_gemmn1_grid_move_slice_window_step_hacks, - nrepeat); - - float perf = static_cast(calculate_convolution_flops( - in_n_c_hi_wi_desc, wei_k_c_y_x_desc, out_n_k_ho_wo_desc)) / - (std::size_t(1000) * 1000 * 1000) / ave_time; - - std::cout << "Average time : " << ave_time << " ms, " << perf << " TFlop/s" << std::endl; - } - - // copy result back to host - out_n_k_ho_wo_device_buf.FromDevice(out_n_k_ho_wo.mData.data()); -} diff --git a/library/include/ck/library/obselete_driver_offline/device_convolution_forward_implicit_gemm_v4r4r2_dlops_nhwc_kyxc_nhwk.hpp b/library/include/ck/library/obselete_driver_offline/device_convolution_forward_implicit_gemm_v4r4r2_dlops_nhwc_kyxc_nhwk.hpp deleted file mode 100644 index 843df27a88..0000000000 --- a/library/include/ck/library/obselete_driver_offline/device_convolution_forward_implicit_gemm_v4r4r2_dlops_nhwc_kyxc_nhwk.hpp +++ /dev/null @@ -1,273 +0,0 @@ -#include -#include "device.hpp" -#include "host_tensor.hpp" -#include "transform_forward_convolution_into_gemm_v4r4r4_nhwc_kyxc_nhwk.hpp" -#include "driver_gemm_dlops_v1r3.hpp" - -template -void device_convolution_forward_implicit_gemm_v4r4r2_dlops_nhwc_kyxc_nhwk( - const InLengths& in_n_hi_wi_c_lengths, - const WeiLengths& wei_k_y_x_c_lengths, - const OutLengths& out_n_ho_wo_k_lengths, - const ConvStrides& conv_strides, - const ConvDilations& conv_dilations, - const InLeftPads& in_left_pads, - const InRightPads& in_right_pads, - const Tensor& in_n_hi_wi_c, - const Tensor& wei_k_y_x_c, - Tensor& out_n_ho_wo_k, - ck::index_t nrepeat) -{ - using namespace ck; - - std::cout << __func__ << std::endl; - - constexpr auto I0 = Number<0>{}; - constexpr auto I1 = Number<1>{}; - constexpr auto I2 = Number<2>{}; - constexpr auto I3 = Number<3>{}; - - DeviceMem in_n_hi_wi_c_device_buf(sizeof(TInWei) * in_n_hi_wi_c.mDesc.GetElementSpace()); - DeviceMem wei_k_y_x_c_device_buf(sizeof(TInWei) * wei_k_y_x_c.mDesc.GetElementSpace()); - DeviceMem out_n_ho_wo_k_device_buf(sizeof(TOut) * out_n_ho_wo_k.mDesc.GetElementSpace()); - - in_n_hi_wi_c_device_buf.ToDevice(in_n_hi_wi_c.mData.data()); - wei_k_y_x_c_device_buf.ToDevice(wei_k_y_x_c.mData.data()); - out_n_ho_wo_k_device_buf.ToDevice(out_n_ho_wo_k.mData.data()); - - const auto in_n_hi_wi_c_desc = make_naive_tensor_descriptor_packed(in_n_hi_wi_c_lengths); - const auto wei_k_y_x_c_desc = make_naive_tensor_descriptor_packed(wei_k_y_x_c_lengths); - const auto out_n_ho_wo_k_desc = make_naive_tensor_descriptor_packed(out_n_ho_wo_k_lengths); - -#if 0 - // [M, N, K0, K1] = [128, 128, 8, 1] for fp32 - // cdata = 64, BlockSize = 256 - constexpr index_t BlockSize = 256; - - constexpr index_t GemmMPerBlockM1 = 128; - constexpr index_t GemmNPerBlockN1 = 128; - constexpr index_t GemmKPerBlock = 8; - constexpr index_t GemmK1 = 1; - - constexpr index_t GemmM1PerThreadM111 = 4; - constexpr index_t GemmN1PerThreadN111 = 4; - constexpr index_t GemmKPerThread = 1; - - using GemmM11N11ThreadClusterM110Xs = Sequence<8, 2>; - using GemmM11N11ThreadClusterN110Xs = Sequence<8, 2>; - - using GemmABlockTransferThreadSliceLengths_K0_M0_M1_K1 = Sequence<4, 1, 1, 1>; - using GemmABlockTransferThreadClusterLengths_K0_M0_M1_K1 = Sequence<2, 1, 128, 1>; - - using GemmABlockTransferSrcVectorTensorLengths_K0_M0_M1_K1 = Sequence<4, 1, 1, 1>; - using GemmABlockTransferDstVectorTensorLengths_K0_M0_M1_K1 = Sequence<1, 1, 1, 1>; - - using GemmBBlockTransferThreadSliceLengths_K0_N0_N1_K1 = Sequence<4, 1, 1, 1>; - using GemmBBlockTransferThreadClusterLengths_K0_N0_N1_K1 = Sequence<2, 1, 128, 1>; - - using GemmBBlockTransferSrcVectorTensorLengths_K0_N0_N1_K1 = Sequence<4, 1, 1, 1>; - using GemmBBlockTransferDstVectorTensorLengths_K0_N0_N1_K1 = Sequence<1, 1, 1, 1>; - - constexpr index_t GemmCThreadTransferDstScalarPerVector_N11 = 4; -#elif 1 - // [M, N, K0, K1] = [128, 128, 8, 2] for fp16 - // cdata = 64, BlockSize = 256 - constexpr index_t BlockSize = 256; - - constexpr index_t GemmMPerBlockM1 = 128; - constexpr index_t GemmNPerBlockN1 = 128; - constexpr index_t GemmKPerBlock = 8; - constexpr index_t GemmK1 = 2; - - constexpr index_t GemmM1PerThreadM111 = 4; - constexpr index_t GemmN1PerThreadN111 = 4; - constexpr index_t GemmKPerThread = 1; - - using GemmM11N11ThreadClusterM110Xs = Sequence<8, 2>; - using GemmM11N11ThreadClusterN110Xs = Sequence<8, 2>; - - using GemmABlockTransferThreadSliceLengths_K0_M0_M1_K1 = Sequence<4, 1, 1, 2>; - using GemmABlockTransferThreadClusterLengths_K0_M0_M1_K1 = Sequence<2, 1, 128, 1>; - - using GemmABlockTransferSrcVectorTensorLengths_K0_M0_M1_K1 = Sequence<4, 1, 1, 2>; - using GemmABlockTransferDstVectorTensorLengths_K0_M0_M1_K1 = Sequence<1, 1, 1, 2>; - - using GemmBBlockTransferThreadSliceLengths_K0_N0_N1_K1 = Sequence<4, 1, 1, 2>; - using GemmBBlockTransferThreadClusterLengths_K0_N0_N1_K1 = Sequence<2, 1, 128, 1>; - - using GemmBBlockTransferSrcVectorTensorLengths_K0_N0_N1_K1 = Sequence<4, 1, 1, 2>; - using GemmBBlockTransferDstVectorTensorLengths_K0_N0_N1_K1 = Sequence<1, 1, 1, 2>; - - constexpr index_t GemmCThreadTransferDstScalarPerVector_N11 = 4; -#elif 1 - // [M, N, K0, K1] = [128, 128, 8, 4] for i8 - // cdata = 64, BlockSize = 256 - constexpr index_t BlockSize = 256; - - constexpr index_t GemmMPerBlockM1 = 128; - constexpr index_t GemmNPerBlockN1 = 128; - constexpr index_t GemmKPerBlock = 8; - constexpr index_t GemmK1 = 4; - - constexpr index_t GemmM1PerThreadM111 = 4; - constexpr index_t GemmN1PerThreadN111 = 4; - constexpr index_t GemmKPerThread = 1; - - using GemmM11N11ThreadClusterM110Xs = Sequence<8, 2>; - using GemmM11N11ThreadClusterN110Xs = Sequence<8, 2>; - - using GemmABlockTransferThreadSliceLengths_K0_M0_M1_K1 = Sequence<4, 1, 1, 4>; - using GemmABlockTransferThreadClusterLengths_K0_M0_M1_K1 = Sequence<2, 1, 128, 1>; - - using GemmABlockTransferSrcVectorTensorLengths_K0_M0_M1_K1 = Sequence<4, 1, 1, 4>; - using GemmABlockTransferDstVectorTensorLengths_K0_M0_M1_K1 = Sequence<1, 1, 1, 4>; - - using GemmBBlockTransferThreadSliceLengths_K0_N0_N1_K1 = Sequence<4, 1, 1, 4>; - using GemmBBlockTransferThreadClusterLengths_K0_N0_N1_K1 = Sequence<2, 1, 128, 1>; - - using GemmBBlockTransferSrcVectorTensorLengths_K0_N0_N1_K1 = Sequence<4, 1, 1, 4>; - using GemmBBlockTransferDstVectorTensorLengths_K0_N0_N1_K1 = Sequence<1, 1, 1, 4>; - - constexpr index_t GemmCThreadTransferDstScalarPerVector_N11 = 4; -#endif - - const auto descs = - transform_forward_convolution_into_gemm_v4r4r4_nhwc_kyxc_nhwk(in_n_hi_wi_c_desc, - wei_k_y_x_c_desc, - out_n_ho_wo_k_desc, - conv_strides, - conv_dilations, - in_left_pads, - in_right_pads, - Number{}); - - const auto in_gemmk0_gemmm_gemmk1_grid_desc = descs[I0]; - const auto wei_gemmk0_gemmn_gemmk1_grid_desc = descs[I1]; - const auto out_gemmm_gemmn_grid_desc = descs[I2]; - - // HACK: hacks that control index calculation when iterating over A, B, C matrix - constexpr auto in_gemmk0_gemmm0_gemmm1_gemmk1_grid_step_hacks = make_tuple( - make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0>{}, // 0+: GemmK0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0>{}, // 1+: GemmM0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0>{}, // 2+: GemmM1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0>{}), // 3+: GemmK1 - make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0>{}, // 0-: GemmK0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0>{}, // 1-: GemmM0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0>{}, // 3-: GemmM1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0>{})); // 3-: GemmK1 - - constexpr auto wei_gemmk0_gemmn0_gemmn1_gemmk1_grid_step_hacks = - make_tuple(make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0>{}, // 0+: GemmK0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0>{}, // 1+: GemmN0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0>{}, // 2+: GemmN1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0>{}), // 3+: GemmK1 - make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0>{}, // 0-: GemmK0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0>{}, // 1-: GemmN0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0>{}, // 2-: GemmN1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0>{})); // 3-: GemmK1 - - constexpr auto out_gemmm0_gemmm10_gemmm11_gemmn0_gemmn10_gemmn11_grid_step_hacks = - make_tuple(make_tuple(Sequence<0, 0, 0, 0, 0>{}, // 0+: GemmM0 - Sequence<0, 0, 0, 0, 0>{}, // 1+: GemmM10 - Sequence<0, 0, 0, 0, 0>{}, // 2+: GemmM11 - Sequence<0, 0, 0, 0, 0>{}, // 3+: GemmN0 - Sequence<0, 0, 0, 0, 0>{}, // 4+: GemmN10 - Sequence<0, 0, 0, 0, 0>{}), // 5+: GemmN11 - make_tuple(Sequence<0, 0, 0, 0, 0>{}, // 0-: GemmM0 - Sequence<0, 0, 0, 0, 0>{}, // 1-: GemmM10 - Sequence<0, 0, 0, 0, 0>{}, // 2-: GemmM11 - Sequence<0, 0, 0, 0, 0>{}, // 3-: GemmN0 - Sequence<0, 0, 0, 0, 0>{}, // 4-: GemmN10 - Sequence<0, 0, 0, 0, 0>{})); // 5-: GemmN11 - - constexpr auto in_gemmk0_gemmm0_gemmm1_gemmk1_grid_move_slice_window_step_hacks = - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0>{}; - - constexpr auto wei_gemmk0_gemmn0_gemmn1_gemmk1_grid_move_slice_window_step_hacks = - Sequence<0, 0, 0, 0, 0, 0, 0, 0>{}; - - for(index_t i = 0; i < 5; ++i) - { - float ave_time = driver_gemm_dlops_v1r3< - BlockSize, - TInWei, - TAcc, - TOut, - InMemoryDataOperationEnum::Set, - decltype(in_gemmk0_gemmm_gemmk1_grid_desc), - decltype(wei_gemmk0_gemmn_gemmk1_grid_desc), - decltype(out_gemmm_gemmn_grid_desc), - GemmMPerBlockM1, - GemmNPerBlockN1, - GemmKPerBlock, - GemmM1PerThreadM111, - GemmN1PerThreadN111, - GemmKPerThread, - GemmM11N11ThreadClusterM110Xs, - GemmM11N11ThreadClusterN110Xs, - GemmABlockTransferThreadSliceLengths_K0_M0_M1_K1, - GemmABlockTransferThreadClusterLengths_K0_M0_M1_K1, - Sequence<1, 2, 0, 3>, // ABlockTransferThreadClusterArrangeOrder - Sequence<1, 2, 0, 3>, // ABlockTransferSrcAccessOrder - GemmABlockTransferSrcVectorTensorLengths_K0_M0_M1_K1, - Sequence<1, 2, 0, 3>, // ABlockTransferSrcVectorTensorContiguousDimOrder - GemmABlockTransferDstVectorTensorLengths_K0_M0_M1_K1, - GemmBBlockTransferThreadSliceLengths_K0_N0_N1_K1, - GemmBBlockTransferThreadClusterLengths_K0_N0_N1_K1, - Sequence<1, 2, 0, 3>, // BBlockTransferThreadClusterArrangeOrder - Sequence<1, 2, 0, 3>, // BBlockTransferSrcAccessOrder - GemmBBlockTransferSrcVectorTensorLengths_K0_N0_N1_K1, - Sequence<1, 2, 0, 3>, // BBlockTransferSrcVectorTensorContiguousDimOrder - GemmBBlockTransferDstVectorTensorLengths_K0_N0_N1_K1, - Sequence<0, 1, 2, 3, 4, 5>, // CThreadTransferSrcDstAccessOrder - 5, // CThreadTransferSrcDstVectorDim - GemmCThreadTransferDstScalarPerVector_N11, - decltype(in_gemmk0_gemmm0_gemmm1_gemmk1_grid_step_hacks), - decltype(wei_gemmk0_gemmn0_gemmn1_gemmk1_grid_step_hacks), - decltype(out_gemmm0_gemmm10_gemmm11_gemmn0_gemmn10_gemmn11_grid_step_hacks), - decltype(in_gemmk0_gemmm0_gemmm1_gemmk1_grid_move_slice_window_step_hacks), - decltype(wei_gemmk0_gemmn0_gemmn1_gemmk1_grid_move_slice_window_step_hacks)>( - static_cast(in_n_hi_wi_c_device_buf.GetDeviceBuffer()), - static_cast(wei_k_y_x_c_device_buf.GetDeviceBuffer()), - static_cast(out_n_ho_wo_k_device_buf.GetDeviceBuffer()), - in_gemmk0_gemmm_gemmk1_grid_desc, - wei_gemmk0_gemmn_gemmk1_grid_desc, - out_gemmm_gemmn_grid_desc, - in_gemmk0_gemmm0_gemmm1_gemmk1_grid_step_hacks, - wei_gemmk0_gemmn0_gemmn1_gemmk1_grid_step_hacks, - out_gemmm0_gemmm10_gemmm11_gemmn0_gemmn10_gemmn11_grid_step_hacks, - in_gemmk0_gemmm0_gemmm1_gemmk1_grid_move_slice_window_step_hacks, - wei_gemmk0_gemmn0_gemmn1_gemmk1_grid_move_slice_window_step_hacks, - nrepeat); - - { - const auto N = out_n_ho_wo_k_lengths[I0]; - const auto K = out_n_ho_wo_k_lengths[I3]; - const auto C = wei_k_y_x_c_lengths[I3]; - - const auto Ho = out_n_ho_wo_k_lengths[I1]; - const auto Wo = out_n_ho_wo_k_lengths[I2]; - - const auto Y = wei_k_y_x_c_lengths[I1]; - const auto X = wei_k_y_x_c_lengths[I2]; - - float perf = static_cast(std::size_t(2) * N * K * Ho * Wo * C * Y * X) / - (std::size_t(1000) * 1000 * 1000) / ave_time; - - std::cout << "Average time : " << ave_time << " ms, " << perf << " TFlop/s" - << std::endl; - } - } - - // copy result back to host - out_n_ho_wo_k_device_buf.FromDevice(out_n_ho_wo_k.mData.data()); -} diff --git a/library/include/ck/library/obselete_driver_offline/device_convolution_forward_implicit_gemm_v4r4r2_xdlops_nchw_kcyx_nkhw.hpp b/library/include/ck/library/obselete_driver_offline/device_convolution_forward_implicit_gemm_v4r4r2_xdlops_nchw_kcyx_nkhw.hpp deleted file mode 100644 index e4cf4dd25c..0000000000 --- a/library/include/ck/library/obselete_driver_offline/device_convolution_forward_implicit_gemm_v4r4r2_xdlops_nchw_kcyx_nkhw.hpp +++ /dev/null @@ -1,228 +0,0 @@ -#include -#include "device.hpp" -#include "host_tensor.hpp" -#include "transform_forward_convolution_into_gemm_v4r4r2_nchw_kcyx_nkhw.hpp" -#include "driver_gemm_xdlops_v2r3.hpp" - -template -void device_convolution_forward_implicit_gemm_v4r4r2_xdlops_nchw_kcyx_nkhw( - const InLengths& in_n_c_hi_wi_lengths, - const WeiLengths& wei_k_c_y_x_lengths, - const OutLengths& out_n_k_ho_wo_lengths, - const ConvStrides& conv_strides, - const ConvDilations& conv_dilations, - const InLeftPads& in_left_pads, - const InRightPads& in_right_pads, - const Tensor& in_n_c_hi_wi, - const Tensor& wei_k_c_y_x, - Tensor& out_n_k_ho_wo, - ck::index_t nrepeat) -{ - using namespace ck; - - std::cout << __func__ << std::endl; - - constexpr auto I0 = Number<0>{}; - constexpr auto I1 = Number<1>{}; - constexpr auto I2 = Number<2>{}; - - DeviceMem in_n_c_hi_wi_device_buf(sizeof(TInWei) * in_n_c_hi_wi.mDesc.GetElementSpace()); - DeviceMem wei_k_c_y_x_device_buf(sizeof(TInWei) * wei_k_c_y_x.mDesc.GetElementSpace()); - DeviceMem out_n_k_ho_wo_device_buf(sizeof(TOut) * out_n_k_ho_wo.mDesc.GetElementSpace()); - - in_n_c_hi_wi_device_buf.ToDevice(in_n_c_hi_wi.mData.data()); - wei_k_c_y_x_device_buf.ToDevice(wei_k_c_y_x.mData.data()); - out_n_k_ho_wo_device_buf.ToDevice(out_n_k_ho_wo.mData.data()); - - const auto in_n_c_hi_wi_desc = make_naive_tensor_descriptor_packed(in_n_c_hi_wi_lengths); - const auto wei_k_c_y_x_desc = make_naive_tensor_descriptor_packed(wei_k_c_y_x_lengths); - const auto out_n_k_ho_wo_desc = make_naive_tensor_descriptor_packed(out_n_k_ho_wo_lengths); - -#if 0 - // [M, N, K0, K1] = [128, 128, 4, 8] for fp16 - constexpr index_t BlockSize = 256; - - constexpr index_t GemmMPerBlock = 128; - constexpr index_t GemmNPerBlock = 128; - constexpr index_t GemmKPerBlock = 4; - - constexpr index_t GemmMPerWave = 32; - constexpr index_t GemmNPerWave = 32; - constexpr index_t GemmK1 = 8; - - constexpr index_t MRepeat = 2; - constexpr index_t NRepeat = 2; - - using GemmABlockTransferThreadSliceLengths_GemmK0_GemmM_GemmK1 = Sequence<1, 2, 8>; - using GemmABlockTransferThreadClusterLengths_GemmK0_GemmM_GemmK1 = Sequence<4, 64, 1>; - - constexpr index_t GemmABlockTransferSrcScalarPerVector_GemmK1 = 8; - constexpr index_t GemmABlockTransferDstScalarPerVector_GemmK1 = 8; - - using GemmBBlockTransferThreadSliceLengths_GemmK0_GemmN_GemmK1 = Sequence<1, 2, 8>; - using GemmBBlockTransferThreadClusterLengths_GemmK0_GemmN_GemmK1 = Sequence<4, 64, 1>; - - constexpr index_t GemmBBlockTransferSrcScalarPerVector_GemmN = 1; - constexpr index_t GemmBBlockTransferDstScalarPerVector_GemmK1 = 8; - - constexpr index_t GemmCThreadTransferDstScalarPerVector = 1; -#elif 1 - // [M, N, K0, K1] = [256, 128, 4, 8] for fp16 - constexpr index_t BlockSize = 256; - - constexpr index_t GemmMPerBlock = 256; - constexpr index_t GemmNPerBlock = 128; - constexpr index_t GemmKPerBlock = 4; - - constexpr index_t GemmMPerWave = 32; - constexpr index_t GemmNPerWave = 32; - constexpr index_t GemmK1 = 8; - - constexpr index_t MRepeat = 4; - constexpr index_t NRepeat = 2; - - using GemmABlockTransferThreadSliceLengths_GemmK0_GemmM_GemmK1 = Sequence<1, 4, 8>; - using GemmABlockTransferThreadClusterLengths_GemmK0_GemmM_GemmK1 = Sequence<4, 64, 1>; - - constexpr index_t GemmABlockTransferSrcScalarPerVector_GemmK1 = 8; - constexpr index_t GemmABlockTransferDstScalarPerVector_GemmK1 = 8; - - using GemmBBlockTransferThreadSliceLengths_GemmK0_GemmN_GemmK1 = Sequence<1, 2, 8>; - using GemmBBlockTransferThreadClusterLengths_GemmK0_GemmN_GemmK1 = Sequence<4, 64, 1>; - - constexpr index_t GemmBBlockTransferSrcScalarPerVector_GemmN = 1; - constexpr index_t GemmBBlockTransferDstScalarPerVector_GemmK1 = 8; - - constexpr index_t GemmCThreadTransferDstScalarPerVector = 1; -#endif - - const auto descs = - transform_forward_convolution_into_gemm_v4r4r2_nchw_kcyx_nkhw_pad(wei_k_c_y_x_desc, - in_n_c_hi_wi_desc, - out_n_k_ho_wo_desc, - conv_strides, - conv_dilations, - in_left_pads, - in_right_pads, - Number{}); - - const auto wei_gemmk0_gemmm_gemmk1_grid_desc = descs[I0]; - const auto in_gemmk0_gemmn_gemmk1_grid_desc = descs[I1]; - const auto out_gemmm_gemmn_grid_desc = descs[I2]; - - // HACK: hacks that control index calculation when iterating over A, B, C matrix - constexpr auto wei_gemmk0_gemmm_gemmk1_grid_step_hacks = - make_tuple(make_tuple(Sequence<0, 0, 0, 0, 0>{}, // 0+: GemmK0 - Sequence<0, 0, 0, 0, 0>{}, // 1+: GemmM - Sequence<0, 0, 0, 0, 0>{}), // 2+: GemmK1 - make_tuple(Sequence<0, 0, 0, 0, 0>{}, // 0-: GemmK0 - Sequence<0, 0, 0, 0, 0>{}, // 1-: GemmM - Sequence<0, 0, 0, 0, 0>{})); // 2-: GemmK1 - - constexpr auto in_gemmk0_gemmn_gemmk1_grid_step_hacks = - make_tuple(make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0>{}, // 0+: GemmK0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0>{}, // 1+: GemmN - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0>{}), // 2+: GemmK1 - make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0>{}, // 0-: GemmK0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0>{}, // 1-: GemmN - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0>{})); // 2-: GemmK1 - - constexpr auto out_m0_n0_m1_n1_m2_m3_m4_n2_grid_step_hacks = - make_tuple(make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 0+: M0 - Sequence<0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 1+: N0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 2+: M1 - Sequence<0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 3+: N1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 4+: M2 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 5+: M3 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 6+: M4 - Sequence<0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0>{}), // 7+: N2 - make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 0-: M0 - Sequence<0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 1-: N0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 2-: M1 - Sequence<0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 3-: N1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 4-: M2 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 5-: M3 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 6-: M4 - Sequence<0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0>{})); // 7-: N2 - - constexpr auto wei_gemmk0_gemmm_gemmk1_grid_move_slice_window_step_hacks = - Sequence<0, 0, 0, 0, 0>{}; - - constexpr auto in_gemmk0_gemmn_gemmk1_grid_move_slice_window_step_hacks = - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0>{}; - - for(index_t i = 0; i < 5; ++i) - { - float ave_time = driver_gemm_xdlops_v2r3< - BlockSize, - TInWei, - TAcc, - TOut, - InMemoryDataOperationEnum::Set, - decltype(wei_gemmk0_gemmm_gemmk1_grid_desc), - decltype(in_gemmk0_gemmn_gemmk1_grid_desc), - decltype(out_gemmm_gemmn_grid_desc), - GemmMPerBlock, - GemmNPerBlock, - GemmKPerBlock, - GemmMPerWave, - GemmNPerWave, - GemmK1, - MRepeat, - NRepeat, - GemmABlockTransferThreadSliceLengths_GemmK0_GemmM_GemmK1, - GemmABlockTransferThreadClusterLengths_GemmK0_GemmM_GemmK1, - Sequence<1, 0, 2>, - Sequence<1, 0, 2>, - 2, - GemmABlockTransferSrcScalarPerVector_GemmK1, - GemmABlockTransferDstScalarPerVector_GemmK1, - false, // don't move back src coordinate after threadwise copy - GemmBBlockTransferThreadSliceLengths_GemmK0_GemmN_GemmK1, - GemmBBlockTransferThreadClusterLengths_GemmK0_GemmN_GemmK1, - Sequence<0, 2, 1>, - Sequence<1, 0, 2>, - 1, - GemmBBlockTransferSrcScalarPerVector_GemmN, - GemmBBlockTransferDstScalarPerVector_GemmK1, - false, // don't move back src coordinate after threadwise copy - Sequence<3, 0, 1, 2, 7, 5, 4, 6>, - 7, - GemmCThreadTransferDstScalarPerVector, - decltype(wei_gemmk0_gemmm_gemmk1_grid_step_hacks), - decltype(in_gemmk0_gemmn_gemmk1_grid_step_hacks), - decltype(out_m0_n0_m1_n1_m2_m3_m4_n2_grid_step_hacks), - decltype(wei_gemmk0_gemmm_gemmk1_grid_move_slice_window_step_hacks), - decltype(in_gemmk0_gemmn_gemmk1_grid_move_slice_window_step_hacks), - false>(static_cast(wei_k_c_y_x_device_buf.GetDeviceBuffer()), - static_cast(in_n_c_hi_wi_device_buf.GetDeviceBuffer()), - static_cast(out_n_k_ho_wo_device_buf.GetDeviceBuffer()), - wei_gemmk0_gemmm_gemmk1_grid_desc, - in_gemmk0_gemmn_gemmk1_grid_desc, - out_gemmm_gemmn_grid_desc, - wei_gemmk0_gemmm_gemmk1_grid_step_hacks, - in_gemmk0_gemmn_gemmk1_grid_step_hacks, - out_m0_n0_m1_n1_m2_m3_m4_n2_grid_step_hacks, - wei_gemmk0_gemmm_gemmk1_grid_move_slice_window_step_hacks, - in_gemmk0_gemmn_gemmk1_grid_move_slice_window_step_hacks, - nrepeat); - - float perf = static_cast(calculate_convolution_flops( - in_n_c_hi_wi_desc, wei_k_c_y_x_desc, out_n_k_ho_wo_desc)) / - (std::size_t(1000) * 1000 * 1000) / ave_time; - - std::cout << "Average time : " << ave_time << " ms, " << perf << " TFlop/s" << std::endl; - } - - // copy result back to host - out_n_k_ho_wo_device_buf.FromDevice(out_n_k_ho_wo.mData.data()); -} diff --git a/library/include/ck/library/obselete_driver_offline/device_convolution_forward_implicit_gemm_v4r4r4_xdlops_nhwc_kyxc_nhwk.hpp b/library/include/ck/library/obselete_driver_offline/device_convolution_forward_implicit_gemm_v4r4r4_xdlops_nhwc_kyxc_nhwk.hpp deleted file mode 100644 index 18e712fb47..0000000000 --- a/library/include/ck/library/obselete_driver_offline/device_convolution_forward_implicit_gemm_v4r4r4_xdlops_nhwc_kyxc_nhwk.hpp +++ /dev/null @@ -1,600 +0,0 @@ -#include -#include "device.hpp" -#include "host_tensor.hpp" -#include "transform_forward_convolution_into_gemm_v4r4r4_nhwc_kyxc_nhwk.hpp" -#include "driver_gemm_xdlops_v2r3.hpp" - -#if 0 -__host__ __device__ static constexpr auto -MakePaddedGridDescriptors(const AGridDesc_K0Raw_MRaw_K1& a_grid_desc_k0raw_mraw_k1, - const BGridDesc_K0Raw_NRaw_K1& b_grid_desc_k0raw_nraw_k1, - const CGridDesc_MRaw_NRaw& c_grid_desc_mraw_nraw) -{ - const auto K0Raw = a_grid_desc_k0raw_mraw_k1.GetLength(I0); - const auto K1 = a_grid_desc_k0raw_mraw_k1.GetLength(I2); - const auto MRaw = c_grid_desc_mraw_nraw.GetLength(I0); - const auto NRaw = c_grid_desc_mraw_nraw.GetLength(I1); - - const auto K0Pad = math::integer_least_multiple(K0Raw, K0PerBlock) - K0Raw; - const auto MPad = math::integer_least_multiple(MRaw, MPerBlock) - MRaw; - const auto NPad = math::integer_least_multiple(NRaw, NPerBlock) - NRaw; - - // A - const auto a_grid_desc_k0_m_k1 = [&]() { - if constexpr(DoPad_K0 && DoPad_M) - { - return transform_tensor_descriptor( - a_grid_desc_k0_m_k1, - make_tuple(make_right_pad_transform(K0Raw, K0Pad), - make_right_pad_transform(MRaw, MPad), - make_pass_through_transform(K1)), - make_tuple(Sequence<0>{}, Sequence<1>{}, Sequence<2>{}), - make_tuple(Sequence<0>{}, Sequence<1>{}, Sequence<2>{})); - } - else if constexpr(DoPad_K0 && !DoPad_M) - { - return transform_tensor_descriptor( - a_grid_desc_k0_m_k1, - make_tuple(make_right_pad_transform(K0Raw, K0Pad), - make_pass_through_transform(MRaw), - make_pass_through_transform(K1)), - make_tuple(Sequence<0>{}, Sequence<1>{}, Sequence<2>{}), - make_tuple(Sequence<0>{}, Sequence<1>{}, Sequence<2>{})); - } - else if constexpr(!DoPad_K0 && DoPad_M) - { - return transform_tensor_descriptor( - a_grid_desc_k0_m_k1, - make_tuple(make_pass_through_transform(K0Raw), - make_right_pad_transform(MRaw, MPad), - make_pass_through_transform(K1)), - make_tuple(Sequence<0>{}, Sequence<1>{}, Sequence<2>{}), - make_tuple(Sequence<0>{}, Sequence<1>{}, Sequence<2>{})); - } - else - { - return a_grid_desc_k0raw_mraw_k1; - } - }(); - - // B - const auto b_grid_desc_k0_n_k1 = [&]() { - if constexpr(DoPad_K0 && DoPad_N) - { - return transform_tensor_descriptor( - b_grid_desc_k0_n_k1, - make_tuple(make_right_pad_transform(K0Raw, K0Pad), - make_right_pad_transform(NRaw, NPad), - make_pass_through_transform(K1)), - make_tuple(Sequence<0>{}, Sequence<1>{}, Sequence<2>{}), - make_tuple(Sequence<0>{}, Sequence<1>{}, Sequence<2>{})); - } - else if constexpr(DoPad_K0 && !DoPad_N) - { - return transform_tensor_descriptor( - b_grid_desc_k0_n_k1, - make_tuple(make_right_pad_transform(K0Raw, K0Pad), - make_pass_through_transform(NRaw), - make_pass_through_transform(K1)), - make_tuple(Sequence<0>{}, Sequence<1>{}, Sequence<2>{}), - make_tuple(Sequence<0>{}, Sequence<1>{}, Sequence<2>{})); - } - else if constexpr(!DoPad_K0 && DoPad_N) - { - return transform_tensor_descriptor( - b_grid_desc_k0_n_k1, - make_tuple(make_pass_through_transform(K0Raw), - make_right_pad_transform(NRaw, NPad), - make_pass_through_transform(K1)), - make_tuple(Sequence<0>{}, Sequence<1>{}, Sequence<2>{}), - make_tuple(Sequence<0>{}, Sequence<1>{}, Sequence<2>{})); - } - else - { - return b_grid_desc_k0raw_nraw_k1; - } - }(); - - // C - const auto c_grid_desc_m_n = [&]() { - if constexpr(DoPad_M && DoPad_N) - { - return transform_tensor_descriptor(c_grid_desc_m_n, - make_tuple(make_right_pad_transform(MRaw, MPad), - make_right_pad_transform(NRaw, NPad)), - make_tuple(Sequence<0>{}, Sequence<1>{}), - make_tuple(Sequence<0>{}, Sequence<1>{})); - } - else if constexpr(DoPad_M && !DoPad_N) - { - return transform_tensor_descriptor( - c_grid_desc_m_n, - make_tuple(make_right_pad_transform(MRaw, MPad), make_pass_through_transform(NRaw)), - make_tuple(Sequence<0>{}, Sequence<1>{}), - make_tuple(Sequence<0>{}, Sequence<1>{})); - } - else if constexpr(!DoPad_M && DoPad_N) - { - return transform_tensor_descriptor( - c_grid_desc_m_n, - make_tuple(make_pass_through_transform(MRaw), make_right_pad_transform(NRaw, NPad)), - make_tuple(Sequence<0>{}, Sequence<1>{}), - make_tuple(Sequence<0>{}, Sequence<1>{})); - } - else - { - reutnr c_grid_desc_m_n; - } - }(); -} -#endif - -template -void device_convolution_forward_implicit_gemm_v4r4r4_xdlops_nhwc_kyxc_nhwk( - const InLengths& in_n_hi_wi_c_lengths, - const WeiLengths& wei_k_y_x_c_lengths, - const OutLengths& out_n_ho_wo_k_lengths, - const ConvStrides& conv_strides, - const ConvDilations& conv_dilations, - const InLeftPads& in_left_pads, - const InRightPads& in_right_pads, - const Tensor& in_n_hi_wi_c, - const Tensor& wei_k_y_x_c, - Tensor& out_n_ho_wo_k, - ck::index_t nrepeat) -{ - using namespace ck; - - std::cout << __func__ << std::endl; - - constexpr auto I0 = Number<0>{}; - constexpr auto I1 = Number<1>{}; - constexpr auto I2 = Number<2>{}; - constexpr auto I3 = Number<3>{}; - - DeviceMem in_n_hi_wi_c_device_buf(sizeof(TInWei) * in_n_hi_wi_c.mDesc.GetElementSpace()); - DeviceMem wei_k_y_x_c_device_buf(sizeof(TInWei) * wei_k_y_x_c.mDesc.GetElementSpace()); - DeviceMem out_n_ho_wo_k_device_buf(sizeof(TOut) * out_n_ho_wo_k.mDesc.GetElementSpace()); - - in_n_hi_wi_c_device_buf.ToDevice(in_n_hi_wi_c.mData.data()); - wei_k_y_x_c_device_buf.ToDevice(wei_k_y_x_c.mData.data()); - out_n_ho_wo_k_device_buf.ToDevice(out_n_ho_wo_k.mData.data()); - - const auto in_n_hi_wi_c_desc = make_naive_tensor_descriptor_packed(in_n_hi_wi_c_lengths); - const auto wei_k_y_x_c_desc = make_naive_tensor_descriptor_packed(wei_k_y_x_c_lengths); - const auto out_n_ho_wo_k_desc = make_naive_tensor_descriptor_packed(out_n_ho_wo_k_lengths); - -#if 0 - // [M, N, K0, K1] = [256, 128, 4, 4], C = 128, for fp32 - constexpr index_t BlockSize = 256; - - constexpr index_t GemmMPerBlock = 256; - constexpr index_t GemmNPerBlock = 128; - constexpr index_t GemmKPerBlock = 4; - - constexpr index_t GemmMPerXDL = 32; - constexpr index_t GemmNPerXDL = 32; - constexpr index_t GemmK1 = 4; - - constexpr index_t MRepeat = 4; - constexpr index_t NRepeat = 2; - - using GemmABlockTransferThreadSliceLengths_GemmK0_GemmM_GemmK1 = Sequence<1, 4, 4>; - using GemmABlockTransferThreadClusterLengths_GemmK0_GemmM_GemmK1 = Sequence<4, 64, 1>; - - constexpr index_t GemmABlockTransferSrcScalarPerVector_GemmK1 = 4; - constexpr index_t GemmABlockTransferDstScalarPerVector_GemmK1 = 4; - - using GemmBBlockTransferThreadSliceLengths_GemmK0_GemmN_GemmK1 = Sequence<1, 2, 4>; - using GemmBBlockTransferThreadClusterLengths_GemmK0_GemmN_GemmK1 = Sequence<4, 64, 1>; - - constexpr index_t GemmBBlockTransferSrcScalarPerVector_GemmK1 = 4; - constexpr index_t GemmBBlockTransferDstScalarPerVector_GemmK1 = 4; - - constexpr index_t GemmCThreadTransferDstScalarPerVector = 1; -#elif 0 - // [M, N, K0, K1] = [128, 128, 4, 4], C = 128, for fp32 - constexpr index_t BlockSize = 256; - - constexpr index_t GemmMPerBlock = 128; - constexpr index_t GemmNPerBlock = 128; - constexpr index_t GemmKPerBlock = 4; - - constexpr index_t GemmMPerXDL = 32; - constexpr index_t GemmNPerXDL = 32; - constexpr index_t GemmK1 = 4; - - constexpr index_t MRepeat = 2; - constexpr index_t NRepeat = 2; - - using GemmABlockTransferThreadSliceLengths_GemmK0_GemmM_GemmK1 = Sequence<1, 2, 4>; - using GemmABlockTransferThreadClusterLengths_GemmK0_GemmM_GemmK1 = Sequence<4, 64, 1>; - - constexpr index_t GemmABlockTransferSrcScalarPerVector_GemmK1 = 4; - constexpr index_t GemmABlockTransferDstScalarPerVector_GemmK1 = 4; - - using GemmBBlockTransferThreadSliceLengths_GemmK0_GemmN_GemmK1 = Sequence<1, 2, 4>; - using GemmBBlockTransferThreadClusterLengths_GemmK0_GemmN_GemmK1 = Sequence<4, 64, 1>; - - constexpr index_t GemmBBlockTransferSrcScalarPerVector_GemmK1 = 4; - constexpr index_t GemmBBlockTransferDstScalarPerVector_GemmK1 = 4; - - constexpr index_t GemmCThreadTransferDstScalarPerVector = 1; -#elif 0 - // [M, N, K0, K1] = [256, 256, 4, 8], C = 256, for fp16 - constexpr index_t BlockSize = 256; - - constexpr index_t GemmMPerBlock = 256; - constexpr index_t GemmNPerBlock = 256; - constexpr index_t GemmKPerBlock = 4; - - constexpr index_t GemmMPerXDL = 32; - constexpr index_t GemmNPerXDL = 32; - constexpr index_t GemmK1 = 8; - - constexpr index_t MRepeat = 4; - constexpr index_t NRepeat = 4; - - using GemmABlockTransferThreadSliceLengths_GemmK0_GemmM_GemmK1 = Sequence<1, 4, 8>; - using GemmABlockTransferThreadClusterLengths_GemmK0_GemmM_GemmK1 = Sequence<4, 64, 1>; - - constexpr index_t GemmABlockTransferSrcScalarPerVector_GemmK1 = 8; - constexpr index_t GemmABlockTransferDstScalarPerVector_GemmK1 = 8; - - using GemmBBlockTransferThreadSliceLengths_GemmK0_GemmN_GemmK1 = Sequence<1, 4, 8>; - using GemmBBlockTransferThreadClusterLengths_GemmK0_GemmN_GemmK1 = Sequence<4, 64, 1>; - - constexpr index_t GemmBBlockTransferSrcScalarPerVector_GemmK1 = 8; - constexpr index_t GemmBBlockTransferDstScalarPerVector_GemmK1 = 8; - - constexpr index_t GemmCThreadTransferDstScalarPerVector = 1; -#elif 0 - // [M, N, K0, K1] = [256, 128, 4, 8], C = 128, for fp16 - constexpr index_t BlockSize = 256; - - constexpr index_t GemmMPerBlock = 256; - constexpr index_t GemmNPerBlock = 128; - constexpr index_t GemmKPerBlock = 4; - - constexpr index_t GemmMPerXDL = 32; - constexpr index_t GemmNPerXDL = 32; - constexpr index_t GemmK1 = 8; - - constexpr index_t MRepeat = 4; - constexpr index_t NRepeat = 2; - - using GemmABlockTransferThreadSliceLengths_GemmK0_GemmM_GemmK1 = Sequence<1, 4, 8>; - using GemmABlockTransferThreadClusterLengths_GemmK0_GemmM_GemmK1 = Sequence<4, 64, 1>; - - constexpr index_t GemmABlockTransferSrcScalarPerVector_GemmK1 = 8; - constexpr index_t GemmABlockTransferDstScalarPerVector_GemmK1 = 8; - - using GemmBBlockTransferThreadSliceLengths_GemmK0_GemmN_GemmK1 = Sequence<1, 2, 8>; - using GemmBBlockTransferThreadClusterLengths_GemmK0_GemmN_GemmK1 = Sequence<4, 64, 1>; - - constexpr index_t GemmBBlockTransferSrcScalarPerVector_GemmK1 = 8; - constexpr index_t GemmBBlockTransferDstScalarPerVector_GemmK1 = 8; - - constexpr index_t GemmCThreadTransferDstScalarPerVector = 1; -#elif 1 - // [M, N, K0, K1] = [128, 256, 4, 8], C = 128, for fp16 - constexpr index_t BlockSize = 256; - - constexpr index_t GemmMPerBlock = 128; - constexpr index_t GemmNPerBlock = 256; - constexpr index_t GemmKPerBlock = 4; - - constexpr index_t GemmMPerXDL = 32; - constexpr index_t GemmNPerXDL = 32; - constexpr index_t GemmK1 = 8; - - constexpr index_t MRepeat = 2; - constexpr index_t NRepeat = 4; - - using GemmABlockTransferThreadSliceLengths_GemmK0_GemmM_GemmK1 = Sequence<1, 2, 8>; - using GemmABlockTransferThreadClusterLengths_GemmK0_GemmM_GemmK1 = Sequence<4, 64, 1>; - - constexpr index_t GemmABlockTransferSrcScalarPerVector_GemmK1 = 8; - constexpr index_t GemmABlockTransferDstScalarPerVector_GemmK1 = 8; - - using GemmBBlockTransferThreadSliceLengths_GemmK0_GemmN_GemmK1 = Sequence<1, 4, 8>; - using GemmBBlockTransferThreadClusterLengths_GemmK0_GemmN_GemmK1 = Sequence<4, 64, 1>; - - constexpr index_t GemmBBlockTransferSrcScalarPerVector_GemmK1 = 8; - constexpr index_t GemmBBlockTransferDstScalarPerVector_GemmK1 = 8; - - constexpr index_t GemmCThreadTransferDstScalarPerVector = 1; -#elif 0 - // [M, N, K0, K1] = [128, 128, 4, 8], C = 64, for fp16 - constexpr index_t BlockSize = 256; - - constexpr index_t GemmMPerBlock = 128; - constexpr index_t GemmNPerBlock = 128; - constexpr index_t GemmKPerBlock = 4; - - constexpr index_t GemmMPerXDL = 32; - constexpr index_t GemmNPerXDL = 32; - constexpr index_t GemmK1 = 8; - - constexpr index_t MRepeat = 2; - constexpr index_t NRepeat = 2; - - using GemmABlockTransferThreadSliceLengths_GemmK0_GemmM_GemmK1 = Sequence<1, 2, 8>; - using GemmABlockTransferThreadClusterLengths_GemmK0_GemmM_GemmK1 = Sequence<4, 64, 1>; - - constexpr index_t GemmABlockTransferSrcScalarPerVector_GemmK1 = 8; - constexpr index_t GemmABlockTransferDstScalarPerVector_GemmK1 = 8; - - using GemmBBlockTransferThreadSliceLengths_GemmK0_GemmN_GemmK1 = Sequence<1, 2, 8>; - using GemmBBlockTransferThreadClusterLengths_GemmK0_GemmN_GemmK1 = Sequence<4, 64, 1>; - - constexpr index_t GemmBBlockTransferSrcScalarPerVector_GemmK1 = 8; - constexpr index_t GemmBBlockTransferDstScalarPerVector_GemmK1 = 8; - - constexpr index_t GemmCThreadTransferDstScalarPerVector = 1; -#elif 0 - // [M, N, K0, K1] = [128, 64, 4, 8], C = 64, for fp16 - constexpr index_t BlockSize = 128; - - constexpr index_t GemmMPerBlock = 128; - constexpr index_t GemmNPerBlock = 64; - constexpr index_t GemmKPerBlock = 4; - - constexpr index_t GemmMPerXDL = 32; - constexpr index_t GemmNPerXDL = 32; - constexpr index_t GemmK1 = 8; - - constexpr index_t MRepeat = 2; - constexpr index_t NRepeat = 2; - - using GemmABlockTransferThreadSliceLengths_GemmK0_GemmM_GemmK1 = Sequence<1, 4, 8>; - using GemmABlockTransferThreadClusterLengths_GemmK0_GemmM_GemmK1 = Sequence<4, 32, 1>; - - constexpr index_t GemmABlockTransferSrcScalarPerVector_GemmK1 = 8; - constexpr index_t GemmABlockTransferDstScalarPerVector_GemmK1 = 8; - - using GemmBBlockTransferThreadSliceLengths_GemmK0_GemmN_GemmK1 = Sequence<1, 2, 8>; - using GemmBBlockTransferThreadClusterLengths_GemmK0_GemmN_GemmK1 = Sequence<4, 32, 1>; - - constexpr index_t GemmBBlockTransferSrcScalarPerVector_GemmK1 = 8; - constexpr index_t GemmBBlockTransferDstScalarPerVector_GemmK1 = 8; - - constexpr index_t GemmCThreadTransferDstScalarPerVector = 1; -#elif 1 - // [M, N, K0, K1] = [128, 64, 4, 8], C = 32, for fp16 - constexpr index_t BlockSize = 256; - - constexpr index_t GemmMPerBlock = 128; - constexpr index_t GemmNPerBlock = 64; - constexpr index_t GemmKPerBlock = 4; - - constexpr index_t GemmMPerXDL = 32; - constexpr index_t GemmNPerXDL = 32; - constexpr index_t GemmK1 = 8; - - constexpr index_t MRepeat = 2; - constexpr index_t NRepeat = 1; - - using GemmABlockTransferThreadSliceLengths_GemmK0_GemmM_GemmK1 = Sequence<1, 2, 8>; - using GemmABlockTransferThreadClusterLengths_GemmK0_GemmM_GemmK1 = Sequence<4, 64, 1>; - - constexpr index_t GemmABlockTransferSrcScalarPerVector_GemmK1 = 8; - constexpr index_t GemmABlockTransferDstScalarPerVector_GemmK1 = 8; - - using GemmBBlockTransferThreadSliceLengths_GemmK0_GemmN_GemmK1 = Sequence<1, 1, 8>; - using GemmBBlockTransferThreadClusterLengths_GemmK0_GemmN_GemmK1 = Sequence<4, 64, 1>; - - constexpr index_t GemmBBlockTransferSrcScalarPerVector_GemmK1 = 8; - constexpr index_t GemmBBlockTransferDstScalarPerVector_GemmK1 = 8; - - constexpr index_t GemmCThreadTransferDstScalarPerVector = 1; -#endif - - const auto descs = - transform_forward_convolution_into_gemm_v4r4r4_nhwc_kyxc_nhwk(in_n_hi_wi_c_desc, - wei_k_y_x_c_desc, - out_n_ho_wo_k_desc, - conv_strides, - conv_dilations, - in_left_pads, - in_right_pads, - Number{}); - -#if 0 // debug - const auto in_gemmk0_gemmm_gemmk1_grid_desc = descs[I0]; - - // HACK: hacks that control index calculation when iterating over A matrix - constexpr auto in_gemmk0_gemmm_gemmk1_grid_step_hacks = - make_tuple(make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0>{}, // 0+: GemmK0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0>{}, // 1+: GemmM - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0>{}), // 2+: GemmK1 - make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0>{}, // 0-: GemmK0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0>{}, // 1-: GemmM - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0>{})); // 2-: GemmK1 - - constexpr auto in_gemmk0_gemmm_gemmk1_grid_move_slice_window_step_hacks = - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0>{}; -#else - const auto in_gemmk0_gemmmraw_gemmk1_grid_desc = descs[I0]; - - const auto GemmK0 = in_gemmk0_gemmmraw_gemmk1_grid_desc.GetLength(I0); - const auto GemmMRaw = in_gemmk0_gemmmraw_gemmk1_grid_desc.GetLength(I1); - const auto GemmMPad = math::integer_least_multiple(GemmMRaw, GemmMPerBlock) - GemmMRaw; - - const auto in_gemmk0_gemmm_gemmk1_grid_desc = - transform_tensor_descriptor(in_gemmk0_gemmmraw_gemmk1_grid_desc, - make_tuple(make_pass_through_transform(GemmK0), - make_right_pad_transform(GemmMRaw, GemmMPad), - make_pass_through_transform(GemmK1)), - make_tuple(Sequence<0>{}, Sequence<1>{}, Sequence<2>{}), - make_tuple(Sequence<0>{}, Sequence<1>{}, Sequence<2>{})); - - // HACK: hacks that control index calculation when iterating over A matrix - constexpr auto in_gemmk0_gemmm_gemmk1_grid_step_hacks = make_tuple( - make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0>{}, // 0+: GemmK0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0>{}, // 1+: GemmM - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0>{}), // 2+: GemmK1 - make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0>{}, // 0-: GemmK0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0>{}, // 1-: GemmM - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0>{})); // 2-: GemmK1 - - constexpr auto in_gemmk0_gemmm_gemmk1_grid_move_slice_window_step_hacks = - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0>{}; -#endif - - const auto wei_gemmk0_gemmn_gemmk1_grid_desc = descs[I1]; - - const auto wei_gemmk0_gemmn_gemmk1_grid_step_hacks = - make_tuple(make_tuple(Sequence<0, 0, 0, 0, 0>{}, // 0+: GemmK0 - Sequence<0, 0, 0, 0, 0>{}, // 1+: GemmN - Sequence<0, 0, 0, 0, 0>{}), // 2+: GemmK1 - make_tuple(Sequence<0, 0, 0, 0, 0>{}, // 0-: GemmK0 - Sequence<0, 0, 0, 0, 0>{}, // 1-: GemmN - Sequence<0, 0, 0, 0, 0>{})); // 2-: GemmK1 - - constexpr auto wei_gemmk0_gemmn_gemmk1_grid_move_slice_window_step_hacks = - Sequence<0, 0, 0, 0, 0>{}; - -#if 0 - const auto out_gemmm_gemmn_grid_desc = descs[I2]; - - constexpr auto out_m0_n0_m1_n1_m2_m3_m4_n2_grid_step_hacks = - make_tuple(make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 0+: M0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 1+: N0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 2+: M1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 3+: N1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 4+: M2 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 5+: M3 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 6+: M4 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}), // 7+: N2 - make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 0-: M0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 1-: N0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 2-: M1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 3-: N1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 4-: M2 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 5-: M3 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 6-: M4 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{})); // 7-: N2 -#else - const auto out_gemmmraw_gemmn_grid_desc = descs[I2]; - - const auto GemmN = out_gemmmraw_gemmn_grid_desc.GetLength(I1); - - const auto out_gemmm_gemmn_grid_desc = - transform_tensor_descriptor(out_gemmmraw_gemmn_grid_desc, - make_tuple(make_right_pad_transform(GemmMRaw, GemmMPad), - make_pass_through_transform(GemmN)), - make_tuple(Sequence<0>{}, Sequence<1>{}), - make_tuple(Sequence<0>{}, Sequence<1>{})); - - constexpr auto out_m0_n0_m1_n1_m2_m3_m4_n2_grid_step_hacks = - make_tuple(make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 0+: M0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 1+: N0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 2+: M1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 3+: N1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 4+: M2 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 5+: M3 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 6+: M4 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}), // 7+: N2 - make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 0-: M0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 1-: N0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 2-: M1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 3-: N1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 4-: M2 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 5-: M3 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 6-: M4 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{})); // 7-: N2 -#endif - - for(index_t i = 0; i < 5; ++i) - { - float ave_time = driver_gemm_xdlops_v2r3< - BlockSize, - TInWei, - TAcc, - TOut, - InMemoryDataOperationEnum::Set, - decltype(in_gemmk0_gemmm_gemmk1_grid_desc), - decltype(wei_gemmk0_gemmn_gemmk1_grid_desc), - decltype(out_gemmm_gemmn_grid_desc), - GemmMPerBlock, - GemmNPerBlock, - GemmKPerBlock, - GemmMPerXDL, - GemmNPerXDL, - GemmK1, - MRepeat, - NRepeat, - GemmABlockTransferThreadSliceLengths_GemmK0_GemmM_GemmK1, - GemmABlockTransferThreadClusterLengths_GemmK0_GemmM_GemmK1, - Sequence<1, 0, 2>, - Sequence<1, 0, 2>, - 2, - GemmABlockTransferSrcScalarPerVector_GemmK1, - GemmABlockTransferDstScalarPerVector_GemmK1, - false, // don't move back src coordinate after threadwise copy - GemmBBlockTransferThreadSliceLengths_GemmK0_GemmN_GemmK1, - GemmBBlockTransferThreadClusterLengths_GemmK0_GemmN_GemmK1, - Sequence<1, 0, 2>, - Sequence<1, 0, 2>, - 2, - GemmBBlockTransferSrcScalarPerVector_GemmK1, - GemmBBlockTransferDstScalarPerVector_GemmK1, - false, // don't move back src coordinate after threadwise copy - Sequence<2, 3, 0, 1, 7, 5, 4, 6>, - 7, - GemmCThreadTransferDstScalarPerVector, - decltype(in_gemmk0_gemmm_gemmk1_grid_step_hacks), - decltype(wei_gemmk0_gemmn_gemmk1_grid_step_hacks), - decltype(out_m0_n0_m1_n1_m2_m3_m4_n2_grid_step_hacks), - decltype(in_gemmk0_gemmm_gemmk1_grid_move_slice_window_step_hacks), - decltype(wei_gemmk0_gemmn_gemmk1_grid_move_slice_window_step_hacks), - false, // CAccessOrderMRepeatNRepeat - true, // ABlockLdsExtraM - true // BBlockLdsExtraN - >(static_cast(in_n_hi_wi_c_device_buf.GetDeviceBuffer()), - static_cast(wei_k_y_x_c_device_buf.GetDeviceBuffer()), - static_cast(out_n_ho_wo_k_device_buf.GetDeviceBuffer()), - in_gemmk0_gemmm_gemmk1_grid_desc, - wei_gemmk0_gemmn_gemmk1_grid_desc, - out_gemmm_gemmn_grid_desc, - debug::debug_driver_gemm_xdlops_v2r3::M01, - debug::debug_driver_gemm_xdlops_v2r3::N01, - in_gemmk0_gemmm_gemmk1_grid_step_hacks, - wei_gemmk0_gemmn_gemmk1_grid_step_hacks, - out_m0_n0_m1_n1_m2_m3_m4_n2_grid_step_hacks, - in_gemmk0_gemmm_gemmk1_grid_move_slice_window_step_hacks, - wei_gemmk0_gemmn_gemmk1_grid_move_slice_window_step_hacks, - nrepeat); - - { - const auto N = out_n_ho_wo_k_lengths[I0]; - const auto K = out_n_ho_wo_k_lengths[I3]; - const auto C = wei_k_y_x_c_lengths[I3]; - - const auto Ho = out_n_ho_wo_k_lengths[I1]; - const auto Wo = out_n_ho_wo_k_lengths[I2]; - - const auto Y = wei_k_y_x_c_lengths[I1]; - const auto X = wei_k_y_x_c_lengths[I2]; - - float perf = static_cast((std::size_t(2) * N * K * Ho * Wo * C * Y * X)) / - (std::size_t(1000) * 1000 * 1000) / ave_time; - - std::cout << "Average time : " << ave_time << " ms, " << perf << " TFlop/s" - << std::endl; - } - } - - // copy result back to host - out_n_ho_wo_k_device_buf.FromDevice(out_n_ho_wo_k.mData.data()); -} diff --git a/library/include/ck/library/obselete_driver_offline/device_convolution_forward_implicit_gemm_v5r1_dlops_nc0hwc1_kc0yxc1_nk0hwk1.hpp b/library/include/ck/library/obselete_driver_offline/device_convolution_forward_implicit_gemm_v5r1_dlops_nc0hwc1_kc0yxc1_nk0hwk1.hpp deleted file mode 100644 index af4676f2a2..0000000000 --- a/library/include/ck/library/obselete_driver_offline/device_convolution_forward_implicit_gemm_v5r1_dlops_nc0hwc1_kc0yxc1_nk0hwk1.hpp +++ /dev/null @@ -1,196 +0,0 @@ -#include -#include "device.hpp" -#include "host_tensor.hpp" -#include "driver_convolution_forward_implicit_gemm_v5r1_dlops_nc0hwc1_kc0yxc1_nk0hwk1.hpp" - -template -void device_convolution_forward_implicit_gemm_v5r1_dlops_nc0hwc1_kc0yxc1_nk0hwk1( - const InLengths& in_n_c0_hi_wi_c1_lengths, - const WeiLengths& wei_k_c0_y_x_c1_lengths, - const OutLengths& out_n_k0_ho_wo_k1_lengths, - const ConvStrides& conv_strides, - const ConvDilations& conv_dilations, - const InLeftPads& in_left_pads, - const InRightPads& in_right_pads, - const Tensor& in_n_c0_hi_wi_c1, - const Tensor& wei_k_c0_y_x_c1, - const Tensor& bias_k0_k1, - Tensor& out_n_k0_ho_wo_k1, - ck::index_t nrepeat) -{ - using namespace ck; - - std::cout << __func__ << std::endl; - - constexpr auto I0 = Number<0>{}; - constexpr auto I1 = Number<1>{}; - constexpr auto I2 = Number<2>{}; - constexpr auto I3 = Number<3>{}; - constexpr auto I4 = Number<4>{}; - - const auto N = out_n_k0_ho_wo_k1_lengths[I0]; - const auto K0 = out_n_k0_ho_wo_k1_lengths[I1]; - const auto Ho = out_n_k0_ho_wo_k1_lengths[I2]; - const auto Wo = out_n_k0_ho_wo_k1_lengths[I3]; - const auto K1 = out_n_k0_ho_wo_k1_lengths[I4]; - - const auto C0 = in_n_c0_hi_wi_c1_lengths[I1]; - const auto Hi = in_n_c0_hi_wi_c1_lengths[I2]; - const auto Wi = in_n_c0_hi_wi_c1_lengths[I3]; - const auto C1 = in_n_c0_hi_wi_c1_lengths[I4]; - - const auto K = wei_k_c0_y_x_c1_lengths[I0]; - const auto Y = wei_k_c0_y_x_c1_lengths[I2]; - const auto X = wei_k_c0_y_x_c1_lengths[I3]; - - DeviceMem in_n_c0_hi_wi_c1_device_buf(sizeof(TInWei) * - in_n_c0_hi_wi_c1.mDesc.GetElementSpace()); - DeviceMem wei_k_c0_y_x_c1_device_buf(sizeof(TInWei) * wei_k_c0_y_x_c1.mDesc.GetElementSpace()); - DeviceMem bias_k0_k1_device_buf(sizeof(TOut) * bias_k0_k1.mDesc.GetElementSpace()); - DeviceMem out_n_k0_ho_wo_k1_device_buf(sizeof(TOut) * - out_n_k0_ho_wo_k1.mDesc.GetElementSpace()); - in_n_c0_hi_wi_c1_device_buf.ToDevice(in_n_c0_hi_wi_c1.mData.data()); - wei_k_c0_y_x_c1_device_buf.ToDevice(wei_k_c0_y_x_c1.mData.data()); - bias_k0_k1_device_buf.ToDevice(bias_k0_k1.mData.data()); - - constexpr index_t InWeiVectorSize = 8; - - if(C1 % InWeiVectorSize != 0) - { - throw std::runtime_error("wrong! C1 cannot be divided by InWeiVectorSize"); - } - -#if 0 - constexpr index_t BlockSize = 256; - - constexpr index_t KPerBlock = 32; - constexpr index_t HoPerBlock = 8; - constexpr index_t WoPerBlock = 64; - - constexpr index_t E1 = C0 * 9; - constexpr index_t E2 = 1; - constexpr index_t E1PerBlock = C0; - - constexpr index_t KPerThread = 16; - constexpr index_t HoPerThread = 2; - constexpr index_t WoPerThread = 2; - constexpr index_t EPerThread = 1; - - using ABlockTransferThreadSliceLengths_E0_E1_K0_K1_E2 = Sequence<1, 9, 1, E2>; - using ABlockTransferThreadClusterLengths_E0_E1_K0_K1_E2 = Sequence<1, E1PerBlock, KPerBlock, 1>; - - constexpr index_t ABlockTransferSrcScalarPerVector_E2 = E2; - constexpr index_t ABlockTransferDstScalarPerVector_E2 = E2; - - constexpr index_t BThreadTransferSrcScalarPerVector_E2 = E2; - - constexpr index_t CThreadTransferDstScalarPerVector_K = K1; -#elif 1 - constexpr index_t BlockSize = 64; - - constexpr index_t KPerBlock = 8; - constexpr index_t HoPerBlock = 8; - constexpr index_t WoPerBlock = 32; - - constexpr index_t E1 = 2 * 9; - constexpr index_t E2 = 1; - constexpr index_t K2 = 2; - constexpr index_t E1PerBlock = 2; - - constexpr index_t KPerThread = KPerBlock; - constexpr index_t HoPerThread = 2; - constexpr index_t WoPerThread = 2; - constexpr index_t EPerThread = 1; - - using ABlockTransferThreadSliceLengths_E0_E1_K0_K1_E2 = Sequence<1, 9, 1, 1, E2>; - using ABlockTransferThreadClusterLengths_E0_E1_K0_K1_E2 = - Sequence<1, E1PerBlock, 1, KPerBlock, 1>; - - constexpr index_t ABlockTransferSrcScalarPerVector_E2 = E2; - constexpr index_t ABlockTransferDstScalarPerVector_E2 = E2; - constexpr index_t BThreadTransferSrcScalarPerVector_E2 = E2; - constexpr index_t CThreadTransferDstScalarPerVector_K = InWeiVectorSize; -#endif - - if(KPerThread % InWeiVectorSize != 0) - { - throw std::runtime_error("wrong! C1 cannot be divided by InWeiVectorSize"); - } - - const auto in_n_c0_hi_wi_c1_desc = - make_naive_tensor_descriptor_packed(make_tuple(N, C0, Hi, Wi, E2)); - const auto wei_k_c0_y_x_c1_desc = - make_naive_tensor_descriptor_packed(make_tuple(K, C0, Y, X, E2)); - const auto out_n_k0_ho_wo_k1_desc = - make_naive_tensor_descriptor_packed(make_tuple(N, K0, Ho, Wo, K1)); - - constexpr auto conv_driver = - DriverDynamicConvolutionForwardImplicitGemmDlops_v5r1_nc0hwc1_kc0yxc1_nk0hwk1_outpad< - BlockSize, - typename vector_type::type, - TAcc, - TOut, - E1, - E2, - K2, - KPerBlock, - HoPerBlock, - WoPerBlock, - E1PerBlock, - KPerThread, - HoPerThread, - WoPerThread, - EPerThread, - ABlockTransferThreadSliceLengths_E0_E1_K0_K1_E2, - ABlockTransferThreadClusterLengths_E0_E1_K0_K1_E2, - ABlockTransferSrcScalarPerVector_E2, - ABlockTransferDstScalarPerVector_E2, - BThreadTransferSrcScalarPerVector_E2, - CThreadTransferDstScalarPerVector_K, - activ_type>{}; - - std::cerr << "conv_bias_activ_input_" - << "n" << N << "c" << C0 << "h" << Hi << "w" << Wi << "c" << C1 << "_filter_k" << K - << "c" << C0 << "y" << Y << "x" << X << "c" << C1 << "_convout_n" << N << "k" << K0 - << "h" << Ho << "w" << Wo << "k" << K1 << std::endl; - - for(int i = 0; i < 5; i++) - { - - const auto ave_time = - conv_driver.Run(wei_k_c0_y_x_c1_desc, - in_n_c0_hi_wi_c1_desc, - out_n_k0_ho_wo_k1_desc, - conv_strides, - conv_dilations, - in_left_pads, - in_right_pads, - static_cast::type*>( - wei_k_c0_y_x_c1_device_buf.GetDeviceBuffer()), - static_cast::type*>( - in_n_c0_hi_wi_c1_device_buf.GetDeviceBuffer()), - static_cast(bias_k0_k1_device_buf.GetDeviceBuffer()), - static_cast(out_n_k0_ho_wo_k1_device_buf.GetDeviceBuffer()), - nrepeat); - - { - float perf = static_cast(std::size_t(2) * N * K * Ho * Wo * C0 * C1 * Y * X) / - (std::size_t(1000) * 1000 * 1000) / ave_time; - - std::cout << "Average time : " << ave_time << " ms, " << perf << " TFlop/s" - << std::endl; - } - } - - out_n_k0_ho_wo_k1_device_buf.FromDevice(out_n_k0_ho_wo_k1.mData.data()); -} diff --git a/library/include/ck/library/obselete_driver_offline/device_convolution_forward_implicit_gemm_v6r1_dlops_nchw_kcyx_nkhw.hpp b/library/include/ck/library/obselete_driver_offline/device_convolution_forward_implicit_gemm_v6r1_dlops_nchw_kcyx_nkhw.hpp deleted file mode 100644 index 31925f0511..0000000000 --- a/library/include/ck/library/obselete_driver_offline/device_convolution_forward_implicit_gemm_v6r1_dlops_nchw_kcyx_nkhw.hpp +++ /dev/null @@ -1,241 +0,0 @@ -#pragma once -#include -#include "device.hpp" -#include "host_tensor.hpp" -#include "transform_forward_convolution_into_gemm_v6r1_nchw_kcyx_nkhw.hpp" -#include "driver_contraction_dlops_v1r2.hpp" - -template -void device_convolution_forward_implicit_gemm_v6r1_dlops_nchw_kcyx_nkhw( - const InLengths& in_n_c_hi_wi_lengths, - const WeiLengths& wei_k_c_y_x_lengths, - const OutLengths& out_n_k_ho_wo_lengths, - const ConvStrides& conv_strides, - const ConvDilations& conv_dilations, - const InLeftPads& in_left_pads, - const InRightPads& in_right_pads, - const Tensor& in_n_c_hi_wi, - const Tensor& wei_k_c_y_x, - Tensor& out_n_k_ho_wo, - ck::index_t nrepeat) -{ - using namespace ck; - - std::cout << __func__ << std::endl; - - constexpr auto I0 = Number<0>{}; - constexpr auto I1 = Number<1>{}; - constexpr auto I2 = Number<2>{}; - - DeviceMem in_n_c_hi_wi_device_buf(sizeof(TInWei) * in_n_c_hi_wi.mDesc.GetElementSpace()); - DeviceMem wei_k_c_y_x_device_buf(sizeof(TInWei) * wei_k_c_y_x.mDesc.GetElementSpace()); - DeviceMem out_n_k_ho_wo_device_buf(sizeof(TOut) * out_n_k_ho_wo.mDesc.GetElementSpace()); - - in_n_c_hi_wi_device_buf.ToDevice(in_n_c_hi_wi.mData.data()); - wei_k_c_y_x_device_buf.ToDevice(wei_k_c_y_x.mData.data()); - out_n_k_ho_wo_device_buf.ToDevice(out_n_k_ho_wo.mData.data()); - - const auto in_desc_n_c_hi_wi = make_naive_tensor_descriptor_packed(in_n_c_hi_wi_lengths); - const auto wei_desc_k_c_y_x = make_naive_tensor_descriptor_packed(wei_k_c_y_x_lengths); - const auto out_desc_n_k_ho_wo = make_naive_tensor_descriptor_packed(out_n_k_ho_wo_lengths); - -#if 1 - // [8, 1, 128, 1] * [8, 4, 32, 1] = [1, 128, 4, 32] for fp32 - // cdata = 64, BlockSize = 256 - constexpr index_t BlockSize = 256; - - constexpr index_t GN0 = 4; - constexpr index_t GK1 = 1; - - constexpr index_t GM1PerBlockGM11 = 128; - constexpr index_t GN1PerBlockGN11 = 32; - constexpr index_t GK0PerBlock = 8; - - constexpr index_t BM1PerThreadBM11 = 4; - constexpr index_t BN1PerThreadBN11 = 4; - constexpr index_t BK0PerThread = 1; - - using BM10BN10ThreadClusterBM10Xs = Sequence<8, 2>; - using BM10BN10ThreadClusterBN10Xs = Sequence<8, 2>; - - using ABlockTransferThreadSliceLengths_GK0_GM0_GM10_GM11_GK1 = Sequence<4, 1, 1, 1, 1>; - using ABlockTransferThreadClusterLengths_GK0_GM0_GM10_GM11_GK1 = Sequence<2, 1, 1, 128, 1>; - - using ABlockTransferSrcVectorTensorLengths_GK0_GM0_GM10_GM11_GK1 = Sequence<4, 1, 1, 1, 1>; - using ABlockTransferDstVectorTensorLengths_GK0_GM0_GM10_GM11_GK1 = Sequence<1, 1, 1, 1, 1>; - - using BBlockTransferThreadSliceLengths_GK0_GN0_GN10_GN11_GK1 = Sequence<1, 4, 1, 1, 1>; - using BBlockTransferThreadClusterLengths_GK0_GN0_GN10_GN11_GK1 = Sequence<8, 1, 1, 32, 1>; - - using BBlockTransferSrcVectorTensorLengths_GK0_GN0_GN10_GN11_GK1 = Sequence<1, 1, 1, 1, 1>; - using BBlockTransferDstVectorTensorLengths_GK0_GN0_GN10_GN11_GK1 = Sequence<1, 1, 1, 1, 1>; - - constexpr index_t CThreadTransferDstScalarPerVector_BN1 = 1; -#elif 1 - // [8, 1, 128, 2] * [8, 4, 32, 2] = [1, 128, 4, 32] for fp16 - // cdata = 64, BlockSize = 256 - constexpr index_t BlockSize = 256; - - constexpr index_t GN0 = 4; - constexpr index_t GK1 = 2; - - constexpr index_t GM1PerBlockGM11 = 128; - constexpr index_t GN1PerBlockGN11 = 32; - constexpr index_t GK0PerBlock = 8; - - constexpr index_t BM1PerThreadBM11 = 4; - constexpr index_t BN1PerThreadBN11 = 4; - constexpr index_t BK0PerThread = 1; - - using BM10BN10ThreadClusterBM10Xs = Sequence<8, 2>; - using BM10BN10ThreadClusterBN10Xs = Sequence<8, 2>; - - using ABlockTransferThreadSliceLengths_GK0_GM0_GM10_GM11_GK1 = Sequence<4, 1, 1, 1, 2>; - using ABlockTransferThreadClusterLengths_GK0_GM0_GM10_GM11_GK1 = Sequence<2, 1, 1, 128, 1>; - - using ABlockTransferSrcVectorTensorLengths_GK0_GM0_GM10_GM11_GK1 = Sequence<4, 1, 1, 1, 1>; - using ABlockTransferDstVectorTensorLengths_GK0_GM0_GM10_GM11_GK1 = Sequence<1, 1, 1, 1, 2>; - - using BBlockTransferThreadSliceLengths_GK0_GN0_GN10_GN11_GK1 = Sequence<1, 4, 1, 1, 2>; - using BBlockTransferThreadClusterLengths_GK0_GN0_GN10_GN11_GK1 = Sequence<8, 1, 1, 32, 1>; - - using BBlockTransferSrcVectorTensorLengths_GK0_GN0_GN10_GN11_GK1 = Sequence<1, 1, 1, 1, 1>; - using BBlockTransferDstVectorTensorLengths_GK0_GN0_GN10_GN11_GK1 = Sequence<1, 1, 1, 1, 2>; - - constexpr index_t CThreadTransferDstScalarPerVector_BN1 = 1; -#endif - - const auto descs = - transform_forward_convolution_into_contraction_v6r1_nchw_kcyx_nkhw_pad(wei_desc_k_c_y_x, - in_desc_n_c_hi_wi, - out_desc_n_k_ho_wo, - conv_strides, - conv_dilations, - in_left_pads, - in_right_pads, - Number{}, - Number{}); - - const auto wei_grid_desc_gk0_gm0_gm1_gk1 = descs[I0]; - const auto in_grid_desc_gk0_gn0_gn1_gk1 = descs[I1]; - const auto out_grid_desc_gm0_gm1_gn0_gn1 = descs[I2]; - - // HACK: hacks that control index calculation when iterating over A, B, C matrix - constexpr auto wei_grid_step_hacks = - make_tuple(make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0>{}, // 0+: GK0 - Sequence<0, 0, 0, 0, 0, 0, 0>{}, // 1+: GM0 - Sequence<0, 0, 0, 0, 0, 0, 0>{}, // 2+: GM10 - Sequence<0, 0, 0, 0, 0, 0, 0>{}, // 3+: GM11 - Sequence<0, 0, 0, 0, 0, 0, 0>{}), // 4+: GK1 - make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0>{}, // 0-: GK0 - Sequence<0, 0, 0, 0, 0, 0, 0>{}, // 1-: GM0 - Sequence<0, 0, 0, 0, 0, 0, 0>{}, // 2-: GM10 - Sequence<0, 0, 0, 0, 0, 0, 0>{}, // 3-: GM11 - Sequence<0, 0, 0, 0, 0, 0, 0>{})); // 4-: GK1 - - constexpr auto in_grid_step_hacks = make_tuple( - make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0>{}, // 0+: GK0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0>{}, // 1+: GN0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0>{}, // 2+: GN10 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0>{}, // 3+: GN11 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}), // 4+: GK1 - make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0>{}, // 0-: GK0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0>{}, // 1-: GN0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0>{}, // 2-: GN10 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0>{}, // 3-: GN11 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{})); // 4-: GK1 - - constexpr auto out_grid_step_hacks = make_tuple( - make_tuple( - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 0+: GM10 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0>{}, // 1+: BM0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0>{}, // 2+: BM1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 3+: GN10 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0>{}, // 4+: BN0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0>{}), // 5+: GN1 - make_tuple( - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 0-: GM10 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0>{}, // 1-: BM0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0>{}, // 2-: BM1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 3-: GN10 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0>{}, // 4-: BN0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0>{})); // 5-: GN1 - - constexpr auto wei_grid_move_slice_window_step_hacks = Sequence<0, 0, 0, 0, 0, 0, 0>{}; - - constexpr auto in_grid_move_slice_window_step_hacks = - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0>{}; - - for(index_t i = 0; i < 5; ++i) - { - float ave_time = driver_contraction_dlops_v1r2< - BlockSize, - TInWei, - TAcc, - TOut, - InMemoryDataOperationEnum::Set, - decltype(wei_grid_desc_gk0_gm0_gm1_gk1), - decltype(in_grid_desc_gk0_gn0_gn1_gk1), - decltype(out_grid_desc_gm0_gm1_gn0_gn1), - GM1PerBlockGM11, - GN1PerBlockGN11, - GK0PerBlock, - BM1PerThreadBM11, - BN1PerThreadBN11, - BK0PerThread, - BM10BN10ThreadClusterBM10Xs, - BM10BN10ThreadClusterBN10Xs, - ABlockTransferThreadSliceLengths_GK0_GM0_GM10_GM11_GK1, - ABlockTransferThreadClusterLengths_GK0_GM0_GM10_GM11_GK1, - Sequence<1, 2, 3, 0, 4>, // ABlockTransferThreadClusterArrangeOrder - Sequence<3, 2, 1, 0, 4>, // ABlockTransferSrcAccessOrder - ABlockTransferSrcVectorTensorLengths_GK0_GM0_GM10_GM11_GK1, - ABlockTransferDstVectorTensorLengths_GK0_GM0_GM10_GM11_GK1, - Sequence<0, 1, 2, 3, 4>, // ABlockTransferSrcVectorTensorContiguousDimOrder - BBlockTransferThreadSliceLengths_GK0_GN0_GN10_GN11_GK1, - BBlockTransferThreadClusterLengths_GK0_GN0_GN10_GN11_GK1, - Sequence<0, 4, 1, 2, 3>, // BBlockTransferThreadClusterArrangeOrder - Sequence<4, 3, 2, 0, 1>, // BBlockTransferSrcAccessOrder - BBlockTransferSrcVectorTensorLengths_GK0_GN0_GN10_GN11_GK1, - BBlockTransferDstVectorTensorLengths_GK0_GN0_GN10_GN11_GK1, - Sequence<0, 1, 2, 3, 4>, // BBlockTransferSrcVectorTensorContiguousDimOrder - Sequence<3, 4, 5, 0, 1, 2>, // CThreadTransferSrcDstAccessOrder - 5, // CThreadTransferSrcDstVectorDim - CThreadTransferDstScalarPerVector_BN1, - decltype(wei_grid_step_hacks), - decltype(in_grid_step_hacks), - decltype(out_grid_step_hacks), - decltype(wei_grid_move_slice_window_step_hacks), - decltype(in_grid_move_slice_window_step_hacks)>( - static_cast(wei_k_c_y_x_device_buf.GetDeviceBuffer()), - static_cast(in_n_c_hi_wi_device_buf.GetDeviceBuffer()), - static_cast(out_n_k_ho_wo_device_buf.GetDeviceBuffer()), - wei_grid_desc_gk0_gm0_gm1_gk1, - in_grid_desc_gk0_gn0_gn1_gk1, - out_grid_desc_gm0_gm1_gn0_gn1, - wei_grid_step_hacks, - in_grid_step_hacks, - out_grid_step_hacks, - wei_grid_move_slice_window_step_hacks, - in_grid_move_slice_window_step_hacks, - nrepeat); - - float perf = static_cast(calculate_convolution_flops( - in_desc_n_c_hi_wi, wei_desc_k_c_y_x, out_desc_n_k_ho_wo)) / - (std::size_t(1000) * 1000 * 1000) / ave_time; - - std::cout << "Average time : " << ave_time << " ms, " << perf << " TFlop/s" << std::endl; - } - - // copy result back to host - out_n_k_ho_wo_device_buf.FromDevice(out_n_k_ho_wo.mData.data()); -} diff --git a/library/include/ck/library/obselete_driver_offline/device_convolution_maxpool_forward_implicit_gemm_v5r1_dlops_nc0hwc1_kc0yxc1_nk0hwk1.hpp b/library/include/ck/library/obselete_driver_offline/device_convolution_maxpool_forward_implicit_gemm_v5r1_dlops_nc0hwc1_kc0yxc1_nk0hwk1.hpp deleted file mode 100644 index 2cb2e10915..0000000000 --- a/library/include/ck/library/obselete_driver_offline/device_convolution_maxpool_forward_implicit_gemm_v5r1_dlops_nc0hwc1_kc0yxc1_nk0hwk1.hpp +++ /dev/null @@ -1,212 +0,0 @@ -#include -#include "device.hpp" -#include "host_tensor.hpp" -#include "driver_convolution_maxpool_forward_implicit_gemm_v5r1_dlops_nc0hwc1_kc0yxc1_nk0hwk1.hpp" - -template -void device_convolution_maxpool_forward_implicit_gemm_v5r1_dlops_nc0hwc1_kc0yxc1_nk0hwk1( - const InLengths& in_n_c0_hi_wi_c1_lengths, - const WeiLengths& wei_k_c0_y_x_c1_lengths, - const MaxLengths& max_n_k0_hx_wx_k1_lengths, - const OutLengths& out_n_k0_ho_wo_k1_lengths, - const ConvStrides& conv_strides, - const ConvDilations& conv_dilations, - const InLeftPads& in_left_pads, - const InRightPads& in_right_pads, - const Tensor& in_n_c0_hi_wi_c1, - const Tensor& wei_k_c0_y_x_c1, - const Tensor& bias_k0_k1, - Tensor& out_n_k0_ho_wo_k1, - Tensor& max_n_k0_hx_wx_k1, - ck::index_t nrepeat) -{ - using namespace ck; - - std::cout << __func__ << std::endl; - - constexpr auto I0 = Number<0>{}; - constexpr auto I1 = Number<1>{}; - constexpr auto I2 = Number<2>{}; - constexpr auto I3 = Number<3>{}; - constexpr auto I4 = Number<4>{}; - - const auto N = out_n_k0_ho_wo_k1_lengths[I0]; - const auto K0 = out_n_k0_ho_wo_k1_lengths[I1]; - const auto Ho = out_n_k0_ho_wo_k1_lengths[I2]; - const auto Wo = out_n_k0_ho_wo_k1_lengths[I3]; - const auto K1 = out_n_k0_ho_wo_k1_lengths[I4]; - - const auto C0 = in_n_c0_hi_wi_c1_lengths[I1]; - const auto Hi = in_n_c0_hi_wi_c1_lengths[I2]; - const auto Wi = in_n_c0_hi_wi_c1_lengths[I3]; - const auto C1 = in_n_c0_hi_wi_c1_lengths[I4]; - - const auto K = wei_k_c0_y_x_c1_lengths[I0]; - const auto Y = wei_k_c0_y_x_c1_lengths[I2]; - const auto X = wei_k_c0_y_x_c1_lengths[I3]; - - const auto Hx = max_n_k0_hx_wx_k1_lengths[I2]; - const auto Wx = max_n_k0_hx_wx_k1_lengths[I3]; - - DeviceMem in_n_c0_hi_wi_c1_device_buf(sizeof(TInWei) * - in_n_c0_hi_wi_c1.mDesc.GetElementSpace()); - DeviceMem wei_k_c0_y_x_c1_device_buf(sizeof(TInWei) * wei_k_c0_y_x_c1.mDesc.GetElementSpace()); - DeviceMem bias_k0_k1_device_buf(sizeof(TOut) * bias_k0_k1.mDesc.GetElementSpace()); - DeviceMem out_n_k0_ho_wo_k1_device_buf(sizeof(TOut) * - out_n_k0_ho_wo_k1.mDesc.GetElementSpace()); - DeviceMem max_n_k0_hx_wx_k1_device_buf(sizeof(TOut) * - max_n_k0_hx_wx_k1.mDesc.GetElementSpace()); - - in_n_c0_hi_wi_c1_device_buf.ToDevice(in_n_c0_hi_wi_c1.mData.data()); - wei_k_c0_y_x_c1_device_buf.ToDevice(wei_k_c0_y_x_c1.mData.data()); - bias_k0_k1_device_buf.ToDevice(bias_k0_k1.mData.data()); - max_n_k0_hx_wx_k1_device_buf.ToDevice(max_n_k0_hx_wx_k1.mData.data()); - - constexpr index_t InWeiVectorSize = 8; - - if(C1 % InWeiVectorSize != 0) - { - throw std::runtime_error("wrong! C1 cannot be divided by InWeiVectorSize"); - } - -#if 0 - constexpr index_t BlockSize = 256; - - constexpr index_t KPerBlock = 32; - constexpr index_t HoPerBlock = 8; - constexpr index_t WoPerBlock = 64; - - constexpr index_t E1 = C0 * 9; - constexpr index_t E2 = 1; - constexpr index_t E1PerBlock = C0; - - constexpr index_t KPerThread = 16; - constexpr index_t HoPerThread = 2; - constexpr index_t WoPerThread = 2; - constexpr index_t EPerThread = 1; - - using ABlockTransferThreadSliceLengths_E0_E1_K0_K1_E2 = Sequence<1, 9, 1, E2>; - using ABlockTransferThreadClusterLengths_E0_E1_K0_K1_E2 = Sequence<1, E1PerBlock, KPerBlock, 1>; - - constexpr index_t ABlockTransferSrcScalarPerVector_E2 = E2; - constexpr index_t ABlockTransferDstScalarPerVector_E2 = E2; - - constexpr index_t BThreadTransferSrcScalarPerVector_E2 = E2; - - constexpr index_t CThreadTransferDstScalarPerVector_K = K1; -#elif 1 - constexpr index_t BlockSize = 64; - - constexpr index_t KPerBlock = 8; - constexpr index_t HoPerBlock = 8; - constexpr index_t WoPerBlock = 32; - - constexpr index_t E1 = 2 * 9; - constexpr index_t E2 = 1; - constexpr index_t K2 = 2; - constexpr index_t E1PerBlock = 2; - - constexpr index_t KPerThread = KPerBlock; - constexpr index_t HoPerThread = 2; - constexpr index_t WoPerThread = 2; - constexpr index_t EPerThread = 1; - - using ABlockTransferThreadSliceLengths_E0_E1_K0_K1_E2 = Sequence<1, 9, 1, 1, E2>; - using ABlockTransferThreadClusterLengths_E0_E1_K0_K1_E2 = - Sequence<1, E1PerBlock, 1, KPerBlock, 1>; - - constexpr index_t ABlockTransferSrcScalarPerVector_E2 = E2; - constexpr index_t ABlockTransferDstScalarPerVector_E2 = E2; - constexpr index_t BThreadTransferSrcScalarPerVector_E2 = E2; - constexpr index_t CThreadTransferDstScalarPerVector_K = InWeiVectorSize; -#endif - - if(KPerThread % InWeiVectorSize != 0) - { - throw std::runtime_error("wrong! C1 cannot be divided by InWeiVectorSize"); - } - - const auto in_n_c0_hi_wi_c1_desc = - make_naive_tensor_descriptor_packed(make_tuple(N, C0, Hi, Wi, E2)); - const auto wei_k_c0_y_x_c1_desc = - make_naive_tensor_descriptor_packed(make_tuple(K, C0, Y, X, E2)); - const auto max_n_k0_hx_wx_k1_desc = - make_naive_tensor_descriptor_packed(make_tuple(N, K0, Hx, Wx, K1)); - const auto out_n_k0_ho_wo_k1_desc = - make_naive_tensor_descriptor_packed(make_tuple(N, K0, Ho, Wo, K1)); - - constexpr auto conv_driver = - DriverDynamicConvolutionForwardImplicitGemmDlops_v5r1_nc0hwc1_kc0yxc1_nk0hwk1_maxpool< - BlockSize, - typename vector_type::type, - TAcc, - TOut, - E1, - E2, - K2, - KPerBlock, - HoPerBlock, - WoPerBlock, - E1PerBlock, - KPerThread, - HoPerThread, - WoPerThread, - EPerThread, - ABlockTransferThreadSliceLengths_E0_E1_K0_K1_E2, - ABlockTransferThreadClusterLengths_E0_E1_K0_K1_E2, - ABlockTransferSrcScalarPerVector_E2, - ABlockTransferDstScalarPerVector_E2, - BThreadTransferSrcScalarPerVector_E2, - CThreadTransferDstScalarPerVector_K, - activ_type>{}; - - std::cerr << "conv_bias_activ_maxpool_input_" - << "n" << N << "c" << C0 << "h" << Hi << "w" << Wi << "c" << C1 << "_filter_k" << K - << "c" << C0 << "y" << Y << "x" << X << "c" << C1 << "_convout_n" << N << "k" << K0 - << "h" << Ho << "w" << Wo << "k" << K1 << "_maxpoolout_n" << N << "k" << K0 << "h" - << Ho / 2 << "w" << Wo / 2 << "k" << K1 << std::endl; - - for(int i = 0; i < 5; i++) - { - - const auto ave_time = - conv_driver.Run(wei_k_c0_y_x_c1_desc, - in_n_c0_hi_wi_c1_desc, - out_n_k0_ho_wo_k1_desc, - max_n_k0_hx_wx_k1_desc, - conv_strides, - conv_dilations, - in_left_pads, - in_right_pads, - static_cast::type*>( - wei_k_c0_y_x_c1_device_buf.GetDeviceBuffer()), - static_cast::type*>( - in_n_c0_hi_wi_c1_device_buf.GetDeviceBuffer()), - static_cast(bias_k0_k1_device_buf.GetDeviceBuffer()), - static_cast(out_n_k0_ho_wo_k1_device_buf.GetDeviceBuffer()), - static_cast(max_n_k0_hx_wx_k1_device_buf.GetDeviceBuffer()), - nrepeat); - - { - float perf = static_cast(std::size_t(2) * N * K * Ho * Wo * C0 * C1 * Y * X) / - (std::size_t(1000) * 1000 * 1000) / ave_time; - - std::cout << "Average time : " << ave_time << " ms, " << perf << " TFlop/s" - << std::endl; - } - } - - out_n_k0_ho_wo_k1_device_buf.FromDevice(out_n_k0_ho_wo_k1.mData.data()); - max_n_k0_hx_wx_k1_device_buf.FromDevice(max_n_k0_hx_wx_k1.mData.data()); -} diff --git a/library/include/ck/library/obselete_driver_offline/device_gemm_xdlops_km_kn_mn.hpp b/library/include/ck/library/obselete_driver_offline/device_gemm_xdlops_km_kn_mn.hpp deleted file mode 100644 index f54ff181dd..0000000000 --- a/library/include/ck/library/obselete_driver_offline/device_gemm_xdlops_km_kn_mn.hpp +++ /dev/null @@ -1,463 +0,0 @@ -#pragma once -#include -#include "device.hpp" -#include "host_tensor.hpp" -#include "driver_gemm_xdlops_v2r3.hpp" - -template -void device_gemm_xdlops_km_kn_mn(const Tensor& a_k_m, - const Tensor& b_k_n, - Tensor& c_m_n, - ck::index_t nrepeat) -{ - using namespace ck; - - std::cout << __func__ << std::endl; - - DeviceMem a_k_m_device_buf(sizeof(ABType) * a_k_m.mDesc.GetElementSpace()); - DeviceMem b_k_n_device_buf(sizeof(ABType) * b_k_n.mDesc.GetElementSpace()); - DeviceMem c_m_n_device_buf(sizeof(CType) * c_m_n.mDesc.GetElementSpace()); - - a_k_m_device_buf.ToDevice(a_k_m.mData.data()); - b_k_n_device_buf.ToDevice(b_k_n.mData.data()); - c_m_n_device_buf.ToDevice(c_m_n.mData.data()); - -#if 0 - // [M, N, K0, K1] = [256, 128, 4, 4] for fp32 - constexpr index_t BlockSize = 256; - - constexpr index_t MPerBlock = 256; - constexpr index_t NPerBlock = 128; - constexpr index_t KPerBlock = 4; - - constexpr index_t MPerXDL = 32; - constexpr index_t NPerXDL = 32; - constexpr index_t K1 = 4; - - constexpr index_t MRepeat = 4; - constexpr index_t NRepeat = 2; - - using ABlockTransferThreadSliceLengths_K0_M_K1 = Sequence<1, 4, 4>; - using ABlockTransferThreadClusterLengths_K0_M_K1 = Sequence<4, 64, 1>; - - constexpr index_t ABlockTransferSrcScalarPerVector_M = 4; - constexpr index_t ABlockTransferDstScalarPerVector_K1 = 4; - - using BBlockTransferThreadSliceLengths_K0_N_K1 = Sequence<1, 2, 4>; - using BBlockTransferThreadClusterLengths_K0_N_K1 = Sequence<4, 64, 1>; - - constexpr index_t BBlockTransferSrcScalarPerVector_N = 2; - constexpr index_t BBlockTransferDstScalarPerVector_K1 = 4; - - constexpr index_t CThreadTransferDstScalarPerVector = 1; -#elif 0 - // [M, N, K0, K1] = [128, 256, 4, 4], C = 128, for fp32 - constexpr index_t BlockSize = 256; - - constexpr index_t MPerBlock = 128; - constexpr index_t NPerBlock = 256; - constexpr index_t KPerBlock = 4; - - constexpr index_t MPerXDL = 32; - constexpr index_t NPerXDL = 32; - constexpr index_t K1 = 4; - - constexpr index_t MRepeat = 2; - constexpr index_t NRepeat = 4; - - using ABlockTransferThreadSliceLengths_K0_M_K1 = Sequence<1, 2, 4>; - using ABlockTransferThreadClusterLengths_K0_M_K1 = Sequence<4, 64, 1>; - - constexpr index_t ABlockTransferSrcScalarPerVector_M = 2; - constexpr index_t ABlockTransferDstScalarPerVector_K1 = 4; - - using BBlockTransferThreadSliceLengths_K0_N_K1 = Sequence<1, 4, 4>; - using BBlockTransferThreadClusterLengths_K0_N_K1 = Sequence<4, 64, 1>; - - constexpr index_t BBlockTransferSrcScalarPerVector_N = 4; - constexpr index_t BBlockTransferDstScalarPerVector_K1 = 4; - - constexpr index_t CThreadTransferDstScalarPerVector = 1; -#elif 0 - // [M, N, K0, K1] = [128, 128, 4, 4], C = 64, for fp32 - constexpr index_t BlockSize = 256; - - constexpr index_t MPerBlock = 128; - constexpr index_t NPerBlock = 128; - constexpr index_t KPerBlock = 4; - - constexpr index_t MPerXDL = 32; - constexpr index_t NPerXDL = 32; - constexpr index_t K1 = 4; - - constexpr index_t MRepeat = 2; - constexpr index_t NRepeat = 2; - - using ABlockTransferThreadSliceLengths_K0_M_K1 = Sequence<1, 2, 4>; - using ABlockTransferThreadClusterLengths_K0_M_K1 = Sequence<4, 64, 1>; - - constexpr index_t ABlockTransferSrcScalarPerVector_M = 2; - constexpr index_t ABlockTransferDstScalarPerVector_K1 = 4; - - using BBlockTransferThreadSliceLengths_K0_N_K1 = Sequence<1, 2, 4>; - using BBlockTransferThreadClusterLengths_K0_N_K1 = Sequence<4, 64, 1>; - - constexpr index_t BBlockTransferSrcScalarPerVector_N = 2; - constexpr index_t BBlockTransferDstScalarPerVector_K1 = 4; - - constexpr index_t CThreadTransferDstScalarPerVector = 1; -#elif 0 - // [M, N, K0, K1] = [128, 64, 4, 4], C = 32, for fp32 - constexpr index_t BlockSize = 256; - - constexpr index_t MPerBlock = 128; - constexpr index_t NPerBlock = 64; - constexpr index_t KPerBlock = 4; - - constexpr index_t MPerXDL = 32; - constexpr index_t NPerXDL = 32; - constexpr index_t K1 = 4; - - constexpr index_t MRepeat = 2; - constexpr index_t NRepeat = 1; - - using ABlockTransferThreadSliceLengths_K0_M_K1 = Sequence<1, 2, 4>; - using ABlockTransferThreadClusterLengths_K0_M_K1 = Sequence<4, 64, 1>; - - constexpr index_t ABlockTransferSrcScalarPerVector_M = 2; - constexpr index_t ABlockTransferDstScalarPerVector_K1 = 4; - - using BBlockTransferThreadSliceLengths_K0_N_K1 = Sequence<1, 1, 4>; - using BBlockTransferThreadClusterLengths_K0_N_K1 = Sequence<4, 64, 1>; - - constexpr index_t BBlockTransferSrcScalarPerVector_N = 1; - constexpr index_t BBlockTransferDstScalarPerVector_K1 = 4; - - constexpr index_t CThreadTransferDstScalarPerVector = 1; -#elif 0 - // [M, N, K0, K1] = [64, 128, 4, 4], C = 32, for fp32 - constexpr index_t BlockSize = 256; - - constexpr index_t MPerBlock = 64; - constexpr index_t NPerBlock = 128; - constexpr index_t KPerBlock = 4; - - constexpr index_t MPerXDL = 32; - constexpr index_t NPerXDL = 32; - constexpr index_t K1 = 4; - - constexpr index_t MRepeat = 1; - constexpr index_t NRepeat = 2; - - using ABlockTransferThreadSliceLengths_K0_M_K1 = Sequence<1, 1, 4>; - using ABlockTransferThreadClusterLengths_K0_M_K1 = Sequence<4, 64, 1>; - - constexpr index_t ABlockTransferSrcScalarPerVector_M = 1; - constexpr index_t ABlockTransferDstScalarPerVector_K1 = 4; - - using BBlockTransferThreadSliceLengths_K0_N_K1 = Sequence<1, 2, 4>; - using BBlockTransferThreadClusterLengths_K0_N_K1 = Sequence<4, 64, 1>; - - constexpr index_t BBlockTransferSrcScalarPerVector_N = 2; - constexpr index_t BBlockTransferDstScalarPerVector_K1 = 4; - - constexpr index_t CThreadTransferDstScalarPerVector = 1; -#elif 1 - // [M, N, K0, K1] = [256, 128, 4, 8], C = 128, for fp16 - constexpr index_t BlockSize = 256; - - constexpr index_t MPerBlock = 256; - constexpr index_t NPerBlock = 128; - constexpr index_t KPerBlock = 4; - - constexpr index_t MPerXDL = 32; - constexpr index_t NPerXDL = 32; - constexpr index_t K1 = 8; - - constexpr index_t MRepeat = 4; - constexpr index_t NRepeat = 2; - - using ABlockTransferThreadSliceLengths_K0_M_K1 = Sequence<1, 4, 8>; - using ABlockTransferThreadClusterLengths_K0_M_K1 = Sequence<4, 64, 1>; - - constexpr index_t ABlockTransferSrcScalarPerVector_M = 4; - constexpr index_t ABlockTransferDstScalarPerVector_K1 = 8; - - using BBlockTransferThreadSliceLengths_K0_N_K1 = Sequence<1, 2, 8>; - using BBlockTransferThreadClusterLengths_K0_N_K1 = Sequence<4, 64, 1>; - - constexpr index_t BBlockTransferSrcScalarPerVector_N = 2; - constexpr index_t BBlockTransferDstScalarPerVector_K1 = 8; - - constexpr index_t CThreadTransferDstScalarPerVector = 1; -#elif 0 - // [M, N, K0, K1] = [128, 256, 4, 8] for fp16 - constexpr index_t BlockSize = 256; - - constexpr index_t MPerBlock = 128; - constexpr index_t NPerBlock = 256; - constexpr index_t KPerBlock = 4; - - constexpr index_t MPerXDL = 32; - constexpr index_t NPerXDL = 32; - constexpr index_t K1 = 8; - - constexpr index_t MRepeat = 2; - constexpr index_t NRepeat = 4; - - using ABlockTransferThreadSliceLengths_K0_M_K1 = Sequence<1, 2, 8>; - using ABlockTransferThreadClusterLengths_K0_M_K1 = Sequence<4, 64, 1>; - - constexpr index_t ABlockTransferSrcScalarPerVector_M = 2; - constexpr index_t ABlockTransferDstScalarPerVector_K1 = 8; - - using BBlockTransferThreadSliceLengths_K0_N_K1 = Sequence<1, 4, 8>; - using BBlockTransferThreadClusterLengths_K0_N_K1 = Sequence<4, 64, 1>; - - constexpr index_t BBlockTransferSrcScalarPerVector_N = 4; - constexpr index_t BBlockTransferDstScalarPerVector_K1 = 8; - - constexpr index_t CThreadTransferDstScalarPerVector = 1; -#elif 0 - // [M, N, K0, K1] = [128, 128, 4, 8], C = 128, for fp16 - constexpr index_t BlockSize = 128; - - constexpr index_t MPerBlock = 128; - constexpr index_t NPerBlock = 128; - constexpr index_t KPerBlock = 4; - - constexpr index_t MPerXDL = 32; - constexpr index_t NPerXDL = 32; - constexpr index_t K1 = 8; - - constexpr index_t MRepeat = 4; - constexpr index_t NRepeat = 2; - - using ABlockTransferThreadSliceLengths_K0_M_K1 = Sequence<1, 4, 8>; - using ABlockTransferThreadClusterLengths_K0_M_K1 = Sequence<4, 32, 1>; - - constexpr index_t ABlockTransferSrcScalarPerVector_M = 4; - constexpr index_t ABlockTransferDstScalarPerVector_K1 = 8; - - using BBlockTransferThreadSliceLengths_K0_N_K1 = Sequence<1, 4, 8>; - using BBlockTransferThreadClusterLengths_K0_N_K1 = Sequence<4, 32, 1>; - - constexpr index_t BBlockTransferSrcScalarPerVector_N = 4; - constexpr index_t BBlockTransferDstScalarPerVector_K1 = 8; - - constexpr index_t CThreadTransferDstScalarPerVector = 1; -#elif 0 - // [M, N, K0, K1] = [128, 128, 4, 8], C = 64, for fp16 - constexpr index_t BlockSize = 256; - - constexpr index_t MPerBlock = 128; - constexpr index_t NPerBlock = 128; - constexpr index_t KPerBlock = 4; - - constexpr index_t MPerXDL = 32; - constexpr index_t NPerXDL = 32; - constexpr index_t K1 = 8; - - constexpr index_t MRepeat = 2; - constexpr index_t NRepeat = 2; - - using ABlockTransferThreadSliceLengths_K0_M_K1 = Sequence<1, 2, 8>; - using ABlockTransferThreadClusterLengths_K0_M_K1 = Sequence<4, 64, 1>; - - constexpr index_t ABlockTransferSrcScalarPerVector_M = 2; - constexpr index_t ABlockTransferDstScalarPerVector_K1 = 8; - - using BBlockTransferThreadSliceLengths_K0_N_K1 = Sequence<1, 2, 8>; - using BBlockTransferThreadClusterLengths_K0_N_K1 = Sequence<4, 64, 1>; - - constexpr index_t BBlockTransferSrcScalarPerVector_N = 2; - constexpr index_t BBlockTransferDstScalarPerVector_K1 = 8; - - constexpr index_t CThreadTransferDstScalarPerVector = 1; -#elif 1 - // [M, N, K0, K1] = [128, 64, 4, 8], C = 32, for fp16 - constexpr index_t BlockSize = 256; - - constexpr index_t MPerBlock = 128; - constexpr index_t NPerBlock = 64; - constexpr index_t KPerBlock = 4; - - constexpr index_t MPerXDL = 32; - constexpr index_t NPerXDL = 32; - constexpr index_t K1 = 8; - - constexpr index_t MRepeat = 2; - constexpr index_t NRepeat = 1; - - using ABlockTransferThreadSliceLengths_K0_M_K1 = Sequence<1, 2, 8>; - using ABlockTransferThreadClusterLengths_K0_M_K1 = Sequence<4, 64, 1>; - - constexpr index_t ABlockTransferSrcScalarPerVector_M = 2; - constexpr index_t ABlockTransferDstScalarPerVector_K1 = 8; - - using BBlockTransferThreadSliceLengths_K0_N_K1 = Sequence<1, 1, 8>; - using BBlockTransferThreadClusterLengths_K0_N_K1 = Sequence<4, 64, 1>; - - constexpr index_t BBlockTransferSrcScalarPerVector_N = 1; - constexpr index_t BBlockTransferDstScalarPerVector_K1 = 8; - - constexpr index_t CThreadTransferDstScalarPerVector = 1; -#elif 0 - // [M, N, K0, K1] = [64, 128, 4, 8], C = 32, for fp16 - constexpr index_t BlockSize = 256; - - constexpr index_t MPerBlock = 64; - constexpr index_t NPerBlock = 128; - constexpr index_t KPerBlock = 4; - - constexpr index_t MPerXDL = 32; - constexpr index_t NPerXDL = 32; - constexpr index_t K1 = 8; - - constexpr index_t MRepeat = 1; - constexpr index_t NRepeat = 2; - - using ABlockTransferThreadSliceLengths_K0_M_K1 = Sequence<1, 1, 8>; - using ABlockTransferThreadClusterLengths_K0_M_K1 = Sequence<4, 64, 1>; - - constexpr index_t ABlockTransferSrcScalarPerVector_M = 1; - constexpr index_t ABlockTransferDstScalarPerVector_K1 = 8; - - using BBlockTransferThreadSliceLengths_K0_N_K1 = Sequence<1, 2, 8>; - using BBlockTransferThreadClusterLengths_K0_N_K1 = Sequence<4, 64, 1>; - - constexpr index_t BBlockTransferSrcScalarPerVector_N = 2; - constexpr index_t BBlockTransferDstScalarPerVector_K1 = 8; - - constexpr index_t CThreadTransferDstScalarPerVector = 1; -#endif - - const auto K = a_k_m.mDesc.GetLengths()[0]; - const auto M = a_k_m.mDesc.GetLengths()[1]; - const auto N = b_k_n.mDesc.GetLengths()[1]; - - constexpr auto K1Number = Number{}; - const auto K0 = K / K1Number; - - const auto a_k0_m_k1_grid_desc = - make_naive_tensor_descriptor(make_tuple(K0, M, K1Number), - make_tuple(K1Number * a_k_m.mDesc.GetStrides()[0], - a_k_m.mDesc.GetStrides()[1], - a_k_m.mDesc.GetStrides()[0])); - - const auto b_k0_n_k1_grid_desc = - make_naive_tensor_descriptor(make_tuple(K0, N, K1Number), - make_tuple(K1Number * b_k_n.mDesc.GetStrides()[0], - b_k_n.mDesc.GetStrides()[1], - b_k_n.mDesc.GetStrides()[0])); - - const auto c_m_n_grid_desc = make_naive_tensor_descriptor( - make_tuple(M, N), make_tuple(c_m_n.mDesc.GetStrides()[0], c_m_n.mDesc.GetStrides()[1])); - - // HACK: hacks that control index calculation when iterating over A, B, C matrix - constexpr auto a_k0_m_k1_grid_step_hacks = make_tuple(make_tuple(Sequence<0>{}, // 0+: K0 - Sequence<0>{}, // 1+: M - Sequence<0>{}), // 2+: K1 - make_tuple(Sequence<0>{}, // 0-: K0 - Sequence<0>{}, // 1-: M - Sequence<0>{})); // 2-: K1 - - constexpr auto b_k0_n_k1_grid_step_hacks = make_tuple(make_tuple(Sequence<0>{}, // 0+: K0 - Sequence<0>{}, // 1+: N - Sequence<0>{}), // 2+: K1 - make_tuple(Sequence<0>{}, // 0-: K0 - Sequence<0>{}, // 1-: N - Sequence<0>{})); // 2-: K1 - - constexpr auto c_m0_n0_m1_n1_m2_m3_m4_n2_grid_step_hacks = - make_tuple(make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 0+: M0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 1+: N0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 2+: M1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 3+: N1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 4+: M2 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 5+: M3 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 6+: M4 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}), // 7+: N2 - make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 0-: M0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 1-: N0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 2-: M1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 3-: N1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 4-: M2 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 5-: M3 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 6-: M4 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{})); // 7-: N2 - - constexpr auto a_k0_m_k1_grid_move_slice_window_step_hacks = Sequence<0>{}; - - constexpr auto b_k0_n_k1_grid_move_slice_window_step_hacks = Sequence<0>{}; - - for(index_t i = 0; i < 5; ++i) - { - float ave_time = - driver_gemm_xdlops_v2r3, - Sequence<0, 2, 1>, - 1, - ABlockTransferSrcScalarPerVector_M, - ABlockTransferDstScalarPerVector_K1, - false, // don't move back src coordinate after threadwise copy - BBlockTransferThreadSliceLengths_K0_N_K1, - BBlockTransferThreadClusterLengths_K0_N_K1, - Sequence<0, 2, 1>, - Sequence<0, 2, 1>, - 1, - BBlockTransferSrcScalarPerVector_N, - BBlockTransferDstScalarPerVector_K1, - false, // don't move back src coordinate after threadwise copy - Sequence<0, 2, 4, 5, 6, 1, 3, 7>, - 7, - CThreadTransferDstScalarPerVector, - decltype(a_k0_m_k1_grid_step_hacks), - decltype(b_k0_n_k1_grid_step_hacks), - decltype(c_m0_n0_m1_n1_m2_m3_m4_n2_grid_step_hacks), - decltype(a_k0_m_k1_grid_move_slice_window_step_hacks), - decltype(b_k0_n_k1_grid_move_slice_window_step_hacks), - false, // CAccessOrderMRepeatNRepeat - true, // ABlockLdsExtraM - true // BBlockLdsExtraN - >(static_cast(a_k_m_device_buf.GetDeviceBuffer()), - static_cast(b_k_n_device_buf.GetDeviceBuffer()), - static_cast(c_m_n_device_buf.GetDeviceBuffer()), - a_k0_m_k1_grid_desc, - b_k0_n_k1_grid_desc, - c_m_n_grid_desc, - debug::debug_driver_gemm_xdlops_v2r3::M01, - debug::debug_driver_gemm_xdlops_v2r3::N01, - a_k0_m_k1_grid_step_hacks, - b_k0_n_k1_grid_step_hacks, - c_m0_n0_m1_n1_m2_m3_m4_n2_grid_step_hacks, - a_k0_m_k1_grid_move_slice_window_step_hacks, - b_k0_n_k1_grid_move_slice_window_step_hacks, - nrepeat); - - float perf = static_cast((std::size_t(2) * M * N * K)) / - (std::size_t(1000) * 1000 * 1000) / ave_time; - - std::cout << "Average time : " << ave_time << " ms, " << perf << " TFlop/s" << std::endl; - } - - // copy result back to host - c_m_n_device_buf.FromDevice(c_m_n.mData.data()); -} diff --git a/library/include/ck/library/obselete_driver_offline/device_gemm_xdlops_km_kn_nm.hpp b/library/include/ck/library/obselete_driver_offline/device_gemm_xdlops_km_kn_nm.hpp deleted file mode 100644 index eb78ba96d8..0000000000 --- a/library/include/ck/library/obselete_driver_offline/device_gemm_xdlops_km_kn_nm.hpp +++ /dev/null @@ -1,263 +0,0 @@ -#pragma once -#include -#include "device.hpp" -#include "host_tensor.hpp" -#include "driver_gemm_xdlops_v2r3.hpp" - -template -void device_gemm_xdlops_km_kn_nm(const Tensor& a_k_m, - const Tensor& b_k_n, - Tensor& c_n_m, - ck::index_t nrepeat) -{ - using namespace ck; - - std::cout << __func__ << std::endl; - - DeviceMem a_k_m_device_buf(sizeof(ABType) * a_k_m.mDesc.GetElementSpace()); - DeviceMem b_k_n_device_buf(sizeof(ABType) * b_k_n.mDesc.GetElementSpace()); - DeviceMem c_n_m_device_buf(sizeof(CType) * c_n_m.mDesc.GetElementSpace()); - - a_k_m_device_buf.ToDevice(a_k_m.mData.data()); - b_k_n_device_buf.ToDevice(b_k_n.mData.data()); - c_n_m_device_buf.ToDevice(c_n_m.mData.data()); - -#if 0 - // [M, N, K0, K1] = [256, 128, 4, 4] for fp32 - constexpr index_t BlockSize = 256; - - constexpr index_t MPerBlock = 256; - constexpr index_t NPerBlock = 128; - constexpr index_t KPerBlock = 4; - - constexpr index_t MPerXDL = 32; - constexpr index_t NPerXDL = 32; - constexpr index_t K1 = 4; - - constexpr index_t MRepeat = 4; - constexpr index_t NRepeat = 2; - - using ABlockTransferThreadSliceLengths_K0_M_K1 = Sequence<1, 4, 4>; - using ABlockTransferThreadClusterLengths_K0_M_K1 = Sequence<4, 64, 1>; - - constexpr index_t ABlockTransferSrcScalarPerVector_M = 4; - constexpr index_t ABlockTransferDstScalarPerVector_K1 = 4; - - using BBlockTransferThreadSliceLengths_K0_N_K1 = Sequence<1, 2, 4>; - using BBlockTransferThreadClusterLengths_K0_N_K1 = Sequence<4, 64, 1>; - - constexpr index_t BBlockTransferSrcScalarPerVector_N = 2; - constexpr index_t BBlockTransferDstScalarPerVector_K1 = 4; - - constexpr index_t CThreadTransferDstScalarPerVector = 4; -#elif 0 - // [M, N, K0, K1] = [128, 256, 4, 4] for fp32 - constexpr index_t BlockSize = 256; - - constexpr index_t MPerBlock = 128; - constexpr index_t NPerBlock = 256; - constexpr index_t KPerBlock = 4; - - constexpr index_t MPerXDL = 32; - constexpr index_t NPerXDL = 32; - constexpr index_t K1 = 4; - - constexpr index_t MRepeat = 2; - constexpr index_t NRepeat = 4; - - using ABlockTransferThreadSliceLengths_K0_M_K1 = Sequence<1, 2, 4>; - using ABlockTransferThreadClusterLengths_K0_M_K1 = Sequence<4, 64, 1>; - - constexpr index_t ABlockTransferSrcScalarPerVector_M = 2; - constexpr index_t ABlockTransferDstScalarPerVector_K1 = 4; - - using BBlockTransferThreadSliceLengths_K0_N_K1 = Sequence<1, 4, 4>; - using BBlockTransferThreadClusterLengths_K0_N_K1 = Sequence<4, 64, 1>; - - constexpr index_t BBlockTransferSrcScalarPerVector_N = 4; - constexpr index_t BBlockTransferDstScalarPerVector_K1 = 4; - - constexpr index_t CThreadTransferDstScalarPerVector = 4; -#elif 1 - // [M, N, K0, K1] = [256, 128, 4, 8] for fp16 - constexpr index_t BlockSize = 256; - - constexpr index_t MPerBlock = 256; - constexpr index_t NPerBlock = 128; - constexpr index_t KPerBlock = 4; - - constexpr index_t MPerXDL = 32; - constexpr index_t NPerXDL = 32; - constexpr index_t K1 = 8; - - constexpr index_t MRepeat = 4; - constexpr index_t NRepeat = 2; - - using ABlockTransferThreadSliceLengths_K0_M_K1 = Sequence<1, 4, 8>; - using ABlockTransferThreadClusterLengths_K0_M_K1 = Sequence<4, 64, 1>; - - constexpr index_t ABlockTransferSrcScalarPerVector_M = 4; - constexpr index_t ABlockTransferDstScalarPerVector_K1 = 8; - - using BBlockTransferThreadSliceLengths_K0_N_K1 = Sequence<1, 2, 8>; - using BBlockTransferThreadClusterLengths_K0_N_K1 = Sequence<4, 64, 1>; - - constexpr index_t BBlockTransferSrcScalarPerVector_N = 2; - constexpr index_t BBlockTransferDstScalarPerVector_K1 = 8; - - constexpr index_t CThreadTransferDstScalarPerVector = 4; -#elif 1 - // [M, N, K0, K1] = [128, 128, 4, 8] for fp16 - constexpr index_t BlockSize = 128; - - constexpr index_t MPerBlock = 128; - constexpr index_t NPerBlock = 128; - constexpr index_t KPerBlock = 4; - - constexpr index_t MPerXDL = 32; - constexpr index_t NPerXDL = 32; - constexpr index_t K1 = 8; - - constexpr index_t MRepeat = 4; - constexpr index_t NRepeat = 2; - - using ABlockTransferThreadSliceLengths_K0_M_K1 = Sequence<1, 4, 8>; - using ABlockTransferThreadClusterLengths_K0_M_K1 = Sequence<4, 32, 1>; - - constexpr index_t ABlockTransferSrcScalarPerVector_M = 4; - constexpr index_t ABlockTransferDstScalarPerVector_K1 = 8; - - using BBlockTransferThreadSliceLengths_K0_N_K1 = Sequence<1, 4, 8>; - using BBlockTransferThreadClusterLengths_K0_N_K1 = Sequence<4, 32, 1>; - - constexpr index_t BBlockTransferSrcScalarPerVector_N = 4; - constexpr index_t BBlockTransferDstScalarPerVector_K1 = 8; - - constexpr index_t CThreadTransferDstScalarPerVector = 4; -#endif - - const auto K = a_k_m.mDesc.GetLengths()[0]; - const auto M = a_k_m.mDesc.GetLengths()[1]; - const auto N = b_k_n.mDesc.GetLengths()[1]; - - constexpr auto K1Number = Number{}; - const auto K0 = K / K1Number; - - const auto a_k0_m_k1_grid_desc = - make_naive_tensor_descriptor(make_tuple(K0, M, K1Number), - make_tuple(K1Number * a_k_m.mDesc.GetStrides()[0], - a_k_m.mDesc.GetStrides()[1], - a_k_m.mDesc.GetStrides()[0])); - - const auto b_k0_n_k1_grid_desc = - make_naive_tensor_descriptor(make_tuple(K0, N, K1Number), - make_tuple(K1Number * b_k_n.mDesc.GetStrides()[0], - b_k_n.mDesc.GetStrides()[1], - b_k_n.mDesc.GetStrides()[0])); - - const auto c_m_n_grid_desc = make_naive_tensor_descriptor( - make_tuple(M, N), make_tuple(c_n_m.mDesc.GetStrides()[1], c_n_m.mDesc.GetStrides()[0])); - - // HACK: hacks that control index calculation when iterating over A, B, C matrix - constexpr auto a_k0_m_k1_grid_step_hacks = make_tuple(make_tuple(Sequence<0>{}, // 0+: K0 - Sequence<0>{}, // 1+: M - Sequence<0>{}), // 2+: K1 - make_tuple(Sequence<0>{}, // 0-: K0 - Sequence<0>{}, // 1-: M - Sequence<0>{})); // 2-: K1 - - constexpr auto b_k0_n_k1_grid_step_hacks = make_tuple(make_tuple(Sequence<0>{}, // 0+: K0 - Sequence<0>{}, // 1+: N - Sequence<0>{}), // 2+: K1 - make_tuple(Sequence<0>{}, // 0-: K0 - Sequence<0>{}, // 1-: N - Sequence<0>{})); // 2-: K1 - - constexpr auto c_m0_n0_m1_n1_m2_m3_m4_n2_grid_step_hacks = - make_tuple(make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 0+: M0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 1+: N0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 2+: M1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 3+: N1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 4+: M2 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 5+: M3 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 6+: M4 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}), // 7+: N2 - make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 0-: M0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 1-: N0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 2-: M1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 3-: N1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 4-: M2 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 5-: M3 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 6-: M4 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{})); // 7-: N2 - - constexpr auto a_k0_m_k1_grid_move_slice_window_step_hacks = Sequence<0>{}; - - constexpr auto b_k0_n_k1_grid_move_slice_window_step_hacks = Sequence<0>{}; - - for(index_t i = 0; i < 5; ++i) - { - float ave_time = - driver_gemm_xdlops_v2r3, - Sequence<0, 2, 1>, - 1, - ABlockTransferSrcScalarPerVector_M, - ABlockTransferDstScalarPerVector_K1, - false, // don't move back src coordinate after threadwise copy - BBlockTransferThreadSliceLengths_K0_N_K1, - BBlockTransferThreadClusterLengths_K0_N_K1, - Sequence<0, 2, 1>, - Sequence<0, 2, 1>, - 1, - BBlockTransferSrcScalarPerVector_N, - BBlockTransferDstScalarPerVector_K1, - false, // don't move back src coordinate after threadwise copy - Sequence<2, 3, 0, 1, 7, 5, 4, 6>, - 6, - CThreadTransferDstScalarPerVector, - decltype(a_k0_m_k1_grid_step_hacks), - decltype(b_k0_n_k1_grid_step_hacks), - decltype(c_m0_n0_m1_n1_m2_m3_m4_n2_grid_step_hacks), - decltype(a_k0_m_k1_grid_move_slice_window_step_hacks), - decltype(b_k0_n_k1_grid_move_slice_window_step_hacks), - false // CAccessOrderMRepeatNRepeat - >(static_cast(a_k_m_device_buf.GetDeviceBuffer()), - static_cast(b_k_n_device_buf.GetDeviceBuffer()), - static_cast(c_n_m_device_buf.GetDeviceBuffer()), - a_k0_m_k1_grid_desc, - b_k0_n_k1_grid_desc, - c_m_n_grid_desc, - a_k0_m_k1_grid_step_hacks, - b_k0_n_k1_grid_step_hacks, - c_m0_n0_m1_n1_m2_m3_m4_n2_grid_step_hacks, - a_k0_m_k1_grid_move_slice_window_step_hacks, - b_k0_n_k1_grid_move_slice_window_step_hacks, - nrepeat); - - float perf = static_cast((std::size_t(2) * M * N * K)) / - (std::size_t(1000) * 1000 * 1000) / ave_time; - - std::cout << "Average time : " << ave_time << " ms, " << perf << " TFlop/s" << std::endl; - } - - // copy result back to host - c_n_m_device_buf.FromDevice(c_n_m.mData.data()); -} diff --git a/library/include/ck/library/obselete_driver_offline/device_gemm_xdlops_km_nk_mn.hpp b/library/include/ck/library/obselete_driver_offline/device_gemm_xdlops_km_nk_mn.hpp deleted file mode 100644 index dbd318ce4d..0000000000 --- a/library/include/ck/library/obselete_driver_offline/device_gemm_xdlops_km_nk_mn.hpp +++ /dev/null @@ -1,463 +0,0 @@ -#pragma once -#include -#include "device.hpp" -#include "host_tensor.hpp" -#include "driver_gemm_xdlops_v2r3.hpp" - -template -void device_gemm_xdlops_km_nk_mn(const Tensor& a_k_m, - const Tensor& b_n_k, - Tensor& c_m_n, - ck::index_t nrepeat) -{ - using namespace ck; - - std::cout << __func__ << std::endl; - - DeviceMem a_k_m_device_buf(sizeof(ABType) * a_k_m.mDesc.GetElementSpace()); - DeviceMem b_n_k_device_buf(sizeof(ABType) * b_n_k.mDesc.GetElementSpace()); - DeviceMem c_m_n_device_buf(sizeof(CType) * c_m_n.mDesc.GetElementSpace()); - - a_k_m_device_buf.ToDevice(a_k_m.mData.data()); - b_n_k_device_buf.ToDevice(b_n_k.mData.data()); - c_m_n_device_buf.ToDevice(c_m_n.mData.data()); - -#if 0 - // [M, N, K0, K1] = [256, 128, 4, 4] for fp32 - constexpr index_t BlockSize = 256; - - constexpr index_t MPerBlock = 256; - constexpr index_t NPerBlock = 128; - constexpr index_t KPerBlock = 4; - - constexpr index_t MPerXDL = 32; - constexpr index_t NPerXDL = 32; - constexpr index_t K1 = 4; - - constexpr index_t MRepeat = 4; - constexpr index_t NRepeat = 2; - - using ABlockTransferThreadSliceLengths_K0_M_K1 = Sequence<1, 4, 4>; - using ABlockTransferThreadClusterLengths_K0_M_K1 = Sequence<4, 64, 1>; - - constexpr index_t ABlockTransferSrcScalarPerVector_M = 4; - constexpr index_t ABlockTransferDstScalarPerVector_K1 = 4; - - using BBlockTransferThreadSliceLengths_K0_N_K1 = Sequence<1, 2, 4>; - using BBlockTransferThreadClusterLengths_K0_N_K1 = Sequence<4, 64, 1>; - - constexpr index_t BBlockTransferSrcScalarPerVector_K1 = 4; - constexpr index_t BBlockTransferDstScalarPerVector_K1 = 4; - - constexpr index_t CThreadTransferDstScalarPerVector = 1; -#elif 0 - // [M, N, K0, K1] = [128, 256, 4, 4] for fp32 - constexpr index_t BlockSize = 256; - - constexpr index_t MPerBlock = 128; - constexpr index_t NPerBlock = 256; - constexpr index_t KPerBlock = 4; - - constexpr index_t MPerXDL = 32; - constexpr index_t NPerXDL = 32; - constexpr index_t K1 = 4; - - constexpr index_t MRepeat = 2; - constexpr index_t NRepeat = 4; - - using ABlockTransferThreadSliceLengths_K0_M_K1 = Sequence<1, 2, 4>; - using ABlockTransferThreadClusterLengths_K0_M_K1 = Sequence<4, 64, 1>; - - constexpr index_t ABlockTransferSrcScalarPerVector_M = 2; - constexpr index_t ABlockTransferDstScalarPerVector_K1 = 4; - - using BBlockTransferThreadSliceLengths_K0_N_K1 = Sequence<1, 4, 4>; - using BBlockTransferThreadClusterLengths_K0_N_K1 = Sequence<4, 64, 1>; - - constexpr index_t BBlockTransferSrcScalarPerVector_K1 = 4; - constexpr index_t BBlockTransferDstScalarPerVector_K1 = 4; - - constexpr index_t CThreadTransferDstScalarPerVector = 1; -#elif 0 - // [M, N, K0, K1] = [128, 128, 4, 4], C = 64, for fp32 - constexpr index_t BlockSize = 256; - - constexpr index_t MPerBlock = 128; - constexpr index_t NPerBlock = 128; - constexpr index_t KPerBlock = 4; - - constexpr index_t MPerXDL = 32; - constexpr index_t NPerXDL = 32; - constexpr index_t K1 = 4; - - constexpr index_t MRepeat = 2; - constexpr index_t NRepeat = 2; - - using ABlockTransferThreadSliceLengths_K0_M_K1 = Sequence<1, 2, 4>; - using ABlockTransferThreadClusterLengths_K0_M_K1 = Sequence<4, 64, 1>; - - constexpr index_t ABlockTransferSrcScalarPerVector_M = 2; - constexpr index_t ABlockTransferDstScalarPerVector_K1 = 4; - - using BBlockTransferThreadSliceLengths_K0_N_K1 = Sequence<1, 2, 4>; - using BBlockTransferThreadClusterLengths_K0_N_K1 = Sequence<4, 64, 1>; - - constexpr index_t BBlockTransferSrcScalarPerVector_K1 = 4; - constexpr index_t BBlockTransferDstScalarPerVector_K1 = 4; - - constexpr index_t CThreadTransferDstScalarPerVector = 1; -#elif 0 - // [M, N, K0, K1] = [128, 64, 4, 4], C = 32, for fp32 - constexpr index_t BlockSize = 256; - - constexpr index_t MPerBlock = 128; - constexpr index_t NPerBlock = 64; - constexpr index_t KPerBlock = 4; - - constexpr index_t MPerXDL = 32; - constexpr index_t NPerXDL = 32; - constexpr index_t K1 = 4; - - constexpr index_t MRepeat = 2; - constexpr index_t NRepeat = 1; - - using ABlockTransferThreadSliceLengths_K0_M_K1 = Sequence<1, 2, 4>; - using ABlockTransferThreadClusterLengths_K0_M_K1 = Sequence<4, 64, 1>; - - constexpr index_t ABlockTransferSrcScalarPerVector_M = 2; - constexpr index_t ABlockTransferDstScalarPerVector_K1 = 4; - - using BBlockTransferThreadSliceLengths_K0_N_K1 = Sequence<1, 1, 4>; - using BBlockTransferThreadClusterLengths_K0_N_K1 = Sequence<4, 64, 1>; - - constexpr index_t BBlockTransferSrcScalarPerVector_K1 = 4; - constexpr index_t BBlockTransferDstScalarPerVector_K1 = 4; - - constexpr index_t CThreadTransferDstScalarPerVector = 1; -#elif 0 - // [M, N, K0, K1] = [64, 128, 4, 4], C = 32, for fp32 - constexpr index_t BlockSize = 256; - - constexpr index_t MPerBlock = 64; - constexpr index_t NPerBlock = 128; - constexpr index_t KPerBlock = 4; - - constexpr index_t MPerXDL = 32; - constexpr index_t NPerXDL = 32; - constexpr index_t K1 = 4; - - constexpr index_t MRepeat = 1; - constexpr index_t NRepeat = 2; - - using ABlockTransferThreadSliceLengths_K0_M_K1 = Sequence<1, 1, 4>; - using ABlockTransferThreadClusterLengths_K0_M_K1 = Sequence<4, 64, 1>; - - constexpr index_t ABlockTransferSrcScalarPerVector_M = 1; - constexpr index_t ABlockTransferDstScalarPerVector_K1 = 4; - - using BBlockTransferThreadSliceLengths_K0_N_K1 = Sequence<1, 2, 4>; - using BBlockTransferThreadClusterLengths_K0_N_K1 = Sequence<4, 64, 1>; - - constexpr index_t BBlockTransferSrcScalarPerVector_K1 = 4; - constexpr index_t BBlockTransferDstScalarPerVector_K1 = 4; - - constexpr index_t CThreadTransferDstScalarPerVector = 1; -#elif 1 - // [M, N, K0, K1] = [256, 128, 4, 8], C = 128, for fp16 - constexpr index_t BlockSize = 256; - - constexpr index_t MPerBlock = 256; - constexpr index_t NPerBlock = 128; - constexpr index_t KPerBlock = 4; - - constexpr index_t MPerXDL = 32; - constexpr index_t NPerXDL = 32; - constexpr index_t K1 = 8; - - constexpr index_t MRepeat = 4; - constexpr index_t NRepeat = 2; - - using ABlockTransferThreadSliceLengths_K0_M_K1 = Sequence<1, 4, 8>; - using ABlockTransferThreadClusterLengths_K0_M_K1 = Sequence<4, 64, 1>; - - constexpr index_t ABlockTransferSrcScalarPerVector_M = 4; - constexpr index_t ABlockTransferDstScalarPerVector_K1 = 8; - - using BBlockTransferThreadSliceLengths_K0_N_K1 = Sequence<1, 2, 8>; - using BBlockTransferThreadClusterLengths_K0_N_K1 = Sequence<4, 64, 1>; - - constexpr index_t BBlockTransferSrcScalarPerVector_K1 = 8; - constexpr index_t BBlockTransferDstScalarPerVector_K1 = 8; - - constexpr index_t CThreadTransferDstScalarPerVector = 1; -#elif 0 - // [M, N, K0, K1] = [128, 256, 4, 8] for fp16 - constexpr index_t BlockSize = 256; - - constexpr index_t MPerBlock = 128; - constexpr index_t NPerBlock = 256; - constexpr index_t KPerBlock = 4; - - constexpr index_t MPerXDL = 32; - constexpr index_t NPerXDL = 32; - constexpr index_t K1 = 8; - - constexpr index_t MRepeat = 2; - constexpr index_t NRepeat = 4; - - using ABlockTransferThreadSliceLengths_K0_M_K1 = Sequence<1, 2, 8>; - using ABlockTransferThreadClusterLengths_K0_M_K1 = Sequence<4, 64, 1>; - - constexpr index_t ABlockTransferSrcScalarPerVector_M = 2; - constexpr index_t ABlockTransferDstScalarPerVector_K1 = 8; - - using BBlockTransferThreadSliceLengths_K0_N_K1 = Sequence<1, 4, 8>; - using BBlockTransferThreadClusterLengths_K0_N_K1 = Sequence<4, 64, 1>; - - constexpr index_t BBlockTransferSrcScalarPerVector_K1 = 8; - constexpr index_t BBlockTransferDstScalarPerVector_K1 = 8; - - constexpr index_t CThreadTransferDstScalarPerVector = 1; -#elif 0 - // [M, N, K0, K1] = [128, 128, 4, 8], C = 128, for fp16 - constexpr index_t BlockSize = 128; - - constexpr index_t MPerBlock = 128; - constexpr index_t NPerBlock = 128; - constexpr index_t KPerBlock = 4; - - constexpr index_t MPerXDL = 32; - constexpr index_t NPerXDL = 32; - constexpr index_t K1 = 8; - - constexpr index_t MRepeat = 4; - constexpr index_t NRepeat = 2; - - using ABlockTransferThreadSliceLengths_K0_M_K1 = Sequence<1, 4, 8>; - using ABlockTransferThreadClusterLengths_K0_M_K1 = Sequence<4, 32, 1>; - - constexpr index_t ABlockTransferSrcScalarPerVector_M = 4; - constexpr index_t ABlockTransferDstScalarPerVector_K1 = 8; - - using BBlockTransferThreadSliceLengths_K0_N_K1 = Sequence<1, 4, 8>; - using BBlockTransferThreadClusterLengths_K0_N_K1 = Sequence<4, 32, 1>; - - constexpr index_t BBlockTransferSrcScalarPerVector_K1 = 8; - constexpr index_t BBlockTransferDstScalarPerVector_K1 = 8; - - constexpr index_t CThreadTransferDstScalarPerVector = 1; -#elif 0 - // [M, N, K0, K1] = [128, 128, 4, 8], C = 64, for fp16 - constexpr index_t BlockSize = 256; - - constexpr index_t MPerBlock = 128; - constexpr index_t NPerBlock = 128; - constexpr index_t KPerBlock = 4; - - constexpr index_t MPerXDL = 32; - constexpr index_t NPerXDL = 32; - constexpr index_t K1 = 8; - - constexpr index_t MRepeat = 2; - constexpr index_t NRepeat = 2; - - using ABlockTransferThreadSliceLengths_K0_M_K1 = Sequence<1, 2, 8>; - using ABlockTransferThreadClusterLengths_K0_M_K1 = Sequence<4, 64, 1>; - - constexpr index_t ABlockTransferSrcScalarPerVector_M = 2; - constexpr index_t ABlockTransferDstScalarPerVector_K1 = 8; - - using BBlockTransferThreadSliceLengths_K0_N_K1 = Sequence<1, 2, 8>; - using BBlockTransferThreadClusterLengths_K0_N_K1 = Sequence<4, 64, 1>; - - constexpr index_t BBlockTransferSrcScalarPerVector_K1 = 8; - constexpr index_t BBlockTransferDstScalarPerVector_K1 = 8; - - constexpr index_t CThreadTransferDstScalarPerVector = 1; -#elif 1 - // [M, N, K0, K1] = [128, 64, 4, 8], C = 32, for fp16 - constexpr index_t BlockSize = 256; - - constexpr index_t MPerBlock = 128; - constexpr index_t NPerBlock = 64; - constexpr index_t KPerBlock = 4; - - constexpr index_t MPerXDL = 32; - constexpr index_t NPerXDL = 32; - constexpr index_t K1 = 8; - - constexpr index_t MRepeat = 2; - constexpr index_t NRepeat = 1; - - using ABlockTransferThreadSliceLengths_K0_M_K1 = Sequence<1, 2, 8>; - using ABlockTransferThreadClusterLengths_K0_M_K1 = Sequence<4, 64, 1>; - - constexpr index_t ABlockTransferSrcScalarPerVector_M = 2; - constexpr index_t ABlockTransferDstScalarPerVector_K1 = 8; - - using BBlockTransferThreadSliceLengths_K0_N_K1 = Sequence<1, 1, 8>; - using BBlockTransferThreadClusterLengths_K0_N_K1 = Sequence<4, 64, 1>; - - constexpr index_t BBlockTransferSrcScalarPerVector_K1 = 8; - constexpr index_t BBlockTransferDstScalarPerVector_K1 = 8; - - constexpr index_t CThreadTransferDstScalarPerVector = 1; -#elif 0 - // [M, N, K0, K1] = [64, 128, 4, 8], C = 32, for fp16 - constexpr index_t BlockSize = 256; - - constexpr index_t MPerBlock = 64; - constexpr index_t NPerBlock = 128; - constexpr index_t KPerBlock = 4; - - constexpr index_t MPerXDL = 32; - constexpr index_t NPerXDL = 32; - constexpr index_t K1 = 8; - - constexpr index_t MRepeat = 1; - constexpr index_t NRepeat = 2; - - using ABlockTransferThreadSliceLengths_K0_M_K1 = Sequence<1, 1, 8>; - using ABlockTransferThreadClusterLengths_K0_M_K1 = Sequence<4, 64, 1>; - - constexpr index_t ABlockTransferSrcScalarPerVector_M = 1; - constexpr index_t ABlockTransferDstScalarPerVector_K1 = 8; - - using BBlockTransferThreadSliceLengths_K0_N_K1 = Sequence<1, 2, 8>; - using BBlockTransferThreadClusterLengths_K0_N_K1 = Sequence<4, 64, 1>; - - constexpr index_t BBlockTransferSrcScalarPerVector_K1 = 8; - constexpr index_t BBlockTransferDstScalarPerVector_K1 = 8; - - constexpr index_t CThreadTransferDstScalarPerVector = 1; -#endif - - const auto K = a_k_m.mDesc.GetLengths()[0]; - const auto M = a_k_m.mDesc.GetLengths()[1]; - const auto N = b_n_k.mDesc.GetLengths()[0]; - - constexpr auto K1Number = Number{}; - const auto K0 = K / K1Number; - - const auto a_k0_m_k1_grid_desc = - make_naive_tensor_descriptor(make_tuple(K0, M, K1Number), - make_tuple(K1Number * a_k_m.mDesc.GetStrides()[0], - a_k_m.mDesc.GetStrides()[1], - a_k_m.mDesc.GetStrides()[0])); - - const auto b_k0_n_k1_grid_desc = - make_naive_tensor_descriptor(make_tuple(K0, N, K1Number), - make_tuple(K1Number * b_n_k.mDesc.GetStrides()[1], - b_n_k.mDesc.GetStrides()[0], - b_n_k.mDesc.GetStrides()[1])); - - const auto c_m_n_grid_desc = make_naive_tensor_descriptor( - make_tuple(M, N), make_tuple(c_m_n.mDesc.GetStrides()[0], c_m_n.mDesc.GetStrides()[1])); - - // HACK: hacks that control index calculation when iterating over A, B, C matrix - constexpr auto a_k0_m_k1_grid_step_hacks = make_tuple(make_tuple(Sequence<0>{}, // 0+: K0 - Sequence<0>{}, // 1+: M - Sequence<0>{}), // 2+: K1 - make_tuple(Sequence<0>{}, // 0-: K0 - Sequence<0>{}, // 1-: M - Sequence<0>{})); // 2-: K1 - - constexpr auto b_k0_n_k1_grid_step_hacks = make_tuple(make_tuple(Sequence<0>{}, // 0+: K0 - Sequence<0>{}, // 1+: N - Sequence<0>{}), // 2+: K1 - make_tuple(Sequence<0>{}, // 0-: K0 - Sequence<0>{}, // 1-: N - Sequence<0>{})); // 2-: K1 - - constexpr auto c_m0_n0_m1_n1_m2_m3_m4_n2_grid_step_hacks = - make_tuple(make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 0+: M0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 1+: N0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 2+: M1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 3+: N1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 4+: M2 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 5+: M3 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 6+: M4 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}), // 7+: N2 - make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 0-: M0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 1-: N0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 2-: M1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 3-: N1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 4-: M2 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 5-: M3 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 6-: M4 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{})); // 7-: N2 - - constexpr auto a_k0_m_k1_grid_move_slice_window_step_hacks = Sequence<0>{}; - - constexpr auto b_k0_n_k1_grid_move_slice_window_step_hacks = Sequence<0>{}; - - for(index_t i = 0; i < 5; ++i) - { - float ave_time = - driver_gemm_xdlops_v2r3, - Sequence<0, 2, 1>, - 1, - ABlockTransferSrcScalarPerVector_M, - ABlockTransferDstScalarPerVector_K1, - false, // don't move back src coordinate after threadwise copy - BBlockTransferThreadSliceLengths_K0_N_K1, - BBlockTransferThreadClusterLengths_K0_N_K1, - Sequence<1, 0, 2>, - Sequence<1, 0, 2>, - 2, - BBlockTransferSrcScalarPerVector_K1, - BBlockTransferDstScalarPerVector_K1, - false, // don't move back src coordinate after threadwise copy - Sequence<0, 2, 4, 5, 6, 1, 3, 7>, - 7, - CThreadTransferDstScalarPerVector, - decltype(a_k0_m_k1_grid_step_hacks), - decltype(b_k0_n_k1_grid_step_hacks), - decltype(c_m0_n0_m1_n1_m2_m3_m4_n2_grid_step_hacks), - decltype(a_k0_m_k1_grid_move_slice_window_step_hacks), - decltype(b_k0_n_k1_grid_move_slice_window_step_hacks), - false, // CAccessOrderMRepeatNRepeat - true, // ABlockLdsExtraM - true // BBlockLdsExtraN - >(static_cast(a_k_m_device_buf.GetDeviceBuffer()), - static_cast(b_n_k_device_buf.GetDeviceBuffer()), - static_cast(c_m_n_device_buf.GetDeviceBuffer()), - a_k0_m_k1_grid_desc, - b_k0_n_k1_grid_desc, - c_m_n_grid_desc, - debug::debug_driver_gemm_xdlops_v2r3::M01, - debug::debug_driver_gemm_xdlops_v2r3::N01, - a_k0_m_k1_grid_step_hacks, - b_k0_n_k1_grid_step_hacks, - c_m0_n0_m1_n1_m2_m3_m4_n2_grid_step_hacks, - a_k0_m_k1_grid_move_slice_window_step_hacks, - b_k0_n_k1_grid_move_slice_window_step_hacks, - nrepeat); - - float perf = static_cast((std::size_t(2) * M * N * K)) / - (std::size_t(1000) * 1000 * 1000) / ave_time; - - std::cout << "Average time : " << ave_time << " ms, " << perf << " TFlop/s" << std::endl; - } - - // copy result back to host - c_m_n_device_buf.FromDevice(c_m_n.mData.data()); -} diff --git a/library/include/ck/library/obselete_driver_offline/device_gemm_xdlops_km_nk_nm.hpp b/library/include/ck/library/obselete_driver_offline/device_gemm_xdlops_km_nk_nm.hpp deleted file mode 100644 index 5b819fd1af..0000000000 --- a/library/include/ck/library/obselete_driver_offline/device_gemm_xdlops_km_nk_nm.hpp +++ /dev/null @@ -1,263 +0,0 @@ -#pragma once -#include -#include "device.hpp" -#include "host_tensor.hpp" -#include "driver_gemm_xdlops_v2r3.hpp" - -template -void device_gemm_xdlops_km_nk_nm(const Tensor& a_k_m, - const Tensor& b_n_k, - Tensor& c_n_m, - ck::index_t nrepeat) -{ - using namespace ck; - - std::cout << __func__ << std::endl; - - DeviceMem a_k_m_device_buf(sizeof(ABType) * a_k_m.mDesc.GetElementSpace()); - DeviceMem b_n_k_device_buf(sizeof(ABType) * b_n_k.mDesc.GetElementSpace()); - DeviceMem c_n_m_device_buf(sizeof(CType) * c_n_m.mDesc.GetElementSpace()); - - a_k_m_device_buf.ToDevice(a_k_m.mData.data()); - b_n_k_device_buf.ToDevice(b_n_k.mData.data()); - c_n_m_device_buf.ToDevice(c_n_m.mData.data()); - -#if 0 - // [M, N, K0, K1] = [256, 128, 4, 4] for fp32 - constexpr index_t BlockSize = 256; - - constexpr index_t MPerBlock = 256; - constexpr index_t NPerBlock = 128; - constexpr index_t KPerBlock = 4; - - constexpr index_t MPerXDL = 32; - constexpr index_t NPerXDL = 32; - constexpr index_t K1 = 4; - - constexpr index_t MRepeat = 4; - constexpr index_t NRepeat = 2; - - using ABlockTransferThreadSliceLengths_K0_M_K1 = Sequence<1, 4, 4>; - using ABlockTransferThreadClusterLengths_K0_M_K1 = Sequence<4, 64, 1>; - - constexpr index_t ABlockTransferSrcScalarPerVector_M = 4; - constexpr index_t ABlockTransferDstScalarPerVector_K1 = 4; - - using BBlockTransferThreadSliceLengths_K0_N_K1 = Sequence<1, 2, 4>; - using BBlockTransferThreadClusterLengths_K0_N_K1 = Sequence<4, 64, 1>; - - constexpr index_t BBlockTransferSrcScalarPerVector_K1 = 4; - constexpr index_t BBlockTransferDstScalarPerVector_K1 = 4; - - constexpr index_t CThreadTransferDstScalarPerVector = 4; -#elif 0 - // [M, N, K0, K1] = [128, 256, 4, 4] for fp32 - constexpr index_t BlockSize = 256; - - constexpr index_t MPerBlock = 128; - constexpr index_t NPerBlock = 256; - constexpr index_t KPerBlock = 4; - - constexpr index_t MPerXDL = 32; - constexpr index_t NPerXDL = 32; - constexpr index_t K1 = 4; - - constexpr index_t MRepeat = 2; - constexpr index_t NRepeat = 4; - - using ABlockTransferThreadSliceLengths_K0_M_K1 = Sequence<1, 2, 4>; - using ABlockTransferThreadClusterLengths_K0_M_K1 = Sequence<4, 64, 1>; - - constexpr index_t ABlockTransferSrcScalarPerVector_M = 2; - constexpr index_t ABlockTransferDstScalarPerVector_K1 = 4; - - using BBlockTransferThreadSliceLengths_K0_N_K1 = Sequence<1, 4, 4>; - using BBlockTransferThreadClusterLengths_K0_N_K1 = Sequence<4, 64, 1>; - - constexpr index_t BBlockTransferSrcScalarPerVector_K1 = 4; - constexpr index_t BBlockTransferDstScalarPerVector_K1 = 4; - - constexpr index_t CThreadTransferDstScalarPerVector = 4; -#elif 1 - // [M, N, K0, K1] = [256, 128, 4, 8] for fp16 - constexpr index_t BlockSize = 256; - - constexpr index_t MPerBlock = 256; - constexpr index_t NPerBlock = 128; - constexpr index_t KPerBlock = 4; - - constexpr index_t MPerXDL = 32; - constexpr index_t NPerXDL = 32; - constexpr index_t K1 = 8; - - constexpr index_t MRepeat = 4; - constexpr index_t NRepeat = 2; - - using ABlockTransferThreadSliceLengths_K0_M_K1 = Sequence<1, 4, 8>; - using ABlockTransferThreadClusterLengths_K0_M_K1 = Sequence<4, 64, 1>; - - constexpr index_t ABlockTransferSrcScalarPerVector_M = 4; - constexpr index_t ABlockTransferDstScalarPerVector_K1 = 8; - - using BBlockTransferThreadSliceLengths_K0_N_K1 = Sequence<1, 2, 8>; - using BBlockTransferThreadClusterLengths_K0_N_K1 = Sequence<4, 64, 1>; - - constexpr index_t BBlockTransferSrcScalarPerVector_K1 = 8; - constexpr index_t BBlockTransferDstScalarPerVector_K1 = 8; - - constexpr index_t CThreadTransferDstScalarPerVector = 4; -#elif 1 - // [M, N, K0, K1] = [128, 128, 4, 8] for fp16 - constexpr index_t BlockSize = 128; - - constexpr index_t MPerBlock = 128; - constexpr index_t NPerBlock = 128; - constexpr index_t KPerBlock = 4; - - constexpr index_t MPerXDL = 32; - constexpr index_t NPerXDL = 32; - constexpr index_t K1 = 8; - - constexpr index_t MRepeat = 4; - constexpr index_t NRepeat = 2; - - using ABlockTransferThreadSliceLengths_K0_M_K1 = Sequence<1, 4, 8>; - using ABlockTransferThreadClusterLengths_K0_M_K1 = Sequence<4, 32, 1>; - - constexpr index_t ABlockTransferSrcScalarPerVector_M = 4; - constexpr index_t ABlockTransferDstScalarPerVector_K1 = 8; - - using BBlockTransferThreadSliceLengths_K0_N_K1 = Sequence<1, 4, 8>; - using BBlockTransferThreadClusterLengths_K0_N_K1 = Sequence<4, 32, 1>; - - constexpr index_t BBlockTransferSrcScalarPerVector_K1 = 8; - constexpr index_t BBlockTransferDstScalarPerVector_K1 = 8; - - constexpr index_t CThreadTransferDstScalarPerVector = 4; -#endif - - const auto K = a_k_m.mDesc.GetLengths()[0]; - const auto M = a_k_m.mDesc.GetLengths()[1]; - const auto N = b_n_k.mDesc.GetLengths()[0]; - - constexpr auto K1Number = Number{}; - const auto K0 = K / K1Number; - - const auto a_k0_m_k1_grid_desc = - make_naive_tensor_descriptor(make_tuple(K0, M, K1Number), - make_tuple(K1Number * a_k_m.mDesc.GetStrides()[0], - a_k_m.mDesc.GetStrides()[1], - a_k_m.mDesc.GetStrides()[0])); - - const auto b_k0_n_k1_grid_desc = - make_naive_tensor_descriptor(make_tuple(K0, N, K1Number), - make_tuple(K1Number * b_n_k.mDesc.GetStrides()[1], - b_n_k.mDesc.GetStrides()[0], - b_n_k.mDesc.GetStrides()[1])); - - const auto c_m_n_grid_desc = make_naive_tensor_descriptor( - make_tuple(M, N), make_tuple(c_n_m.mDesc.GetStrides()[1], c_n_m.mDesc.GetStrides()[0])); - - // HACK: hacks that control index calculation when iterating over A, B, C matrix - constexpr auto a_k0_m_k1_grid_step_hacks = make_tuple(make_tuple(Sequence<0>{}, // 0+: K0 - Sequence<0>{}, // 1+: M - Sequence<0>{}), // 2+: K1 - make_tuple(Sequence<0>{}, // 0-: K0 - Sequence<0>{}, // 1-: M - Sequence<0>{})); // 2-: K1 - - constexpr auto b_k0_n_k1_grid_step_hacks = make_tuple(make_tuple(Sequence<0>{}, // 0+: K0 - Sequence<0>{}, // 1+: N - Sequence<0>{}), // 2+: K1 - make_tuple(Sequence<0>{}, // 0-: K0 - Sequence<0>{}, // 1-: N - Sequence<0>{})); // 2-: K1 - - constexpr auto c_m0_n0_m1_n1_m2_m3_m4_n2_grid_step_hacks = - make_tuple(make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 0+: M0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 1+: N0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 2+: M1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 3+: N1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 4+: M2 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 5+: M3 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 6+: M4 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}), // 7+: N2 - make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 0-: M0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 1-: N0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 2-: M1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 3-: N1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 4-: M2 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 5-: M3 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 6-: M4 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{})); // 7-: N2 - - constexpr auto a_k0_m_k1_grid_move_slice_window_step_hacks = Sequence<0>{}; - - constexpr auto b_k0_n_k1_grid_move_slice_window_step_hacks = Sequence<0>{}; - - for(index_t i = 0; i < 5; ++i) - { - float ave_time = - driver_gemm_xdlops_v2r3, - Sequence<0, 2, 1>, - 1, - ABlockTransferSrcScalarPerVector_M, - ABlockTransferDstScalarPerVector_K1, - false, // don't move back src coordinate after threadwise copy - BBlockTransferThreadSliceLengths_K0_N_K1, - BBlockTransferThreadClusterLengths_K0_N_K1, - Sequence<1, 0, 2>, - Sequence<1, 0, 2>, - 2, - BBlockTransferSrcScalarPerVector_K1, - BBlockTransferDstScalarPerVector_K1, - false, // don't move back src coordinate after threadwise copy - Sequence<2, 3, 0, 1, 7, 5, 4, 6>, - 6, - CThreadTransferDstScalarPerVector, - decltype(a_k0_m_k1_grid_step_hacks), - decltype(b_k0_n_k1_grid_step_hacks), - decltype(c_m0_n0_m1_n1_m2_m3_m4_n2_grid_step_hacks), - decltype(a_k0_m_k1_grid_move_slice_window_step_hacks), - decltype(b_k0_n_k1_grid_move_slice_window_step_hacks), - false // CAccessOrderMRepeatNRepeat - >(static_cast(a_k_m_device_buf.GetDeviceBuffer()), - static_cast(b_n_k_device_buf.GetDeviceBuffer()), - static_cast(c_n_m_device_buf.GetDeviceBuffer()), - a_k0_m_k1_grid_desc, - b_k0_n_k1_grid_desc, - c_m_n_grid_desc, - a_k0_m_k1_grid_step_hacks, - b_k0_n_k1_grid_step_hacks, - c_m0_n0_m1_n1_m2_m3_m4_n2_grid_step_hacks, - a_k0_m_k1_grid_move_slice_window_step_hacks, - b_k0_n_k1_grid_move_slice_window_step_hacks, - nrepeat); - - float perf = static_cast((std::size_t(2) * M * N * K)) / - (std::size_t(1000) * 1000 * 1000) / ave_time; - - std::cout << "Average time : " << ave_time << " ms, " << perf << " TFlop/s" << std::endl; - } - - // copy result back to host - c_n_m_device_buf.FromDevice(c_n_m.mData.data()); -} diff --git a/library/include/ck/library/obselete_driver_offline/device_gemm_xdlops_mk_kn_mn.hpp b/library/include/ck/library/obselete_driver_offline/device_gemm_xdlops_mk_kn_mn.hpp deleted file mode 100644 index 4b041777c3..0000000000 --- a/library/include/ck/library/obselete_driver_offline/device_gemm_xdlops_mk_kn_mn.hpp +++ /dev/null @@ -1,463 +0,0 @@ -#pragma once -#include -#include "device.hpp" -#include "host_tensor.hpp" -#include "driver_gemm_xdlops_v2r3.hpp" - -template -void device_gemm_xdlops_mk_kn_mn(const Tensor& a_m_k, - const Tensor& b_k_n, - Tensor& c_m_n, - ck::index_t nrepeat) -{ - using namespace ck; - - std::cout << __func__ << std::endl; - - DeviceMem a_m_k_device_buf(sizeof(ABType) * a_m_k.mDesc.GetElementSpace()); - DeviceMem b_k_n_device_buf(sizeof(ABType) * b_k_n.mDesc.GetElementSpace()); - DeviceMem c_m_n_device_buf(sizeof(CType) * c_m_n.mDesc.GetElementSpace()); - - a_m_k_device_buf.ToDevice(a_m_k.mData.data()); - b_k_n_device_buf.ToDevice(b_k_n.mData.data()); - c_m_n_device_buf.ToDevice(c_m_n.mData.data()); - -#if 0 - // [M, N, K0, K1] = [256, 128, 4, 4] for fp32 - constexpr index_t BlockSize = 256; - - constexpr index_t MPerBlock = 256; - constexpr index_t NPerBlock = 128; - constexpr index_t KPerBlock = 4; - - constexpr index_t MPerXDL = 32; - constexpr index_t NPerXDL = 32; - constexpr index_t K1 = 4; - - constexpr index_t MRepeat = 4; - constexpr index_t NRepeat = 2; - - using ABlockTransferThreadSliceLengths_K0_M_K1 = Sequence<1, 4, 4>; - using ABlockTransferThreadClusterLengths_K0_M_K1 = Sequence<4, 64, 1>; - - constexpr index_t ABlockTransferSrcScalarPerVector_K1 = 4; - constexpr index_t ABlockTransferDstScalarPerVector_K1 = 4; - - using BBlockTransferThreadSliceLengths_K0_N_K1 = Sequence<1, 2, 4>; - using BBlockTransferThreadClusterLengths_K0_N_K1 = Sequence<4, 64, 1>; - - constexpr index_t BBlockTransferSrcScalarPerVector_N = 2; - constexpr index_t BBlockTransferDstScalarPerVector_K1 = 4; - - constexpr index_t CThreadTransferDstScalarPerVector = 1; -#elif 0 - // [M, N, K0, K1] = [128, 256, 4, 4] for fp32 - constexpr index_t BlockSize = 256; - - constexpr index_t MPerBlock = 128; - constexpr index_t NPerBlock = 256; - constexpr index_t KPerBlock = 4; - - constexpr index_t MPerXDL = 32; - constexpr index_t NPerXDL = 32; - constexpr index_t K1 = 4; - - constexpr index_t MRepeat = 2; - constexpr index_t NRepeat = 4; - - using ABlockTransferThreadSliceLengths_K0_M_K1 = Sequence<1, 2, 4>; - using ABlockTransferThreadClusterLengths_K0_M_K1 = Sequence<4, 64, 1>; - - constexpr index_t ABlockTransferSrcScalarPerVector_K1 = 4; - constexpr index_t ABlockTransferDstScalarPerVector_K1 = 4; - - using BBlockTransferThreadSliceLengths_K0_N_K1 = Sequence<1, 4, 4>; - using BBlockTransferThreadClusterLengths_K0_N_K1 = Sequence<4, 64, 1>; - - constexpr index_t BBlockTransferSrcScalarPerVector_N = 4; - constexpr index_t BBlockTransferDstScalarPerVector_K1 = 4; - - constexpr index_t CThreadTransferDstScalarPerVector = 1; -#elif 0 - // [M, N, K0, K1] = [128, 128, 4, 4], C = 64, for fp32 - constexpr index_t BlockSize = 256; - - constexpr index_t MPerBlock = 128; - constexpr index_t NPerBlock = 128; - constexpr index_t KPerBlock = 4; - - constexpr index_t MPerXDL = 32; - constexpr index_t NPerXDL = 32; - constexpr index_t K1 = 4; - - constexpr index_t MRepeat = 2; - constexpr index_t NRepeat = 2; - - using ABlockTransferThreadSliceLengths_K0_M_K1 = Sequence<1, 2, 4>; - using ABlockTransferThreadClusterLengths_K0_M_K1 = Sequence<4, 64, 1>; - - constexpr index_t ABlockTransferSrcScalarPerVector_K1 = 4; - constexpr index_t ABlockTransferDstScalarPerVector_K1 = 4; - - using BBlockTransferThreadSliceLengths_K0_N_K1 = Sequence<1, 2, 4>; - using BBlockTransferThreadClusterLengths_K0_N_K1 = Sequence<4, 64, 1>; - - constexpr index_t BBlockTransferSrcScalarPerVector_N = 2; - constexpr index_t BBlockTransferDstScalarPerVector_K1 = 4; - - constexpr index_t CThreadTransferDstScalarPerVector = 1; -#elif 0 - // [M, N, K0, K1] = [128, 64, 4, 4], C = 32, for fp32 - constexpr index_t BlockSize = 256; - - constexpr index_t MPerBlock = 128; - constexpr index_t NPerBlock = 64; - constexpr index_t KPerBlock = 4; - - constexpr index_t MPerXDL = 32; - constexpr index_t NPerXDL = 32; - constexpr index_t K1 = 4; - - constexpr index_t MRepeat = 2; - constexpr index_t NRepeat = 1; - - using ABlockTransferThreadSliceLengths_K0_M_K1 = Sequence<1, 2, 4>; - using ABlockTransferThreadClusterLengths_K0_M_K1 = Sequence<4, 64, 1>; - - constexpr index_t ABlockTransferSrcScalarPerVector_K1 = 4; - constexpr index_t ABlockTransferDstScalarPerVector_K1 = 4; - - using BBlockTransferThreadSliceLengths_K0_N_K1 = Sequence<1, 1, 4>; - using BBlockTransferThreadClusterLengths_K0_N_K1 = Sequence<4, 64, 1>; - - constexpr index_t BBlockTransferSrcScalarPerVector_N = 1; - constexpr index_t BBlockTransferDstScalarPerVector_K1 = 4; - - constexpr index_t CThreadTransferDstScalarPerVector = 1; -#elif 0 - // [M, N, K0, K1] = [64, 128, 4, 4], C = 32, for fp32 - constexpr index_t BlockSize = 256; - - constexpr index_t MPerBlock = 64; - constexpr index_t NPerBlock = 128; - constexpr index_t KPerBlock = 4; - - constexpr index_t MPerXDL = 32; - constexpr index_t NPerXDL = 32; - constexpr index_t K1 = 4; - - constexpr index_t MRepeat = 1; - constexpr index_t NRepeat = 2; - - using ABlockTransferThreadSliceLengths_K0_M_K1 = Sequence<1, 1, 4>; - using ABlockTransferThreadClusterLengths_K0_M_K1 = Sequence<4, 64, 1>; - - constexpr index_t ABlockTransferSrcScalarPerVector_K1 = 4; - constexpr index_t ABlockTransferDstScalarPerVector_K1 = 4; - - using BBlockTransferThreadSliceLengths_K0_N_K1 = Sequence<1, 2, 4>; - using BBlockTransferThreadClusterLengths_K0_N_K1 = Sequence<4, 64, 1>; - - constexpr index_t BBlockTransferSrcScalarPerVector_N = 2; - constexpr index_t BBlockTransferDstScalarPerVector_K1 = 4; - - constexpr index_t CThreadTransferDstScalarPerVector = 1; -#elif 1 - // [M, N, K0, K1] = [256, 128, 4, 8], C = 128, for fp16 - constexpr index_t BlockSize = 256; - - constexpr index_t MPerBlock = 256; - constexpr index_t NPerBlock = 128; - constexpr index_t KPerBlock = 4; - - constexpr index_t MPerXDL = 32; - constexpr index_t NPerXDL = 32; - constexpr index_t K1 = 8; - - constexpr index_t MRepeat = 4; - constexpr index_t NRepeat = 2; - - using ABlockTransferThreadSliceLengths_K0_M_K1 = Sequence<1, 4, 8>; - using ABlockTransferThreadClusterLengths_K0_M_K1 = Sequence<4, 64, 1>; - - constexpr index_t ABlockTransferSrcScalarPerVector_K1 = 8; - constexpr index_t ABlockTransferDstScalarPerVector_K1 = 8; - - using BBlockTransferThreadSliceLengths_K0_N_K1 = Sequence<1, 2, 8>; - using BBlockTransferThreadClusterLengths_K0_N_K1 = Sequence<4, 64, 1>; - - constexpr index_t BBlockTransferSrcScalarPerVector_N = 2; - constexpr index_t BBlockTransferDstScalarPerVector_K1 = 8; - - constexpr index_t CThreadTransferDstScalarPerVector = 1; -#elif 0 - // [M, N, K0, K1] = [128, 256, 4, 8] for fp16 - constexpr index_t BlockSize = 256; - - constexpr index_t MPerBlock = 128; - constexpr index_t NPerBlock = 256; - constexpr index_t KPerBlock = 4; - - constexpr index_t MPerXDL = 32; - constexpr index_t NPerXDL = 32; - constexpr index_t K1 = 8; - - constexpr index_t MRepeat = 2; - constexpr index_t NRepeat = 4; - - using ABlockTransferThreadSliceLengths_K0_M_K1 = Sequence<1, 2, 8>; - using ABlockTransferThreadClusterLengths_K0_M_K1 = Sequence<4, 64, 1>; - - constexpr index_t ABlockTransferSrcScalarPerVector_K1 = 8; - constexpr index_t ABlockTransferDstScalarPerVector_K1 = 8; - - using BBlockTransferThreadSliceLengths_K0_N_K1 = Sequence<1, 4, 8>; - using BBlockTransferThreadClusterLengths_K0_N_K1 = Sequence<4, 64, 1>; - - constexpr index_t BBlockTransferSrcScalarPerVector_N = 4; - constexpr index_t BBlockTransferDstScalarPerVector_K1 = 8; - - constexpr index_t CThreadTransferDstScalarPerVector = 1; -#elif 0 - // [M, N, K0, K1] = [128, 128, 4, 8], C = 128, for fp16 - constexpr index_t BlockSize = 128; - - constexpr index_t MPerBlock = 128; - constexpr index_t NPerBlock = 128; - constexpr index_t KPerBlock = 4; - - constexpr index_t MPerXDL = 32; - constexpr index_t NPerXDL = 32; - constexpr index_t K1 = 8; - - constexpr index_t MRepeat = 4; - constexpr index_t NRepeat = 2; - - using ABlockTransferThreadSliceLengths_K0_M_K1 = Sequence<1, 4, 8>; - using ABlockTransferThreadClusterLengths_K0_M_K1 = Sequence<4, 32, 1>; - - constexpr index_t ABlockTransferSrcScalarPerVector_K1 = 8; - constexpr index_t ABlockTransferDstScalarPerVector_K1 = 8; - - using BBlockTransferThreadSliceLengths_K0_N_K1 = Sequence<1, 4, 8>; - using BBlockTransferThreadClusterLengths_K0_N_K1 = Sequence<4, 32, 1>; - - constexpr index_t BBlockTransferSrcScalarPerVector_N = 4; - constexpr index_t BBlockTransferDstScalarPerVector_K1 = 8; - - constexpr index_t CThreadTransferDstScalarPerVector = 1; -#elif 0 - // [M, N, K0, K1] = [128, 128, 4, 8], C = 64, for fp16 - constexpr index_t BlockSize = 256; - - constexpr index_t MPerBlock = 128; - constexpr index_t NPerBlock = 128; - constexpr index_t KPerBlock = 4; - - constexpr index_t MPerXDL = 32; - constexpr index_t NPerXDL = 32; - constexpr index_t K1 = 8; - - constexpr index_t MRepeat = 2; - constexpr index_t NRepeat = 2; - - using ABlockTransferThreadSliceLengths_K0_M_K1 = Sequence<1, 2, 8>; - using ABlockTransferThreadClusterLengths_K0_M_K1 = Sequence<4, 64, 1>; - - constexpr index_t ABlockTransferSrcScalarPerVector_K1 = 8; - constexpr index_t ABlockTransferDstScalarPerVector_K1 = 8; - - using BBlockTransferThreadSliceLengths_K0_N_K1 = Sequence<1, 2, 8>; - using BBlockTransferThreadClusterLengths_K0_N_K1 = Sequence<4, 64, 1>; - - constexpr index_t BBlockTransferSrcScalarPerVector_N = 2; - constexpr index_t BBlockTransferDstScalarPerVector_K1 = 8; - - constexpr index_t CThreadTransferDstScalarPerVector = 1; -#elif 1 - // [M, N, K0, K1] = [128, 64, 4, 8], C = 32, for fp16 - constexpr index_t BlockSize = 256; - - constexpr index_t MPerBlock = 128; - constexpr index_t NPerBlock = 64; - constexpr index_t KPerBlock = 4; - - constexpr index_t MPerXDL = 32; - constexpr index_t NPerXDL = 32; - constexpr index_t K1 = 8; - - constexpr index_t MRepeat = 2; - constexpr index_t NRepeat = 1; - - using ABlockTransferThreadSliceLengths_K0_M_K1 = Sequence<1, 2, 8>; - using ABlockTransferThreadClusterLengths_K0_M_K1 = Sequence<4, 64, 1>; - - constexpr index_t ABlockTransferSrcScalarPerVector_K1 = 8; - constexpr index_t ABlockTransferDstScalarPerVector_K1 = 8; - - using BBlockTransferThreadSliceLengths_K0_N_K1 = Sequence<1, 1, 8>; - using BBlockTransferThreadClusterLengths_K0_N_K1 = Sequence<4, 64, 1>; - - constexpr index_t BBlockTransferSrcScalarPerVector_N = 1; - constexpr index_t BBlockTransferDstScalarPerVector_K1 = 8; - - constexpr index_t CThreadTransferDstScalarPerVector = 1; -#elif 1 - // [M, N, K0, K1] = [64, 128, 4, 8], C = 32, for fp16 - constexpr index_t BlockSize = 256; - - constexpr index_t MPerBlock = 64; - constexpr index_t NPerBlock = 128; - constexpr index_t KPerBlock = 4; - - constexpr index_t MPerXDL = 32; - constexpr index_t NPerXDL = 32; - constexpr index_t K1 = 8; - - constexpr index_t MRepeat = 1; - constexpr index_t NRepeat = 2; - - using ABlockTransferThreadSliceLengths_K0_M_K1 = Sequence<1, 1, 8>; - using ABlockTransferThreadClusterLengths_K0_M_K1 = Sequence<4, 64, 1>; - - constexpr index_t ABlockTransferSrcScalarPerVector_K1 = 8; - constexpr index_t ABlockTransferDstScalarPerVector_K1 = 8; - - using BBlockTransferThreadSliceLengths_K0_N_K1 = Sequence<1, 2, 8>; - using BBlockTransferThreadClusterLengths_K0_N_K1 = Sequence<4, 64, 1>; - - constexpr index_t BBlockTransferSrcScalarPerVector_N = 2; - constexpr index_t BBlockTransferDstScalarPerVector_K1 = 8; - - constexpr index_t CThreadTransferDstScalarPerVector = 1; -#endif - - const auto K = a_m_k.mDesc.GetLengths()[1]; - const auto M = a_m_k.mDesc.GetLengths()[0]; - const auto N = b_k_n.mDesc.GetLengths()[1]; - - constexpr auto K1Number = Number{}; - const auto K0 = K / K1Number; - - const auto a_k0_m_k1_grid_desc = - make_naive_tensor_descriptor(make_tuple(K0, M, K1Number), - make_tuple(K1Number * a_m_k.mDesc.GetStrides()[1], - a_m_k.mDesc.GetStrides()[0], - a_m_k.mDesc.GetStrides()[1])); - - const auto b_k0_n_k1_grid_desc = - make_naive_tensor_descriptor(make_tuple(K0, N, K1Number), - make_tuple(K1Number * b_k_n.mDesc.GetStrides()[0], - b_k_n.mDesc.GetStrides()[1], - b_k_n.mDesc.GetStrides()[0])); - - const auto c_m_n_grid_desc = make_naive_tensor_descriptor( - make_tuple(M, N), make_tuple(c_m_n.mDesc.GetStrides()[0], c_m_n.mDesc.GetStrides()[1])); - - // HACK: hacks that control index calculation when iterating over A, B, C matrix - constexpr auto a_k0_m_k1_grid_step_hacks = make_tuple(make_tuple(Sequence<0>{}, // 0+: K0 - Sequence<0>{}, // 1+: M - Sequence<0>{}), // 2+: K1 - make_tuple(Sequence<0>{}, // 0-: K0 - Sequence<0>{}, // 1-: M - Sequence<0>{})); // 2-: K1 - - constexpr auto b_k0_n_k1_grid_step_hacks = make_tuple(make_tuple(Sequence<0>{}, // 0+: K0 - Sequence<0>{}, // 1+: N - Sequence<0>{}), // 2+: K1 - make_tuple(Sequence<0>{}, // 0-: K0 - Sequence<0>{}, // 1-: N - Sequence<0>{})); // 2-: K1 - - constexpr auto c_m0_n0_m1_n1_m2_m3_m4_n2_grid_step_hacks = - make_tuple(make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 0+: M0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 1+: N0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 2+: M1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 3+: N1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 4+: M2 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 5+: M3 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 6+: M4 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}), // 7+: N2 - make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 0-: M0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 1-: N0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 2-: M1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 3-: N1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 4-: M2 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 5-: M3 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 6-: M4 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{})); // 7-: N2 - - constexpr auto a_k0_m_k1_grid_move_slice_window_step_hacks = Sequence<0>{}; - - constexpr auto b_k0_n_k1_grid_move_slice_window_step_hacks = Sequence<0>{}; - - for(index_t i = 0; i < 5; ++i) - { - float ave_time = - driver_gemm_xdlops_v2r3, - Sequence<1, 0, 2>, - 2, - ABlockTransferSrcScalarPerVector_K1, - ABlockTransferDstScalarPerVector_K1, - false, // don't move back src coordinate after threadwise copy - BBlockTransferThreadSliceLengths_K0_N_K1, - BBlockTransferThreadClusterLengths_K0_N_K1, - Sequence<0, 2, 1>, - Sequence<0, 2, 1>, - 1, - BBlockTransferSrcScalarPerVector_N, - BBlockTransferDstScalarPerVector_K1, - false, // don't move back src coordinate after threadwise copy - Sequence<0, 2, 4, 5, 6, 1, 3, 7>, - 7, - CThreadTransferDstScalarPerVector, - decltype(a_k0_m_k1_grid_step_hacks), - decltype(b_k0_n_k1_grid_step_hacks), - decltype(c_m0_n0_m1_n1_m2_m3_m4_n2_grid_step_hacks), - decltype(a_k0_m_k1_grid_move_slice_window_step_hacks), - decltype(b_k0_n_k1_grid_move_slice_window_step_hacks), - false, // CAccessOrderMRepeatNRepeat - true, // ABlockLdsExtraM - true // BBlockLdsExtraN - >(static_cast(a_m_k_device_buf.GetDeviceBuffer()), - static_cast(b_k_n_device_buf.GetDeviceBuffer()), - static_cast(c_m_n_device_buf.GetDeviceBuffer()), - a_k0_m_k1_grid_desc, - b_k0_n_k1_grid_desc, - c_m_n_grid_desc, - debug::debug_driver_gemm_xdlops_v2r3::M01, - debug::debug_driver_gemm_xdlops_v2r3::N01, - a_k0_m_k1_grid_step_hacks, - b_k0_n_k1_grid_step_hacks, - c_m0_n0_m1_n1_m2_m3_m4_n2_grid_step_hacks, - a_k0_m_k1_grid_move_slice_window_step_hacks, - b_k0_n_k1_grid_move_slice_window_step_hacks, - nrepeat); - - float perf = static_cast((std::size_t(2) * M * N * K)) / - (std::size_t(1000) * 1000 * 1000) / ave_time; - - std::cout << "Average time : " << ave_time << " ms, " << perf << " TFlop/s" << std::endl; - } - - // copy result back to host - c_m_n_device_buf.FromDevice(c_m_n.mData.data()); -} diff --git a/library/include/ck/library/obselete_driver_offline/device_gemm_xdlops_mk_kn_nm.hpp b/library/include/ck/library/obselete_driver_offline/device_gemm_xdlops_mk_kn_nm.hpp deleted file mode 100644 index c848cd7936..0000000000 --- a/library/include/ck/library/obselete_driver_offline/device_gemm_xdlops_mk_kn_nm.hpp +++ /dev/null @@ -1,291 +0,0 @@ -#pragma once -#include -#include "device.hpp" -#include "host_tensor.hpp" -#include "driver_gemm_xdlops_v2r3.hpp" - -template -void device_gemm_xdlops_mk_kn_nm(const Tensor& a_m_k, - const Tensor& b_k_n, - Tensor& c_n_m, - ck::index_t nrepeat) -{ - using namespace ck; - - std::cout << __func__ << std::endl; - - DeviceMem a_m_k_device_buf(sizeof(ABType) * a_m_k.mDesc.GetElementSpace()); - DeviceMem b_k_n_device_buf(sizeof(ABType) * b_k_n.mDesc.GetElementSpace()); - DeviceMem c_n_m_device_buf(sizeof(CType) * c_n_m.mDesc.GetElementSpace()); - - a_m_k_device_buf.ToDevice(a_m_k.mData.data()); - b_k_n_device_buf.ToDevice(b_k_n.mData.data()); - c_n_m_device_buf.ToDevice(c_n_m.mData.data()); - -#if 0 - // [M, N, K0, K1] = [256, 128, 4, 4] for fp32 - constexpr index_t BlockSize = 256; - - constexpr index_t MPerBlock = 256; - constexpr index_t NPerBlock = 128; - constexpr index_t KPerBlock = 4; - - constexpr index_t MPerXDL = 32; - constexpr index_t NPerXDL = 32; - constexpr index_t K1 = 4; - - constexpr index_t MRepeat = 4; - constexpr index_t NRepeat = 2; - - using ABlockTransferThreadSliceLengths_K0_M_K1 = Sequence<1, 4, 4>; - using ABlockTransferThreadClusterLengths_K0_M_K1 = Sequence<4, 64, 1>; - - constexpr index_t ABlockTransferSrcScalarPerVector_K1 = 4; - constexpr index_t ABlockTransferDstScalarPerVector_K1 = 4; - - using BBlockTransferThreadSliceLengths_K0_N_K1 = Sequence<1, 2, 4>; - using BBlockTransferThreadClusterLengths_K0_N_K1 = Sequence<4, 64, 1>; - - constexpr index_t BBlockTransferSrcScalarPerVector_N = 2; - constexpr index_t BBlockTransferDstScalarPerVector_K1 = 4; - - constexpr index_t CThreadTransferDstScalarPerVector = 4; -#elif 0 - // [M, N, K0, K1] = [128, 256, 4, 4] for fp32 - constexpr index_t BlockSize = 256; - - constexpr index_t MPerBlock = 128; - constexpr index_t NPerBlock = 256; - constexpr index_t KPerBlock = 4; - - constexpr index_t MPerXDL = 32; - constexpr index_t NPerXDL = 32; - constexpr index_t K1 = 4; - - constexpr index_t MRepeat = 2; - constexpr index_t NRepeat = 4; - - using ABlockTransferThreadSliceLengths_K0_M_K1 = Sequence<1, 2, 4>; - using ABlockTransferThreadClusterLengths_K0_M_K1 = Sequence<4, 64, 1>; - - constexpr index_t ABlockTransferSrcScalarPerVector_K1 = 4; - constexpr index_t ABlockTransferDstScalarPerVector_K1 = 4; - - using BBlockTransferThreadSliceLengths_K0_N_K1 = Sequence<1, 4, 4>; - using BBlockTransferThreadClusterLengths_K0_N_K1 = Sequence<4, 64, 1>; - - constexpr index_t BBlockTransferSrcScalarPerVector_N = 4; - constexpr index_t BBlockTransferDstScalarPerVector_K1 = 4; - - constexpr index_t CThreadTransferDstScalarPerVector = 4; -#elif 1 - // [M, N, K0, K1] = [256, 128, 4, 8] for fp16 - constexpr index_t BlockSize = 256; - - constexpr index_t MPerBlock = 256; - constexpr index_t NPerBlock = 128; - constexpr index_t KPerBlock = 4; - - constexpr index_t MPerXDL = 32; - constexpr index_t NPerXDL = 32; - constexpr index_t K1 = 8; - - constexpr index_t MRepeat = 4; - constexpr index_t NRepeat = 2; - - using ABlockTransferThreadSliceLengths_K0_M_K1 = Sequence<1, 4, 8>; - using ABlockTransferThreadClusterLengths_K0_M_K1 = Sequence<4, 64, 1>; - - constexpr index_t ABlockTransferSrcScalarPerVector_K1 = 8; - constexpr index_t ABlockTransferDstScalarPerVector_K1 = 8; - - using BBlockTransferThreadSliceLengths_K0_N_K1 = Sequence<1, 2, 8>; - using BBlockTransferThreadClusterLengths_K0_N_K1 = Sequence<4, 64, 1>; - - constexpr index_t BBlockTransferSrcScalarPerVector_N = 2; - constexpr index_t BBlockTransferDstScalarPerVector_K1 = 8; - - constexpr index_t CThreadTransferDstScalarPerVector = 4; -#elif 1 - // [M, N, K0, K1] = [128, 256, 4, 8] for fp16 - constexpr index_t BlockSize = 256; - - constexpr index_t MPerBlock = 128; - constexpr index_t NPerBlock = 256; - constexpr index_t KPerBlock = 4; - - constexpr index_t MPerXDL = 32; - constexpr index_t NPerXDL = 32; - constexpr index_t K1 = 8; - - constexpr index_t MRepeat = 2; - constexpr index_t NRepeat = 4; - - using ABlockTransferThreadSliceLengths_K0_M_K1 = Sequence<1, 2, 8>; - using ABlockTransferThreadClusterLengths_K0_M_K1 = Sequence<4, 64, 1>; - - constexpr index_t ABlockTransferSrcScalarPerVector_K1 = 8; - constexpr index_t ABlockTransferDstScalarPerVector_K1 = 8; - - using BBlockTransferThreadSliceLengths_K0_N_K1 = Sequence<1, 4, 8>; - using BBlockTransferThreadClusterLengths_K0_N_K1 = Sequence<4, 64, 1>; - - constexpr index_t BBlockTransferSrcScalarPerVector_N = 4; - constexpr index_t BBlockTransferDstScalarPerVector_K1 = 8; - - constexpr index_t CThreadTransferDstScalarPerVector = 4; -#elif 1 - // [M, N, K0, K1] = [128, 128, 4, 8] for fp16 - constexpr index_t BlockSize = 128; - - constexpr index_t MPerBlock = 128; - constexpr index_t NPerBlock = 128; - constexpr index_t KPerBlock = 4; - - constexpr index_t MPerXDL = 32; - constexpr index_t NPerXDL = 32; - constexpr index_t K1 = 8; - - constexpr index_t MRepeat = 4; - constexpr index_t NRepeat = 2; - - using ABlockTransferThreadSliceLengths_K0_M_K1 = Sequence<1, 4, 8>; - using ABlockTransferThreadClusterLengths_K0_M_K1 = Sequence<4, 32, 1>; - - constexpr index_t ABlockTransferSrcScalarPerVector_K1 = 8; - constexpr index_t ABlockTransferDstScalarPerVector_K1 = 8; - - using BBlockTransferThreadSliceLengths_K0_N_K1 = Sequence<1, 4, 8>; - using BBlockTransferThreadClusterLengths_K0_N_K1 = Sequence<4, 32, 1>; - - constexpr index_t BBlockTransferSrcScalarPerVector_N = 4; - constexpr index_t BBlockTransferDstScalarPerVector_K1 = 8; - - constexpr index_t CThreadTransferDstScalarPerVector = 4; -#endif - - const auto K = a_m_k.mDesc.GetLengths()[1]; - const auto M = a_m_k.mDesc.GetLengths()[0]; - const auto N = b_k_n.mDesc.GetLengths()[1]; - - constexpr auto K1Number = Number{}; - const auto K0 = K / K1Number; - - const auto a_k0_m_k1_grid_desc = - make_naive_tensor_descriptor(make_tuple(K0, M, K1Number), - make_tuple(K1Number * a_m_k.mDesc.GetStrides()[1], - a_m_k.mDesc.GetStrides()[0], - a_m_k.mDesc.GetStrides()[1])); - - const auto b_k0_n_k1_grid_desc = - make_naive_tensor_descriptor(make_tuple(K0, N, K1Number), - make_tuple(K1Number * b_k_n.mDesc.GetStrides()[0], - b_k_n.mDesc.GetStrides()[1], - b_k_n.mDesc.GetStrides()[0])); - - const auto c_m_n_grid_desc = make_naive_tensor_descriptor( - make_tuple(M, N), make_tuple(c_n_m.mDesc.GetStrides()[1], c_n_m.mDesc.GetStrides()[0])); - - // HACK: hacks that control index calculation when iterating over A, B, C matrix - constexpr auto a_k0_m_k1_grid_step_hacks = make_tuple(make_tuple(Sequence<0>{}, // 0+: K0 - Sequence<0>{}, // 1+: M - Sequence<0>{}), // 2+: K1 - make_tuple(Sequence<0>{}, // 0-: K0 - Sequence<0>{}, // 1-: M - Sequence<0>{})); // 2-: K1 - - constexpr auto b_k0_n_k1_grid_step_hacks = make_tuple(make_tuple(Sequence<0>{}, // 0+: K0 - Sequence<0>{}, // 1+: N - Sequence<0>{}), // 2+: K1 - make_tuple(Sequence<0>{}, // 0-: K0 - Sequence<0>{}, // 1-: N - Sequence<0>{})); // 2-: K1 - - constexpr auto c_m0_n0_m1_n1_m2_m3_m4_n2_grid_step_hacks = - make_tuple(make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 0+: M0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 1+: N0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 2+: M1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 3+: N1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 4+: M2 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 5+: M3 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 6+: M4 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}), // 7+: N2 - make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 0-: M0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 1-: N0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 2-: M1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 3-: N1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 4-: M2 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 5-: M3 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 6-: M4 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{})); // 7-: N2 - - constexpr auto a_k0_m_k1_grid_move_slice_window_step_hacks = Sequence<0>{}; - - constexpr auto b_k0_n_k1_grid_move_slice_window_step_hacks = Sequence<0>{}; - - for(index_t i = 0; i < 5; ++i) - { - float ave_time = - driver_gemm_xdlops_v2r3, - Sequence<1, 0, 2>, - 2, - ABlockTransferSrcScalarPerVector_K1, - ABlockTransferDstScalarPerVector_K1, - false, // don't move back src coordinate after threadwise copy - BBlockTransferThreadSliceLengths_K0_N_K1, - BBlockTransferThreadClusterLengths_K0_N_K1, - Sequence<0, 2, 1>, - Sequence<0, 2, 1>, - 1, - BBlockTransferSrcScalarPerVector_N, - BBlockTransferDstScalarPerVector_K1, - false, // don't move back src coordinate after threadwise copy - Sequence<2, 3, 0, 1, 7, 5, 4, 6>, - 6, - CThreadTransferDstScalarPerVector, - decltype(a_k0_m_k1_grid_step_hacks), - decltype(b_k0_n_k1_grid_step_hacks), - decltype(c_m0_n0_m1_n1_m2_m3_m4_n2_grid_step_hacks), - decltype(a_k0_m_k1_grid_move_slice_window_step_hacks), - decltype(b_k0_n_k1_grid_move_slice_window_step_hacks), - false // CAccessOrderMRepeatNRepeat - >(static_cast(a_m_k_device_buf.GetDeviceBuffer()), - static_cast(b_k_n_device_buf.GetDeviceBuffer()), - static_cast(c_n_m_device_buf.GetDeviceBuffer()), - a_k0_m_k1_grid_desc, - b_k0_n_k1_grid_desc, - c_m_n_grid_desc, - a_k0_m_k1_grid_step_hacks, - b_k0_n_k1_grid_step_hacks, - c_m0_n0_m1_n1_m2_m3_m4_n2_grid_step_hacks, - a_k0_m_k1_grid_move_slice_window_step_hacks, - b_k0_n_k1_grid_move_slice_window_step_hacks, - nrepeat); - - float perf = static_cast((std::size_t(2) * M * N * K)) / - (std::size_t(1000) * 1000 * 1000) / ave_time; - - std::cout << "Average time : " << ave_time << " ms, " << perf << " TFlop/s" << std::endl; - } - - // copy result back to host - c_n_m_device_buf.FromDevice(c_n_m.mData.data()); -} diff --git a/library/include/ck/library/obselete_driver_offline/device_gemm_xdlops_mk_nk_mn.hpp b/library/include/ck/library/obselete_driver_offline/device_gemm_xdlops_mk_nk_mn.hpp deleted file mode 100644 index 557624026d..0000000000 --- a/library/include/ck/library/obselete_driver_offline/device_gemm_xdlops_mk_nk_mn.hpp +++ /dev/null @@ -1,564 +0,0 @@ -#pragma once -#include -#include "device.hpp" -#include "host_tensor.hpp" -#include "driver_gemm_xdlops_v2r3.hpp" - -template -void device_gemm_xdlops_mk_nk_mn(const Tensor& a_m_k, - const Tensor& b_n_k, - Tensor& c_m_n, - ck::index_t nrepeat) -{ - using namespace ck; - - std::cout << __func__ << std::endl; - - DeviceMem a_m_k_device_buf(sizeof(ABType) * a_m_k.mDesc.GetElementSpace()); - DeviceMem b_n_k_device_buf(sizeof(ABType) * b_n_k.mDesc.GetElementSpace()); - DeviceMem c_m_n_device_buf(sizeof(CType) * c_m_n.mDesc.GetElementSpace()); - - a_m_k_device_buf.ToDevice(a_m_k.mData.data()); - b_n_k_device_buf.ToDevice(b_n_k.mData.data()); - c_m_n_device_buf.ToDevice(c_m_n.mData.data()); - -#if 0 - // [M, N, K0, K1] = [256, 128, 4, 4] for fp32 - constexpr index_t BlockSize = 256; - - constexpr index_t MPerBlock = 256; - constexpr index_t NPerBlock = 128; - constexpr index_t KPerBlock = 4; - - constexpr index_t MPerXDL = 32; - constexpr index_t NPerXDL = 32; - constexpr index_t K1 = 4; - - constexpr index_t MRepeat = 4; - constexpr index_t NRepeat = 2; - - using ABlockTransferThreadSliceLengths_K0_M_K1 = Sequence<1, 4, 4>; - using ABlockTransferThreadClusterLengths_K0_M_K1 = Sequence<4, 64, 1>; - - constexpr index_t ABlockTransferSrcScalarPerVector_K1 = 4; - constexpr index_t ABlockTransferDstScalarPerVector_K1 = 4; - - using BBlockTransferThreadSliceLengths_K0_N_K1 = Sequence<1, 2, 4>; - using BBlockTransferThreadClusterLengths_K0_N_K1 = Sequence<4, 64, 1>; - - constexpr index_t BBlockTransferSrcScalarPerVector_K1 = 4; - constexpr index_t BBlockTransferDstScalarPerVector_K1 = 4; - - constexpr index_t CThreadTransferDstScalarPerVector = 1; -#elif 0 - // [M, N, K0, K1] = [128, 256, 4, 4] for fp32 - constexpr index_t BlockSize = 256; - - constexpr index_t MPerBlock = 128; - constexpr index_t NPerBlock = 256; - constexpr index_t KPerBlock = 4; - - constexpr index_t MPerXDL = 32; - constexpr index_t NPerXDL = 32; - constexpr index_t K1 = 4; - - constexpr index_t MRepeat = 2; - constexpr index_t NRepeat = 4; - - using ABlockTransferThreadSliceLengths_K0_M_K1 = Sequence<1, 2, 4>; - using ABlockTransferThreadClusterLengths_K0_M_K1 = Sequence<4, 64, 1>; - - constexpr index_t ABlockTransferSrcScalarPerVector_K1 = 4; - constexpr index_t ABlockTransferDstScalarPerVector_K1 = 4; - - using BBlockTransferThreadSliceLengths_K0_N_K1 = Sequence<1, 4, 4>; - using BBlockTransferThreadClusterLengths_K0_N_K1 = Sequence<4, 64, 1>; - - constexpr index_t BBlockTransferSrcScalarPerVector_K1 = 4; - constexpr index_t BBlockTransferDstScalarPerVector_K1 = 4; - - constexpr index_t CThreadTransferDstScalarPerVector = 1; -#elif 0 - // [M, N, K0, K1] = [128, 128, 4, 4], C = 64, for fp32 - constexpr index_t BlockSize = 256; - - constexpr index_t MPerBlock = 128; - constexpr index_t NPerBlock = 128; - constexpr index_t KPerBlock = 4; - - constexpr index_t MPerXDL = 32; - constexpr index_t NPerXDL = 32; - constexpr index_t K1 = 4; - - constexpr index_t MRepeat = 2; - constexpr index_t NRepeat = 2; - - using ABlockTransferThreadSliceLengths_K0_M_K1 = Sequence<1, 2, 4>; - using ABlockTransferThreadClusterLengths_K0_M_K1 = Sequence<4, 64, 1>; - - constexpr index_t ABlockTransferSrcScalarPerVector_K1 = 4; - constexpr index_t ABlockTransferDstScalarPerVector_K1 = 4; - - using BBlockTransferThreadSliceLengths_K0_N_K1 = Sequence<1, 2, 4>; - using BBlockTransferThreadClusterLengths_K0_N_K1 = Sequence<4, 64, 1>; - - constexpr index_t BBlockTransferSrcScalarPerVector_K1 = 4; - constexpr index_t BBlockTransferDstScalarPerVector_K1 = 4; - - constexpr index_t CThreadTransferDstScalarPerVector = 1; -#elif 0 - // [M, N, K0, K1] = [128, 64, 4, 4], C = 32, for fp32 - constexpr index_t BlockSize = 256; - - constexpr index_t MPerBlock = 128; - constexpr index_t NPerBlock = 64; - constexpr index_t KPerBlock = 4; - - constexpr index_t MPerXDL = 32; - constexpr index_t NPerXDL = 32; - constexpr index_t K1 = 4; - - constexpr index_t MRepeat = 2; - constexpr index_t NRepeat = 1; - - using ABlockTransferThreadSliceLengths_K0_M_K1 = Sequence<1, 2, 4>; - using ABlockTransferThreadClusterLengths_K0_M_K1 = Sequence<4, 64, 1>; - - constexpr index_t ABlockTransferSrcScalarPerVector_K1 = 4; - constexpr index_t ABlockTransferDstScalarPerVector_K1 = 4; - - using BBlockTransferThreadSliceLengths_K0_N_K1 = Sequence<1, 1, 4>; - using BBlockTransferThreadClusterLengths_K0_N_K1 = Sequence<4, 64, 1>; - - constexpr index_t BBlockTransferSrcScalarPerVector_K1 = 4; - constexpr index_t BBlockTransferDstScalarPerVector_K1 = 4; - - constexpr index_t CThreadTransferDstScalarPerVector = 1; -#elif 0 - // [M, N, K0, K1] = [64, 128, 4, 4], C = 32, for fp32 - constexpr index_t BlockSize = 256; - - constexpr index_t MPerBlock = 64; - constexpr index_t NPerBlock = 128; - constexpr index_t KPerBlock = 4; - - constexpr index_t MPerXDL = 32; - constexpr index_t NPerXDL = 32; - constexpr index_t K1 = 4; - - constexpr index_t MRepeat = 1; - constexpr index_t NRepeat = 2; - - using ABlockTransferThreadSliceLengths_K0_M_K1 = Sequence<1, 1, 4>; - using ABlockTransferThreadClusterLengths_K0_M_K1 = Sequence<4, 64, 1>; - - constexpr index_t ABlockTransferSrcScalarPerVector_K1 = 4; - constexpr index_t ABlockTransferDstScalarPerVector_K1 = 4; - - using BBlockTransferThreadSliceLengths_K0_N_K1 = Sequence<1, 2, 4>; - using BBlockTransferThreadClusterLengths_K0_N_K1 = Sequence<4, 64, 1>; - - constexpr index_t BBlockTransferSrcScalarPerVector_K1 = 4; - constexpr index_t BBlockTransferDstScalarPerVector_K1 = 4; - - constexpr index_t CThreadTransferDstScalarPerVector = 1; -#elif 1 - // [M, N, K0, K1] = [256, 128, 4, 8], C = 128, for fp16 - constexpr index_t BlockSize = 256; - - constexpr index_t MPerBlock = 256; - constexpr index_t NPerBlock = 128; - constexpr index_t KPerBlock = 4; - - constexpr index_t MPerXDL = 32; - constexpr index_t NPerXDL = 32; - constexpr index_t K1 = 8; - - constexpr index_t MRepeat = 4; - constexpr index_t NRepeat = 2; - - using ABlockTransferThreadSliceLengths_K0_M_K1 = Sequence<1, 4, 8>; - using ABlockTransferThreadClusterLengths_K0_M_K1 = Sequence<4, 64, 1>; - - constexpr index_t ABlockTransferSrcScalarPerVector_K1 = 8; - constexpr index_t ABlockTransferDstScalarPerVector_K1 = 8; - - using BBlockTransferThreadSliceLengths_K0_N_K1 = Sequence<1, 2, 8>; - using BBlockTransferThreadClusterLengths_K0_N_K1 = Sequence<4, 64, 1>; - - constexpr index_t BBlockTransferSrcScalarPerVector_K1 = 8; - constexpr index_t BBlockTransferDstScalarPerVector_K1 = 8; - - constexpr index_t CThreadTransferDstScalarPerVector = 1; -#elif 0 - // [M, N, K0, K1] = [128, 256, 4, 8], C = 128, for fp16 - constexpr index_t BlockSize = 256; - - constexpr index_t MPerBlock = 128; - constexpr index_t NPerBlock = 256; - constexpr index_t KPerBlock = 4; - - constexpr index_t MPerXDL = 32; - constexpr index_t NPerXDL = 32; - constexpr index_t K1 = 8; - - constexpr index_t MRepeat = 2; - constexpr index_t NRepeat = 4; - - using ABlockTransferThreadSliceLengths_K0_M_K1 = Sequence<1, 2, 8>; - using ABlockTransferThreadClusterLengths_K0_M_K1 = Sequence<4, 64, 1>; - - constexpr index_t ABlockTransferSrcScalarPerVector_K1 = 8; - constexpr index_t ABlockTransferDstScalarPerVector_K1 = 8; - - using BBlockTransferThreadSliceLengths_K0_N_K1 = Sequence<1, 4, 8>; - using BBlockTransferThreadClusterLengths_K0_N_K1 = Sequence<4, 64, 1>; - - constexpr index_t BBlockTransferSrcScalarPerVector_K1 = 8; - constexpr index_t BBlockTransferDstScalarPerVector_K1 = 8; - - constexpr index_t CThreadTransferDstScalarPerVector = 1; -#elif 0 - // [M, N, K0, K1] = [128, 128, 4, 8], C = 128, for fp16 - constexpr index_t BlockSize = 128; - - constexpr index_t MPerBlock = 128; - constexpr index_t NPerBlock = 128; - constexpr index_t KPerBlock = 4; - - constexpr index_t MPerXDL = 32; - constexpr index_t NPerXDL = 32; - constexpr index_t K1 = 8; - - constexpr index_t MRepeat = 4; - constexpr index_t NRepeat = 2; - - using ABlockTransferThreadSliceLengths_K0_M_K1 = Sequence<1, 4, 8>; - using ABlockTransferThreadClusterLengths_K0_M_K1 = Sequence<4, 32, 1>; - - constexpr index_t ABlockTransferSrcScalarPerVector_K1 = 8; - constexpr index_t ABlockTransferDstScalarPerVector_K1 = 8; - - using BBlockTransferThreadSliceLengths_K0_N_K1 = Sequence<1, 4, 8>; - using BBlockTransferThreadClusterLengths_K0_N_K1 = Sequence<4, 32, 1>; - - constexpr index_t BBlockTransferSrcScalarPerVector_K1 = 8; - constexpr index_t BBlockTransferDstScalarPerVector_K1 = 8; - - constexpr index_t CThreadTransferDstScalarPerVector = 1; -#elif 0 - // [M, N, K0, K1] = [128, 128, 4, 8], C = 64, for fp16 - constexpr index_t BlockSize = 256; - - constexpr index_t MPerBlock = 128; - constexpr index_t NPerBlock = 128; - constexpr index_t KPerBlock = 4; - - constexpr index_t MPerXDL = 32; - constexpr index_t NPerXDL = 32; - constexpr index_t K1 = 8; - - constexpr index_t MRepeat = 2; - constexpr index_t NRepeat = 2; - - using ABlockTransferThreadSliceLengths_K0_M_K1 = Sequence<1, 2, 8>; - using ABlockTransferThreadClusterLengths_K0_M_K1 = Sequence<4, 64, 1>; - - constexpr index_t ABlockTransferSrcScalarPerVector_K1 = 8; - constexpr index_t ABlockTransferDstScalarPerVector_K1 = 8; - - using BBlockTransferThreadSliceLengths_K0_N_K1 = Sequence<1, 2, 8>; - using BBlockTransferThreadClusterLengths_K0_N_K1 = Sequence<4, 64, 1>; - - constexpr index_t BBlockTransferSrcScalarPerVector_K1 = 8; - constexpr index_t BBlockTransferDstScalarPerVector_K1 = 8; - - constexpr index_t CThreadTransferDstScalarPerVector = 1; -#elif 0 - // [M, N, K0, K1] = [64, 128, 4, 8], C = 64, for fp16 - constexpr index_t BlockSize = 128; - - constexpr index_t MPerBlock = 64; - constexpr index_t NPerBlock = 128; - constexpr index_t KPerBlock = 4; - - constexpr index_t MPerXDL = 32; - constexpr index_t NPerXDL = 32; - constexpr index_t K1 = 8; - - constexpr index_t MRepeat = 2; - constexpr index_t NRepeat = 2; - - using ABlockTransferThreadSliceLengths_K0_M_K1 = Sequence<1, 2, 8>; - using ABlockTransferThreadClusterLengths_K0_M_K1 = Sequence<4, 32, 1>; - - constexpr index_t ABlockTransferSrcScalarPerVector_K1 = 8; - constexpr index_t ABlockTransferDstScalarPerVector_K1 = 8; - - using BBlockTransferThreadSliceLengths_K0_N_K1 = Sequence<1, 4, 8>; - using BBlockTransferThreadClusterLengths_K0_N_K1 = Sequence<4, 32, 1>; - - constexpr index_t BBlockTransferSrcScalarPerVector_K1 = 8; - constexpr index_t BBlockTransferDstScalarPerVector_K1 = 8; - - constexpr index_t CThreadTransferDstScalarPerVector = 1; -#elif 1 - // [M, N, K0, K1] = [128, 64, 4, 8], C = 32, for fp16 - constexpr index_t BlockSize = 256; - - constexpr index_t MPerBlock = 128; - constexpr index_t NPerBlock = 64; - constexpr index_t KPerBlock = 4; - - constexpr index_t MPerXDL = 32; - constexpr index_t NPerXDL = 32; - constexpr index_t K1 = 8; - - constexpr index_t MRepeat = 2; - constexpr index_t NRepeat = 1; - - using ABlockTransferThreadSliceLengths_K0_M_K1 = Sequence<1, 2, 8>; - using ABlockTransferThreadClusterLengths_K0_M_K1 = Sequence<4, 64, 1>; - - constexpr index_t ABlockTransferSrcScalarPerVector_K1 = 8; - constexpr index_t ABlockTransferDstScalarPerVector_K1 = 8; - - using BBlockTransferThreadSliceLengths_K0_N_K1 = Sequence<1, 1, 8>; - using BBlockTransferThreadClusterLengths_K0_N_K1 = Sequence<4, 64, 1>; - - constexpr index_t BBlockTransferSrcScalarPerVector_K1 = 8; - constexpr index_t BBlockTransferDstScalarPerVector_K1 = 8; - - constexpr index_t CThreadTransferDstScalarPerVector = 1; -#elif 1 - // [M, N, K0, K1] = [64, 128, 4, 8], C = 32, for fp16 - constexpr index_t BlockSize = 256; - - constexpr index_t MPerBlock = 64; - constexpr index_t NPerBlock = 128; - constexpr index_t KPerBlock = 4; - - constexpr index_t MPerXDL = 32; - constexpr index_t NPerXDL = 32; - constexpr index_t K1 = 8; - - constexpr index_t MRepeat = 1; - constexpr index_t NRepeat = 2; - - using ABlockTransferThreadSliceLengths_K0_M_K1 = Sequence<1, 1, 8>; - using ABlockTransferThreadClusterLengths_K0_M_K1 = Sequence<4, 64, 1>; - - constexpr index_t ABlockTransferSrcScalarPerVector_K1 = 8; - constexpr index_t ABlockTransferDstScalarPerVector_K1 = 8; - - using BBlockTransferThreadSliceLengths_K0_N_K1 = Sequence<1, 2, 8>; - using BBlockTransferThreadClusterLengths_K0_N_K1 = Sequence<4, 64, 1>; - - constexpr index_t BBlockTransferSrcScalarPerVector_K1 = 8; - constexpr index_t BBlockTransferDstScalarPerVector_K1 = 8; - - constexpr index_t CThreadTransferDstScalarPerVector = 1; -#endif - - const auto K = a_m_k.mDesc.GetLengths()[1]; - const auto M = a_m_k.mDesc.GetLengths()[0]; - const auto N = b_n_k.mDesc.GetLengths()[0]; - - constexpr auto K1Number = Number{}; - const auto K0 = K / K1Number; - -#if 1 - // non-padded GEMM - const auto a_k0_m_k1_grid_desc = - make_naive_tensor_descriptor(make_tuple(K0, M, K1Number), - make_tuple(K1Number * a_m_k.mDesc.GetStrides()[1], - a_m_k.mDesc.GetStrides()[0], - a_m_k.mDesc.GetStrides()[1])); - - const auto b_k0_n_k1_grid_desc = - make_naive_tensor_descriptor(make_tuple(K0, N, K1Number), - make_tuple(K1Number * b_n_k.mDesc.GetStrides()[1], - b_n_k.mDesc.GetStrides()[0], - b_n_k.mDesc.GetStrides()[1])); - - const auto c_m_n_grid_desc = make_naive_tensor_descriptor( - make_tuple(M, N), make_tuple(c_m_n.mDesc.GetStrides()[0], c_m_n.mDesc.GetStrides()[1])); - - // HACK: hacks that control index calculation when iterating over A, B, C matrix - constexpr auto a_k0_m_k1_grid_step_hacks = make_tuple(make_tuple(Sequence<0>{}, // 0+: K0 - Sequence<0>{}, // 1+: M - Sequence<0>{}), // 2+: K1 - make_tuple(Sequence<0>{}, // 0-: K0 - Sequence<0>{}, // 1-: M - Sequence<0>{})); // 2-: K1 - - constexpr auto b_k0_n_k1_grid_step_hacks = make_tuple(make_tuple(Sequence<0>{}, // 0+: K0 - Sequence<0>{}, // 1+: N - Sequence<0>{}), // 2+: K1 - make_tuple(Sequence<0>{}, // 0-: K0 - Sequence<0>{}, // 1-: N - Sequence<0>{})); // 2-: K1 - - constexpr auto c_m0_n0_m1_n1_m2_m3_m4_n2_grid_step_hacks = - make_tuple(make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 0+: M0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 1+: N0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 2+: M1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 3+: N1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 4+: M2 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 5+: M3 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 6+: M4 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}), // 7+: N2 - make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 0-: M0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 1-: N0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 2-: M1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 3-: N1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 4-: M2 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 5-: M3 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 6-: M4 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{})); // 7-: N2 - - constexpr auto a_k0_m_k1_grid_move_slice_window_step_hacks = Sequence<0>{}; - - constexpr auto b_k0_n_k1_grid_move_slice_window_step_hacks = Sequence<0>{}; -#else - // padded GEMM - const auto a_k0_m_k1_grid_desc_tmp = - make_naive_tensor_descriptor(make_tuple(K0, M, K1Number), - make_tuple(K1Number * a_m_k.mDesc.GetStrides()[1], - a_m_k.mDesc.GetStrides()[0], - a_m_k.mDesc.GetStrides()[1])); - - const auto MRightPad = math::integer_divide_ceil(M, MPerBlock) * MPerBlock - M; - - const auto a_k0_m_k1_grid_desc = - transform_tensor_descriptor(a_k0_m_k1_grid_desc_tmp, - make_tuple(make_pass_through_transform(K0), - make_right_pad_transform(M, MRightPad), - make_pass_through_transform(K1Number)), - make_tuple(Sequence<0>{}, Sequence<1>{}, Sequence<2>{}), - make_tuple(Sequence<0>{}, Sequence<1>{}, Sequence<2>{})); - - const auto b_k0_n_k1_grid_desc = - make_naive_tensor_descriptor(make_tuple(K0, N, K1Number), - make_tuple(K1Number * b_n_k.mDesc.GetStrides()[1], - b_n_k.mDesc.GetStrides()[0], - b_n_k.mDesc.GetStrides()[1])); - - const auto c_m_n_grid_desc_tmp = make_naive_tensor_descriptor( - make_tuple(M, N), make_tuple(c_m_n.mDesc.GetStrides()[0], c_m_n.mDesc.GetStrides()[1])); - - const auto c_m_n_grid_desc = transform_tensor_descriptor( - c_m_n_grid_desc_tmp, - make_tuple(make_right_pad_transform(M, MRightPad), make_pass_through_transform(N)), - make_tuple(Sequence<0>{}, Sequence<1>{}), - make_tuple(Sequence<0>{}, Sequence<1>{})); - - // HACK: hacks that control index calculation when iterating over A, B, C matrix - constexpr auto a_k0_m_k1_grid_step_hacks = - make_tuple(make_tuple(Sequence<0, 0, 0, 0>{}, // 0+: K0 - Sequence<0, 0, 0, 0>{}, // 1+: M - Sequence<0, 0, 0, 0>{}), // 2+: K1 - make_tuple(Sequence<0, 0, 0, 0>{}, // 0-: K0 - Sequence<0, 0, 0, 0>{}, // 1-: M - Sequence<0, 0, 0, 0>{})); // 2-: K1 - - constexpr auto b_k0_n_k1_grid_step_hacks = make_tuple(make_tuple(Sequence<0>{}, // 0+: K0 - Sequence<0>{}, // 1+: N - Sequence<0>{}), // 2+: K1 - make_tuple(Sequence<0>{}, // 0-: K0 - Sequence<0>{}, // 1-: N - Sequence<0>{})); // 2-: K1 - - constexpr auto c_m0_n0_m1_n1_m2_m3_m4_n2_grid_step_hacks = - make_tuple(make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 0+: M0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 1+: N0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 2+: M1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 3+: N1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 4+: M2 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 5+: M3 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 6+: M4 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}), // 7+: N2 - make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 0-: M0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 1-: N0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 2-: M1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 3-: N1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 4-: M2 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 5-: M3 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 6-: M4 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{})); // 7-: N2 - - constexpr auto a_k0_m_k1_grid_move_slice_window_step_hacks = Sequence<0, 0, 0, 0>{}; - - constexpr auto b_k0_n_k1_grid_move_slice_window_step_hacks = Sequence<0>{}; -#endif - - for(index_t i = 0; i < 5; ++i) - { - float ave_time = - driver_gemm_xdlops_v2r3, - Sequence<1, 0, 2>, - 2, - ABlockTransferSrcScalarPerVector_K1, - ABlockTransferDstScalarPerVector_K1, - false, // don't move back src coordinate after threadwise copy - BBlockTransferThreadSliceLengths_K0_N_K1, - BBlockTransferThreadClusterLengths_K0_N_K1, - Sequence<1, 0, 2>, - Sequence<1, 0, 2>, - 2, - BBlockTransferSrcScalarPerVector_K1, - BBlockTransferDstScalarPerVector_K1, - false, // don't move back src coordinate after threadwise copy - Sequence<0, 2, 4, 5, 6, 1, 3, 7>, - 7, - CThreadTransferDstScalarPerVector, - decltype(a_k0_m_k1_grid_step_hacks), - decltype(b_k0_n_k1_grid_step_hacks), - decltype(c_m0_n0_m1_n1_m2_m3_m4_n2_grid_step_hacks), - decltype(a_k0_m_k1_grid_move_slice_window_step_hacks), - decltype(b_k0_n_k1_grid_move_slice_window_step_hacks), - false, // CAccessOrderMRepeatNRepeat - true, // ABlockLdsExtraM - true // BBlockLdsExtraN - >(static_cast(a_m_k_device_buf.GetDeviceBuffer()), - static_cast(b_n_k_device_buf.GetDeviceBuffer()), - static_cast(c_m_n_device_buf.GetDeviceBuffer()), - a_k0_m_k1_grid_desc, - b_k0_n_k1_grid_desc, - c_m_n_grid_desc, - debug::debug_driver_gemm_xdlops_v2r3::M01, - debug::debug_driver_gemm_xdlops_v2r3::N01, - a_k0_m_k1_grid_step_hacks, - b_k0_n_k1_grid_step_hacks, - c_m0_n0_m1_n1_m2_m3_m4_n2_grid_step_hacks, - a_k0_m_k1_grid_move_slice_window_step_hacks, - b_k0_n_k1_grid_move_slice_window_step_hacks, - nrepeat); - - float perf = static_cast((std::size_t(2) * M * N * K)) / - (std::size_t(1000) * 1000 * 1000) / ave_time; - - std::cout << "Average time : " << ave_time << " ms, " << perf << " TFlop/s" << std::endl; - } - - // copy result back to host - c_m_n_device_buf.FromDevice(c_m_n.mData.data()); -} diff --git a/library/include/ck/library/obselete_driver_offline/device_gemm_xdlops_mk_nk_nm.hpp b/library/include/ck/library/obselete_driver_offline/device_gemm_xdlops_mk_nk_nm.hpp deleted file mode 100644 index 06d8ed2940..0000000000 --- a/library/include/ck/library/obselete_driver_offline/device_gemm_xdlops_mk_nk_nm.hpp +++ /dev/null @@ -1,347 +0,0 @@ -#pragma once -#include -#include "device.hpp" -#include "host_tensor.hpp" -#include "driver_gemm_xdlops_v2r3.hpp" - -template -void device_gemm_xdlops_mk_nk_nm(const Tensor& a_m_k, - const Tensor& b_n_k, - Tensor& c_n_m, - ck::index_t nrepeat) -{ - using namespace ck; - - std::cout << __func__ << std::endl; - - DeviceMem a_m_k_device_buf(sizeof(ABType) * a_m_k.mDesc.GetElementSpace()); - DeviceMem b_n_k_device_buf(sizeof(ABType) * b_n_k.mDesc.GetElementSpace()); - DeviceMem c_n_m_device_buf(sizeof(CType) * c_n_m.mDesc.GetElementSpace()); - - a_m_k_device_buf.ToDevice(a_m_k.mData.data()); - b_n_k_device_buf.ToDevice(b_n_k.mData.data()); - c_n_m_device_buf.ToDevice(c_n_m.mData.data()); - -#if 0 - // [M, N, K0, K1] = [256, 128, 4, 4] for fp32 - constexpr index_t BlockSize = 256; - - constexpr index_t MPerBlock = 256; - constexpr index_t NPerBlock = 128; - constexpr index_t KPerBlock = 4; - - constexpr index_t MPerXDL = 32; - constexpr index_t NPerXDL = 32; - constexpr index_t K1 = 4; - - constexpr index_t MRepeat = 4; - constexpr index_t NRepeat = 2; - - using ABlockTransferThreadSliceLengths_K0_M_K1 = Sequence<1, 4, 4>; - using ABlockTransferThreadClusterLengths_K0_M_K1 = Sequence<4, 64, 1>; - - constexpr index_t ABlockTransferSrcScalarPerVector_K1 = 4; - constexpr index_t ABlockTransferDstScalarPerVector_K1 = 4; - - using BBlockTransferThreadSliceLengths_K0_N_K1 = Sequence<1, 2, 4>; - using BBlockTransferThreadClusterLengths_K0_N_K1 = Sequence<4, 64, 1>; - - constexpr index_t BBlockTransferSrcScalarPerVector_K1 = 4; - constexpr index_t BBlockTransferDstScalarPerVector_K1 = 4; - - constexpr index_t CThreadTransferDstScalarPerVector = 4; -#elif 0 - // [M, N, K0, K1] = [128, 256, 4, 4] for fp32 - constexpr index_t BlockSize = 256; - - constexpr index_t MPerBlock = 128; - constexpr index_t NPerBlock = 256; - constexpr index_t KPerBlock = 4; - - constexpr index_t MPerXDL = 32; - constexpr index_t NPerXDL = 32; - constexpr index_t K1 = 4; - - constexpr index_t MRepeat = 2; - constexpr index_t NRepeat = 4; - - using ABlockTransferThreadSliceLengths_K0_M_K1 = Sequence<1, 2, 4>; - using ABlockTransferThreadClusterLengths_K0_M_K1 = Sequence<4, 64, 1>; - - constexpr index_t ABlockTransferSrcScalarPerVector_K1 = 4; - constexpr index_t ABlockTransferDstScalarPerVector_K1 = 4; - - using BBlockTransferThreadSliceLengths_K0_N_K1 = Sequence<1, 4, 4>; - using BBlockTransferThreadClusterLengths_K0_N_K1 = Sequence<4, 64, 1>; - - constexpr index_t BBlockTransferSrcScalarPerVector_K1 = 4; - constexpr index_t BBlockTransferDstScalarPerVector_K1 = 4; - - constexpr index_t CThreadTransferDstScalarPerVector = 4; -#elif 0 - // [M, N, K0, K1] = [256, 128, 4, 8] for fp16 - constexpr index_t BlockSize = 256; - - constexpr index_t MPerBlock = 256; - constexpr index_t NPerBlock = 128; - constexpr index_t KPerBlock = 4; - - constexpr index_t MPerXDL = 32; - constexpr index_t NPerXDL = 32; - constexpr index_t K1 = 8; - - constexpr index_t MRepeat = 4; - constexpr index_t NRepeat = 2; - - using ABlockTransferThreadSliceLengths_K0_M_K1 = Sequence<1, 4, 8>; - using ABlockTransferThreadClusterLengths_K0_M_K1 = Sequence<4, 64, 1>; - - constexpr index_t ABlockTransferSrcScalarPerVector_K1 = 8; - constexpr index_t ABlockTransferDstScalarPerVector_K1 = 8; - - using BBlockTransferThreadSliceLengths_K0_N_K1 = Sequence<1, 2, 8>; - using BBlockTransferThreadClusterLengths_K0_N_K1 = Sequence<4, 64, 1>; - - constexpr index_t BBlockTransferSrcScalarPerVector_K1 = 8; - constexpr index_t BBlockTransferDstScalarPerVector_K1 = 8; - - constexpr index_t CThreadTransferDstScalarPerVector = 4; -#elif 0 - // [M, N, K0, K1] = [128, 256, 4, 8] for fp16 - constexpr index_t BlockSize = 256; - - constexpr index_t MPerBlock = 128; - constexpr index_t NPerBlock = 256; - constexpr index_t KPerBlock = 4; - - constexpr index_t MPerXDL = 32; - constexpr index_t NPerXDL = 32; - constexpr index_t K1 = 8; - - constexpr index_t MRepeat = 2; - constexpr index_t NRepeat = 4; - - using ABlockTransferThreadSliceLengths_K0_M_K1 = Sequence<1, 2, 8>; - using ABlockTransferThreadClusterLengths_K0_M_K1 = Sequence<4, 64, 1>; - - constexpr index_t ABlockTransferSrcScalarPerVector_K1 = 8; - constexpr index_t ABlockTransferDstScalarPerVector_K1 = 8; - - using BBlockTransferThreadSliceLengths_K0_N_K1 = Sequence<1, 4, 8>; - using BBlockTransferThreadClusterLengths_K0_N_K1 = Sequence<4, 64, 1>; - - constexpr index_t BBlockTransferSrcScalarPerVector_K1 = 8; - constexpr index_t BBlockTransferDstScalarPerVector_K1 = 8; - - constexpr index_t CThreadTransferDstScalarPerVector = 4; -#elif 0 - // [M, N, K0, K1] = [128, 128, 4, 8], C = 128, for fp16 - constexpr index_t BlockSize = 128; - - constexpr index_t MPerBlock = 128; - constexpr index_t NPerBlock = 128; - constexpr index_t KPerBlock = 4; - - constexpr index_t MPerXDL = 32; - constexpr index_t NPerXDL = 32; - constexpr index_t K1 = 8; - - constexpr index_t MRepeat = 4; - constexpr index_t NRepeat = 2; - - using ABlockTransferThreadSliceLengths_K0_M_K1 = Sequence<1, 4, 8>; - using ABlockTransferThreadClusterLengths_K0_M_K1 = Sequence<4, 32, 1>; - - constexpr index_t ABlockTransferSrcScalarPerVector_K1 = 8; - constexpr index_t ABlockTransferDstScalarPerVector_K1 = 8; - - using BBlockTransferThreadSliceLengths_K0_N_K1 = Sequence<1, 4, 8>; - using BBlockTransferThreadClusterLengths_K0_N_K1 = Sequence<4, 32, 1>; - - constexpr index_t BBlockTransferSrcScalarPerVector_K1 = 8; - constexpr index_t BBlockTransferDstScalarPerVector_K1 = 8; - - constexpr index_t CThreadTransferDstScalarPerVector = 4; -#elif 0 - // [M, N, K0, K1] = [128, 128, 4, 8], C = 64, for fp16 - constexpr index_t BlockSize = 256; - - constexpr index_t MPerBlock = 128; - constexpr index_t NPerBlock = 128; - constexpr index_t KPerBlock = 4; - - constexpr index_t MPerXDL = 32; - constexpr index_t NPerXDL = 32; - constexpr index_t K1 = 8; - - constexpr index_t MRepeat = 2; - constexpr index_t NRepeat = 2; - - using ABlockTransferThreadSliceLengths_K0_M_K1 = Sequence<1, 2, 8>; - using ABlockTransferThreadClusterLengths_K0_M_K1 = Sequence<4, 64, 1>; - - constexpr index_t ABlockTransferSrcScalarPerVector_K1 = 8; - constexpr index_t ABlockTransferDstScalarPerVector_K1 = 8; - - using BBlockTransferThreadSliceLengths_K0_N_K1 = Sequence<1, 2, 8>; - using BBlockTransferThreadClusterLengths_K0_N_K1 = Sequence<4, 64, 1>; - - constexpr index_t BBlockTransferSrcScalarPerVector_K1 = 8; - constexpr index_t BBlockTransferDstScalarPerVector_K1 = 8; - - constexpr index_t CThreadTransferDstScalarPerVector = 4; -#elif 1 - // [M, N, K0, K1] = [64, 128, 4, 8], C = 32, for fp16 - constexpr index_t BlockSize = 256; - - constexpr index_t MPerBlock = 64; - constexpr index_t NPerBlock = 128; - constexpr index_t KPerBlock = 4; - - constexpr index_t MPerXDL = 32; - constexpr index_t NPerXDL = 32; - constexpr index_t K1 = 8; - - constexpr index_t MRepeat = 1; - constexpr index_t NRepeat = 2; - - using ABlockTransferThreadSliceLengths_K0_M_K1 = Sequence<1, 1, 8>; - using ABlockTransferThreadClusterLengths_K0_M_K1 = Sequence<4, 64, 1>; - - constexpr index_t ABlockTransferSrcScalarPerVector_K1 = 8; - constexpr index_t ABlockTransferDstScalarPerVector_K1 = 8; - - using BBlockTransferThreadSliceLengths_K0_N_K1 = Sequence<1, 2, 8>; - using BBlockTransferThreadClusterLengths_K0_N_K1 = Sequence<4, 64, 1>; - - constexpr index_t BBlockTransferSrcScalarPerVector_K1 = 8; - constexpr index_t BBlockTransferDstScalarPerVector_K1 = 8; - - constexpr index_t CThreadTransferDstScalarPerVector = 4; -#endif - - const auto K = a_m_k.mDesc.GetLengths()[1]; - const auto M = a_m_k.mDesc.GetLengths()[0]; - const auto N = b_n_k.mDesc.GetLengths()[0]; - - constexpr auto K1Number = Number{}; - const auto K0 = K / K1Number; - - const auto a_k0_m_k1_grid_desc = - make_naive_tensor_descriptor(make_tuple(K0, M, K1Number), - make_tuple(K1Number * a_m_k.mDesc.GetStrides()[1], - a_m_k.mDesc.GetStrides()[0], - a_m_k.mDesc.GetStrides()[1])); - - const auto b_k0_n_k1_grid_desc = - make_naive_tensor_descriptor(make_tuple(K0, N, K1Number), - make_tuple(K1Number * b_n_k.mDesc.GetStrides()[1], - b_n_k.mDesc.GetStrides()[0], - b_n_k.mDesc.GetStrides()[1])); - - const auto c_m_n_grid_desc = make_naive_tensor_descriptor( - make_tuple(M, N), make_tuple(c_n_m.mDesc.GetStrides()[1], c_n_m.mDesc.GetStrides()[0])); - - // HACK: hacks that control index calculation when iterating over A, B, C matrix - constexpr auto a_k0_m_k1_grid_step_hacks = make_tuple(make_tuple(Sequence<0>{}, // 0+: K0 - Sequence<0>{}, // 1+: M - Sequence<0>{}), // 2+: K1 - make_tuple(Sequence<0>{}, // 0-: K0 - Sequence<0>{}, // 1-: M - Sequence<0>{})); // 2-: K1 - - constexpr auto b_k0_n_k1_grid_step_hacks = make_tuple(make_tuple(Sequence<0>{}, // 0+: K0 - Sequence<0>{}, // 1+: N - Sequence<0>{}), // 2+: K1 - make_tuple(Sequence<0>{}, // 0-: K0 - Sequence<0>{}, // 1-: N - Sequence<0>{})); // 2-: K1 - - constexpr auto c_m0_n0_m1_n1_m2_m3_m4_n2_grid_step_hacks = - make_tuple(make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 0+: M0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 1+: N0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 2+: M1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 3+: N1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 4+: M2 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 5+: M3 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 6+: M4 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}), // 7+: N2 - make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 0-: M0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 1-: N0 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 2-: M1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 3-: N1 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 4-: M2 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 5-: M3 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, // 6-: M4 - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{})); // 7-: N2 - - constexpr auto a_k0_m_k1_grid_move_slice_window_step_hacks = Sequence<0>{}; - - constexpr auto b_k0_n_k1_grid_move_slice_window_step_hacks = Sequence<0>{}; - - for(index_t i = 0; i < 5; ++i) - { - float ave_time = - driver_gemm_xdlops_v2r3, - Sequence<1, 0, 2>, - 2, - ABlockTransferSrcScalarPerVector_K1, - ABlockTransferDstScalarPerVector_K1, - false, // don't move back src coordinate after threadwise copy - BBlockTransferThreadSliceLengths_K0_N_K1, - BBlockTransferThreadClusterLengths_K0_N_K1, - Sequence<1, 0, 2>, - Sequence<1, 0, 2>, - 2, - BBlockTransferSrcScalarPerVector_K1, - BBlockTransferDstScalarPerVector_K1, - false, // don't move back src coordinate after threadwise copy - Sequence<2, 3, 0, 1, 7, 5, 4, 6>, - 6, - CThreadTransferDstScalarPerVector, - decltype(a_k0_m_k1_grid_step_hacks), - decltype(b_k0_n_k1_grid_step_hacks), - decltype(c_m0_n0_m1_n1_m2_m3_m4_n2_grid_step_hacks), - decltype(a_k0_m_k1_grid_move_slice_window_step_hacks), - decltype(b_k0_n_k1_grid_move_slice_window_step_hacks), - false // CAccessOrderMRepeatNRepeat - >(static_cast(a_m_k_device_buf.GetDeviceBuffer()), - static_cast(b_n_k_device_buf.GetDeviceBuffer()), - static_cast(c_n_m_device_buf.GetDeviceBuffer()), - a_k0_m_k1_grid_desc, - b_k0_n_k1_grid_desc, - c_m_n_grid_desc, - a_k0_m_k1_grid_step_hacks, - b_k0_n_k1_grid_step_hacks, - c_m0_n0_m1_n1_m2_m3_m4_n2_grid_step_hacks, - a_k0_m_k1_grid_move_slice_window_step_hacks, - b_k0_n_k1_grid_move_slice_window_step_hacks, - nrepeat); - - float perf = static_cast((std::size_t(2) * M * N * K)) / - (std::size_t(1000) * 1000 * 1000) / ave_time; - - std::cout << "Average time : " << ave_time << " ms, " << perf << " TFlop/s" << std::endl; - } - - // copy result back to host - c_n_m_device_buf.FromDevice(c_n_m.mData.data()); -} diff --git a/library/include/ck/library/obselete_driver_offline/driver_contraction_dlops_v1r2.hpp b/library/include/ck/library/obselete_driver_offline/driver_contraction_dlops_v1r2.hpp deleted file mode 100644 index 000098f4fc..0000000000 --- a/library/include/ck/library/obselete_driver_offline/driver_contraction_dlops_v1r2.hpp +++ /dev/null @@ -1,286 +0,0 @@ -#ifndef DRIVER_CONTRACTION_DLOPS_V1R2_HPP -#define DRIVER_CONTRACTION_DLOPS_V1R2_HPP - -#include "common_header.hpp" -#include "tensor_descriptor.hpp" -#include "tensor_descriptor_helper.hpp" -#include "gridwise_contraction_dlops_v1r2.hpp" - -template -__host__ float -driver_contraction_dlops_v1r2(const FloatAB* p_a_grid, - const FloatAB* p_b_grid, - FloatC* p_c_grid, - const AGridDesc_GK0_GM0_GM1_GK1& a_grid_desc_gk0_gm0_gm1_gk1, - const BGridDesc_GK0_GN0_GN1_GK1& b_grid_desc_gk0_gn0_gn1_gk1, - const CGridDesc_GM0_GM1_GN0_GN1& c_grid_desc_gm0_gm1_gn0_gn1, - AGridStepHacks, - BGridStepHacks, - CGridStepHacks, - AGridMoveSliceWindowStepHacks, - BGridMoveSliceWindowStepHacks, - ck::index_t nrepeat) - -{ - using namespace ck; - - constexpr auto I0 = Number<0>{}; - constexpr auto I1 = Number<1>{}; - constexpr auto I2 = Number<2>{}; - constexpr auto I3 = Number<3>{}; - constexpr auto I4 = Number<4>{}; - constexpr auto I5 = Number<5>{}; - - // GEMM - using GridwiseContraction = - GridwiseContractionDlops_A_GK0_GM0_GM1_GK1_B_GK0_GN0_GN1_GK1_C_GM0_GM1_GN0_GN1< - BlockSize, - FloatAB, - FloatAcc, - FloatC, - CGlobalMemoryDataOperation, - AGridDesc_GK0_GM0_GM1_GK1, - BGridDesc_GK0_GN0_GN1_GK1, - CGridDesc_GM0_GM1_GN0_GN1, - GM1PerBlockGM11, - GN1PerBlockGN11, - GK0PerBlock, - BM1PerThreadBM11, - BN1PerThreadBN11, - BK0PerThread, - BM10BN10ThreadClusterBM10Xs, - BM10BN10ThreadClusterBN10Xs, - ABlockTransferThreadSliceLengths_GK0_GM0_GM10_GM11_GK1, - ABlockTransferThreadClusterLengths_GK0_GM0_GM10_GM11_GK1, - ABlockTransferThreadClusterArrangeOrder, - ABlockTransferSrcAccessOrder, - ABlockTransferSrcVectorTensorLengths_GK0_GM0_GM10_GM11_GK1, - ABlockTransferDstVectorTensorLengths_GK0_GM0_GM10_GM11_GK1, - ABlockTransferSrcVectorTensorContiguousDimOrder, - BBlockTransferThreadSliceLengths_GK0_GN0_GN10_GN11_GK1, - BBlockTransferThreadClusterLengths_GK0_GN0_GN10_GN11_GK1, - BBlockTransferThreadClusterArrangeOrder, - BBlockTransferSrcAccessOrder, - BBlockTransferSrcVectorTensorLengths_GK0_GN0_GN10_GN11_GK1, - BBlockTransferDstVectorTensorLengths_GK0_GN0_GN10_GN11_GK1, - BBlockTransferSrcVectorTensorContiguousDimOrder, - CThreadTransferSrcDstAccessOrder, - CThreadTransferSrcDstVectorDim, - CThreadTransferDstScalarPerVector, - AGridStepHacks, - BGridStepHacks, - CGridStepHacks, - AGridMoveSliceWindowStepHacks, - BGridMoveSliceWindowStepHacks>; - - const auto GK0 = a_grid_desc_gk0_gm0_gm1_gk1.GetLength(I0); - - if(!GridwiseContraction::CheckValidity( - a_grid_desc_gk0_gm0_gm1_gk1, b_grid_desc_gk0_gn0_gn1_gk1, c_grid_desc_gm0_gm1_gn0_gn1)) - { - throw std::runtime_error("wrong! " - "GridwiseContraction_A_GK0_GM0_GM1_GK1_B_GK0_GN0_GN1_GK1_C_" - "GM0_GM1_GN0_GN1 has invalid setting"); - } - - const auto a_grid_desc_gk0_gm0_gm10_gm11_gk1 = - GridwiseContraction::MakeAGridDescriptor_GK0_GM0_GM10_GM11_GK1(a_grid_desc_gk0_gm0_gm1_gk1); - const auto b_grid_desc_gk0_gn0_gn10_gn11_gk1 = - GridwiseContraction::MakeBGridDescriptor_GK0_GN0_GN10_GN11_GK1(b_grid_desc_gk0_gn0_gn1_gk1); - - using AGridDesc_GK0_GM0_GM10_GM11_GK1 = decltype(a_grid_desc_gk0_gm0_gm10_gm11_gk1); - using BGridDesc_GK0_GN0_GN10_GN11_GK1 = decltype(b_grid_desc_gk0_gn0_gn10_gn11_gk1); - - // c_grid_desc_gm10_bm0_bm1_gn10_bn0_bn1 - const auto c_grid_desc_gm10_bm0_bm1_gn10_bn0_bn1 = - GridwiseContraction::MakeCGridDescriptor_GM10_BM0_BM1_GN10_BN0_BN1( - c_grid_desc_gm0_gm1_gn0_gn1); - - using CGridDesc_GM10_BM0_BM1_GN10_BN0_BN1 = decltype(c_grid_desc_gm10_bm0_bm1_gn10_bn0_bn1); - - // c_grid_block_cluster_blockid_to_gm10_gn10 - const auto c_grid_block_cluster_blockid_to_gm10_gn10 = - GridwiseContraction::MakeCGridBlockCluster_BlockId_To_GM10_GN10( - c_grid_desc_gm0_gm1_gn0_gn1); - - using CGridBlockCluster_BlockId_To_GM10_GN10 = - decltype(c_grid_block_cluster_blockid_to_gm10_gn10); - - const index_t grid_size = GridwiseContraction::CalculateGridSize(c_grid_desc_gm0_gm1_gn0_gn1); - - const bool has_main_k_block_loop = GridwiseContraction::CalculateHasMainKBlockLoop(GK0); - - const bool has_double_tail_k_block_loop = - GridwiseContraction::CalculateHasDoubleTailKBlockLoop(GK0); - - { - std::cout << "a_grid_desc_gk0_gm0_gm10_gm11_gk1{" - << a_grid_desc_gk0_gm0_gm10_gm11_gk1.GetLength(I0) << ", " - << a_grid_desc_gk0_gm0_gm10_gm11_gk1.GetLength(I1) << ", " - << a_grid_desc_gk0_gm0_gm10_gm11_gk1.GetLength(I2) << ", " - << a_grid_desc_gk0_gm0_gm10_gm11_gk1.GetLength(I3) << ", " - << a_grid_desc_gk0_gm0_gm10_gm11_gk1.GetLength(I4) << "}" << std::endl; - - std::cout << "b_grid_desc_gk0_gn0_gn10_gn11_gk1{" - << b_grid_desc_gk0_gn0_gn10_gn11_gk1.GetLength(I0) << ", " - << b_grid_desc_gk0_gn0_gn10_gn11_gk1.GetLength(I1) << ", " - << b_grid_desc_gk0_gn0_gn10_gn11_gk1.GetLength(I2) << ", " - << b_grid_desc_gk0_gn0_gn10_gn11_gk1.GetLength(I3) << ", " - << b_grid_desc_gk0_gn0_gn10_gn11_gk1.GetLength(I4) << "}" << std::endl; - - std::cout << "c_grid_desc_gm10_bm0_bm1_gn10_bn0_bn1{ " - << c_grid_desc_gm10_bm0_bm1_gn10_bn0_bn1.GetLength(I0) << ", " - << c_grid_desc_gm10_bm0_bm1_gn10_bn0_bn1.GetLength(I1) << ", " - << c_grid_desc_gm10_bm0_bm1_gn10_bn0_bn1.GetLength(I2) << ", " - << c_grid_desc_gm10_bm0_bm1_gn10_bn0_bn1.GetLength(I3) << ", " - << c_grid_desc_gm10_bm0_bm1_gn10_bn0_bn1.GetLength(I4) << ", " - << c_grid_desc_gm10_bm0_bm1_gn10_bn0_bn1.GetLength(I5) << "}" << std::endl; - } - - float ave_time = 0; - - if(has_main_k_block_loop && has_double_tail_k_block_loop) - { - const auto kernel = kernel_contraction_dlops_v1r2< - GridwiseContraction, - FloatAB, - FloatC, - remove_reference_t, - remove_reference_t, - remove_reference_t, - remove_reference_t, - true, - true>; - - ave_time = launch_and_time_kernel(kernel, - nrepeat, - dim3(grid_size), - dim3(BlockSize), - 0, - p_a_grid, - p_b_grid, - p_c_grid, - a_grid_desc_gk0_gm0_gm10_gm11_gk1, - b_grid_desc_gk0_gn0_gn10_gn11_gk1, - c_grid_desc_gm10_bm0_bm1_gn10_bn0_bn1, - c_grid_block_cluster_blockid_to_gm10_gn10); - } - else if(has_main_k_block_loop && !has_double_tail_k_block_loop) - { - const auto kernel = kernel_contraction_dlops_v1r2< - GridwiseContraction, - FloatAB, - FloatC, - remove_reference_t, - remove_reference_t, - remove_reference_t, - remove_reference_t, - true, - false>; - - ave_time = launch_and_time_kernel(kernel, - nrepeat, - dim3(grid_size), - dim3(BlockSize), - 0, - p_a_grid, - p_b_grid, - p_c_grid, - a_grid_desc_gk0_gm0_gm10_gm11_gk1, - b_grid_desc_gk0_gn0_gn10_gn11_gk1, - c_grid_desc_gm10_bm0_bm1_gn10_bn0_bn1, - c_grid_block_cluster_blockid_to_gm10_gn10); - } - else if(!has_main_k_block_loop && has_double_tail_k_block_loop) - { - const auto kernel = kernel_contraction_dlops_v1r2< - GridwiseContraction, - FloatAB, - FloatC, - remove_reference_t, - remove_reference_t, - remove_reference_t, - remove_reference_t, - false, - true>; - - ave_time = launch_and_time_kernel(kernel, - nrepeat, - dim3(grid_size), - dim3(BlockSize), - 0, - p_a_grid, - p_b_grid, - p_c_grid, - a_grid_desc_gk0_gm0_gm10_gm11_gk1, - b_grid_desc_gk0_gn0_gn10_gn11_gk1, - c_grid_desc_gm10_bm0_bm1_gn10_bn0_bn1, - c_grid_block_cluster_blockid_to_gm10_gn10); - } - else - { - const auto kernel = kernel_contraction_dlops_v1r2< - GridwiseContraction, - FloatAB, - FloatC, - remove_reference_t, - remove_reference_t, - remove_reference_t, - remove_reference_t, - false, - false>; - - ave_time = launch_and_time_kernel(kernel, - nrepeat, - dim3(grid_size), - dim3(BlockSize), - 0, - p_a_grid, - p_b_grid, - p_c_grid, - a_grid_desc_gk0_gm0_gm10_gm11_gk1, - b_grid_desc_gk0_gn0_gn10_gn11_gk1, - c_grid_desc_gm10_bm0_bm1_gn10_bn0_bn1, - c_grid_block_cluster_blockid_to_gm10_gn10); - } - - return ave_time; -} -#endif diff --git a/library/include/ck/library/obselete_driver_offline/driver_convolution_add_forward_implicit_gemm_v5r1_dlops_nc0hwc1_kc0yxc1_nk0hwk1.hpp b/library/include/ck/library/obselete_driver_offline/driver_convolution_add_forward_implicit_gemm_v5r1_dlops_nc0hwc1_kc0yxc1_nk0hwk1.hpp deleted file mode 100644 index ec16a97f6f..0000000000 --- a/library/include/ck/library/obselete_driver_offline/driver_convolution_add_forward_implicit_gemm_v5r1_dlops_nc0hwc1_kc0yxc1_nk0hwk1.hpp +++ /dev/null @@ -1,429 +0,0 @@ -#ifndef DRIVER_CONVOLUTION_ADD_FORWARD_IMPLICIT_GEMM_V5R1_DLOPS_NC0HWc1_KC0YXC1_NK0HWK1_HPP -#define DRIVER_CONVOLUTION_ADD_FORWARD_IMPLICIT_GEMM_V5R1_DLOPS_NC0HWc1_KC0YXC1_NK0HWK1_HPP - -#include "common_header.hpp" -#include "tensor_descriptor.hpp" -#include "tensor_descriptor_helper.hpp" -#include "gridwise_gemm_dlops_v3.hpp" - -template -struct DriverDynamicConvolutionForwardImplicitGemmDlops_v5r1_nc0hwc1_kc0yxc1_nk0hwk1_add -{ - template - __host__ float Run(const ck::TensorDescriptor& wei_k_c0_y_x_c1_global_desc, - const ck::TensorDescriptor& in_n_c0_hi_wi_c1_global_desc, - const ck::TensorDescriptor& out_n_k0_ho_wo_k1_global_desc, - const ck::TensorDescriptor& add_n_k0_hox2_wox2_k1_global_desc, - const ConvStrides& conv_strides, - const ConvDilations& conv_dilations, - const InLeftPads& in_left_pads, - const InRightPads& in_right_pads, - const FloatAB* __restrict__ p_a_grid, - const FloatAB* __restrict__ p_b_grid, - const FloatC* __restrict__ p_bias_grid, - FloatC* __restrict__ p_d_grid, - const int nrepeat) const - { - using namespace ck; - - constexpr auto I0 = Number<0>{}; - constexpr auto I1 = Number<1>{}; - constexpr auto I2 = Number<2>{}; - constexpr auto I3 = Number<3>{}; - constexpr auto I4 = Number<4>{}; - - const auto N = in_n_c0_hi_wi_c1_global_desc.GetLength(I0); - const auto C0 = in_n_c0_hi_wi_c1_global_desc.GetLength(I1); - const auto Hi = in_n_c0_hi_wi_c1_global_desc.GetLength(I2); - const auto Wi = in_n_c0_hi_wi_c1_global_desc.GetLength(I3); - // const auto C1 = in_n_c0_hi_wi_c1_global_desc.GetLength(I4); - - const auto K0 = out_n_k0_ho_wo_k1_global_desc.GetLength(I1); - const auto Ho = out_n_k0_ho_wo_k1_global_desc.GetLength(I2); - const auto Wo = out_n_k0_ho_wo_k1_global_desc.GetLength(I3); - const auto K1 = out_n_k0_ho_wo_k1_global_desc.GetLength(I4); - - const auto Hox2 = add_n_k0_hox2_wox2_k1_global_desc.GetLength(I2); - const auto Wox2 = add_n_k0_hox2_wox2_k1_global_desc.GetLength(I3); - - const auto K = wei_k_c0_y_x_c1_global_desc.GetLength(I0); - const auto Y = wei_k_c0_y_x_c1_global_desc.GetLength(I2); - const auto X = wei_k_c0_y_x_c1_global_desc.GetLength(I3); - - const auto ConvStrideH = conv_strides[I0]; - const auto ConvStrideW = conv_strides[I1]; - - const auto ConvDilationH = conv_dilations[I0]; - const auto ConvDilationW = conv_dilations[I1]; - - const auto Hop = (Ho + HoPerBlock - 1) / HoPerBlock * HoPerBlock; - const auto Wop = (Wo + WoPerBlock - 1) / WoPerBlock * WoPerBlock; - - const auto OutRightPadH = Hop - Ho; - const auto OutRightPadW = Wop - Wo; - - const auto OutRightPadHx = OutRightPadH * 2; - const auto OutRightPadWx = OutRightPadW * 2; - - const auto InLeftPadH = in_left_pads[I0]; - const auto InLeftPadW = in_left_pads[I1]; - - const auto InRightPadH = in_right_pads[I0] + OutRightPadH * ConvStrideH; - const auto InRightPadW = in_right_pads[I1] + OutRightPadW * ConvStrideW; - - const auto E = C0 * Y * X; - - constexpr auto E1 = Number{}; - constexpr auto E2 = Number{}; - constexpr auto K2 = Number{}; - - const auto E0 = E / E1; - - // weight tensor - const auto a_e_k_e2_grid_desc = transform_tensor_descriptor( - make_naive_tensor_descriptor_packed(make_tuple(K, C0 * Y * X, E2)), - make_tuple(make_pass_through_transform(K), - make_pass_through_transform(C0 * Y * X), - make_pass_through_transform(E2)), - make_tuple(Sequence<0>{}, Sequence<1>{}, Sequence<2>{}), - make_tuple(Sequence<1>{}, Sequence<0>{}, Sequence<2>{})); - - const auto a_e0_e1_k_e2_grid_desc = - transform_tensor_descriptor(a_e_k_e2_grid_desc, - make_tuple(make_unmerge_transform(make_tuple(E0, E1)), - make_pass_through_transform(K), - make_pass_through_transform(E2)), - make_tuple(Sequence<0>{}, Sequence<1>{}, Sequence<2>{}), - make_tuple(Sequence<0, 1>{}, Sequence<2>{}, Sequence<3>{})); - - // input tensor - const auto in_n_c0_hip_wip_e2_global_desc = transform_tensor_descriptor( - make_naive_tensor_descriptor_packed(make_tuple(N, C0, Hi, Wi, E2)), - make_tuple(make_pass_through_transform(N), - make_pass_through_transform(C0), - make_pad_transform(Hi, InLeftPadH, InRightPadH), - make_pad_transform(Wi, InLeftPadW, InRightPadW), - make_pass_through_transform(E2)), - make_tuple(Sequence<0>{}, Sequence<1>{}, Sequence<2>{}, Sequence<3>{}, Sequence<4>{}), - make_tuple(Sequence<0>{}, Sequence<1>{}, Sequence<2>{}, Sequence<3>{}, Sequence<4>{})); - - const auto in_n_c0_y_ho_x_wo_e2_global_desc = transform_tensor_descriptor( - in_n_c0_hip_wip_e2_global_desc, - make_tuple( - make_pass_through_transform(N), - make_pass_through_transform(C0), - make_embed_transform(make_tuple(Y, Hop), make_tuple(ConvDilationH, ConvStrideH)), - make_embed_transform(make_tuple(X, Wop), make_tuple(ConvDilationW, ConvStrideW)), - make_pass_through_transform(E2)), - make_tuple(Sequence<0>{}, Sequence<1>{}, Sequence<2>{}, Sequence<3>{}, Sequence<4>{}), - make_tuple( - Sequence<0>{}, Sequence<1>{}, Sequence<2, 3>{}, Sequence<4, 5>{}, Sequence<6>{})); - - const auto in_e_n_ho_wo_e2_grid_desc = transform_tensor_descriptor( - in_n_c0_y_ho_x_wo_e2_global_desc, - make_tuple(make_merge_transform(make_tuple(C0, Y, X)), - make_pass_through_transform(N), - make_pass_through_transform(Hop), - make_pass_through_transform(Wop), - make_pass_through_transform(E2)), - make_tuple( - Sequence<1, 2, 4>{}, Sequence<0>{}, Sequence<3>{}, Sequence<5>{}, Sequence<6>{}), - make_tuple(Sequence<0>{}, Sequence<1>{}, Sequence<2>{}, Sequence<3>{}, Sequence<4>{})); - - const auto b_e0_e1_n_ho_wo_e2_grid_desc = transform_tensor_descriptor( - in_e_n_ho_wo_e2_grid_desc, - make_tuple(make_unmerge_transform(make_tuple(E0, E1)), - make_pass_through_transform(N), - make_pass_through_transform(Hop), - make_pass_through_transform(Wop), - make_pass_through_transform(E2)), - make_tuple(Sequence<0>{}, Sequence<1>{}, Sequence<2>{}, Sequence<3>{}, Sequence<4>{}), - make_tuple( - Sequence<0, 1>{}, Sequence<2>{}, Sequence<3>{}, Sequence<4>{}, Sequence<5>{})); - - // output tensor - const auto c_k_n_hop_wop_grid_desc = transform_tensor_descriptor( - make_naive_tensor_descriptor_packed(make_tuple(N, K0, Ho, Wo, K1)), - make_tuple(make_merge_transform(make_tuple(K0, K1)), - make_pass_through_transform(N), - make_pad_transform(Ho, I0, OutRightPadH), - make_pad_transform(Wo, I0, OutRightPadW)), - make_tuple(Sequence<1, 4>{}, Sequence<0>{}, Sequence<2>{}, Sequence<3>{}), - make_tuple(Sequence<0>{}, Sequence<1>{}, Sequence<2>{}, Sequence<3>{})); - - // add tensor - const auto d_k_n_hopx2_wopx2_grid_desc = transform_tensor_descriptor( - make_naive_tensor_descriptor_packed(make_tuple(N, K0, Hox2, Wox2, K1)), - make_tuple(make_merge_transform(make_tuple(K0, K1)), - make_pass_through_transform(N), - make_pad_transform(Hox2, I0, OutRightPadHx), - make_pad_transform(Wox2, I0, OutRightPadWx)), - make_tuple(Sequence<1, 4>{}, Sequence<0>{}, Sequence<2>{}, Sequence<3>{}), - make_tuple(Sequence<0>{}, Sequence<1>{}, Sequence<2>{}, Sequence<3>{})); - - std::cerr << "Hop = " << Hop << " Wop = " << Wop << std::endl; - - if(!((K % KPerBlock) == 0 && (Hop % HoPerBlock) == 0 && (Wop % WoPerBlock) == 0 && - (E1 % E1PerBlock) == 0)) - { - throw std::runtime_error("wrong! GEMM size no divisible"); - } - - // clang-format off - - // hack to control index calculation when iterating over a_e0_e1_k_e2_global tensor - constexpr auto a_e0_e1_k_e2_global_step_hacks = - make_tuple(make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}), - make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{})); - - constexpr auto a_e0_e1_k_e2_global_move_slice_window_step_hack = - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}; - - // hack to control index calculation when iterating over b_e0_e1_n_h0_h1_h2_w0_w1_w2_e2_global tensor - constexpr auto b_e0_e1_n_h0_h1_h2_w0_w1_w2_e2_global_step_hacks = - make_tuple( - make_tuple( - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}), - make_tuple( - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}) - ); - - constexpr auto b_e0_e1_n_h0_h1_h2_w0_w1_w2_e2_global_move_slice_window_step_hack = - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}; - - // hack to control index calculation when iterating over c_k0_k1_n_h0_h1_h2_w0_w1_w2_global tensor - constexpr auto c_k0_k1_n_h0_h1_h2_w0_w1_w2_global_tensor_step_hacks = - make_tuple(make_tuple(Sequence<0, 1, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 1, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}), - make_tuple(Sequence<0, 2, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 2, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{})); - - constexpr auto d_k0_k1_n_h0_h1_h2x2_w0_w1_w2x2_global_tensor_step_hacks = - make_tuple(make_tuple(Sequence<0, 1, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 1, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}), - make_tuple(Sequence<0, 2, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 2, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{})); - - // clang-format on - - // GEMM - using GridwiseGemm = GridwiseGemmDlops_km_kn_mn_v3< - BlockSize, - FloatAB, - FloatAcc, - FloatC, - InMemoryDataOperationEnum::Set, - decltype(a_e0_e1_k_e2_grid_desc), - decltype(b_e0_e1_n_ho_wo_e2_grid_desc), - decltype(c_k_n_hop_wop_grid_desc), - decltype(d_k_n_hopx2_wopx2_grid_desc), - E1, - E2, - K2, - KPerBlock, - HoPerBlock, - WoPerBlock, - E1PerBlock, - KPerThread, - HoPerThread, - WoPerThread, - EPerThread, - ABlockTransferThreadSliceLengths_E0_E1_K0_K1_E2, - ABlockTransferThreadClusterLengths_E0_E1_K0_K1_E2, - Sequence<2, 3, 0, 1, 4>, - Sequence<0, 1, 2, 3, 4>, - 4, - ABlockTransferSrcScalarPerVector_E2, - ABlockTransferDstScalarPerVector_E2, - false, // don't move back src coordinate after threadwise copy - Sequence<0, 1, 2, 3, 4, 5, 6, 7, 8, 9>, // E0, E1, N, H0, H1, H2, W0, W1, W2, E2 - 9, - BThreadTransferSrcScalarPerVector_E2, - false, // don't move back src coordinate after threadwise copy, which will be fused with - // MoveSrcSliceWindow() to save addr computation - Sequence<0, 1, 2, 3, 4, 5, 6, 7, 8>, // K0, K1, N, H0, H1, I2, H2, W0, W1, I2, W2 - 1, - CThreadTransferDstScalarPerVector_K, - decltype(a_e0_e1_k_e2_global_step_hacks), - decltype(b_e0_e1_n_h0_h1_h2_w0_w1_w2_e2_global_step_hacks), - decltype(c_k0_k1_n_h0_h1_h2_w0_w1_w2_global_tensor_step_hacks), - decltype(d_k0_k1_n_h0_h1_h2x2_w0_w1_w2x2_global_tensor_step_hacks), - decltype(a_e0_e1_k_e2_global_move_slice_window_step_hack), - decltype(b_e0_e1_n_h0_h1_h2_w0_w1_w2_e2_global_move_slice_window_step_hack)>; - - const auto a_e0_e1_k0_k1_e2_grid_desc = - GridwiseGemm::MakeAE0E1K0K1E2GridDescriptor(a_e0_e1_k_e2_grid_desc); - const auto b_e0_e1_n_h0_h1_h2_w0_w1_w2_e2_grid_desc = - GridwiseGemm::MakeBE0E1NH0H1H2W0W1W2E2GridDescriptor(b_e0_e1_n_ho_wo_e2_grid_desc); - const auto c_k0_k1_n_h0_h1_h2_w0_w1_w2_grid_desc = - GridwiseGemm::MakeCK0K1NH0H1H2W0W1W2GridDescriptor(c_k_n_hop_wop_grid_desc); - const auto d_k0_k1_n_h0_h1_h2x2_w0_w1_w2x2_grid_desc = - GridwiseGemm::MakeDK0K1NH0H1HxW0W1WxGridDescriptorResizeAdd( - d_k_n_hopx2_wopx2_grid_desc); - - using AGridDesc_E0_E1_K0_K1_E2 = decltype(a_e0_e1_k0_k1_e2_grid_desc); - using BGridDesc_E0_E1_N_H0_H1_H2_W0_W1_W2_E2 = - decltype(b_e0_e1_n_h0_h1_h2_w0_w1_w2_e2_grid_desc); - using CGridDesc_K0_K1_N_H0_H1_H2_W0_W1_W2 = decltype(c_k0_k1_n_h0_h1_h2_w0_w1_w2_grid_desc); - using DGridDesc_K0_K1_N_H0_H1_H2x2_W0_W1_W2x2 = - decltype(d_k0_k1_n_h0_h1_h2x2_w0_w1_w2x2_grid_desc); - - const auto grid_size = (K / KPerBlock) * (Hop / HoPerBlock) * (Wop / WoPerBlock) * N; - - const bool has_main_e0_block_loop = E0 > 1; - - std::cerr << "has_main_e0_block_loop = " << has_main_e0_block_loop << std::endl; - - const auto cblockid_to_k_n_h_w_block_cluster_adaptor = - GridwiseGemm::MakeCBlockIdToKNHoWoBlockClusterAdaptor(c_k_n_hop_wop_grid_desc); - - using CBlockIdToBlockClusterAdaptor_K_N_H_W = - decltype(cblockid_to_k_n_h_w_block_cluster_adaptor); - - float ave_time = 0; - - if(has_main_e0_block_loop) - { - const auto kernel = kernel_gemm_dlops_v3_resize_add< - GridwiseGemm, - FloatAB, - FloatC, - remove_reference_t, - remove_reference_t, - remove_reference_t, - remove_reference_t, - remove_reference_t, - true, - activ_type>; - - ave_time = launch_and_time_kernel(kernel, - nrepeat, - dim3(grid_size), - dim3(BlockSize), - 0, - p_a_grid, - p_b_grid, - p_bias_grid, - p_d_grid, - a_e0_e1_k0_k1_e2_grid_desc, - b_e0_e1_n_h0_h1_h2_w0_w1_w2_e2_grid_desc, - c_k0_k1_n_h0_h1_h2_w0_w1_w2_grid_desc, - d_k0_k1_n_h0_h1_h2x2_w0_w1_w2x2_grid_desc, - cblockid_to_k_n_h_w_block_cluster_adaptor); - } - else - { - const auto kernel = kernel_gemm_dlops_v3_resize_add< - GridwiseGemm, - FloatAB, - FloatC, - remove_reference_t, - remove_reference_t, - remove_reference_t, - remove_reference_t, - remove_reference_t, - false, - activ_type>; - - ave_time = launch_and_time_kernel(kernel, - nrepeat, - dim3(grid_size), - dim3(BlockSize), - 0, - p_a_grid, - p_b_grid, - p_bias_grid, - p_d_grid, - a_e0_e1_k0_k1_e2_grid_desc, - b_e0_e1_n_h0_h1_h2_w0_w1_w2_e2_grid_desc, - c_k0_k1_n_h0_h1_h2_w0_w1_w2_grid_desc, - d_k0_k1_n_h0_h1_h2x2_w0_w1_w2x2_grid_desc, - cblockid_to_k_n_h_w_block_cluster_adaptor); - } - - return ave_time; - } -}; -#endif diff --git a/library/include/ck/library/obselete_driver_offline/driver_convolution_forward_implicit_gemm_v5r1_dlops_nc0hwc1_kc0yxc1_nk0hwk1.hpp b/library/include/ck/library/obselete_driver_offline/driver_convolution_forward_implicit_gemm_v5r1_dlops_nc0hwc1_kc0yxc1_nk0hwk1.hpp deleted file mode 100644 index 34296405d4..0000000000 --- a/library/include/ck/library/obselete_driver_offline/driver_convolution_forward_implicit_gemm_v5r1_dlops_nc0hwc1_kc0yxc1_nk0hwk1.hpp +++ /dev/null @@ -1,386 +0,0 @@ -#ifndef DRIVER_CONVOLUTION_FORWARD_IMPLICIT_GEMM_V5R1_DLOPS_NC0HWc1_KC0YXC1_NK0HWK1_HPP -#define DRIVER_CONVOLUTION_FORWARD_IMPLICIT_GEMM_V5R1_DLOPS_NC0HWc1_KC0YXC1_NK0HWK1_HPP - -#include "common_header.hpp" -#include "tensor_descriptor.hpp" -#include "tensor_descriptor_helper.hpp" -#include "gridwise_gemm_dlops_v3.hpp" - -template -struct DriverDynamicConvolutionForwardImplicitGemmDlops_v5r1_nc0hwc1_kc0yxc1_nk0hwk1_outpad -{ - template - __host__ float Run(const ck::TensorDescriptor& wei_k_c0_y_x_c1_global_desc, - const ck::TensorDescriptor& in_n_c0_hi_wi_c1_global_desc, - const ck::TensorDescriptor& out_n_k0_ho_wo_k1_global_desc, - const ConvStrides& conv_strides, - const ConvDilations& conv_dilations, - const InLeftPads& in_left_pads, - const InRightPads& in_right_pads, - const FloatAB* __restrict__ p_a_grid, - const FloatAB* __restrict__ p_b_grid, - const FloatC* __restrict__ p_bias_grid, - FloatC* __restrict__ p_c_grid, - const int nrepeat) const - { - using namespace ck; - - constexpr auto I0 = Number<0>{}; - constexpr auto I1 = Number<1>{}; - constexpr auto I2 = Number<2>{}; - constexpr auto I3 = Number<3>{}; - constexpr auto I4 = Number<4>{}; - - const auto N = in_n_c0_hi_wi_c1_global_desc.GetLength(I0); - const auto C0 = in_n_c0_hi_wi_c1_global_desc.GetLength(I1); - const auto Hi = in_n_c0_hi_wi_c1_global_desc.GetLength(I2); - const auto Wi = in_n_c0_hi_wi_c1_global_desc.GetLength(I3); - // const auto C1 = in_n_c0_hi_wi_c1_global_desc.GetLength(I4); - - const auto K0 = out_n_k0_ho_wo_k1_global_desc.GetLength(I1); - const auto Ho = out_n_k0_ho_wo_k1_global_desc.GetLength(I2); - const auto Wo = out_n_k0_ho_wo_k1_global_desc.GetLength(I3); - const auto K1 = out_n_k0_ho_wo_k1_global_desc.GetLength(I4); - - const auto K = wei_k_c0_y_x_c1_global_desc.GetLength(I0); - const auto Y = wei_k_c0_y_x_c1_global_desc.GetLength(I2); - const auto X = wei_k_c0_y_x_c1_global_desc.GetLength(I3); - - const auto ConvStrideH = conv_strides[I0]; - const auto ConvStrideW = conv_strides[I1]; - - const auto ConvDilationH = conv_dilations[I0]; - const auto ConvDilationW = conv_dilations[I1]; - -#if CK_EXPERIMENTAL_STATIC_TENSOR_DESCRIPTOR - const auto Hop = Number<(Ho + HoPerBlock - 1) / HoPerBlock * HoPerBlock>{}; - const auto Wop = Number<(Wo + WoPerBlock - 1) / WoPerBlock * WoPerBlock>{}; -#else - const auto Hop = (Ho + HoPerBlock - 1) / HoPerBlock * HoPerBlock; - const auto Wop = (Wo + WoPerBlock - 1) / WoPerBlock * WoPerBlock; -#endif - - const auto OutRightPadH = Hop - Ho; - const auto OutRightPadW = Wop - Wo; - - const auto InLeftPadH = in_left_pads[I0]; - const auto InLeftPadW = in_left_pads[I1]; - - const auto InRightPadH = in_right_pads[I0] + OutRightPadH * ConvStrideH; - const auto InRightPadW = in_right_pads[I1] + OutRightPadW * ConvStrideW; - - const auto E = C0 * Y * X; - - constexpr auto E1 = Number{}; - constexpr auto E2 = Number{}; - constexpr auto K2 = Number{}; - - const auto E0 = E / E1; - - // weight tensor - const auto a_e_k_e2_grid_desc = transform_tensor_descriptor( - make_naive_tensor_descriptor_packed(make_tuple(K, C0 * Y * X, E2)), - make_tuple(make_pass_through_transform(K), - make_pass_through_transform(C0 * Y * X), - make_pass_through_transform(E2)), - make_tuple(Sequence<0>{}, Sequence<1>{}, Sequence<2>{}), - make_tuple(Sequence<1>{}, Sequence<0>{}, Sequence<2>{})); - - const auto a_e0_e1_k_e2_grid_desc = - transform_tensor_descriptor(a_e_k_e2_grid_desc, - make_tuple(make_unmerge_transform(make_tuple(E0, E1)), - make_pass_through_transform(K), - make_pass_through_transform(E2)), - make_tuple(Sequence<0>{}, Sequence<1>{}, Sequence<2>{}), - make_tuple(Sequence<0, 1>{}, Sequence<2>{}, Sequence<3>{})); - - // input tensor - const auto in_n_c0_hip_wip_e2_global_desc = transform_tensor_descriptor( - make_naive_tensor_descriptor_packed(make_tuple(N, C0, Hi, Wi, E2)), - make_tuple(make_pass_through_transform(N), - make_pass_through_transform(C0), - make_pad_transform(Hi, InLeftPadH, InRightPadH), - make_pad_transform(Wi, InLeftPadW, InRightPadW), - make_pass_through_transform(E2)), - make_tuple(Sequence<0>{}, Sequence<1>{}, Sequence<2>{}, Sequence<3>{}, Sequence<4>{}), - make_tuple(Sequence<0>{}, Sequence<1>{}, Sequence<2>{}, Sequence<3>{}, Sequence<4>{})); - - const auto in_n_c0_y_ho_x_wo_e2_global_desc = transform_tensor_descriptor( - in_n_c0_hip_wip_e2_global_desc, - make_tuple( - make_pass_through_transform(N), - make_pass_through_transform(C0), - make_embed_transform(make_tuple(Y, Hop), make_tuple(ConvDilationH, ConvStrideH)), - make_embed_transform(make_tuple(X, Wop), make_tuple(ConvDilationW, ConvStrideW)), - make_pass_through_transform(E2)), - make_tuple(Sequence<0>{}, Sequence<1>{}, Sequence<2>{}, Sequence<3>{}, Sequence<4>{}), - make_tuple( - Sequence<0>{}, Sequence<1>{}, Sequence<2, 3>{}, Sequence<4, 5>{}, Sequence<6>{})); - - const auto in_e_n_ho_wo_e2_grid_desc = transform_tensor_descriptor( - in_n_c0_y_ho_x_wo_e2_global_desc, - make_tuple(make_merge_transform(make_tuple(C0, Y, X)), - make_pass_through_transform(N), - make_pass_through_transform(Hop), - make_pass_through_transform(Wop), - make_pass_through_transform(E2)), - make_tuple( - Sequence<1, 2, 4>{}, Sequence<0>{}, Sequence<3>{}, Sequence<5>{}, Sequence<6>{}), - make_tuple(Sequence<0>{}, Sequence<1>{}, Sequence<2>{}, Sequence<3>{}, Sequence<4>{})); - - const auto b_e0_e1_n_ho_wo_e2_grid_desc = transform_tensor_descriptor( - in_e_n_ho_wo_e2_grid_desc, - make_tuple(make_unmerge_transform(make_tuple(E0, E1)), - make_pass_through_transform(N), - make_pass_through_transform(Hop), - make_pass_through_transform(Wop), - make_pass_through_transform(E2)), - make_tuple(Sequence<0>{}, Sequence<1>{}, Sequence<2>{}, Sequence<3>{}, Sequence<4>{}), - make_tuple( - Sequence<0, 1>{}, Sequence<2>{}, Sequence<3>{}, Sequence<4>{}, Sequence<5>{})); - - // output tensor - const auto c_k_n_hop_wop_grid_desc = transform_tensor_descriptor( - make_naive_tensor_descriptor_packed(make_tuple(N, K0, Ho, Wo, K1)), - make_tuple(make_merge_transform(make_tuple(K0, K1)), - make_pass_through_transform(N), - make_pad_transform(Ho, I0, OutRightPadH), - make_pad_transform(Wo, I0, OutRightPadW)), - make_tuple(Sequence<1, 4>{}, Sequence<0>{}, Sequence<2>{}, Sequence<3>{}), - make_tuple(Sequence<0>{}, Sequence<1>{}, Sequence<2>{}, Sequence<3>{})); - - std::cerr << "Hop = " << Hop << " Wop = " << Wop << std::endl; - - if(!((K % KPerBlock) == 0 && (Hop % HoPerBlock) == 0 && (Wop % WoPerBlock) == 0 && - (E1 % E1PerBlock) == 0)) - { - throw std::runtime_error("wrong! GEMM size no divisible"); - } - - // clang-format off - - // hack to control index calculation when iterating over a_e0_e1_k_e2_global tensor - constexpr auto a_e0_e1_k_e2_global_step_hacks = - make_tuple(make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}), - make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{})); - - constexpr auto a_e0_e1_k_e2_global_move_slice_window_step_hack = - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}; - - // hack to control index calculation when iterating over b_e0_e1_n_h0_h1_h2_w0_w1_w2_e2_global tensor - constexpr auto b_e0_e1_n_h0_h1_h2_w0_w1_w2_e2_global_step_hacks = - make_tuple( - make_tuple( - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}), - make_tuple( - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}) - ); - - constexpr auto b_e0_e1_n_h0_h1_h2_w0_w1_w2_e2_global_move_slice_window_step_hack = - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}; - - // hack to control index calculation when iterating over c_k0_k1_n_h0_h1_h2_w0_w1_w2_global tensor - constexpr auto c_k0_k1_n_h0_h1_h2_w0_w1_w2_global_tensor_step_hacks = - make_tuple(make_tuple(Sequence<0, 1, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 1, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}), - make_tuple(Sequence<0, 2, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 2, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{})); - // clang-format on - - // GEMM - using GridwiseGemm = GridwiseGemmDlops_km_kn_mn_v3< - BlockSize, - FloatAB, - FloatAcc, - FloatC, - InMemoryDataOperationEnum::Set, - decltype(a_e0_e1_k_e2_grid_desc), - decltype(b_e0_e1_n_ho_wo_e2_grid_desc), - decltype(c_k_n_hop_wop_grid_desc), - decltype(c_k_n_hop_wop_grid_desc), - E1, - E2, - K2, - KPerBlock, - HoPerBlock, - WoPerBlock, - E1PerBlock, - KPerThread, - HoPerThread, - WoPerThread, - EPerThread, - ABlockTransferThreadSliceLengths_E0_E1_K0_K1_E2, - ABlockTransferThreadClusterLengths_E0_E1_K0_K1_E2, - Sequence<2, 3, 0, 1, 4>, - Sequence<0, 1, 2, 3, 4>, - 4, - ABlockTransferSrcScalarPerVector_E2, - ABlockTransferDstScalarPerVector_E2, - false, // don't move back src coordinate after threadwise copy - Sequence<0, 1, 2, 3, 4, 5, 6, 7, 8, 9>, // E0, E1, N, H0, H1, H2, W0, W1, W2, E2 - 9, - BThreadTransferSrcScalarPerVector_E2, - false, // don't move back src coordinate after threadwise copy, which will be fused with - // MoveSrcSliceWindow() to save addr computation - Sequence<0, 1, 2, 3, 4, 5, 6, 7, 8>, // K0, K1, N, H0, H1, H2, W0, W1, W2 - 1, - CThreadTransferDstScalarPerVector_K, - decltype(a_e0_e1_k_e2_global_step_hacks), - decltype(b_e0_e1_n_h0_h1_h2_w0_w1_w2_e2_global_step_hacks), - decltype(c_k0_k1_n_h0_h1_h2_w0_w1_w2_global_tensor_step_hacks), - decltype(c_k0_k1_n_h0_h1_h2_w0_w1_w2_global_tensor_step_hacks), - decltype(a_e0_e1_k_e2_global_move_slice_window_step_hack), - decltype(b_e0_e1_n_h0_h1_h2_w0_w1_w2_e2_global_move_slice_window_step_hack)>; - - const auto a_e0_e1_k0_k1_e2_grid_desc = - GridwiseGemm::MakeAE0E1K0K1E2GridDescriptor(a_e0_e1_k_e2_grid_desc); - const auto b_e0_e1_n_h0_h1_h2_w0_w1_w2_e2_grid_desc = - GridwiseGemm::MakeBE0E1NH0H1H2W0W1W2E2GridDescriptor(b_e0_e1_n_ho_wo_e2_grid_desc); - const auto c_k0_k1_n_h0_h1_h2_w0_w1_w2_grid_desc = - GridwiseGemm::MakeCK0K1NH0H1H2W0W1W2GridDescriptor(c_k_n_hop_wop_grid_desc); - - using AGridDesc_E0_E1_K0_K1_E2 = decltype(a_e0_e1_k0_k1_e2_grid_desc); - using BGridDesc_E0_E1_N_H0_H1_H2_W0_W1_W2_E2 = - decltype(b_e0_e1_n_h0_h1_h2_w0_w1_w2_e2_grid_desc); - using CGridDesc_K0_K1_N_H0_H1_H2_W0_W1_W2 = decltype(c_k0_k1_n_h0_h1_h2_w0_w1_w2_grid_desc); - - const auto grid_size = (K / KPerBlock) * (Hop / HoPerBlock) * (Wop / WoPerBlock) * N; - - const bool has_main_e0_block_loop = E0 > 1; - - std::cerr << "has_main_e0_block_loop = " << has_main_e0_block_loop << std::endl; - - const auto cblockid_to_k_n_h_w_block_cluster_adaptor = - GridwiseGemm::MakeCBlockIdToKNHoWoBlockClusterAdaptor(c_k_n_hop_wop_grid_desc); - - using CBlockIdToBlockClusterAdaptor_K_N_H_W = - decltype(cblockid_to_k_n_h_w_block_cluster_adaptor); - - float ave_time = 0; - - if(has_main_e0_block_loop) - { - const auto kernel = - kernel_gemm_dlops_v3, - remove_reference_t, - remove_reference_t, - remove_reference_t, - true, - activ_type>; - - ave_time = launch_and_time_kernel(kernel, - nrepeat, - dim3(grid_size), - dim3(BlockSize), - 0, - p_a_grid, - p_b_grid, - p_bias_grid, - p_c_grid, - a_e0_e1_k0_k1_e2_grid_desc, - b_e0_e1_n_h0_h1_h2_w0_w1_w2_e2_grid_desc, - c_k0_k1_n_h0_h1_h2_w0_w1_w2_grid_desc, - cblockid_to_k_n_h_w_block_cluster_adaptor); - } - else - { - const auto kernel = - kernel_gemm_dlops_v3, - remove_reference_t, - remove_reference_t, - remove_reference_t, - false, - activ_type>; - - ave_time = launch_and_time_kernel(kernel, - nrepeat, - dim3(grid_size), - dim3(BlockSize), - 0, - p_a_grid, - p_b_grid, - p_bias_grid, - p_c_grid, - a_e0_e1_k0_k1_e2_grid_desc, - b_e0_e1_n_h0_h1_h2_w0_w1_w2_e2_grid_desc, - c_k0_k1_n_h0_h1_h2_w0_w1_w2_grid_desc, - cblockid_to_k_n_h_w_block_cluster_adaptor); - } - - return ave_time; - } -}; -#endif diff --git a/library/include/ck/library/obselete_driver_offline/driver_convolution_maxpool_forward_implicit_gemm_v5r1_dlops_nc0hwc1_kc0yxc1_nk0hwk1.hpp b/library/include/ck/library/obselete_driver_offline/driver_convolution_maxpool_forward_implicit_gemm_v5r1_dlops_nc0hwc1_kc0yxc1_nk0hwk1.hpp deleted file mode 100644 index 1b8e48e6c1..0000000000 --- a/library/include/ck/library/obselete_driver_offline/driver_convolution_maxpool_forward_implicit_gemm_v5r1_dlops_nc0hwc1_kc0yxc1_nk0hwk1.hpp +++ /dev/null @@ -1,440 +0,0 @@ -#ifndef DRIVER_CONVOLUTION_MAXPOOL_FORWARD_IMPLICIT_GEMM_V5R1_DLOPS_NC0HWc1_KC0YXC1_NK0HWK1_HPP -#define DRIVER_CONVOLUTION_MAXPOOL_FORWARD_IMPLICIT_GEMM_V5R1_DLOPS_NC0HWc1_KC0YXC1_NK0HWK1_HPP - -#include "common_header.hpp" -#include "tensor_descriptor.hpp" -#include "tensor_descriptor_helper.hpp" -#include "gridwise_gemm_dlops_v3.hpp" - -template -struct DriverDynamicConvolutionForwardImplicitGemmDlops_v5r1_nc0hwc1_kc0yxc1_nk0hwk1_maxpool -{ - template - __host__ float Run(const ck::TensorDescriptor& wei_k_c0_y_x_c1_global_desc, - const ck::TensorDescriptor& in_n_c0_hi_wi_c1_global_desc, - const ck::TensorDescriptor& out_n_k0_ho_wo_k1_global_desc, - const ck::TensorDescriptor& max_n_k0_hx_wx_k1_global_desc, - const ConvStrides& conv_strides, - const ConvDilations& conv_dilations, - const InLeftPads& in_left_pads, - const InRightPads& in_right_pads, - const FloatAB* __restrict__ p_a_grid, - const FloatAB* __restrict__ p_b_grid, - const FloatC* __restrict__ p_bias_grid, - FloatC* __restrict__ p_c_grid, - FloatC* __restrict__ p_d_grid, - const int nrepeat) const - { - using namespace ck; - - constexpr auto I0 = Number<0>{}; - constexpr auto I1 = Number<1>{}; - constexpr auto I2 = Number<2>{}; - constexpr auto I3 = Number<3>{}; - constexpr auto I4 = Number<4>{}; - - const auto N = in_n_c0_hi_wi_c1_global_desc.GetLength(I0); - const auto C0 = in_n_c0_hi_wi_c1_global_desc.GetLength(I1); - const auto Hi = in_n_c0_hi_wi_c1_global_desc.GetLength(I2); - const auto Wi = in_n_c0_hi_wi_c1_global_desc.GetLength(I3); - // const auto C1 = in_n_c0_hi_wi_c1_global_desc.GetLength(I4); - - const auto K0 = out_n_k0_ho_wo_k1_global_desc.GetLength(I1); - const auto Ho = out_n_k0_ho_wo_k1_global_desc.GetLength(I2); - const auto Wo = out_n_k0_ho_wo_k1_global_desc.GetLength(I3); - const auto K1 = out_n_k0_ho_wo_k1_global_desc.GetLength(I4); - - const auto Hx = max_n_k0_hx_wx_k1_global_desc.GetLength(I2); - const auto Wx = max_n_k0_hx_wx_k1_global_desc.GetLength(I3); - - const auto K = wei_k_c0_y_x_c1_global_desc.GetLength(I0); - const auto Y = wei_k_c0_y_x_c1_global_desc.GetLength(I2); - const auto X = wei_k_c0_y_x_c1_global_desc.GetLength(I3); - - const auto ConvStrideH = conv_strides[I0]; - const auto ConvStrideW = conv_strides[I1]; - - const auto ConvDilationH = conv_dilations[I0]; - const auto ConvDilationW = conv_dilations[I1]; - -#if CK_EXPERIMENTAL_STATIC_TENSOR_DESCRIPTOR - const auto Hop = Number<(Ho + HoPerBlock - 1) / HoPerBlock * HoPerBlock>{}; - const auto Wop = Number<(Wo + WoPerBlock - 1) / WoPerBlock * WoPerBlock>{}; - - const auto OutRightPadH = Hop - Ho; - const auto OutRightPadW = Wop - Wo; - - const auto OutRightPadHx = Number{}; - const auto OutRightPadWx = Number{}; -#else - const auto Hop = (Ho + HoPerBlock - 1) / HoPerBlock * HoPerBlock; - const auto Wop = (Wo + WoPerBlock - 1) / WoPerBlock * WoPerBlock; - - const auto OutRightPadH = Hop - Ho; - const auto OutRightPadW = Wop - Wo; - - const auto OutRightPadHx = OutRightPadH / 2; - const auto OutRightPadWx = OutRightPadW / 2; -#endif - - const auto InLeftPadH = in_left_pads[I0]; - const auto InLeftPadW = in_left_pads[I1]; - - const auto InRightPadH = in_right_pads[I0] + OutRightPadH * ConvStrideH; - const auto InRightPadW = in_right_pads[I1] + OutRightPadW * ConvStrideW; - - const auto E = C0 * Y * X; - - constexpr auto E1 = Number{}; - constexpr auto E2 = Number{}; - constexpr auto K2 = Number{}; - - const auto E0 = E / E1; - - // weight tensor - const auto a_e_k_e2_grid_desc = transform_tensor_descriptor( - make_naive_tensor_descriptor_packed(make_tuple(K, C0 * Y * X, E2)), - make_tuple(make_pass_through_transform(K), - make_pass_through_transform(C0 * Y * X), - make_pass_through_transform(E2)), - make_tuple(Sequence<0>{}, Sequence<1>{}, Sequence<2>{}), - make_tuple(Sequence<1>{}, Sequence<0>{}, Sequence<2>{})); - - const auto a_e0_e1_k_e2_grid_desc = - transform_tensor_descriptor(a_e_k_e2_grid_desc, - make_tuple(make_unmerge_transform(make_tuple(E0, E1)), - make_pass_through_transform(K), - make_pass_through_transform(E2)), - make_tuple(Sequence<0>{}, Sequence<1>{}, Sequence<2>{}), - make_tuple(Sequence<0, 1>{}, Sequence<2>{}, Sequence<3>{})); - - // input tensor - const auto in_n_c0_hip_wip_e2_global_desc = transform_tensor_descriptor( - make_naive_tensor_descriptor_packed(make_tuple(N, C0, Hi, Wi, E2)), - make_tuple(make_pass_through_transform(N), - make_pass_through_transform(C0), - make_pad_transform(Hi, InLeftPadH, InRightPadH), - make_pad_transform(Wi, InLeftPadW, InRightPadW), - make_pass_through_transform(E2)), - make_tuple(Sequence<0>{}, Sequence<1>{}, Sequence<2>{}, Sequence<3>{}, Sequence<4>{}), - make_tuple(Sequence<0>{}, Sequence<1>{}, Sequence<2>{}, Sequence<3>{}, Sequence<4>{})); - - const auto in_n_c0_y_ho_x_wo_e2_global_desc = transform_tensor_descriptor( - in_n_c0_hip_wip_e2_global_desc, - make_tuple( - make_pass_through_transform(N), - make_pass_through_transform(C0), - make_embed_transform(make_tuple(Y, Hop), make_tuple(ConvDilationH, ConvStrideH)), - make_embed_transform(make_tuple(X, Wop), make_tuple(ConvDilationW, ConvStrideW)), - make_pass_through_transform(E2)), - make_tuple(Sequence<0>{}, Sequence<1>{}, Sequence<2>{}, Sequence<3>{}, Sequence<4>{}), - make_tuple( - Sequence<0>{}, Sequence<1>{}, Sequence<2, 3>{}, Sequence<4, 5>{}, Sequence<6>{})); - - const auto in_e_n_ho_wo_e2_grid_desc = transform_tensor_descriptor( - in_n_c0_y_ho_x_wo_e2_global_desc, - make_tuple(make_merge_transform(make_tuple(C0, Y, X)), - make_pass_through_transform(N), - make_pass_through_transform(Hop), - make_pass_through_transform(Wop), - make_pass_through_transform(E2)), - make_tuple( - Sequence<1, 2, 4>{}, Sequence<0>{}, Sequence<3>{}, Sequence<5>{}, Sequence<6>{}), - make_tuple(Sequence<0>{}, Sequence<1>{}, Sequence<2>{}, Sequence<3>{}, Sequence<4>{})); - - const auto b_e0_e1_n_ho_wo_e2_grid_desc = transform_tensor_descriptor( - in_e_n_ho_wo_e2_grid_desc, - make_tuple(make_unmerge_transform(make_tuple(E0, E1)), - make_pass_through_transform(N), - make_pass_through_transform(Hop), - make_pass_through_transform(Wop), - make_pass_through_transform(E2)), - make_tuple(Sequence<0>{}, Sequence<1>{}, Sequence<2>{}, Sequence<3>{}, Sequence<4>{}), - make_tuple( - Sequence<0, 1>{}, Sequence<2>{}, Sequence<3>{}, Sequence<4>{}, Sequence<5>{})); - - // output tensor - const auto c_k_n_hop_wop_grid_desc = transform_tensor_descriptor( - make_naive_tensor_descriptor_packed(make_tuple(N, K0, Ho, Wo, K1)), - make_tuple(make_merge_transform(make_tuple(K0, K1)), - make_pass_through_transform(N), - make_pad_transform(Ho, I0, OutRightPadH), - make_pad_transform(Wo, I0, OutRightPadW)), - make_tuple(Sequence<1, 4>{}, Sequence<0>{}, Sequence<2>{}, Sequence<3>{}), - make_tuple(Sequence<0>{}, Sequence<1>{}, Sequence<2>{}, Sequence<3>{})); - - // max tensor - const auto d_k_n_hx_wx_grid_desc = transform_tensor_descriptor( - make_naive_tensor_descriptor_packed(make_tuple(N, K0, Hx, Wx, K1)), - make_tuple(make_merge_transform(make_tuple(K0, K1)), - make_pass_through_transform(N), - make_pad_transform(Hx, I0, OutRightPadHx), - make_pad_transform(Wx, I0, OutRightPadWx)), - make_tuple(Sequence<1, 4>{}, Sequence<0>{}, Sequence<2>{}, Sequence<3>{}), - make_tuple(Sequence<0>{}, Sequence<1>{}, Sequence<2>{}, Sequence<3>{})); - - std::cerr << "Hop = " << Hop << " Wop = " << Wop << std::endl; - - if(!((K % KPerBlock) == 0 && (Hop % HoPerBlock) == 0 && (Wop % WoPerBlock) == 0 && - (E1 % E1PerBlock) == 0)) - { - throw std::runtime_error("wrong! GEMM size no divisible"); - } - - // clang-format off - - // hack to control index calculation when iterating over a_e0_e1_k_e2_global tensor - constexpr auto a_e0_e1_k_e2_global_step_hacks = - make_tuple(make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}), - make_tuple(Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{})); - - constexpr auto a_e0_e1_k_e2_global_move_slice_window_step_hack = - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}; - - // hack to control index calculation when iterating over b_e0_e1_n_h0_h1_h2_w0_w1_w2_e2_global tensor - constexpr auto b_e0_e1_n_h0_h1_h2_w0_w1_w2_e2_global_step_hacks = - make_tuple( - make_tuple( - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}), - make_tuple( - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}) - ); - - constexpr auto b_e0_e1_n_h0_h1_h2_w0_w1_w2_e2_global_move_slice_window_step_hack = - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0>{}; - - constexpr auto c_k0_k1_n_h0_h1_h2_w0_w1_w2_global_tensor_step_hacks = - make_tuple(make_tuple(Sequence<0, 1, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 1, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}), - make_tuple(Sequence<0, 2, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 2, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{})); - - constexpr auto d_k0_k1_n_h0_h1_hx_w0_w1_wx_global_tensor_step_hacks = - make_tuple(make_tuple(Sequence<0, 1, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 1, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}), - make_tuple(Sequence<0, 2, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 2, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{}, - Sequence<0, 0, 0, 0, 0, 0, 0, 0, 0>{})); - - // clang-format on - - // GEMM - using GridwiseGemm = GridwiseGemmDlops_km_kn_mn_v3< - BlockSize, - FloatAB, - FloatAcc, - FloatC, - InMemoryDataOperationEnum::Set, - decltype(a_e0_e1_k_e2_grid_desc), - decltype(b_e0_e1_n_ho_wo_e2_grid_desc), - decltype(c_k_n_hop_wop_grid_desc), - decltype(d_k_n_hx_wx_grid_desc), - E1, - E2, - K2, - KPerBlock, - HoPerBlock, - WoPerBlock, - E1PerBlock, - KPerThread, - HoPerThread, - WoPerThread, - EPerThread, - ABlockTransferThreadSliceLengths_E0_E1_K0_K1_E2, - ABlockTransferThreadClusterLengths_E0_E1_K0_K1_E2, - Sequence<2, 3, 0, 1, 4>, - Sequence<0, 1, 2, 3, 4>, - 4, - ABlockTransferSrcScalarPerVector_E2, - ABlockTransferDstScalarPerVector_E2, - false, // don't move back src coordinate after threadwise copy - Sequence<0, 1, 2, 3, 4, 5, 6, 7, 8, 9>, // E0, E1, N, H0, H1, H2, W0, W1, W2, E2 - 9, - BThreadTransferSrcScalarPerVector_E2, - false, // don't move back src coordinate after threadwise copy, which will be fused - // with MoveSrcSliceWindow() to save addr computation - Sequence<0, 1, 2, 3, 4, 5, 6, 7, 8>, // K0, K1, N, H0, H1, I2, H2, W0, W1, I2, W2 - 1, - CThreadTransferDstScalarPerVector_K, - decltype(a_e0_e1_k_e2_global_step_hacks), - decltype(b_e0_e1_n_h0_h1_h2_w0_w1_w2_e2_global_step_hacks), - decltype(c_k0_k1_n_h0_h1_h2_w0_w1_w2_global_tensor_step_hacks), - decltype(d_k0_k1_n_h0_h1_hx_w0_w1_wx_global_tensor_step_hacks), - decltype(a_e0_e1_k_e2_global_move_slice_window_step_hack), - decltype(b_e0_e1_n_h0_h1_h2_w0_w1_w2_e2_global_move_slice_window_step_hack)>; - - const auto a_e0_e1_k0_k1_e2_grid_desc = - GridwiseGemm::MakeAE0E1K0K1E2GridDescriptor(a_e0_e1_k_e2_grid_desc); - const auto b_e0_e1_n_h0_h1_h2_w0_w1_w2_e2_grid_desc = - GridwiseGemm::MakeBE0E1NH0H1H2W0W1W2E2GridDescriptor(b_e0_e1_n_ho_wo_e2_grid_desc); - const auto c_k0_k1_n_h0_h1_h2_w0_w1_w2_grid_desc = - GridwiseGemm::MakeCK0K1NH0H1H2W0W1W2GridDescriptor(c_k_n_hop_wop_grid_desc); - const auto d_k0_k1_n_h0_h1_hx_w0_w1_wx_grid_desc = - GridwiseGemm::MakeDK0K1NH0H1HxW0W1WxGridDescriptorMaxPool(d_k_n_hx_wx_grid_desc); - - using AGridDesc_E0_E1_K0_K1_E2 = decltype(a_e0_e1_k0_k1_e2_grid_desc); - using BGridDesc_E0_E1_N_H0_H1_H2_W0_W1_W2_E2 = - decltype(b_e0_e1_n_h0_h1_h2_w0_w1_w2_e2_grid_desc); - using CGridDesc_K0_K1_N_H0_H1_H2_W0_W1_W2 = decltype(c_k0_k1_n_h0_h1_h2_w0_w1_w2_grid_desc); - using DGridDesc_K0_K1_N_H0_H1_Hx_W0_W1_Wx = decltype(d_k0_k1_n_h0_h1_hx_w0_w1_wx_grid_desc); - - const auto grid_size = (K / KPerBlock) * (Hop / HoPerBlock) * (Wop / WoPerBlock) * N; - - const bool has_main_e0_block_loop = E0 > 1; - - std::cerr << "has_main_e0_block_loop = " << has_main_e0_block_loop << std::endl; - - const auto cblockid_to_k_n_h_w_block_cluster_adaptor = - GridwiseGemm::MakeCBlockIdToKNHoWoBlockClusterAdaptor(c_k_n_hop_wop_grid_desc); - - using CBlockIdToBlockClusterAdaptor_K_N_H_W = - decltype(cblockid_to_k_n_h_w_block_cluster_adaptor); - - float ave_time = 0; - - if(has_main_e0_block_loop) - { - const auto kernel = kernel_gemm_dlops_v3_maxpool< - GridwiseGemm, - FloatAB, - FloatC, - remove_reference_t, - remove_reference_t, - remove_reference_t, - remove_reference_t, - remove_reference_t, - true, - activ_type>; - - ave_time = launch_and_time_kernel(kernel, - nrepeat, - dim3(grid_size), - dim3(BlockSize), - 0, - p_a_grid, - p_b_grid, - p_bias_grid, - p_c_grid, - p_d_grid, - a_e0_e1_k0_k1_e2_grid_desc, - b_e0_e1_n_h0_h1_h2_w0_w1_w2_e2_grid_desc, - c_k0_k1_n_h0_h1_h2_w0_w1_w2_grid_desc, - d_k0_k1_n_h0_h1_hx_w0_w1_wx_grid_desc, - cblockid_to_k_n_h_w_block_cluster_adaptor); - } - else - { - const auto kernel = kernel_gemm_dlops_v3_maxpool< - GridwiseGemm, - FloatAB, - FloatC, - remove_reference_t, - remove_reference_t, - remove_reference_t, - remove_reference_t, - remove_reference_t, - false, - activ_type>; - - ave_time = launch_and_time_kernel(kernel, - nrepeat, - dim3(grid_size), - dim3(BlockSize), - 0, - p_a_grid, - p_b_grid, - p_bias_grid, - p_c_grid, - p_d_grid, - a_e0_e1_k0_k1_e2_grid_desc, - b_e0_e1_n_h0_h1_h2_w0_w1_w2_e2_grid_desc, - c_k0_k1_n_h0_h1_h2_w0_w1_w2_grid_desc, - d_k0_k1_n_h0_h1_hx_w0_w1_wx_grid_desc, - cblockid_to_k_n_h_w_block_cluster_adaptor); - } - - return ave_time; - } -}; -#endif diff --git a/library/include/ck/library/obselete_driver_offline/driver_gemm_dlops_v1r2.hpp b/library/include/ck/library/obselete_driver_offline/driver_gemm_dlops_v1r2.hpp deleted file mode 100644 index ce0530b3fd..0000000000 --- a/library/include/ck/library/obselete_driver_offline/driver_gemm_dlops_v1r2.hpp +++ /dev/null @@ -1,278 +0,0 @@ -#ifndef DRIVER_GEMM_DLOPS_V1R2 -#define DRIVER_GEMM_DLOPS_V1R2 - -#include "common_header.hpp" -#include "tensor_descriptor.hpp" -#include "tensor_descriptor_helper.hpp" -#include "gridwise_gemm_dlops_v1r2.hpp" - -template -__host__ float driver_gemm_dlops_v1r2(const FloatAB* p_a_grid, - const FloatAB* p_b_grid, - FloatC* p_c_grid, - const AKMGridDesc& a_k_m_grid_desc, - const BKNGridDesc& b_k_n_grid_desc, - const CMNGridDesc& c_m_n_grid_desc, - AGridStepHacks, - BGridStepHacks, - CGridStepHacks, - AGridMoveSliceWindowStepHacks, - BGridMoveSliceWindowStepHacks, - ck::index_t nrepeat) - -{ - using namespace ck; - - constexpr auto I0 = Number<0>{}; - constexpr auto I1 = Number<1>{}; - constexpr auto I2 = Number<2>{}; - constexpr auto I3 = Number<3>{}; - constexpr auto I4 = Number<4>{}; - constexpr auto I5 = Number<5>{}; - - // GEMM - using GridwiseGemm = GridwiseGemmDlops_km_kn_mn_v1r2; - - const auto M = a_k_m_grid_desc.GetLength(I1); - const auto N = b_k_n_grid_desc.GetLength(I1); - const auto K = a_k_m_grid_desc.GetLength(I0); - - if(!GridwiseGemm::CheckValidity(a_k_m_grid_desc, b_k_n_grid_desc, c_m_n_grid_desc)) - { - throw std::runtime_error("wrong! GridwiseGemmDlops_km_kn_mn_v1r2 has invalid setting"); - } - - const auto a_k_m0_m1_grid_desc = GridwiseGemm::MakeAKM0M1GridDescriptor(a_k_m_grid_desc); - const auto b_k_n0_n1_grid_desc = GridwiseGemm::MakeBKN0N1GridDescriptor(b_k_n_grid_desc); - - using AKM0M1GridDesc = decltype(a_k_m0_m1_grid_desc); - using BKN0N1GridDesc = decltype(b_k_n0_n1_grid_desc); - - // c_m0_m10_m11_n0_n10_n11_grid_desc - const auto c_m0_m10_m11_n0_n10_n11_grid_desc = - GridwiseGemm::MakeCM0M10M11N0N10N11GridDescriptor(c_m_n_grid_desc); - - using CM0M10M11N0N10N11GridDesc = decltype(c_m0_m10_m11_n0_n10_n11_grid_desc); - - // cblockid_to_m0_n0_block_cluster_adaptor - const auto cblockid_to_m0_n0_block_cluster_adaptor = - GridwiseGemm::MakeCBlockIdToM0N0BlockClusterAdaptor(c_m_n_grid_desc); - - using CBlockIdToM0N0BlockClusterAdaptor = decltype(cblockid_to_m0_n0_block_cluster_adaptor); - - const index_t grid_size = GridwiseGemm::CalculateGridSize(M, N); - - const bool has_main_k_block_loop = GridwiseGemm::CalculateHasMainKBlockLoop(K); - - const bool has_double_tail_k_block_loop = GridwiseGemm::CalculateHasDoubleTailKBlockLoop(K); - - { - std::cout << "a_k_m0_m1_grid_desc{" << a_k_m0_m1_grid_desc.GetLength(I0) << ", " - << a_k_m0_m1_grid_desc.GetLength(I1) << ", " << a_k_m0_m1_grid_desc.GetLength(I2) - << "}" << std::endl; - - std::cout << "b_k_n0_n1_grid_desc{" << b_k_n0_n1_grid_desc.GetLength(I0) << ", " - << b_k_n0_n1_grid_desc.GetLength(I1) << ", " << b_k_n0_n1_grid_desc.GetLength(I2) - << "}" << std::endl; - - std::cout << "c_m0_m10_m11_n0_n10_n11_grid_desc{ " - << c_m0_m10_m11_n0_n10_n11_grid_desc.GetLength(I0) << ", " - << c_m0_m10_m11_n0_n10_n11_grid_desc.GetLength(I1) << ", " - << c_m0_m10_m11_n0_n10_n11_grid_desc.GetLength(I2) << ", " - << c_m0_m10_m11_n0_n10_n11_grid_desc.GetLength(I3) << ", " - << c_m0_m10_m11_n0_n10_n11_grid_desc.GetLength(I4) << ", " - << c_m0_m10_m11_n0_n10_n11_grid_desc.GetLength(I5) << "}" << std::endl; - } - - float ave_time = 0; - - if(has_main_k_block_loop && has_double_tail_k_block_loop) - { - const auto kernel = - kernel_gemm_dlops_v1r2, - remove_reference_t, - remove_reference_t, - remove_reference_t, - true, - true>; - - ave_time = launch_and_time_kernel(kernel, - nrepeat, - dim3(grid_size), - dim3(BlockSize), - 0, - p_a_grid, - p_b_grid, - p_c_grid, - a_k_m0_m1_grid_desc, - b_k_n0_n1_grid_desc, - c_m0_m10_m11_n0_n10_n11_grid_desc, - cblockid_to_m0_n0_block_cluster_adaptor); - } - else if(has_main_k_block_loop && !has_double_tail_k_block_loop) - { - const auto kernel = - kernel_gemm_dlops_v1r2, - remove_reference_t, - remove_reference_t, - remove_reference_t, - true, - false>; - - ave_time = launch_and_time_kernel(kernel, - nrepeat, - dim3(grid_size), - dim3(BlockSize), - 0, - p_a_grid, - p_b_grid, - p_c_grid, - a_k_m0_m1_grid_desc, - b_k_n0_n1_grid_desc, - c_m0_m10_m11_n0_n10_n11_grid_desc, - cblockid_to_m0_n0_block_cluster_adaptor); - } - else if(!has_main_k_block_loop && has_double_tail_k_block_loop) - { - const auto kernel = - kernel_gemm_dlops_v1r2, - remove_reference_t, - remove_reference_t, - remove_reference_t, - false, - true>; - - ave_time = launch_and_time_kernel(kernel, - nrepeat, - dim3(grid_size), - dim3(BlockSize), - 0, - p_a_grid, - p_b_grid, - p_c_grid, - a_k_m0_m1_grid_desc, - b_k_n0_n1_grid_desc, - c_m0_m10_m11_n0_n10_n11_grid_desc, - cblockid_to_m0_n0_block_cluster_adaptor); - } - else - { - const auto kernel = - kernel_gemm_dlops_v1r2, - remove_reference_t, - remove_reference_t, - remove_reference_t, - false, - false>; - - ave_time = launch_and_time_kernel(kernel, - nrepeat, - dim3(grid_size), - dim3(BlockSize), - 0, - p_a_grid, - p_b_grid, - p_c_grid, - a_k_m0_m1_grid_desc, - b_k_n0_n1_grid_desc, - c_m0_m10_m11_n0_n10_n11_grid_desc, - cblockid_to_m0_n0_block_cluster_adaptor); - } - - return ave_time; -} -#endif diff --git a/library/include/ck/library/obselete_driver_offline/driver_gemm_dlops_v1r3.hpp b/library/include/ck/library/obselete_driver_offline/driver_gemm_dlops_v1r3.hpp deleted file mode 100644 index 3fd1a1dbba..0000000000 --- a/library/include/ck/library/obselete_driver_offline/driver_gemm_dlops_v1r3.hpp +++ /dev/null @@ -1,275 +0,0 @@ -#ifndef DRIVER_GEMM_DLOPS_V1R3 -#define DRIVER_GEMM_DLOPS_V1R3 - -#include "common_header.hpp" -#include "tensor_descriptor.hpp" -#include "tensor_descriptor_helper.hpp" -#include "gridwise_gemm_dlops_v1r3.hpp" - -template -__host__ float driver_gemm_dlops_v1r3(const FloatAB* p_a_grid, - const FloatAB* p_b_grid, - FloatC* p_c_grid, - const AK0MK1GridDesc& a_k0_m_k1_grid_desc, - const BK0NK1GridDesc& b_k0_n_k1_grid_desc, - const CMNGridDesc& c_m_n_grid_desc, - AGridStepHacks, - BGridStepHacks, - CGridStepHacks, - AGridMoveSliceWindowStepHacks, - BGridMoveSliceWindowStepHacks, - ck::index_t nrepeat) - -{ - using namespace ck; - - constexpr auto I0 = Number<0>{}; - constexpr auto I1 = Number<1>{}; - constexpr auto I2 = Number<2>{}; - constexpr auto I3 = Number<3>{}; - constexpr auto I4 = Number<4>{}; - constexpr auto I5 = Number<5>{}; - - // GEMM - using GridwiseGemm = - GridwiseGemmDlops_km_kn_mn_v1r3; - - const auto M = a_k0_m_k1_grid_desc.GetLength(I1); - const auto N = b_k0_n_k1_grid_desc.GetLength(I1); - const auto K0 = a_k0_m_k1_grid_desc.GetLength(I0); - - if(!GridwiseGemm::CheckValidity(a_k0_m_k1_grid_desc, b_k0_n_k1_grid_desc, c_m_n_grid_desc)) - { - throw std::runtime_error("wrong! GridwiseGemmDlops_km_kn_mn_v1r3 has invalid setting"); - } - - const auto a_k0_m0_m1_k1_grid_desc = - GridwiseGemm::MakeAK0M0M1K1GridDescriptor(a_k0_m_k1_grid_desc); - const auto b_k0_n0_n1_k1_grid_desc = - GridwiseGemm::MakeBK0N0N1K1GridDescriptor(b_k0_n_k1_grid_desc); - - using AK0M0M1K1GridDesc = decltype(a_k0_m0_m1_k1_grid_desc); - using BK0N0N1K1GridDesc = decltype(b_k0_n0_n1_k1_grid_desc); - - // c_m0_m10_m11_n0_n10_n11_grid_desc - const auto c_m0_m10_m11_n0_n10_n11_grid_desc = - GridwiseGemm::MakeCM0M10M11N0N10N11GridDescriptor(c_m_n_grid_desc); - - using CM0M10M11N0N10N11GridDesc = decltype(c_m0_m10_m11_n0_n10_n11_grid_desc); - - // cblockid_to_m0_n0_block_cluster_adaptor - const auto cblockid_to_m0_n0_block_cluster_adaptor = - GridwiseGemm::MakeCBlockIdToM0N0BlockClusterAdaptor(c_m_n_grid_desc); - - using CBlockIdToM0N0BlockClusterAdaptor = decltype(cblockid_to_m0_n0_block_cluster_adaptor); - - const index_t grid_size = GridwiseGemm::CalculateGridSize(M, N); - - const bool has_main_k_block_loop = GridwiseGemm::CalculateHasMainKBlockLoop(K0); - - const bool has_double_tail_k_block_loop = GridwiseGemm::CalculateHasDoubleTailKBlockLoop(K0); - - { - std::cout << "a_k0_m0_m1_k1_grid_desc{" << a_k0_m0_m1_k1_grid_desc.GetLength(I0) << ", " - << a_k0_m0_m1_k1_grid_desc.GetLength(I1) << ", " - << a_k0_m0_m1_k1_grid_desc.GetLength(I2) << ", " - << a_k0_m0_m1_k1_grid_desc.GetLength(I3) << "}" << std::endl; - - std::cout << "b_k0_n0_n1_k1_grid_desc{" << b_k0_n0_n1_k1_grid_desc.GetLength(I0) << ", " - << b_k0_n0_n1_k1_grid_desc.GetLength(I1) << ", " - << b_k0_n0_n1_k1_grid_desc.GetLength(I2) << ", " - << b_k0_n0_n1_k1_grid_desc.GetLength(I3) << "}" << std::endl; - - std::cout << "c_m0_m10_m11_n0_n10_n11_grid_desc{ " - << c_m0_m10_m11_n0_n10_n11_grid_desc.GetLength(I0) << ", " - << c_m0_m10_m11_n0_n10_n11_grid_desc.GetLength(I1) << ", " - << c_m0_m10_m11_n0_n10_n11_grid_desc.GetLength(I2) << ", " - << c_m0_m10_m11_n0_n10_n11_grid_desc.GetLength(I3) << ", " - << c_m0_m10_m11_n0_n10_n11_grid_desc.GetLength(I4) << ", " - << c_m0_m10_m11_n0_n10_n11_grid_desc.GetLength(I5) << "}" << std::endl; - } - - float ave_time = 0; - - if(has_main_k_block_loop && has_double_tail_k_block_loop) - { - const auto kernel = - kernel_gemm_dlops_v1r3, - remove_reference_t, - remove_reference_t, - remove_reference_t, - true, - true>; - - ave_time = launch_and_time_kernel(kernel, - nrepeat, - dim3(grid_size), - dim3(BlockSize), - 0, - p_a_grid, - p_b_grid, - p_c_grid, - a_k0_m0_m1_k1_grid_desc, - b_k0_n0_n1_k1_grid_desc, - c_m0_m10_m11_n0_n10_n11_grid_desc, - cblockid_to_m0_n0_block_cluster_adaptor); - } - else if(has_main_k_block_loop && !has_double_tail_k_block_loop) - { - const auto kernel = - kernel_gemm_dlops_v1r3, - remove_reference_t, - remove_reference_t, - remove_reference_t, - true, - false>; - - ave_time = launch_and_time_kernel(kernel, - nrepeat, - dim3(grid_size), - dim3(BlockSize), - 0, - p_a_grid, - p_b_grid, - p_c_grid, - a_k0_m0_m1_k1_grid_desc, - b_k0_n0_n1_k1_grid_desc, - c_m0_m10_m11_n0_n10_n11_grid_desc, - cblockid_to_m0_n0_block_cluster_adaptor); - } - else if(!has_main_k_block_loop && has_double_tail_k_block_loop) - { - const auto kernel = - kernel_gemm_dlops_v1r3, - remove_reference_t, - remove_reference_t, - remove_reference_t, - false, - true>; - - ave_time = launch_and_time_kernel(kernel, - nrepeat, - dim3(grid_size), - dim3(BlockSize), - 0, - p_a_grid, - p_b_grid, - p_c_grid, - a_k0_m0_m1_k1_grid_desc, - b_k0_n0_n1_k1_grid_desc, - c_m0_m10_m11_n0_n10_n11_grid_desc, - cblockid_to_m0_n0_block_cluster_adaptor); - } - else - { - const auto kernel = - kernel_gemm_dlops_v1r3, - remove_reference_t, - remove_reference_t, - remove_reference_t, - false, - false>; - - ave_time = launch_and_time_kernel(kernel, - nrepeat, - dim3(grid_size), - dim3(BlockSize), - 0, - p_a_grid, - p_b_grid, - p_c_grid, - a_k0_m0_m1_k1_grid_desc, - b_k0_n0_n1_k1_grid_desc, - c_m0_m10_m11_n0_n10_n11_grid_desc, - cblockid_to_m0_n0_block_cluster_adaptor); - } - - return ave_time; -} -#endif diff --git a/library/include/ck/library/obselete_driver_offline/driver_gemm_xdlops_v2r3.hpp b/library/include/ck/library/obselete_driver_offline/driver_gemm_xdlops_v2r3.hpp deleted file mode 100644 index 5652040250..0000000000 --- a/library/include/ck/library/obselete_driver_offline/driver_gemm_xdlops_v2r3.hpp +++ /dev/null @@ -1,220 +0,0 @@ -#ifndef DRIVER_GEMM_XDLOPS_V2R3_HPP -#define DRIVER_GEMM_XDLOPS_V2R3_HPP - -#include "common_header.hpp" -#include "tensor_descriptor.hpp" -#include "tensor_descriptor_helper.hpp" -#include "gridwise_gemm_xdlops_v2r3.hpp" -#include "element_wise_operation.hpp" - -template -__host__ float driver_gemm_xdlops_v2r3(const FloatAB* p_a_grid, - const FloatAB* p_b_grid, - FloatC* p_c_grid, - const AGridDesc_K0_M_K1& a_grid_desc_k0_m_k1, - const BGridDesc_K0_N_K& b_grid_desc_k0_n_k1, - const CMNGridDesc& c_grid_desc_m_n, - ck::index_t M01, - ck::index_t N01, - AGridStepHacks, - BGridStepHacks, - CGridStepHacks, - AGridMoveSliceWindowStepHacks, - BGridMoveSliceWindowStepHacks, - ck::index_t nrepeat) -{ - using namespace ck; - - constexpr auto I0 = Number<0>{}; - constexpr auto I1 = Number<1>{}; - constexpr auto I2 = Number<2>{}; - - using ElementwiseOperation = ck::tensor_operation::element_wise::PassThrough; - - using GridwiseGemm = - GridwiseGemm_k0mk1_k0nk1_mn_xdlops_v2r3; - - { - std::cout << "a_grid_desc_k0_m_k1{" << a_grid_desc_k0_m_k1.GetLength(I0) << ", " - << a_grid_desc_k0_m_k1.GetLength(I1) << ", " << a_grid_desc_k0_m_k1.GetLength(I2) - << "}" << std::endl; - - std::cout << "b_grid_desc_k0_n_k1{" << b_grid_desc_k0_n_k1.GetLength(I0) << ", " - << b_grid_desc_k0_n_k1.GetLength(I1) << ", " << b_grid_desc_k0_n_k1.GetLength(I2) - << "}" << std::endl; - - std::cout << "c_grid_desc_m_n{ " << c_grid_desc_m_n.GetLength(I0) << ", " - << c_grid_desc_m_n.GetLength(I1) << "}" << std::endl; - } - - if(!GridwiseGemm::CheckValidity( - a_grid_desc_k0_m_k1, b_grid_desc_k0_n_k1, c_grid_desc_m_n, M01, N01)) - { - throw std::runtime_error( - "wrong! GridwiseGemm_km_kn_m0m1n0n1_xdlops_v2r3 has invalid setting"); - } - - const auto c_m0_n0_m1_n1_m2_m3_m4_n2_grid_desc = - GridwiseGemm::MakeCGridDescriptor_M0_N0_M1_N1_M2_M3_M4_N2(c_grid_desc_m_n); - - using CGridDesc_M0_N0_M1_N1_M2_M3_M4_N2 = decltype(c_m0_n0_m1_n1_m2_m3_m4_n2_grid_desc); - - const auto block_2_ctile_map = - GridwiseGemm::MakeDefaultBlock2CTileMap(c_grid_desc_m_n, M01, N01); - - using Block2CTileMap = decltype(block_2_ctile_map); - - const index_t grid_size = GridwiseGemm::CalculateGridSize(c_grid_desc_m_n); - - const auto K0 = a_grid_desc_k0_m_k1.GetLength(I0); - - const bool has_main_k0_block_loop = GridwiseGemm::CalculateHasMainK0BlockLoop(K0); - - float ave_time = 0; - - auto element_op_ = ElementwiseOperation{}; - - if(has_main_k0_block_loop) - { - const auto kernel = - kernel_gemm_xdlops_v2r3, - remove_reference_t, - remove_reference_t, - ElementwiseOperation, - ElementwiseOperation, - ElementwiseOperation, - remove_reference_t, - true>; - - ave_time = launch_and_time_kernel(kernel, - nrepeat, - dim3(grid_size), - dim3(BlockSize), - 0, - p_a_grid, - p_b_grid, - p_c_grid, - a_grid_desc_k0_m_k1, - b_grid_desc_k0_n_k1, - c_m0_n0_m1_n1_m2_m3_m4_n2_grid_desc, - element_op_, - element_op_, - element_op_, - block_2_ctile_map); - } - else - { - const auto kernel = - kernel_gemm_xdlops_v2r3, - remove_reference_t, - remove_reference_t, - ElementwiseOperation, - ElementwiseOperation, - ElementwiseOperation, - remove_reference_t, - false>; - - ave_time = launch_and_time_kernel(kernel, - nrepeat, - dim3(grid_size), - dim3(BlockSize), - 0, - p_a_grid, - p_b_grid, - p_c_grid, - a_grid_desc_k0_m_k1, - b_grid_desc_k0_n_k1, - c_m0_n0_m1_n1_m2_m3_m4_n2_grid_desc, - element_op_, - element_op_, - element_op_, - block_2_ctile_map); - } - return ave_time; -} -#endif diff --git a/library/include/ck/library/obselete_driver_offline/driver_gemm_xdlops_v2r4.hpp b/library/include/ck/library/obselete_driver_offline/driver_gemm_xdlops_v2r4.hpp deleted file mode 100644 index 6e9983b0b5..0000000000 --- a/library/include/ck/library/obselete_driver_offline/driver_gemm_xdlops_v2r4.hpp +++ /dev/null @@ -1,213 +0,0 @@ -#ifndef DRIVER_GEMM_XDLOPS_V2R4 -#define DRIVER_GEMM_XDLOPS_V2R4 - -#include "common_header.hpp" -#include "tensor_descriptor.hpp" -#include "tensor_descriptor_helper.hpp" -#include "gridwise_gemm_xdlops_v2r4.hpp" - -template -__host__ float driver_gemm_xdlops_v2r4(const FloatAB* p_a_grid, - const FloatAB* p_b_grid, - FloatC* p_c_grid, - const ABK0MK1GridDesc& a_b_k0_m_k1_grid_desc, - const BBK0NK1GridDesc& b_b_k0_n_k1_grid_desc, - const CMNGridDesc& c_m_n_grid_desc, - ck::index_t M01, - ck::index_t N01, - AGridStepHacks, - BGridStepHacks, - CGridStepHacks, - AGridMoveSliceWindowStepHacks, - BGridMoveSliceWindowStepHacks, - ck::index_t nrepeat) - -{ - using namespace ck; - - constexpr auto I0 = Number<0>{}; - constexpr auto I1 = Number<1>{}; - constexpr auto I2 = Number<2>{}; - constexpr auto I3 = Number<3>{}; - - using GridwiseGemm = - GridwiseGemm_bk0mk1_bk0nk1_mn_xdlops_v2r4; - - { - std::cout << "a_b_k0_m_k1_grid_desc{" << a_b_k0_m_k1_grid_desc.GetLength(I0) << ", " - << a_b_k0_m_k1_grid_desc.GetLength(I1) << ", " - << a_b_k0_m_k1_grid_desc.GetLength(I2) << ", " - << a_b_k0_m_k1_grid_desc.GetLength(I3) << "}" << std::endl; - - std::cout << "b_b_k0_n_k1_grid_desc{" << b_b_k0_n_k1_grid_desc.GetLength(I0) << ", " - << b_b_k0_n_k1_grid_desc.GetLength(I1) << ", " - << b_b_k0_n_k1_grid_desc.GetLength(I2) << ", " - << b_b_k0_n_k1_grid_desc.GetLength(I3) << "}" << std::endl; - - std::cout << "c_m_n_grid_desc{ " << c_m_n_grid_desc.GetLength(I0) << ", " - << c_m_n_grid_desc.GetLength(I1) << "}" << std::endl; - } - - if(!GridwiseGemm::CheckValidity( - a_b_k0_m_k1_grid_desc, b_b_k0_n_k1_grid_desc, c_m_n_grid_desc, M01, N01)) - { - throw std::runtime_error( - "wrong! GridwiseGemm_km_kn_m0m1n0n1_xdlops_v2r4 has invalid setting"); - } - - const auto c_m0_n0_m1_n1_m2_m3_m4_n2_grid_desc = - GridwiseGemm::MakeCM0N0M1N1M2M3M4N2GridDescriptor(c_m_n_grid_desc); - - using CM0N0M1N1M2M3M4N2GridDesc = decltype(c_m0_n0_m1_n1_m2_m3_m4_n2_grid_desc); - - const auto KBatch = a_b_k0_m_k1_grid_desc.GetLength(I0); - - const auto c_block_cluster_adaptor = - GridwiseGemm::MakeCBlockClusterAdaptor(c_m_n_grid_desc, M01, N01, KBatch); - - using CBlockClusterAdaptor = decltype(c_block_cluster_adaptor); - - const index_t grid_size = GridwiseGemm::CalculateGridSize(c_m_n_grid_desc, KBatch); - { - std::cout << "gridSize : " << grid_size << std::endl; - } - - const auto K0 = a_b_k0_m_k1_grid_desc.GetLength(I1); - - const bool has_main_k0_block_loop = GridwiseGemm::CalculateHasMainK0BlockLoop(K0); - - float ave_time = 0; - if(has_main_k0_block_loop) - { - const auto kernel = kernel_gemm_xdlops_v2r4, - remove_reference_t, - remove_reference_t, - remove_reference_t, - true>; - ave_time = launch_and_time_kernel(kernel, - nrepeat, - dim3(grid_size), - dim3(BlockSize), - 0, - p_a_grid, - p_b_grid, - p_c_grid, - a_b_k0_m_k1_grid_desc, - b_b_k0_n_k1_grid_desc, - c_m0_n0_m1_n1_m2_m3_m4_n2_grid_desc, - c_block_cluster_adaptor); - } - else - { - const auto kernel = kernel_gemm_xdlops_v2r4, - remove_reference_t, - remove_reference_t, - remove_reference_t, - false>; - ave_time = launch_and_time_kernel(kernel, - nrepeat, - dim3(grid_size), - dim3(BlockSize), - 0, - p_a_grid, - p_b_grid, - p_c_grid, - a_b_k0_m_k1_grid_desc, - b_b_k0_n_k1_grid_desc, - c_m0_n0_m1_n1_m2_m3_m4_n2_grid_desc, - c_block_cluster_adaptor); - } - - return ave_time; -} -#endif diff --git a/library/include/ck/library/reference_tensor_operation/cpu/reference_batched_gemm.hpp b/library/include/ck/library/reference_tensor_operation/cpu/reference_batched_gemm.hpp index f4944a28d2..14889e599a 100644 --- a/library/include/ck/library/reference_tensor_operation/cpu/reference_batched_gemm.hpp +++ b/library/include/ck/library/reference_tensor_operation/cpu/reference_batched_gemm.hpp @@ -1,10 +1,10 @@ -#ifndef REFERENCE_BATCHED_GEMM_HPP -#define REFERENCE_BATCHED_GEMM_HPP +#pragma once #include #include -#include "device_base.hpp" -#include "host_tensor.hpp" + +#include "ck/tensor_operation/gpu/device/device_base.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" namespace ck { namespace tensor_operation { @@ -132,4 +132,3 @@ struct ReferenceBatchedGemm : public device::BaseOperator } // namespace host } // namespace tensor_operation } // namespace ck -#endif diff --git a/library/include/ck/library/reference_tensor_operation/cpu/reference_cgemm.hpp b/library/include/ck/library/reference_tensor_operation/cpu/reference_cgemm.hpp index c6a5304766..5ebb6d70d5 100644 --- a/library/include/ck/library/reference_tensor_operation/cpu/reference_cgemm.hpp +++ b/library/include/ck/library/reference_tensor_operation/cpu/reference_cgemm.hpp @@ -1,33 +1,10 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2022 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ #pragma once + #include #include -#include "device_base.hpp" -#include "host_tensor.hpp" + +#include "ck/tensor_operation/gpu/device/device_base.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" namespace ck { namespace tensor_operation { diff --git a/library/include/ck/library/reference_tensor_operation/cpu/reference_conv_backward_weight.hpp b/library/include/ck/library/reference_tensor_operation/cpu/reference_conv_backward_weight.hpp index 4203085dbc..cb655dbd06 100644 --- a/library/include/ck/library/reference_tensor_operation/cpu/reference_conv_backward_weight.hpp +++ b/library/include/ck/library/reference_tensor_operation/cpu/reference_conv_backward_weight.hpp @@ -2,8 +2,9 @@ #include #include -#include "device_base.hpp" -#include "host_tensor.hpp" + +#include "ck/tensor_operation/gpu/device/device_base.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" namespace ck { namespace tensor_operation { diff --git a/library/include/ck/library/reference_tensor_operation/cpu/reference_conv_bwd_data.hpp b/library/include/ck/library/reference_tensor_operation/cpu/reference_conv_bwd_data.hpp index 11252e2398..41c8cad285 100644 --- a/library/include/ck/library/reference_tensor_operation/cpu/reference_conv_bwd_data.hpp +++ b/library/include/ck/library/reference_tensor_operation/cpu/reference_conv_bwd_data.hpp @@ -1,10 +1,11 @@ -#ifndef REFERENCE_CONV_BWD_DATA_HPP -#define REFERENCE_CONV_BWD_DATA_HPP +#pragma once #include #include -#include "device_base.hpp" -#include "host_tensor.hpp" + +#include "ck/tensor_operation/gpu/device/device_base.hpp" + +#include "ck/library/host_tensor/host_tensor.hpp" namespace ck { namespace tensor_operation { @@ -351,4 +352,3 @@ struct ReferenceConvBwdData : public device::BaseOperator } // namespace host } // namespace tensor_operation } // namespace ck -#endif diff --git a/library/include/ck/library/reference_tensor_operation/cpu/reference_conv_fwd.hpp b/library/include/ck/library/reference_tensor_operation/cpu/reference_conv_fwd.hpp index d1afa898e4..bf60577ce7 100644 --- a/library/include/ck/library/reference_tensor_operation/cpu/reference_conv_fwd.hpp +++ b/library/include/ck/library/reference_tensor_operation/cpu/reference_conv_fwd.hpp @@ -4,9 +4,8 @@ #include #include -#include "stream_config.hpp" -#include "device_base.hpp" -#include "host_tensor.hpp" +#include "ck/tensor_operation/gpu/device/device_base.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" namespace ck { namespace tensor_operation { diff --git a/library/include/ck/library/reference_tensor_operation/cpu/reference_conv_fwd_bias_activation.hpp b/library/include/ck/library/reference_tensor_operation/cpu/reference_conv_fwd_bias_activation.hpp index 4be6169c15..d6d49cfbde 100644 --- a/library/include/ck/library/reference_tensor_operation/cpu/reference_conv_fwd_bias_activation.hpp +++ b/library/include/ck/library/reference_tensor_operation/cpu/reference_conv_fwd_bias_activation.hpp @@ -1,10 +1,10 @@ -#ifndef REFERENCE_CONV_FWD_BIAS_ACTIVATION_HPP -#define REFERENCE_CONV_FWD_BIAS_ACTIVATION_HPP +#pragma once #include #include -#include "device_base.hpp" -#include "host_tensor.hpp" + +#include "ck/tensor_operation/gpu/device/device_base.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" namespace ck { namespace tensor_operation { @@ -187,4 +187,3 @@ struct ReferenceConvFwd_Bias_Activation : public device::BaseOperator } // namespace host } // namespace tensor_operation } // namespace ck -#endif diff --git a/library/include/ck/library/reference_tensor_operation/cpu/reference_conv_fwd_bias_activation_add.hpp b/library/include/ck/library/reference_tensor_operation/cpu/reference_conv_fwd_bias_activation_add.hpp index 466537c686..662a08267e 100644 --- a/library/include/ck/library/reference_tensor_operation/cpu/reference_conv_fwd_bias_activation_add.hpp +++ b/library/include/ck/library/reference_tensor_operation/cpu/reference_conv_fwd_bias_activation_add.hpp @@ -1,10 +1,10 @@ -#ifndef REFERENCE_CONV2D_FWD_BIAS_ACTIVATION_ADD_HPP -#define REFERENCE_CONV2D_FWD_BIAS_ACTIVATION_ADD_HPP +#pragma once #include #include -#include "device_base.hpp" -#include "host_tensor.hpp" + +#include "ck/tensor_operation/gpu/device/device_base.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" namespace ck { namespace tensor_operation { @@ -195,4 +195,3 @@ struct ReferenceConvFwd_Bias_Activation_Add : public device::BaseOperator } // namespace host } // namespace tensor_operation } // namespace ck -#endif diff --git a/library/include/ck/library/reference_tensor_operation/cpu/reference_gemm.hpp b/library/include/ck/library/reference_tensor_operation/cpu/reference_gemm.hpp index 6f097c6deb..0b87025c69 100644 --- a/library/include/ck/library/reference_tensor_operation/cpu/reference_gemm.hpp +++ b/library/include/ck/library/reference_tensor_operation/cpu/reference_gemm.hpp @@ -1,8 +1,10 @@ #pragma once + #include #include -#include "device_base.hpp" -#include "host_tensor.hpp" + +#include "ck/tensor_operation/gpu/device/device_base.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" namespace ck { namespace tensor_operation { diff --git a/library/include/ck/library/reference_tensor_operation/cpu/reference_gemm_bias_2d.hpp b/library/include/ck/library/reference_tensor_operation/cpu/reference_gemm_bias_2d.hpp index a0ceb28a11..0502058cfc 100644 --- a/library/include/ck/library/reference_tensor_operation/cpu/reference_gemm_bias_2d.hpp +++ b/library/include/ck/library/reference_tensor_operation/cpu/reference_gemm_bias_2d.hpp @@ -1,10 +1,10 @@ -#ifndef REFERENCE_GEMM_BIAS_BIAS_2D_HPP -#define REFERENCE_GEMM_BIAS_BIAS_2D_HPP +#pragma once #include #include -#include "device_base.hpp" -#include "host_tensor.hpp" + +#include "ck/tensor_operation/gpu/device/device_base.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" namespace ck { namespace tensor_operation { @@ -131,4 +131,3 @@ struct ReferenceGemmBias2D : public device::BaseOperator } // namespace host } // namespace tensor_operation } // namespace ck -#endif diff --git a/library/include/ck/library/reference_tensor_operation/cpu/reference_gemm_bias_activation.hpp b/library/include/ck/library/reference_tensor_operation/cpu/reference_gemm_bias_activation.hpp index 60f72e9e51..b369c6a3d3 100644 --- a/library/include/ck/library/reference_tensor_operation/cpu/reference_gemm_bias_activation.hpp +++ b/library/include/ck/library/reference_tensor_operation/cpu/reference_gemm_bias_activation.hpp @@ -1,10 +1,11 @@ -#ifndef REFERENCE_GEMM_BIAS_ACTIVATION_HPP -#define REFERENCE_GEMM_BIAS_ACTIVATION_HPP +#pragma once #include #include -#include "device_base.hpp" -#include "host_tensor.hpp" + +#include "ck/tensor_operation/gpu/device/device_base.hpp" + +#include "ck/library/host_tensor/host_tensor.hpp" namespace ck { namespace tensor_operation { @@ -134,4 +135,3 @@ struct ReferenceGemmBiasActivation : public device::BaseOperator } // namespace host } // namespace tensor_operation } // namespace ck -#endif diff --git a/library/include/ck/library/reference_tensor_operation/cpu/reference_gemm_bias_activation_add.hpp b/library/include/ck/library/reference_tensor_operation/cpu/reference_gemm_bias_activation_add.hpp index 5e0ec75e5e..37c24bd996 100644 --- a/library/include/ck/library/reference_tensor_operation/cpu/reference_gemm_bias_activation_add.hpp +++ b/library/include/ck/library/reference_tensor_operation/cpu/reference_gemm_bias_activation_add.hpp @@ -1,10 +1,11 @@ -#ifndef REFERENCE_GEMM_BIAS_ACTIVATION_ADD_HPP -#define REFERENCE_GEMM_BIAS_ACTIVATION_ADD_HPP +#pragma once #include #include -#include "device_base.hpp" -#include "host_tensor.hpp" + +#include "ck/tensor_operation/gpu/device/device_base.hpp" + +#include "ck/library/host_tensor/host_tensor.hpp" namespace ck { namespace tensor_operation { @@ -142,4 +143,3 @@ struct ReferenceGemmBiasActivationAdd : public device::BaseOperator } // namespace host } // namespace tensor_operation } // namespace ck -#endif diff --git a/library/include/ck/library/reference_tensor_operation/cpu/reference_softmax.hpp b/library/include/ck/library/reference_tensor_operation/cpu/reference_softmax.hpp index 7271103d54..74695e3b60 100644 --- a/library/include/ck/library/reference_tensor_operation/cpu/reference_softmax.hpp +++ b/library/include/ck/library/reference_tensor_operation/cpu/reference_softmax.hpp @@ -1,11 +1,13 @@ #pragma once + #include #include #include #include -#include "device_base.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" + +#include "ck/tensor_operation/gpu/device/device_base.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" +#include "ck/library/host_tensor/host_tensor_generator.hpp" namespace ck { namespace tensor_operation { diff --git a/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance.hpp b/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance.hpp index 6f0dbe75ff..dab6a59cff 100644 --- a/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance.hpp +++ b/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance.hpp @@ -1,26 +1,23 @@ -#ifndef DEVICE_REDUCE_INSTANTCE_HPP -#define DEVICE_REDUCE_INSTANTCE_HPP +#pragma once -#include "device_reduce_instance_blockwise_f16_f16_f16.hpp" -#include "device_reduce_instance_blockwise_f16_f32_f16.hpp" -#include "device_reduce_instance_blockwise_f32_f32_f32.hpp" -#include "device_reduce_instance_blockwise_f32_f64_f32.hpp" -#include "device_reduce_instance_blockwise_f64_f64_f64.hpp" -#include "device_reduce_instance_blockwise_i8_i8_i8.hpp" -#include "device_reduce_instance_blockwise_i8_i32_i8.hpp" -#include "device_reduce_instance_blockwise_b16_f32_b16.hpp" -#include "device_reduce_instance_multiblock_atomic_add_f16_f32_f32.hpp" -#include "device_reduce_instance_multiblock_atomic_add_f32_f32_f32.hpp" -#include "device_reduce_instance_multiblock_atomic_add_f32_f64_f32.hpp" -#include "device_reduce_instance_multiblock_atomic_add_f64_f64_f64.hpp" -#include "device_reduce_instance_multiblock_atomic_add_b16_f32_f32.hpp" -#include "device_reduce_instance_threadwise_f16_f16_f16.hpp" -#include "device_reduce_instance_threadwise_f16_f32_f16.hpp" -#include "device_reduce_instance_threadwise_f32_f32_f32.hpp" -#include "device_reduce_instance_threadwise_f32_f64_f32.hpp" -#include "device_reduce_instance_threadwise_f64_f64_f64.hpp" -#include "device_reduce_instance_threadwise_i8_i8_i8.hpp" -#include "device_reduce_instance_threadwise_i8_i32_i8.hpp" -#include "device_reduce_instance_threadwise_b16_f32_b16.hpp" - -#endif +#include "ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_f16_f16_f16.hpp" +#include "ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_f16_f32_f16.hpp" +#include "ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_f32_f32_f32.hpp" +#include "ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_f32_f64_f32.hpp" +#include "ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_f64_f64_f64.hpp" +#include "ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_i8_i8_i8.hpp" +#include "ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_i8_i32_i8.hpp" +#include "ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_b16_f32_b16.hpp" +#include "ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_multiblock_atomic_add_f16_f32_f32.hpp" +#include "ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_multiblock_atomic_add_f32_f32_f32.hpp" +#include "ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_multiblock_atomic_add_f32_f64_f32.hpp" +#include "ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_multiblock_atomic_add_f64_f64_f64.hpp" +#include "ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_multiblock_atomic_add_b16_f32_f32.hpp" +#include "ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_f16_f16_f16.hpp" +#include "ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_f16_f32_f16.hpp" +#include "ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_f32_f32_f32.hpp" +#include "ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_f32_f64_f32.hpp" +#include "ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_f64_f64_f64.hpp" +#include "ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_i8_i8_i8.hpp" +#include "ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_i8_i32_i8.hpp" +#include "ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_b16_f32_b16.hpp" diff --git a/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise.hpp b/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise.hpp index 0f8c365007..82b2ae3e1f 100644 --- a/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise.hpp +++ b/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise.hpp @@ -1,9 +1,8 @@ -#ifndef DEVICE_REDUCE_INSTANCE_BLOCKWISE_HPP -#define DEVICE_REDUCE_INSTANCE_BLOCKWISE_HPP +#pragma once -#include "reduction_operator_mapping.hpp" -#include "device_reduce_instance_impl_common.hpp" -#include "device_reduce_multiblock.hpp" +#include "ck/tensor_operation/gpu/device/reduction_operator_mapping.hpp" +#include "ck/tensor_operation/gpu/device/device_reduce_multiblock.hpp" +#include "ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_impl_common.hpp" namespace ck { namespace tensor_operation { @@ -175,7 +174,4 @@ void add_device_reduce_instance_blockwise( } // namespace device_reduce_instance } // namespace device } // namespace tensor_operation - } // namespace ck - -#endif diff --git a/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_b16_f32_b16.hpp b/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_b16_f32_b16.hpp index 3cad45f2e5..d81f0b20f0 100644 --- a/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_b16_f32_b16.hpp +++ b/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_b16_f32_b16.hpp @@ -1,8 +1,8 @@ -#ifndef DEVICE_REDUCE_INSTANCE_BLOCKWISE_B16_F32_B16_HPP -#define DEVICE_REDUCE_INSTANCE_BLOCKWISE_B16_F32_B16_HPP +#pragma once -#include "data_type.hpp" -#include "device_reduce_instance_blockwise.hpp" +#include "ck/utility/data_type.hpp" + +#include "ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise.hpp" namespace ck { namespace tensor_operation { @@ -53,7 +53,4 @@ ADD_BLOCKWISE_INST_REF_BY_ID(bhalf_t, float, bhalf_t, 4, 0, 1, 2, 1); } // namespace device_reduce_instance } // namespace device } // namespace tensor_operation - } // namespace ck - -#endif diff --git a/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_f16_f16_f16.hpp b/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_f16_f16_f16.hpp index 441c1aec3f..ed434aaad4 100644 --- a/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_f16_f16_f16.hpp +++ b/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_f16_f16_f16.hpp @@ -1,8 +1,8 @@ -#ifndef DEVICE_REDUCE_INSTANCE_BLOCKWISE_F16_F16_F16_HPP -#define DEVICE_REDUCE_INSTANCE_BLOCKWISE_F16_F16_F16_HPP +#pragma once -#include "data_type.hpp" -#include "device_reduce_instance_blockwise.hpp" +#include "ck/utility/data_type.hpp" + +#include "ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise.hpp" namespace ck { namespace tensor_operation { @@ -40,7 +40,4 @@ ADD_BLOCKWISE_INST_REF_BY_ID(half_t, half_t, half_t, 4, 0, 1, 2, 1); } // namespace device_reduce_instance } // namespace device } // namespace tensor_operation - } // namespace ck - -#endif diff --git a/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_f16_f32_f16.hpp b/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_f16_f32_f16.hpp index ca8532a458..742371d367 100644 --- a/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_f16_f32_f16.hpp +++ b/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_f16_f32_f16.hpp @@ -1,8 +1,8 @@ -#ifndef DEVICE_REDUCE_INSTANCE_BLOCKWISE_F16_F32_F16_HPP -#define DEVICE_REDUCE_INSTANCE_BLOCKWISE_F16_F32_F16_HPP +#pragma once -#include "data_type.hpp" -#include "device_reduce_instance_blockwise.hpp" +#include "ck/utility/data_type.hpp" + +#include "ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise.hpp" namespace ck { namespace tensor_operation { @@ -28,7 +28,4 @@ ADD_BLOCKWISE_INST_REF_BY_ID(half_t, float, half_t, 7, 0, 0, 2, 1); } // namespace device_reduce_instance } // namespace device } // namespace tensor_operation - } // namespace ck - -#endif diff --git a/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_f32_f32_f32.hpp b/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_f32_f32_f32.hpp index 64f504c9da..de9320e376 100644 --- a/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_f32_f32_f32.hpp +++ b/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_f32_f32_f32.hpp @@ -1,7 +1,8 @@ -#ifndef DEVICE_REDUCE_INSTANCE_BLOCKWISE_F32_F32_F32_HPP -#define DEVICE_REDUCE_INSTANCE_BLOCKWISE_F32_F32_F32_HPP +#pragma once -#include "device_reduce_instance_blockwise.hpp" +#include "ck/utility/data_type.hpp" + +#include "ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise.hpp" namespace ck { namespace tensor_operation { @@ -51,7 +52,4 @@ ADD_BLOCKWISE_INST_REF_BY_ID(float, float, float, 4, 0, 1, 2, 1); } // namespace device_reduce_instance } // namespace device } // namespace tensor_operation - } // namespace ck - -#endif diff --git a/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_f32_f64_f32.hpp b/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_f32_f64_f32.hpp index 9e84ee34fb..045f580262 100644 --- a/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_f32_f64_f32.hpp +++ b/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_f32_f64_f32.hpp @@ -1,7 +1,8 @@ -#ifndef DEVICE_REDUCE_INSTANCE_BLOCKWISE_F32_F64_F32_HPP -#define DEVICE_REDUCE_INSTANCE_BLOCKWISE_F32_F64_F32_HPP +#pragma once -#include "device_reduce_instance_blockwise.hpp" +#include "ck/utility/data_type.hpp" + +#include "ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise.hpp" namespace ck { namespace tensor_operation { @@ -27,7 +28,4 @@ ADD_BLOCKWISE_INST_REF_BY_ID(float, double, float, 7, 0, 0, 2, 1); } // namespace device_reduce_instance } // namespace device } // namespace tensor_operation - } // namespace ck - -#endif diff --git a/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_f64_f64_f64.hpp b/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_f64_f64_f64.hpp index a37e3bdeb9..8018f9a14e 100644 --- a/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_f64_f64_f64.hpp +++ b/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_f64_f64_f64.hpp @@ -1,7 +1,8 @@ -#ifndef DEVICE_REDUCE_INSTANCE_BLOCKWISE_F64_F64_F64_HPP -#define DEVICE_REDUCE_INSTANCE_BLOCKWISE_F64_F64_F64_HPP +#pragma once -#include "device_reduce_instance_blockwise.hpp" +#include "ck/utility/data_type.hpp" + +#include "ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise.hpp" namespace ck { namespace tensor_operation { @@ -51,7 +52,4 @@ ADD_BLOCKWISE_INST_REF_BY_ID(double, double, double, 4, 0, 1, 2, 1); } // namespace device_reduce_instance } // namespace device } // namespace tensor_operation - } // namespace ck - -#endif diff --git a/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_i8_i32_i8.hpp b/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_i8_i32_i8.hpp index 1d8695bbb0..b5f3d88fe2 100644 --- a/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_i8_i32_i8.hpp +++ b/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_i8_i32_i8.hpp @@ -1,7 +1,8 @@ -#ifndef DEVICE_REDUCE_INSTANCE_BLOCKWISE_I8_I32_I8_HPP -#define DEVICE_REDUCE_INSTANCE_BLOCKWISE_I8_I32_I8_HPP +#pragma once -#include "device_reduce_instance_blockwise.hpp" +#include "ck/utility/data_type.hpp" + +#include "ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise.hpp" namespace ck { namespace tensor_operation { @@ -23,7 +24,4 @@ ADD_BLOCKWISE_INST_REF_BY_ID(int8_t, int32_t, int8_t, 5, 0, 0, 2, 1); } // namespace device_reduce_instance } // namespace device } // namespace tensor_operation - } // namespace ck - -#endif diff --git a/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_i8_i8_i8.hpp b/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_i8_i8_i8.hpp index b5c19b7207..105ea6fdd3 100644 --- a/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_i8_i8_i8.hpp +++ b/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_i8_i8_i8.hpp @@ -1,7 +1,8 @@ -#ifndef DEVICE_REDUCE_INSTANCE_BLOCKWISE_I8_I8_I8_HPP -#define DEVICE_REDUCE_INSTANCE_BLOCKWISE_I8_I8_I8_HPP +#pragma once -#include "device_reduce_instance_blockwise.hpp" +#include "ck/utility/data_type.hpp" + +#include "ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise.hpp" namespace ck { namespace tensor_operation { @@ -39,7 +40,4 @@ ADD_BLOCKWISE_INST_REF_BY_ID(int8_t, int8_t, int8_t, 4, 0, 1, 2, 1); } // namespace device_reduce_instance } // namespace device } // namespace tensor_operation - } // namespace ck - -#endif diff --git a/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_impl_common.hpp b/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_impl_common.hpp index 721d98a718..24ff3894b8 100644 --- a/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_impl_common.hpp +++ b/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_impl_common.hpp @@ -1,5 +1,4 @@ -#ifndef DEVICE_REDUCE_INSTANCE_IMPL_COMMON_HPP -#define DEVICE_REDUCE_INSTANCE_IMPL_COMMON_HPP +#pragma once namespace ck { namespace tensor_operation { @@ -35,7 +34,4 @@ struct ReductionConfiguration_2 } // namespace device_reduce_instance } // namespace device } // namespace tensor_operation - } // namespace ck - -#endif diff --git a/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_multiblock_atomic_add.hpp b/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_multiblock_atomic_add.hpp index 9f78933bde..a31bcacf16 100644 --- a/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_multiblock_atomic_add.hpp +++ b/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_multiblock_atomic_add.hpp @@ -1,9 +1,9 @@ -#ifndef DEVICE_REDUCE_INSTANCE_MULTIBLOCK_ATOMIC_ADD_HPP -#define DEVICE_REDUCE_INSTANCE_MULTIBLOCK_ATOMIC_ADD_HPP +#pragma once -#include "reduction_operator_mapping.hpp" -#include "device_reduce_instance_impl_common.hpp" -#include "device_reduce_multiblock.hpp" +#include "ck/tensor_operation/gpu/device/reduction_operator_mapping.hpp" +#include "ck/tensor_operation/gpu/device/device_reduce_multiblock.hpp" + +#include "ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_impl_common.hpp" namespace ck { namespace tensor_operation { @@ -193,7 +193,4 @@ void add_device_reduce_instance_multiblock_atomic_add( } // namespace device_reduce_instance } // namespace device } // namespace tensor_operation - } // namespace ck - -#endif diff --git a/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_multiblock_atomic_add_b16_f32_f32.hpp b/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_multiblock_atomic_add_b16_f32_f32.hpp index 4e39cf49f6..882e08c5e3 100644 --- a/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_multiblock_atomic_add_b16_f32_f32.hpp +++ b/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_multiblock_atomic_add_b16_f32_f32.hpp @@ -1,8 +1,8 @@ -#ifndef DEVICE_REDUCE_INSTANCE_MULTIBLOCK_ATOMIC_ADD_B16_F32_F32_HPP -#define DEVICE_REDUCE_INSTANCE_MULTIBLOCK_ATOMIC_ADD_B16_F32_F32_HPP +#pragma once -#include "data_type.hpp" -#include "device_reduce_instance_multiblock_atomic_add.hpp" +#include "ck/utility/data_type.hpp" + +#include "ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_multiblock_atomic_add.hpp" namespace ck { namespace tensor_operation { @@ -24,7 +24,4 @@ ADD_MULTIBLOCK_ATOMIC_ADD_INST_REF_BY_ID(bhalf_t, float, float, 5, 0, 0, 2, 1); } // namespace device_reduce_instance } // namespace device } // namespace tensor_operation - } // namespace ck - -#endif diff --git a/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_multiblock_atomic_add_f16_f32_f32.hpp b/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_multiblock_atomic_add_f16_f32_f32.hpp index 73424322ae..b68aba5512 100644 --- a/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_multiblock_atomic_add_f16_f32_f32.hpp +++ b/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_multiblock_atomic_add_f16_f32_f32.hpp @@ -1,8 +1,8 @@ -#ifndef DEVICE_REDUCE_INSTANCE_MULTIBLOCK_ATOMIC_ADD_F16_F32_F32_HPP -#define DEVICE_REDUCE_INSTANCE_MULTIBLOCK_ATOMIC_ADD_F16_F32_F32_HPP +#pragma once -#include "data_type.hpp" -#include "device_reduce_instance_multiblock_atomic_add.hpp" +#include "ck/utility/data_type.hpp" + +#include "ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_multiblock_atomic_add.hpp" namespace ck { namespace tensor_operation { @@ -24,7 +24,4 @@ ADD_MULTIBLOCK_ATOMIC_ADD_INST_REF_BY_ID(half_t, float, float, 5, 0, 0, 2, 1); } // namespace device_reduce_instance } // namespace device } // namespace tensor_operation - } // namespace ck - -#endif diff --git a/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_multiblock_atomic_add_f32_f32_f32.hpp b/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_multiblock_atomic_add_f32_f32_f32.hpp index ecc9c4ea87..c252ee0834 100644 --- a/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_multiblock_atomic_add_f32_f32_f32.hpp +++ b/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_multiblock_atomic_add_f32_f32_f32.hpp @@ -1,7 +1,8 @@ -#ifndef DEVICE_REDUCE_INSTANCE_MULTIBLOCK_ATOMIC_ADD_F32_F32_F32_HPP -#define DEVICE_REDUCE_INSTANCE_MULTIBLOCK_ATOMIC_ADD_F32_F32_F32_HPP +#pragma once -#include "device_reduce_instance_multiblock_atomic_add.hpp" +#include "ck/utility/data_type.hpp" + +#include "ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_multiblock_atomic_add.hpp" namespace ck { namespace tensor_operation { @@ -23,7 +24,4 @@ ADD_MULTIBLOCK_ATOMIC_ADD_INST_REF_BY_ID(float, float, float, 5, 0, 0, 2, 1); } // namespace device_reduce_instance } // namespace device } // namespace tensor_operation - } // namespace ck - -#endif diff --git a/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_multiblock_atomic_add_f32_f64_f32.hpp b/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_multiblock_atomic_add_f32_f64_f32.hpp index 41a60d5b70..3b624f677e 100644 --- a/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_multiblock_atomic_add_f32_f64_f32.hpp +++ b/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_multiblock_atomic_add_f32_f64_f32.hpp @@ -1,7 +1,8 @@ -#ifndef DEVICE_REDUCE_INSTANCE_MULTIBLOCK_ATOMIC_ADD_F32_F64_F32_HPP -#define DEVICE_REDUCE_INSTANCE_MULTIBLOCK_ATOMIC_ADD_F32_F64_F32_HPP +#pragma once -#include "device_reduce_instance_multiblock_atomic_add.hpp" +#include "ck/utility/data_type.hpp" + +#include "ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_multiblock_atomic_add.hpp" namespace ck { namespace tensor_operation { @@ -23,7 +24,4 @@ ADD_MULTIBLOCK_ATOMIC_ADD_INST_REF_BY_ID(float, double, float, 5, 0, 0, 2, 1); } // namespace device_reduce_instance } // namespace device } // namespace tensor_operation - } // namespace ck - -#endif diff --git a/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_multiblock_atomic_add_f64_f64_f64.hpp b/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_multiblock_atomic_add_f64_f64_f64.hpp index bdcca274d7..3ae58cfe5d 100644 --- a/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_multiblock_atomic_add_f64_f64_f64.hpp +++ b/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_multiblock_atomic_add_f64_f64_f64.hpp @@ -1,7 +1,8 @@ -#ifndef DEVICE_REDUCE_INSTANCE_MULTIBLOCK_ATOMIC_ADD_F64_F64_F64_HPP -#define DEVICE_REDUCE_INSTANCE_MULTIBLOCK_ATOMIC_ADD_F64_F64_F64_HPP +#pragma once -#include "device_reduce_instance_multiblock_atomic_add.hpp" +#include "ck/utility/data_type.hpp" + +#include "ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_multiblock_atomic_add.hpp" namespace ck { namespace tensor_operation { @@ -23,7 +24,4 @@ ADD_MULTIBLOCK_ATOMIC_ADD_INST_REF_BY_ID(double, double, double, 5, 0, 0, 2, 1); } // namespace device_reduce_instance } // namespace device } // namespace tensor_operation - } // namespace ck - -#endif diff --git a/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise.hpp b/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise.hpp index 563dd09b10..95dfa9d61f 100644 --- a/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise.hpp +++ b/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise.hpp @@ -1,9 +1,8 @@ -#ifndef DEVICE_REDUCE_INSTANCE_THREADWISE_HPP -#define DEVICE_REDUCE_INSTANCE_THREADWISE_HPP +#pragma once -#include "reduction_operator_mapping.hpp" -#include "device_reduce_instance_impl_common.hpp" -#include "device_reduce_threadwise.hpp" +#include "ck/tensor_operation/gpu/device/reduction_operator_mapping.hpp" +#include "ck/tensor_operation/gpu/device/device_reduce_threadwise.hpp" +#include "ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_impl_common.hpp" namespace ck { namespace tensor_operation { @@ -152,7 +151,4 @@ void add_device_reduce_instance_threadwise( } // namespace device_reduce_instance } // namespace device } // namespace tensor_operation - } // namespace ck - -#endif diff --git a/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_b16_f32_b16.hpp b/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_b16_f32_b16.hpp index 0291f33214..75bcea933c 100644 --- a/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_b16_f32_b16.hpp +++ b/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_b16_f32_b16.hpp @@ -1,8 +1,8 @@ -#ifndef DEVICE_REDUCE_INSTANCE_THREADWISE_B16_F32_B16_HPP -#define DEVICE_REDUCE_INSTANCE_THREADWISE_B16_F32_B16_HPP +#pragma once -#include "data_type.hpp" -#include "device_reduce_instance_threadwise.hpp" +#include "ck/utility/data_type.hpp" + +#include "ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise.hpp" namespace ck { namespace tensor_operation { @@ -53,7 +53,4 @@ ADD_THREADWISE_INST_REF_BY_ID(bhalf_t, float, bhalf_t, 4, 0, 1, 2, 1); } // namespace device_reduce_instance } // namespace device } // namespace tensor_operation - } // namespace ck - -#endif diff --git a/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_f16_f16_f16.hpp b/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_f16_f16_f16.hpp index 7ab1bebc5f..c685114661 100644 --- a/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_f16_f16_f16.hpp +++ b/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_f16_f16_f16.hpp @@ -1,8 +1,8 @@ -#ifndef DEVICE_REDUCE_INSTANCE_THREADWISE_F16_F16_F16_HPP -#define DEVICE_REDUCE_INSTANCE_THREADWISE_F16_F16_F16_HPP +#pragma once -#include "data_type.hpp" -#include "device_reduce_instance_threadwise.hpp" +#include "ck/utility/data_type.hpp" + +#include "ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise.hpp" namespace ck { namespace tensor_operation { @@ -40,7 +40,4 @@ ADD_THREADWISE_INST_REF_BY_ID(half_t, half_t, half_t, 4, 0, 1, 2, 1); } // namespace device_reduce_instance } // namespace device } // namespace tensor_operation - } // namespace ck - -#endif diff --git a/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_f16_f32_f16.hpp b/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_f16_f32_f16.hpp index 39c3d10660..f9dee47f9c 100644 --- a/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_f16_f32_f16.hpp +++ b/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_f16_f32_f16.hpp @@ -1,8 +1,8 @@ -#ifndef DEVICE_REDUCE_INSTANCE_THREADWISE_F16_F32_F16_HPP -#define DEVICE_REDUCE_INSTANCE_THREADWISE_F16_F32_F16_HPP +#pragma once -#include "data_type.hpp" -#include "device_reduce_instance_threadwise.hpp" +#include "ck/utility/data_type.hpp" + +#include "ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise.hpp" namespace ck { namespace tensor_operation { @@ -28,7 +28,4 @@ ADD_THREADWISE_INST_REF_BY_ID(half_t, float, half_t, 7, 0, 0, 2, 1); } // namespace device_reduce_instance } // namespace device } // namespace tensor_operation - } // namespace ck - -#endif diff --git a/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_f32_f32_f32.hpp b/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_f32_f32_f32.hpp index 3c47bfd189..7f677037b0 100644 --- a/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_f32_f32_f32.hpp +++ b/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_f32_f32_f32.hpp @@ -1,7 +1,8 @@ -#ifndef DEVICE_REDUCE_INSTANCE_THREADWISE_F32_F32_F32_HPP -#define DEVICE_REDUCE_INSTANCE_THREADWISE_F32_F32_F32_HPP +#pragma once -#include "device_reduce_instance_threadwise.hpp" +#include "ck/utility/data_type.hpp" + +#include "ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise.hpp" namespace ck { namespace tensor_operation { @@ -51,7 +52,4 @@ ADD_THREADWISE_INST_REF_BY_ID(float, float, float, 4, 0, 1, 2, 1); } // namespace device_reduce_instance } // namespace device } // namespace tensor_operation - } // namespace ck - -#endif diff --git a/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_f32_f64_f32.hpp b/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_f32_f64_f32.hpp index 9df9f6f1fa..e82f5875d8 100644 --- a/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_f32_f64_f32.hpp +++ b/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_f32_f64_f32.hpp @@ -1,7 +1,8 @@ -#ifndef DEVICE_REDUCE_INSTANCE_THREADWISE_F32_F64_F32_HPP -#define DEVICE_REDUCE_INSTANCE_THREADWISE_F32_F64_F32_HPP +#pragma once -#include "device_reduce_instance_threadwise.hpp" +#include "ck/utility/data_type.hpp" + +#include "ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise.hpp" namespace ck { namespace tensor_operation { @@ -27,7 +28,4 @@ ADD_THREADWISE_INST_REF_BY_ID(float, double, float, 7, 0, 0, 2, 1); } // namespace device_reduce_instance } // namespace device } // namespace tensor_operation - } // namespace ck - -#endif diff --git a/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_f64_f64_f64.hpp b/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_f64_f64_f64.hpp index 00ab218f20..db49a1bea4 100644 --- a/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_f64_f64_f64.hpp +++ b/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_f64_f64_f64.hpp @@ -1,7 +1,8 @@ -#ifndef DEVICE_REDUCE_INSTANCE_THREADWISE_F64_F64_F64_HPP -#define DEVICE_REDUCE_INSTANCE_THREADWISE_F64_F64_F64_HPP +#pragma once -#include "device_reduce_instance_threadwise.hpp" +#include "ck/utility/data_type.hpp" + +#include "ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise.hpp" namespace ck { namespace tensor_operation { @@ -51,7 +52,4 @@ ADD_THREADWISE_INST_REF_BY_ID(double, double, double, 4, 0, 1, 2, 1); } // namespace device_reduce_instance } // namespace device } // namespace tensor_operation - } // namespace ck - -#endif diff --git a/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_i8_i32_i8.hpp b/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_i8_i32_i8.hpp index de7445b043..2edd9b0fa5 100644 --- a/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_i8_i32_i8.hpp +++ b/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_i8_i32_i8.hpp @@ -1,7 +1,8 @@ -#ifndef DEVICE_REDUCE_INSTANCE_THREADWISE_I8_I32_I8_HPP -#define DEVICE_REDUCE_INSTANCE_THREADWISE_I8_I32_I8_HPP +#pragma once -#include "device_reduce_instance_threadwise.hpp" +#include "ck/utility/data_type.hpp" + +#include "ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise.hpp" namespace ck { namespace tensor_operation { @@ -23,7 +24,4 @@ ADD_THREADWISE_INST_REF_BY_ID(int8_t, int32_t, int8_t, 5, 0, 0, 2, 1); } // namespace device_reduce_instance } // namespace device } // namespace tensor_operation - } // namespace ck - -#endif diff --git a/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_i8_i8_i8.hpp b/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_i8_i8_i8.hpp index 1ea1ee745e..d47bf9d536 100644 --- a/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_i8_i8_i8.hpp +++ b/library/include/ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_i8_i8_i8.hpp @@ -1,7 +1,8 @@ -#ifndef DEVICE_REDUCE_INSTANCE_THREADWISE_I8_I8_I8_HPP -#define DEVICE_REDUCE_INSTANCE_THREADWISE_I8_I8_I8_HPP +#pragma once -#include "device_reduce_instance_threadwise.hpp" +#include "ck/utility/data_type.hpp" + +#include "ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise.hpp" namespace ck { namespace tensor_operation { @@ -39,7 +40,4 @@ ADD_THREADWISE_INST_REF_BY_ID(int8_t, int8_t, int8_t, 4, 0, 1, 2, 1); } // namespace device_reduce_instance } // namespace device } // namespace tensor_operation - } // namespace ck - -#endif diff --git a/library/include/ck/library/utility/check_err.hpp b/library/include/ck/library/utility/check_err.hpp index 368da4d207..c8fcbd01c8 100644 --- a/library/include/ck/library/utility/check_err.hpp +++ b/library/include/ck/library/utility/check_err.hpp @@ -3,7 +3,6 @@ #include #include #include -#include #include #include #include @@ -11,7 +10,7 @@ #include #include -#include "data_type.hpp" +#include "ck/utility/data_type.hpp" namespace ck { namespace utils { @@ -107,8 +106,7 @@ check_err(const std::vector& out, } template -typename std::enable_if::value || std::is_same::value, - bool>::type +typename std::enable_if::value, bool>::type check_err(const std::vector& out, const std::vector& ref, const std::string& msg = "Error: Incorrect results!", diff --git a/library/include/ck/library/utility/conv_util.hpp b/library/include/ck/library/utility/conv_util.hpp index 409fa5aff2..3ab0b3f276 100644 --- a/library/include/ck/library/utility/conv_util.hpp +++ b/library/include/ck/library/utility/conv_util.hpp @@ -9,17 +9,17 @@ #include #include -#include "check_err.hpp" -#include "config.hpp" -#include "device.hpp" -#include "device_conv_fwd.hpp" -#include "device_tensor.hpp" -#include "element_wise_operation.hpp" -#include "fill.hpp" -#include "host_tensor.hpp" -#include "op_instance_engine.hpp" -#include "reference_conv_fwd.hpp" -#include "tensor_layout.hpp" +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_conv_fwd.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" + +#include "ck/library/utility/check_err.hpp" +#include "ck/library/utility/fill.hpp" +#include "ck/library/utility/op_instance_engine.hpp" +#include "ck/library/host_tensor/device_memory.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" +#include "ck/library/reference_tensor_operation/cpu/reference_conv_fwd.hpp" namespace ck { namespace tensor_operation { diff --git a/library/include/ck/library/utility/fill.hpp b/library/include/ck/library/utility/fill.hpp index 8c31e56beb..d530ccfa9e 100644 --- a/library/include/ck/library/utility/fill.hpp +++ b/library/include/ck/library/utility/fill.hpp @@ -4,7 +4,7 @@ #include #include -#include "data_type.hpp" +#include "ck/utility/data_type.hpp" namespace ck { namespace utils { diff --git a/library/include/ck/library/utility/op_instance_engine.hpp b/library/include/ck/library/utility/op_instance_engine.hpp index 1d11b62a4a..fef3dc890a 100644 --- a/library/include/ck/library/utility/op_instance_engine.hpp +++ b/library/include/ck/library/utility/op_instance_engine.hpp @@ -9,9 +9,12 @@ #include #include -#include "check_err.hpp" -#include "device_base.hpp" -#include "functional2.hpp" +#include "ck/utility/functional2.hpp" +#include "ck/tensor_operation/gpu/device/device_base.hpp" + +#include "ck/library/utility/check_err.hpp" +#include "ck/library/host_tensor/device_memory.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" namespace ck { namespace utils { diff --git a/library/src/host_tensor/CMakeLists.txt b/library/src/host_tensor/CMakeLists.txt index 2a020b763d..ae3ecf2eed 100644 --- a/library/src/host_tensor/CMakeLists.txt +++ b/library/src/host_tensor/CMakeLists.txt @@ -1,12 +1,6 @@ ## host_tensor -include_directories(BEFORE - ${PROJECT_SOURCE_DIR}/include/ck - ${PROJECT_SOURCE_DIR}/include/ck/utility - ${PROJECT_SOURCE_DIR}/library/include/ck/library/host_tensor -) - set(HOST_TENSOR_SOURCE - device.cpp + device_memory.cpp host_tensor.cpp ) diff --git a/library/src/host_tensor/device.cpp b/library/src/host_tensor/device.cpp deleted file mode 100644 index 9f0d982dbc..0000000000 --- a/library/src/host_tensor/device.cpp +++ /dev/null @@ -1,70 +0,0 @@ -#include "device.hpp" - -DeviceMem::DeviceMem(std::size_t mem_size) : mMemSize(mem_size) -{ - hip_check_error(hipMalloc(static_cast(&mpDeviceBuf), mMemSize)); -} - -void* DeviceMem::GetDeviceBuffer() { return mpDeviceBuf; } - -std::size_t DeviceMem::GetBufferSize() { return mMemSize; } - -void DeviceMem::ToDevice(const void* p) -{ - hip_check_error(hipMemcpy(mpDeviceBuf, const_cast(p), mMemSize, hipMemcpyHostToDevice)); -} - -void DeviceMem::FromDevice(void* p) -{ - hip_check_error(hipMemcpy(p, mpDeviceBuf, mMemSize, hipMemcpyDeviceToHost)); -} - -void DeviceMem::SetZero() { hip_check_error(hipMemset(mpDeviceBuf, 0, mMemSize)); } - -DeviceMem::~DeviceMem() { hip_check_error(hipFree(mpDeviceBuf)); } - -struct KernelTimerImpl -{ - KernelTimerImpl() - { - hip_check_error(hipEventCreate(&mStart)); - hip_check_error(hipEventCreate(&mEnd)); - } - - ~KernelTimerImpl() - { - hip_check_error(hipEventDestroy(mStart)); - hip_check_error(hipEventDestroy(mEnd)); - } - - void Start() - { - hip_check_error(hipDeviceSynchronize()); - hip_check_error(hipEventRecord(mStart, nullptr)); - } - - void End() - { - hip_check_error(hipEventRecord(mEnd, nullptr)); - hip_check_error(hipEventSynchronize(mEnd)); - } - - float GetElapsedTime() const - { - float time; - hip_check_error(hipEventElapsedTime(&time, mStart, mEnd)); - return time; - } - - hipEvent_t mStart, mEnd; -}; - -KernelTimer::KernelTimer() : impl(new KernelTimerImpl()) {} - -KernelTimer::~KernelTimer() {} - -void KernelTimer::Start() { impl->Start(); } - -void KernelTimer::End() { impl->End(); } - -float KernelTimer::GetElapsedTime() const { return impl->GetElapsedTime(); } diff --git a/library/src/host_tensor/device_memory.cpp b/library/src/host_tensor/device_memory.cpp new file mode 100644 index 0000000000..f425a5c1cd --- /dev/null +++ b/library/src/host_tensor/device_memory.cpp @@ -0,0 +1,25 @@ +#include "ck/device_utility/hip_check_error.hpp" +#include "ck/library/host_tensor/device_memory.hpp" + +DeviceMem::DeviceMem(std::size_t mem_size) : mMemSize(mem_size) +{ + hip_check_error(hipMalloc(static_cast(&mpDeviceBuf), mMemSize)); +} + +void* DeviceMem::GetDeviceBuffer() { return mpDeviceBuf; } + +std::size_t DeviceMem::GetBufferSize() { return mMemSize; } + +void DeviceMem::ToDevice(const void* p) +{ + hip_check_error(hipMemcpy(mpDeviceBuf, const_cast(p), mMemSize, hipMemcpyHostToDevice)); +} + +void DeviceMem::FromDevice(void* p) +{ + hip_check_error(hipMemcpy(p, mpDeviceBuf, mMemSize, hipMemcpyDeviceToHost)); +} + +void DeviceMem::SetZero() { hip_check_error(hipMemset(mpDeviceBuf, 0, mMemSize)); } + +DeviceMem::~DeviceMem() { hip_check_error(hipFree(mpDeviceBuf)); } diff --git a/library/src/host_tensor/host_tensor.cpp b/library/src/host_tensor/host_tensor.cpp index 138e3fc254..8fd22a4c6b 100644 --- a/library/src/host_tensor/host_tensor.cpp +++ b/library/src/host_tensor/host_tensor.cpp @@ -1,5 +1,6 @@ #include -#include "host_tensor.hpp" + +#include "ck/library/host_tensor/host_tensor.hpp" void HostTensorDescriptor::CalculateStrides() { diff --git a/library/src/obselete_driver_offline/CMakeLists.txt b/library/src/obselete_driver_offline/CMakeLists.txt deleted file mode 100644 index 54b1395327..0000000000 --- a/library/src/obselete_driver_offline/CMakeLists.txt +++ /dev/null @@ -1,37 +0,0 @@ -include_directories(BEFORE - include - ${PROJECT_SOURCE_DIR}/host/host_tensor/include - ${PROJECT_SOURCE_DIR}/host/device/include - ${PROJECT_SOURCE_DIR}/host/solver/include - ${PROJECT_SOURCE_DIR}/composable_kernel/include - ${PROJECT_SOURCE_DIR}/composable_kernel/include/utility - ${PROJECT_SOURCE_DIR}/composable_kernel/include/tensor_description - ${PROJECT_SOURCE_DIR}/composable_kernel/include/tensor_operation - ${PROJECT_SOURCE_DIR}/composable_kernel/include/problem_transform - ${PROJECT_SOURCE_DIR}/composable_kernel/include/driver - ${PROJECT_SOURCE_DIR}/external/rocm/include -) - -set(CONV_FWD_DRIVER_OFFLINE_SOURCE src/conv_fwd_driver_offline.cpp) -set(CONV_FWD_DRIVER_OFFLINE_NCHWC_SOURCE src/conv_fwd_driver_offline_nchwc.cpp) -set(CONV_ADD_FWD_DRIVER_OFFLINE_NCHWC_SOURCE src/conv_add_fwd_driver_offline_nchwc.cpp) -set(CONV_MAXPOOL_FWD_DRIVER_OFFLINE_NCHWC_SOURCE src/conv_maxpool_fwd_driver_offline_nchwc.cpp) -set(CONV_BWD_DRIVER_OFFLINE_SOURCE src/conv_bwd_driver_offline.cpp) -set(CONV_WRW_DRIVER_OFFLINE_SOURCE src/conv_wrw_driver_offline.cpp) -set(GEMM_DRIVER_OFFLINE_SOURCE src/gemm_driver_offline.cpp) - -add_executable(conv_fwd_driver_offline ${CONV_FWD_DRIVER_OFFLINE_SOURCE}) -add_executable(conv_fwd_driver_offline_nchwc ${CONV_FWD_DRIVER_OFFLINE_NCHWC_SOURCE}) -add_executable(conv_add_fwd_driver_offline_nchwc ${CONV_ADD_FWD_DRIVER_OFFLINE_NCHWC_SOURCE}) -add_executable(conv_maxpool_fwd_driver_offline_nchwc ${CONV_MAXPOOL_FWD_DRIVER_OFFLINE_NCHWC_SOURCE}) -add_executable(conv_bwd_driver_offline ${CONV_BWD_DRIVER_OFFLINE_SOURCE}) -add_executable(conv_wrw_driver_offline ${CONV_WRW_DRIVER_OFFLINE_SOURCE}) -add_executable(gemm_driver_offline ${GEMM_DRIVER_OFFLINE_SOURCE}) - -target_link_libraries(conv_fwd_driver_offline PRIVATE host_tensor) -target_link_libraries(conv_fwd_driver_offline_nchwc PRIVATE host_tensor) -target_link_libraries(conv_add_fwd_driver_offline_nchwc PRIVATE host_tensor) -target_link_libraries(conv_maxpool_fwd_driver_offline_nchwc PRIVATE host_tensor) -target_link_libraries(conv_bwd_driver_offline PRIVATE host_tensor) -target_link_libraries(conv_wrw_driver_offline PRIVATE host_tensor) -target_link_libraries(gemm_driver_offline PRIVATE host_tensor) diff --git a/library/src/obselete_driver_offline/conv_add_fwd_driver_offline_nchwc.cpp b/library/src/obselete_driver_offline/conv_add_fwd_driver_offline_nchwc.cpp deleted file mode 100644 index a7541f03de..0000000000 --- a/library/src/obselete_driver_offline/conv_add_fwd_driver_offline_nchwc.cpp +++ /dev/null @@ -1,416 +0,0 @@ -#include -#include -#include -#include -#include -#include - -#include "check_err.hpp" -#include "config.hpp" -#include "debug.hpp" -#include "print.hpp" -#include "device.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" -#include "conv_common.hpp" -#include "device_tensor.hpp" -#include "device_convolution_add_forward_implicit_gemm_v5r1_dlops_nc0hwc1_kc0yxc1_nk0hwk1.hpp" - -#define USE_DYNAMIC_MODE 0 -#define USE_CONV_FWD_V5R1_NCHWC 1 - -enum ConvForwardAlgo -{ - V5R1NCHWC // 0 -}; - -template -void host_direct_convolution_add_nchwc(const Tensor& in, - const Tensor& wei, - const Tensor& add, - const Tensor& bias, - Tensor& add_host, - Tensor& out_host, - const ConvStrides& conv_strides, - const ConvDilations& conv_dilations, - const InLeftPads& in_left_pads, - const InRightPads&, - const ck::ActivTypeEnum activ_type) -{ - using namespace ck; - - constexpr auto I0 = Number<0>{}; - constexpr auto I1 = Number<1>{}; - - auto f_nchw = [&](auto n, auto k0, auto ho, auto wo, auto k1) { - double v = 0; - auto k = k0 * out_host.mDesc.GetLengths()[4] + k1; - - for(int c0 = 0; c0 < wei.mDesc.GetLengths()[1]; ++c0) - { - for(int y = 0; y < wei.mDesc.GetLengths()[2]; ++y) - { - int hi = ho * conv_strides[I0] + y * conv_dilations[I0] - in_left_pads[I0]; - for(int x = 0; x < wei.mDesc.GetLengths()[3]; ++x) - { - int wi = wo * conv_strides[I1] + x * conv_dilations[I1] - in_left_pads[I1]; - if(hi >= 0 && hi < in.mDesc.GetLengths()[2] && wi >= 0 && - wi < in.mDesc.GetLengths()[3]) - { - - for(int c1 = 0; c1 < wei.mDesc.GetLengths()[4]; ++c1) - { - v += static_cast(in(n, c0, hi, wi, c1)) * - static_cast(wei(k, c0, y, x, c1)); - } - } - } - } - } - - v += bias(k0, k1); - v = activ(v, activ_type); - - const int hox2 = ho * 2; - const int wox2 = wo * 2; - - out_host(n, k0, ho, wo, k1) = v; - - add_host(n, k0, hox2, wox2, k1) = v + add(n, k0, hox2, wox2, k1); - add_host(n, k0, hox2, wox2 + 1, k1) = v + add(n, k0, hox2, wox2 + 1, k1); - add_host(n, k0, hox2 + 1, wox2, k1) = v + add(n, k0, hox2 + 1, wox2, k1); - add_host(n, k0, hox2 + 1, wox2 + 1, k1) = v + add(n, k0, hox2 + 1, wox2 + 1, k1); - }; - - make_ParallelTensorFunctor(f_nchw, - out_host.mDesc.GetLengths()[0], - out_host.mDesc.GetLengths()[1], - out_host.mDesc.GetLengths()[2], - out_host.mDesc.GetLengths()[3], - out_host.mDesc.GetLengths()[4])(std::thread::hardware_concurrency()); -} - -int main(int argc, char* argv[]) -{ - using namespace ck; - - constexpr auto I0 = Number<0>{}; - constexpr auto I1 = Number<1>{}; - constexpr auto I2 = Number<2>{}; - constexpr auto I3 = Number<3>{}; - constexpr auto I4 = Number<4>{}; - constexpr auto I5 = Number<5>{}; - constexpr auto I6 = Number<6>{}; - constexpr auto I7 = Number<7>{}; - -#if USE_DYNAMIC_MODE - // dynamic mode - if(argc != 23) - { - printf("arg1 to 5: algo, do_verification, init_method, do_log, nrepeat\n"); - printf("rest: N, K0, K1, C0, C1, Y, X, Hi, Wi, Sy, Sx, Dy, Dx, LeftPy, LeftPx, RightPy, " - "RightPx\n"); - exit(1); - } - - constexpr ck::ActivTypeEnum activ_type = ActivTypeEnum::LeakyRelu; - - const ConvForwardAlgo algo = static_cast(std::stoi(argv[1])); - const bool do_verification = std::stoi(argv[2]); - const int init_method = std::stoi(argv[3]); - const bool do_log = std::stoi(argv[4]); - const int nrepeat = std::stoi(argv[5]); - - const index_t N = std::stoi(argv[6]); - const index_t K0 = std::stoi(argv[7]); - const index_t K1 = std::stoi(argv[8]); - const index_t C0 = std::stoi(argv[9]); - const index_t C1 = std::stoi(argv[10]); - const index_t Y = std::stoi(argv[11]); - const index_t X = std::stoi(argv[12]); - const index_t Hi = std::stoi(argv[13]); - const index_t Wi = std::stoi(argv[14]); - - const index_t conv_stride_h = std::stoi(argv[15]); - const index_t conv_stride_w = std::stoi(argv[16]); - const index_t conv_dilation_h = std::stoi(argv[17]); - const index_t conv_dilation_w = std::stoi(argv[18]); - const index_t in_left_pad_h = std::stoi(argv[19]); - const index_t in_left_pad_w = std::stoi(argv[20]); - const index_t in_right_pad_h = std::stoi(argv[21]); - const index_t in_right_pad_w = std::stoi(argv[22]); - - const index_t YEff = (Y - 1) * conv_dilation_h + 1; - const index_t XEff = (X - 1) * conv_dilation_w + 1; - - const index_t Ho = (Hi + in_left_pad_h + in_right_pad_h - YEff) / conv_stride_h + 1; - const index_t Wo = (Wi + in_left_pad_w + in_right_pad_w - XEff) / conv_stride_w + 1; - - const auto Hox2 = Ho * 2; - const auto Wox2 = Wo * 2; -#else - // static mode - if(argc < 6) - { - printf("arg1 to 5: algo, do_verification, init_method, do_log, nrepeat\n"); - exit(1); - } - - const ConvForwardAlgo algo = static_cast(std::stoi(argv[1])); - - const bool do_verification = std::stoi(argv[2]); - const int init_method = std::stoi(argv[3]); - const bool do_log = std::stoi(argv[4]); - const int nrepeat = std::stoi(argv[5]); - - constexpr ck::ActivTypeEnum activ_type = ActivTypeEnum::LeakyRelu; - -#if 0 - constexpr auto N = Number<1>{}; - constexpr auto Hi = Number<1080>{}; - constexpr auto Wi = Number<1920>{}; - constexpr auto Y = Number<3>{}; - constexpr auto X = Number<3>{}; - constexpr auto C0 = Number<2>{}; - constexpr auto C1 = Number<8>{}; - constexpr auto K1 = Number<8>{}; - constexpr auto K0 = Number<8>{}; -#elif 0 - constexpr auto N = Number<1>{}; - constexpr auto Hi = Number<540>{}; - constexpr auto Wi = Number<960>{}; - constexpr auto Y = Number<3>{}; - constexpr auto X = Number<3>{}; - constexpr auto C0 = Number<2>{}; - constexpr auto C1 = Number<8>{}; - constexpr auto K0 = Number<2>{}; - constexpr auto K1 = Number<8>{}; -#elif 0 - constexpr auto N = Number<1>{}; - constexpr auto Hi = Number<270>{}; - constexpr auto Wi = Number<480>{}; - constexpr auto Y = Number<3>{}; - constexpr auto X = Number<3>{}; - constexpr auto C0 = Number<2>{}; - constexpr auto C1 = Number<8>{}; - constexpr auto K0 = Number<2>{}; - constexpr auto K1 = Number<8>{}; -#elif 1 - constexpr auto N = Number<128>{}; - constexpr auto Hi = Number<135>{}; - constexpr auto Wi = Number<240>{}; - constexpr auto Y = Number<3>{}; - constexpr auto X = Number<3>{}; - constexpr auto C0 = Number<2>{}; - constexpr auto C1 = Number<8>{}; - constexpr auto K0 = Number<2>{}; - constexpr auto K1 = Number<8>{}; -#elif 1 - constexpr auto N = Number<1>{}; - constexpr auto Hi = Number<32>{}; - constexpr auto Wi = Number<32>{}; - constexpr auto Y = Number<3>{}; - constexpr auto X = Number<3>{}; - constexpr auto C0 = Number<2>{}; - constexpr auto C1 = Number<8>{}; - constexpr auto K1 = Number<8>{}; - constexpr auto K0 = Number<8>{}; -#endif - - constexpr auto conv_stride_h = I1; - constexpr auto conv_stride_w = I1; - constexpr auto conv_dilation_h = I1; - constexpr auto conv_dilation_w = I1; - constexpr auto in_left_pad_h = I1; - constexpr auto in_left_pad_w = I1; - constexpr auto in_right_pad_h = I1; - constexpr auto in_right_pad_w = I1; - - constexpr auto YEff = (Y - I1) * conv_dilation_h + I1; - constexpr auto XEff = (X - I1) * conv_dilation_w + I1; - - constexpr auto Ho = (Hi + in_left_pad_h + in_right_pad_h - YEff) / conv_stride_h + I1; - constexpr auto Wo = (Wi + in_left_pad_w + in_right_pad_w - XEff) / conv_stride_w + I1; - - constexpr auto Hox2 = Number{}; - constexpr auto Wox2 = Number{}; - -#endif - -#if 0 - using in_data_t = float; - using acc_data_t = float; - using out_data_t = float; -#elif 1 - using in_data_t = half_t; - using acc_data_t = float; - using out_data_t = half_t; -#elif 1 - using in_data_t = int8_t; - using acc_data_t = int32_t; - using out_data_t = int8_t; -#endif - - std::vector in_lengths_host(5), wei_lengths_host(5), out_lengths_host(5), - add_lengths_host(5), bias_lengths_host(2); - - in_lengths_host[0] = static_cast(N); - in_lengths_host[1] = static_cast(C0); - in_lengths_host[2] = static_cast(Hi); - in_lengths_host[3] = static_cast(Wi); - in_lengths_host[4] = static_cast(C1); - - wei_lengths_host[0] = static_cast(K0 * K1); - wei_lengths_host[1] = static_cast(C0); - wei_lengths_host[2] = static_cast(Y); - wei_lengths_host[3] = static_cast(X); - wei_lengths_host[4] = static_cast(C1); - - out_lengths_host[0] = static_cast(N); - out_lengths_host[1] = static_cast(K0); - out_lengths_host[2] = static_cast(Ho); - out_lengths_host[3] = static_cast(Wo); - out_lengths_host[4] = static_cast(K1); - - add_lengths_host[0] = static_cast(N); - add_lengths_host[1] = static_cast(K0); - add_lengths_host[2] = static_cast(Hox2); - add_lengths_host[3] = static_cast(Wox2); - add_lengths_host[4] = static_cast(K1); - - bias_lengths_host[0] = static_cast(K0); - bias_lengths_host[1] = static_cast(K1); - - Tensor in(in_lengths_host); - Tensor wei(wei_lengths_host); - Tensor add(add_lengths_host); - Tensor add_device(add_lengths_host); - Tensor add_host(add_lengths_host); - Tensor bias(bias_lengths_host); - Tensor out_host(out_lengths_host); - - ostream_HostTensorDescriptor(in.mDesc, std::cout << "in: "); - ostream_HostTensorDescriptor(wei.mDesc, std::cout << "wei: "); - ostream_HostTensorDescriptor(add.mDesc, std::cout << "add: "); - - print_array("InLeftPads", make_tuple(in_left_pad_h, in_left_pad_w)); - print_array("InRightPads", make_tuple(in_right_pad_h, in_right_pad_w)); - print_array("ConvStrides", make_tuple(conv_stride_h, conv_stride_w)); - print_array("ConvDilations", make_tuple(conv_dilation_h, conv_dilation_w)); - - std::size_t num_thread = 1; - - switch(init_method) - { - case 0: - // no initialization - break; - case 1: - in.GenerateTensorValue(GeneratorTensor_1{}, num_thread); - wei.GenerateTensorValue(GeneratorTensor_1{}, num_thread); - break; - case 2: - in.GenerateTensorValue(GeneratorTensor_1{}, num_thread); - wei.GenerateTensorValue(GeneratorTensor_2{-5, 5}, num_thread); - break; - case 3: - in.GenerateTensorValue(GeneratorTensor_2{-5, 5}, num_thread); - wei.GenerateTensorValue(GeneratorTensor_1{}, num_thread); - break; - case 4: - in.GenerateTensorValue(GeneratorTensor_2{-5, 5}, num_thread); - wei.GenerateTensorValue(GeneratorTensor_2{-5, 5}, num_thread); - break; - case 5: - in.GenerateTensorValue(GeneratorTensor_3{0.0, 1.0}, num_thread); - wei.GenerateTensorValue(GeneratorTensor_3{-0.5, 0.5}, num_thread); - break; - default: - in.GenerateTensorValue(GeneratorTensor_2{1, 5}, num_thread); - - auto gen_wei = [](auto... is) { - return GeneratorTensor_2{1, 5}(is...) * GeneratorTensor_Checkboard{}(is...); - }; - wei.GenerateTensorValue(gen_wei, num_thread); - } - - bias.GenerateTensorValue(GeneratorTensor_1{}, num_thread); - add.GenerateTensorValue(GeneratorTensor_1{}, num_thread); - - auto f_make_for_device_nchwc = [&]() { - const auto in_lengths_dev = make_tuple(N, C0, Hi, Wi, C1); - const auto wei_lengths_dev = make_tuple(K0 * K1, C0, Y, X, C1); - const auto add_lengths_dev = make_tuple(N, K0, Hox2, Wox2, K1); - const auto out_lengths_dev = make_tuple(N, K0, Ho, Wo, K1); - const auto conv_strides_dev = make_tuple(conv_stride_h, conv_stride_w); - const auto conv_dilations_dev = make_tuple(conv_dilation_h, conv_dilation_w); - const auto in_left_pads_dev = make_tuple(in_left_pad_h, in_left_pad_w); - const auto in_right_pads_dev = make_tuple(in_right_pad_h, in_right_pad_w); - - return make_tuple(in_lengths_dev, - wei_lengths_dev, - add_lengths_dev, - out_lengths_dev, - conv_strides_dev, - conv_dilations_dev, - in_left_pads_dev, - in_right_pads_dev); - }; - -#if USE_CONV_FWD_V5R1_NCHWC - if(algo == ConvForwardAlgo::V5R1NCHWC) - { - const auto tmp = f_make_for_device_nchwc(); - - device_convolution_add_forward_implicit_gemm_v5r1_dlops_nc0hwc1_kc0yxc1_nk0hwk1( - tmp[I0], // in_lengths_dev - tmp[I1], // wei_lengths_dev - tmp[I2], // add_lengths_dev - tmp[I3], // out_lengths_dev - tmp[I4], // conv_strides_dev - tmp[I5], // conv_dilations_dev - tmp[I6], // in_left_pads_dev - tmp[I7], // in_right_pads_dev - in, - wei, - bias, - add, - add_device, - nrepeat); - } -#endif - - if(do_verification) - { - host_direct_convolution_add_nchwc(in, - wei, - add, - bias, - add_host, - out_host, - make_tuple(conv_stride_h, conv_stride_w), - make_tuple(conv_dilation_h, conv_dilation_w), - make_tuple(in_left_pad_h, in_left_pad_w), - make_tuple(in_right_pad_h, in_right_pad_w), - activ_type); - - ck::utils::check_err(add_device.mData, add_host.mData); - - if(do_log) - { - LogRangeAsType(std::cout << "in : ", in.mData, ",") << std::endl; - LogRangeAsType(std::cout << "wei: ", wei.mData, ",") << std::endl; - LogRangeAsType(std::cout << "add_host: ", add_host.mData, ",") << std::endl; - LogRangeAsType(std::cout << "add_device: ", add_device.mData, ",") << std::endl; - } - } -} diff --git a/library/src/obselete_driver_offline/conv_bwd_driver_offline.cpp b/library/src/obselete_driver_offline/conv_bwd_driver_offline.cpp deleted file mode 100644 index c4dcb7c085..0000000000 --- a/library/src/obselete_driver_offline/conv_bwd_driver_offline.cpp +++ /dev/null @@ -1,488 +0,0 @@ -#include -#include -#include -#include -#include -#include - -#include "check_err.hpp" -#include "config.hpp" -#include "debug.hpp" -#include "print.hpp" -#include "device.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" -#include "conv_common.hpp" -#include "device_tensor.hpp" -#include "device_convolution_backward_data_implicit_gemm_v4r1_xdlops_nhwc_kyxc_nhwk.hpp" -#include "device_convolution_backward_data_implicit_gemm_v4r1r2_xdlops_nhwc_kyxc_nhwk.hpp" -#include "device_convolution_backward_data_implicit_gemm_v4r1r2_xdlops_nhwc_kyxc_nhwk_1x1.hpp" - -#define USE_MODE 1 -#define USE_CONV_BWD_V4R1_XDL_NHWC 0 -#define USE_CONV_BWD_V4R1R2_XDL_NHWC 1 - -enum ConvTensorLayout -{ - NCHW, - NHWC, - CHWN, - NCHWc, - NHWCc -}; - -enum ConvBackwardDataAlgo -{ - V4R1XDLNHWC, // 0 - V4R1R2XDLNHWC, // 1 -}; - -template -void host_convolution_backward_data(Tensor& in, - const Tensor& wei, - const Tensor& out, - const ConvStrides& conv_strides, - const ConvDilations& conv_dilations, - const InLeftPads& in_left_pads, - const InRightPads& /* in_right_pads */, - const ConvTensorLayout layout = ConvTensorLayout::NCHW) -{ - using namespace ck; - - constexpr auto I0 = Number<0>{}; - constexpr auto I1 = Number<1>{}; - constexpr auto I2 = Number<2>{}; - constexpr auto I3 = Number<3>{}; - - auto f_nchw = [&](auto n, auto c, auto hi, auto wi) { - std::size_t K = wei.mDesc.GetLengths()[I0]; - std::size_t Y = wei.mDesc.GetLengths()[I2]; - std::size_t X = wei.mDesc.GetLengths()[I3]; - - std::size_t Ho = out.mDesc.GetLengths()[I2]; - std::size_t Wo = out.mDesc.GetLengths()[I3]; - - double v = 0; - - for(int y = 0; y < Y; ++y) - { - int h_tmp = hi + in_left_pads[I0] - y * conv_dilations[I0]; - - if(h_tmp % conv_strides[I0] == 0) - { - int ho = h_tmp / conv_strides[I0]; - - if(ho >= 0 && ho < Ho) - { - for(int x = 0; x < X; ++x) - { - int w_tmp = wi + in_left_pads[I1] - x * conv_dilations[I1]; - - if(w_tmp % conv_strides[I1] == 0) - { - int wo = w_tmp / conv_strides[I1]; - - if(wo >= 0 && wo < Wo) - { - for(int k = 0; k < K; ++k) - { - v += out(n, k, ho, wo) * wei(k, c, y, x); - } - } - } - } - } - } - } - - in(n, c, hi, wi) = v; - }; - - auto f_nhwc = [&](auto n, auto hi, auto wi, auto c) { - std::size_t K = wei.mDesc.GetLengths()[I0]; - std::size_t Y = wei.mDesc.GetLengths()[I1]; - std::size_t X = wei.mDesc.GetLengths()[I2]; - - std::size_t Ho = out.mDesc.GetLengths()[I1]; - std::size_t Wo = out.mDesc.GetLengths()[I2]; - - double v = 0; - - for(int y = 0; y < Y; ++y) - { - int h_tmp = hi + in_left_pads[I0] - y * conv_dilations[I0]; - - if(h_tmp % conv_strides[I0] == 0) - { - int ho = h_tmp / conv_strides[I0]; - - if(ho >= 0 && ho < Ho) - { - for(int x = 0; x < X; ++x) - { - int w_tmp = wi + in_left_pads[I1] - x * conv_dilations[I1]; - - if(w_tmp % conv_strides[I1] == 0) - { - int wo = w_tmp / conv_strides[I1]; - - if(wo >= 0 && wo < Wo) - { - for(int k = 0; k < K; ++k) - { - v += out(n, ho, wo, k) * wei(k, y, x, c); - } - } - } - } - } - } - } - - in(n, hi, wi, c) = v; - }; - - if(layout == ConvTensorLayout::NCHW) - { - make_ParallelTensorFunctor(f_nchw, - in.mDesc.GetLengths()[0], - in.mDesc.GetLengths()[1], - in.mDesc.GetLengths()[2], - in.mDesc.GetLengths()[3])(std::thread::hardware_concurrency()); - } - else if(layout == ConvTensorLayout::NHWC) - { - make_ParallelTensorFunctor(f_nhwc, - in.mDesc.GetLengths()[0], - in.mDesc.GetLengths()[1], - in.mDesc.GetLengths()[2], - in.mDesc.GetLengths()[3])(std::thread::hardware_concurrency()); - } - else - { - throw std::runtime_error("wrong! not supported layout"); - } -} -int main(int argc, char* argv[]) -{ - using namespace ck; - - constexpr auto I0 = Number<0>{}; - constexpr auto I1 = Number<1>{}; - constexpr auto I2 = Number<2>{}; - constexpr auto I3 = Number<3>{}; - constexpr auto I4 = Number<4>{}; - constexpr auto I5 = Number<5>{}; - constexpr auto I6 = Number<6>{}; - -#if USE_MODE - // dynamic mode - if(argc != 22) - { - printf("arg1 to 6: layout, algo, do_verification, init_method, do_log, nrepeat\n"); - printf("rest: N, K, C, Y, X, Hi, Wi, Sy, Sx, Dy, Dx, LeftPy, LeftPx, RightPy, RightPx\n"); - exit(1); - } - - const ConvTensorLayout layout = static_cast(std::stoi(argv[1])); - const ConvBackwardDataAlgo algo = static_cast(std::stoi(argv[2])); - const bool do_verification = std::stoi(argv[3]); - const int init_method = std::stoi(argv[4]); - const bool do_log = std::stoi(argv[5]); - const int nrepeat = std::stoi(argv[6]); - - const index_t N = std::stoi(argv[7]); - const index_t K = std::stoi(argv[8]); - const index_t C = std::stoi(argv[9]); - const index_t Y = std::stoi(argv[10]); - const index_t X = std::stoi(argv[11]); - const index_t Hi = std::stoi(argv[12]); - const index_t Wi = std::stoi(argv[13]); - - const index_t conv_stride_h = std::stoi(argv[14]); - const index_t conv_stride_w = std::stoi(argv[15]); - const index_t conv_dilation_h = std::stoi(argv[16]); - const index_t conv_dilation_w = std::stoi(argv[17]); - const index_t in_left_pad_h = std::stoi(argv[18]); - const index_t in_left_pad_w = std::stoi(argv[19]); - const index_t in_right_pad_h = std::stoi(argv[20]); - const index_t in_right_pad_w = std::stoi(argv[21]); - - const index_t YEff = (Y - 1) * conv_dilation_h + 1; - const index_t XEff = (X - 1) * conv_dilation_w + 1; - - const index_t Ho = (Hi + in_left_pad_h + in_right_pad_h - YEff) / conv_stride_h + 1; - const index_t Wo = (Wi + in_left_pad_w + in_right_pad_w - XEff) / conv_stride_w + 1; -#else - // static mode - if(argc < 7) - { - printf("arg1 to 6: layout, algo, do_verification, init_method, do_log, nrepeat\n"); - exit(1); - } - - const ConvTensorLayout layout = static_cast(std::stoi(argv[1])); - const ConvBackwardDataAlgo algo = static_cast(std::stoi(argv[2])); - const bool do_verification = std::stoi(argv[3]); - const int init_method = std::stoi(argv[4]); - const bool do_log = std::stoi(argv[5]); - const int nrepeat = std::stoi(argv[6]); - - constexpr auto N = Number<128>{}; - constexpr auto C = Number<192>{}; - constexpr auto Hi = Number<71>{}; - constexpr auto Wi = Number<71>{}; - constexpr auto K = Number<256>{}; - constexpr auto Y = Number<3>{}; - constexpr auto X = Number<3>{}; - - constexpr auto conv_stride_h = I2; - constexpr auto conv_stride_w = I2; - constexpr auto conv_dilation_h = I1; - constexpr auto conv_dilation_w = I1; - constexpr auto in_left_pad_h = I1; - constexpr auto in_left_pad_w = I1; - constexpr auto in_right_pad_h = I1; - constexpr auto in_right_pad_w = I1; - - constexpr auto YEff = (Y - I1) * conv_dilation_h + I1; - constexpr auto XEff = (X - I1) * conv_dilation_w + I1; - - constexpr auto Ho = (Hi + in_left_pad_h + in_right_pad_h - YEff) / conv_stride_h + I1; - constexpr auto Wo = (Wi + in_left_pad_w + in_right_pad_w - XEff) / conv_stride_w + I1; -#endif - -#if 0 - using in_data_t = float; - using acc_data_t = float; - using out_data_t = float; -#elif 1 - using in_data_t = half_t; - using acc_data_t = float; - using out_data_t = half_t; -#endif - - std::vector in_lengths_host(4), wei_lengths_host(4), out_lengths_host(4); - - if(layout == ConvTensorLayout::NCHW) - { - in_lengths_host[0] = static_cast(N); - in_lengths_host[1] = static_cast(C); - in_lengths_host[2] = static_cast(Hi); - in_lengths_host[3] = static_cast(Wi); - wei_lengths_host[0] = static_cast(K); - wei_lengths_host[1] = static_cast(C); - wei_lengths_host[2] = static_cast(Y); - wei_lengths_host[3] = static_cast(X); - out_lengths_host[0] = static_cast(N); - out_lengths_host[1] = static_cast(K); - out_lengths_host[2] = static_cast(Ho); - out_lengths_host[3] = static_cast(Wo); - } - else if(layout == ConvTensorLayout::NHWC) - { - in_lengths_host[0] = static_cast(N); - in_lengths_host[1] = static_cast(Hi); - in_lengths_host[2] = static_cast(Wi); - in_lengths_host[3] = static_cast(C); - wei_lengths_host[0] = static_cast(K); - wei_lengths_host[1] = static_cast(Y); - wei_lengths_host[2] = static_cast(X); - wei_lengths_host[3] = static_cast(C); - out_lengths_host[0] = static_cast(N); - out_lengths_host[1] = static_cast(Ho); - out_lengths_host[2] = static_cast(Wo); - out_lengths_host[3] = static_cast(K); - } - else - { - throw std::runtime_error("wrong! not implemented"); - } - - Tensor in_host(in_lengths_host); - Tensor in_device(in_lengths_host); - Tensor wei(wei_lengths_host); - Tensor out(out_lengths_host); - - std::cout << "layout: " << layout << std::endl; - ostream_HostTensorDescriptor(in_host.mDesc, std::cout << "in: "); - ostream_HostTensorDescriptor(wei.mDesc, std::cout << "wei: "); - ostream_HostTensorDescriptor(out.mDesc, std::cout << "out: "); - print_array("InLeftPads", make_tuple(in_left_pad_h, in_left_pad_w)); - print_array("InRightPads", make_tuple(in_right_pad_h, in_right_pad_w)); - print_array("ConvStrides", make_tuple(conv_stride_h, conv_stride_w)); - print_array("ConvDilations", make_tuple(conv_dilation_h, conv_dilation_w)); - - std::size_t num_thread = 1; - - switch(init_method) - { - case 0: - // no initialization - break; - case 1: - out.GenerateTensorValue(GeneratorTensor_1{}, num_thread); - wei.GenerateTensorValue(GeneratorTensor_1{}, num_thread); - break; - case 2: - out.GenerateTensorValue(GeneratorTensor_1{}, num_thread); - wei.GenerateTensorValue(GeneratorTensor_2{-5, 5}, num_thread); - break; - case 3: - out.GenerateTensorValue(GeneratorTensor_2{-5, 5}, num_thread); - wei.GenerateTensorValue(GeneratorTensor_1{}, num_thread); - break; - case 4: - out.GenerateTensorValue(GeneratorTensor_2{-5, 5}, num_thread); - wei.GenerateTensorValue(GeneratorTensor_2{-5, 5}, num_thread); - break; - case 5: - out.GenerateTensorValue(GeneratorTensor_3{0.0, 1.0}, num_thread); - wei.GenerateTensorValue(GeneratorTensor_3{-0.5, 0.5}, num_thread); - break; - default: - out.GenerateTensorValue(GeneratorTensor_2{1, 5}, num_thread); - - auto gen_wei = [](auto... is) { - return GeneratorTensor_2{1, 5}(is...) * GeneratorTensor_Checkboard{}(is...); - }; - wei.GenerateTensorValue(gen_wei, num_thread); - } - - auto f_make_for_device_nhwc = [&]() { -#if USE_MODE - const auto in_lengths_dev = make_tuple(N, Hi, Wi, C); - const auto wei_lengths_dev = make_tuple(K, Y, X, C); - const auto out_lengths_dev = make_tuple(N, Ho, Wo, K); - const auto conv_strides_dev = make_tuple(conv_stride_h, conv_stride_w); - const auto conv_dilations_dev = make_tuple(conv_dilation_h, conv_dilation_w); - const auto in_left_pads_dev = make_tuple(in_left_pad_h, in_left_pad_w); - const auto in_right_pads_dev = make_tuple(in_right_pad_h, in_right_pad_w); -#else - const auto in_lengths_dev = - make_tuple(Number{}, Number{}, Number{}, Number{}); - const auto wei_lengths_dev = make_tuple(Number{}, Number{}, Number{}, Number{}); - const auto out_lengths_dev = - make_tuple(Number{}, Number{}, Number{}, Number{}); - const auto conv_strides_dev = make_tuple(Number{}, Number{}); - const auto conv_dilations_dev = - make_tuple(Number{}, Number{}); - const auto in_left_pads_dev = make_tuple(Number{}, Number{}); - const auto in_right_pads_dev = - make_tuple(Number{}, Number{}); -#endif - - return make_tuple(in_lengths_dev, - wei_lengths_dev, - out_lengths_dev, - conv_strides_dev, - conv_dilations_dev, - in_left_pads_dev, - in_right_pads_dev); - }; - -#if USE_CONV_BWD_V4R1_XDL_NHWC - if(algo == ConvBackwardDataAlgo::V4R1XDLNHWC) - { - if(layout != ConvTensorLayout::NHWC) - { - throw std::runtime_error("wrong! layout"); - } - - const auto tmp = f_make_for_device_nhwc(); - - device_convolution_backward_data_implicit_gemm_v4r1_xdlops_nhwc_kyxc_nhwk( - tmp[I0], - tmp[I1], - tmp[I2], - tmp[I3], - tmp[I4], - tmp[I5], - tmp[I6], - in_device, - wei, - out, - nrepeat); - } -#endif - -#if USE_CONV_BWD_V4R1R2_XDL_NHWC - if(algo == ConvBackwardDataAlgo::V4R1R2XDLNHWC) - { - if(layout != ConvTensorLayout::NHWC) - { - throw std::runtime_error("wrong! layout"); - } - - const auto tmp = f_make_for_device_nhwc(); - - if(Y == 1 && X == 1 && in_left_pad_h == 0 && in_left_pad_w == 0 && in_right_pad_h == 0 && - in_right_pad_w == 0) - { - device_convolution_backward_data_implicit_gemm_v4r1r2_xdlops_nhwc_kyxc_nhwk_1x1< - in_data_t, - acc_data_t, - out_data_t>(tmp[I0], - tmp[I1], - tmp[I2], - tmp[I3], - tmp[I4], - tmp[I5], - tmp[I6], - in_device, - wei, - out, - nrepeat); - } - else - { -#if 1 - device_convolution_backward_data_implicit_gemm_v4r1r2_xdlops_nhwc_kyxc_nhwk( - tmp[I0], - tmp[I1], - tmp[I2], - tmp[I3], - tmp[I4], - tmp[I5], - tmp[I6], - in_device, - wei, - out, - nrepeat); -#endif - } - } -#endif - - if(do_verification) - { - host_convolution_backward_data(in_host, - wei, - out, - make_tuple(conv_stride_h, conv_stride_w), - make_tuple(conv_dilation_h, conv_dilation_w), - make_tuple(in_left_pad_h, in_left_pad_w), - make_tuple(in_right_pad_h, in_right_pad_w), - layout); - - ck::utils::check_err(in_device.mData, in_host.mData); - - if(do_log) - { - LogRangeAsType(std::cout << "out : ", out.mData, ",") << std::endl; - LogRangeAsType(std::cout << "wei: ", wei.mData, ",") << std::endl; - LogRangeAsType(std::cout << "in_host : ", in_host.mData, ",") << std::endl; - LogRangeAsType(std::cout << "in_device: ", in_device.mData, ",") << std::endl; - } - } -} diff --git a/library/src/obselete_driver_offline/conv_fwd_driver_offline.cpp b/library/src/obselete_driver_offline/conv_fwd_driver_offline.cpp deleted file mode 100644 index ab8beec87b..0000000000 --- a/library/src/obselete_driver_offline/conv_fwd_driver_offline.cpp +++ /dev/null @@ -1,549 +0,0 @@ -#include -#include -#include -#include -#include -#include - -#include "check_err.hpp" -#include "config.hpp" -#include "debug.hpp" -#include "print.hpp" -#include "device.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" -#include "conv_common.hpp" -#include "device_tensor.hpp" -#include "device_convolution_forward_implicit_gemm_v4r4_dlops_nchw_kcyx_nkhw.hpp" -#include "device_convolution_forward_implicit_gemm_v4r4r2_dlops_nhwc_kyxc_nhwk.hpp" -#include "device_convolution_forward_implicit_gemm_v6r1_dlops_nchw_kcyx_nkhw.hpp" -#include "device_convolution_forward_implicit_gemm_v4r4r2_xdlops_nchw_kcyx_nkhw.hpp" -#include "device_convolution_forward_implicit_gemm_v4r4r4_xdlops_nhwc_kyxc_nhwk.hpp" - -#define USE_DYNAMIC_MODE 1 -#define USE_CONV_FWD_V4R4_NCHW 0 -#define USE_CONV_FWD_V4R4R2_NHWC 0 -#define USE_CONV_FWD_V6R1_NCHW 0 -#define USE_CONV_FWD_V4R4R2_XDL_NCHW 0 -#define USE_CONV_FWD_V4R4R4_XDL_NHWC 1 - -enum ConvTensorLayout -{ - NCHW, - NHWC, - CHWN, - NCHWc, - NHWCc -}; - -enum ConvForwardAlgo -{ - V4R4NCHW, // 0 - V4R4R2NHWC, // 1 - V6R1NCHW, // 2 - V4R4R2XDLNCHW, // 3 - V4R4R4XDLNHWC // 4 -}; - -template -void host_convolution_forward(const Tensor& in, - const Tensor& wei, - Tensor& out, - const ConvStrides& conv_strides, - const ConvDilations& conv_dilations, - const InLeftPads& in_left_pads, - const InRightPads&, - const ConvTensorLayout layout = ConvTensorLayout::NCHW) -{ - using namespace ck; - - constexpr auto I0 = Number<0>{}; - constexpr auto I1 = Number<1>{}; - - auto f_nchw = [&](auto n, auto k, auto ho, auto wo) { - double v = 0; - for(int c = 0; c < wei.mDesc.GetLengths()[1]; ++c) - { - for(int y = 0; y < wei.mDesc.GetLengths()[2]; ++y) - { - int hi = ho * conv_strides[I0] + y * conv_dilations[I0] - in_left_pads[I0]; - for(int x = 0; x < wei.mDesc.GetLengths()[3]; ++x) - { - int wi = wo * conv_strides[I1] + x * conv_dilations[I1] - in_left_pads[I1]; - if(hi >= 0 && hi < in.mDesc.GetLengths()[2] && wi >= 0 && - wi < in.mDesc.GetLengths()[3]) - { - if constexpr(is_same::value) - { - v += ck::type_convert(in(n, c, hi, wi)) * - ck::type_convert(wei(k, c, y, x)); - } - else - { - v += static_cast(in(n, c, hi, wi)) * - static_cast(wei(k, c, y, x)); - } - } - } - } - } - - if constexpr(is_same::value) - { - out(n, k, ho, wo) = ck::type_convert(static_cast(v)); - } - else - { - out(n, k, ho, wo) = v; - } - }; - - auto f_nhwc = [&](auto n, auto ho, auto wo, auto k) { - double v = 0; - for(int c = 0; c < wei.mDesc.GetLengths()[3]; ++c) - { - for(int y = 0; y < wei.mDesc.GetLengths()[1]; ++y) - { - int hi = ho * conv_strides[I0] + y * conv_dilations[I0] - in_left_pads[I0]; - for(int x = 0; x < wei.mDesc.GetLengths()[2]; ++x) - { - int wi = wo * conv_strides[I1] + x * conv_dilations[I1] - in_left_pads[I1]; - if(hi >= 0 && hi < in.mDesc.GetLengths()[1] && wi >= 0 && - wi < in.mDesc.GetLengths()[2]) - { - if constexpr(is_same::value) - { - v += ck::type_convert(in(n, hi, wi, c)) * - ck::type_convert(wei(k, y, x, c)); - } - else - { - v += static_cast(in(n, hi, wi, c)) * - static_cast(wei(k, y, x, c)); - } - } - } - } - } - if constexpr(is_same::value) - { - out(n, ho, wo, k) = ck::type_convert(static_cast(v)); - } - else - { - out(n, ho, wo, k) = v; - } - }; - - if(layout == ConvTensorLayout::NCHW) - { - make_ParallelTensorFunctor(f_nchw, - out.mDesc.GetLengths()[0], - out.mDesc.GetLengths()[1], - out.mDesc.GetLengths()[2], - out.mDesc.GetLengths()[3])(std::thread::hardware_concurrency()); - } - else if(layout == ConvTensorLayout::NHWC) - { - make_ParallelTensorFunctor(f_nhwc, - out.mDesc.GetLengths()[0], - out.mDesc.GetLengths()[1], - out.mDesc.GetLengths()[2], - out.mDesc.GetLengths()[3])(std::thread::hardware_concurrency()); - } - else - { - throw std::runtime_error("wrong! not supported layout"); - } -} - -int main(int argc, char* argv[]) -{ - using namespace ck; - - constexpr auto I0 = Number<0>{}; - constexpr auto I1 = Number<1>{}; - constexpr auto I2 = Number<2>{}; - constexpr auto I3 = Number<3>{}; - constexpr auto I4 = Number<4>{}; - constexpr auto I5 = Number<5>{}; - constexpr auto I6 = Number<6>{}; - -#if USE_DYNAMIC_MODE - // dynamic mode - if(argc != 22) - { - printf("arg1 to 6: layout, algo, do_verification, init_method, do_log, nrepeat\n"); - printf("rest: N, K, C, Y, X, Hi, Wi, Sy, Sx, Dy, Dx, LeftPy, LeftPx, RightPy, RightPx\n"); - exit(1); - } - - const ConvTensorLayout layout = static_cast(std::stoi(argv[1])); - const ConvForwardAlgo algo = static_cast(std::stoi(argv[2])); - const bool do_verification = std::stoi(argv[3]); - const int init_method = std::stoi(argv[4]); - const bool do_log = std::stoi(argv[5]); - const int nrepeat = std::stoi(argv[6]); - - const index_t N = std::stoi(argv[7]); - const index_t K = std::stoi(argv[8]); - const index_t C = std::stoi(argv[9]); - const index_t Y = std::stoi(argv[10]); - const index_t X = std::stoi(argv[11]); - const index_t Hi = std::stoi(argv[12]); - const index_t Wi = std::stoi(argv[13]); - - const index_t conv_stride_h = std::stoi(argv[14]); - const index_t conv_stride_w = std::stoi(argv[15]); - const index_t conv_dilation_h = std::stoi(argv[16]); - const index_t conv_dilation_w = std::stoi(argv[17]); - const index_t in_left_pad_h = std::stoi(argv[18]); - const index_t in_left_pad_w = std::stoi(argv[19]); - const index_t in_right_pad_h = std::stoi(argv[20]); - const index_t in_right_pad_w = std::stoi(argv[21]); - - const index_t YEff = (Y - 1) * conv_dilation_h + 1; - const index_t XEff = (X - 1) * conv_dilation_w + 1; - - const index_t Ho = (Hi + in_left_pad_h + in_right_pad_h - YEff) / conv_stride_h + 1; - const index_t Wo = (Wi + in_left_pad_w + in_right_pad_w - XEff) / conv_stride_w + 1; -#else - // static mode - if(argc < 7) - { - printf("arg1 to 6: layout, algo, do_verification, init_method, do_log, nrepeat\n"); - exit(1); - } - - const ConvTensorLayout layout = static_cast(std::stoi(argv[1])); - const ConvForwardAlgo algo = static_cast(std::stoi(argv[2])); - const bool do_verification = std::stoi(argv[3]); - const int init_method = std::stoi(argv[4]); - const bool do_log = std::stoi(argv[5]); - const int nrepeat = std::stoi(argv[6]); - - constexpr auto N = Number<128>{}; - constexpr auto C = Number<192>{}; - constexpr auto Hi = Number<71>{}; - constexpr auto Wi = Number<71>{}; - constexpr auto K = Number<256>{}; - constexpr auto Y = Number<3>{}; - constexpr auto X = Number<3>{}; - - constexpr auto conv_stride_h = I1; - constexpr auto conv_stride_w = I1; - constexpr auto conv_dilation_h = I1; - constexpr auto conv_dilation_w = I1; - constexpr auto in_left_pad_h = I1; - constexpr auto in_left_pad_w = I1; - constexpr auto in_right_pad_h = I1; - constexpr auto in_right_pad_w = I1; - - constexpr auto YEff = (Y - I1) * conv_dilation_h + I1; - constexpr auto XEff = (X - I1) * conv_dilation_w + I1; - - constexpr auto Ho = (Hi + in_left_pad_h + in_right_pad_h - YEff) / conv_stride_h + I1; - constexpr auto Wo = (Wi + in_left_pad_w + in_right_pad_w - XEff) / conv_stride_w + I1; -#endif - -#if 1 - using in_data_t = float; - using acc_data_t = float; - using out_data_t = float; -#elif 1 - using in_data_t = half_t; - using acc_data_t = float; - using out_data_t = half_t; -#elif 0 - using in_data_t = bhalf_t; - using acc_data_t = float; - using out_data_t = bhalf_t; -#elif 1 - using in_data_t = int8_t; - using acc_data_t = int32_t; - using out_data_t = int8_t; -#endif - - std::vector in_lengths_host(4), wei_lengths_host(4), out_lengths_host(4); - - if(layout == ConvTensorLayout::NCHW) - { - in_lengths_host[0] = static_cast(N); - in_lengths_host[1] = static_cast(C); - in_lengths_host[2] = static_cast(Hi); - in_lengths_host[3] = static_cast(Wi); - wei_lengths_host[0] = static_cast(K); - wei_lengths_host[1] = static_cast(C); - wei_lengths_host[2] = static_cast(Y); - wei_lengths_host[3] = static_cast(X); - out_lengths_host[0] = static_cast(N); - out_lengths_host[1] = static_cast(K); - out_lengths_host[2] = static_cast(Ho); - out_lengths_host[3] = static_cast(Wo); - } - else if(layout == ConvTensorLayout::NHWC) - { - in_lengths_host[0] = static_cast(N); - in_lengths_host[1] = static_cast(Hi); - in_lengths_host[2] = static_cast(Wi); - in_lengths_host[3] = static_cast(C); - wei_lengths_host[0] = static_cast(K); - wei_lengths_host[1] = static_cast(Y); - wei_lengths_host[2] = static_cast(X); - wei_lengths_host[3] = static_cast(C); - out_lengths_host[0] = static_cast(N); - out_lengths_host[1] = static_cast(Ho); - out_lengths_host[2] = static_cast(Wo); - out_lengths_host[3] = static_cast(K); - } - else - { - std::runtime_error("wrong! not implemented"); - } - - Tensor in(in_lengths_host); - Tensor wei(wei_lengths_host); - Tensor out_host(out_lengths_host); - Tensor out_device(out_lengths_host); - - std::cout << "layout: " << layout << std::endl; - ostream_HostTensorDescriptor(in.mDesc, std::cout << "in: "); - ostream_HostTensorDescriptor(wei.mDesc, std::cout << "wei: "); - ostream_HostTensorDescriptor(out_host.mDesc, std::cout << "out: "); - print_array("InLeftPads", make_tuple(in_left_pad_h, in_left_pad_w)); - print_array("InRightPads", make_tuple(in_right_pad_h, in_right_pad_w)); - print_array("ConvStrides", make_tuple(conv_stride_h, conv_stride_w)); - print_array("ConvDilations", make_tuple(conv_dilation_h, conv_dilation_w)); - - std::size_t num_thread = 1; - - switch(init_method) - { - case 0: - // no initialization - break; - case 1: - in.GenerateTensorValue(GeneratorTensor_1{}, num_thread); - wei.GenerateTensorValue(GeneratorTensor_1{}, num_thread); - break; - case 2: - in.GenerateTensorValue(GeneratorTensor_1{}, num_thread); - wei.GenerateTensorValue(GeneratorTensor_2{-5, 5}, num_thread); - break; - case 3: - in.GenerateTensorValue(GeneratorTensor_2{-5, 5}, num_thread); - wei.GenerateTensorValue(GeneratorTensor_1{}, num_thread); - break; - case 4: - in.GenerateTensorValue(GeneratorTensor_2{-5, 5}, num_thread); - wei.GenerateTensorValue(GeneratorTensor_2{-5, 5}, num_thread); - break; - case 5: - in.GenerateTensorValue(GeneratorTensor_3{0.0, 1.0}, num_thread); - wei.GenerateTensorValue(GeneratorTensor_3{-0.5, 0.5}, num_thread); - break; - default: - in.GenerateTensorValue(GeneratorTensor_2{1, 5}, num_thread); - - auto gen_wei = [](auto... is) { - return GeneratorTensor_2{1, 5}(is...) * GeneratorTensor_Checkboard{}(is...); - }; - wei.GenerateTensorValue(gen_wei, num_thread); - } - - auto f_make_for_device_nchw = [&]() { - const auto in_lengths_dev = make_tuple(N, C, Hi, Wi); - const auto wei_lengths_dev = make_tuple(K, C, Y, X); - const auto out_lengths_dev = make_tuple(N, K, Ho, Wo); - const auto conv_strides_dev = make_tuple(conv_stride_h, conv_stride_w); - const auto conv_dilations_dev = make_tuple(conv_dilation_h, conv_dilation_w); - const auto in_left_pads_dev = make_tuple(in_left_pad_h, in_left_pad_w); - const auto in_right_pads_dev = make_tuple(in_right_pad_h, in_right_pad_w); - - return make_tuple(in_lengths_dev, - wei_lengths_dev, - out_lengths_dev, - conv_strides_dev, - conv_dilations_dev, - in_left_pads_dev, - in_right_pads_dev); - }; - - auto f_make_for_device_nhwc = [&]() { - const auto in_lengths_dev = make_tuple(N, Hi, Wi, C); - const auto wei_lengths_dev = make_tuple(K, Y, X, C); - const auto out_lengths_dev = make_tuple(N, Ho, Wo, K); - const auto conv_strides_dev = make_tuple(conv_stride_h, conv_stride_w); - const auto conv_dilations_dev = make_tuple(conv_dilation_h, conv_dilation_w); - const auto in_left_pads_dev = make_tuple(in_left_pad_h, in_left_pad_w); - const auto in_right_pads_dev = make_tuple(in_right_pad_h, in_right_pad_w); - - return make_tuple(in_lengths_dev, - wei_lengths_dev, - out_lengths_dev, - conv_strides_dev, - conv_dilations_dev, - in_left_pads_dev, - in_right_pads_dev); - }; - -#if USE_CONV_FWD_V4R4_NCHW - if(algo == ConvForwardAlgo::V4R4NCHW) - { - if(layout != ConvTensorLayout::NCHW) - { - throw std::runtime_error("wrong! layout"); - } - - const auto tmp = f_make_for_device_nchw(); - - device_convolution_forward_implicit_gemm_v4r4_dlops_nchw_kcyx_nkhw(tmp[I0], - tmp[I1], - tmp[I2], - tmp[I3], - tmp[I4], - tmp[I5], - tmp[I6], - in, - wei, - out_device, - nrepeat); - } -#endif - -#if USE_CONV_FWD_V4R4R2_NHWC - if(algo == ConvForwardAlgo::V4R4R2NHWC) - { - if(layout != ConvTensorLayout::NHWC) - { - throw std::runtime_error("wrong! layout"); - } - - const auto tmp = f_make_for_device_nhwc(); - - device_convolution_forward_implicit_gemm_v4r4r2_dlops_nhwc_kyxc_nhwk(tmp[I0], - tmp[I1], - tmp[I2], - tmp[I3], - tmp[I4], - tmp[I5], - tmp[I6], - in, - wei, - out_device, - nrepeat); - } -#endif - -#if USE_CONV_FWD_V6R1_NCHW - if(algo == ConvForwardAlgo::V6R1NCHW) - { - if(layout != ConvTensorLayout::NCHW) - { - throw std::runtime_error("wrong! layout"); - } - - const auto tmp = f_make_for_device_nchw(); - - device_convolution_forward_implicit_gemm_v6r1_dlops_nchw_kcyx_nkhw(tmp[I0], - tmp[I1], - tmp[I2], - tmp[I3], - tmp[I4], - tmp[I5], - tmp[I6], - in, - wei, - out_device, - nrepeat); - } -#endif - -#if USE_CONV_FWD_V4R4R2_XDL_NCHW - if(algo == ConvForwardAlgo::V4R4R2XDLNCHW) - { - if(layout != ConvTensorLayout::NCHW) - { - throw std::runtime_error("wrong! layout"); - } - - const auto tmp = f_make_for_device_nchw(); - - device_convolution_forward_implicit_gemm_v4r4r2_xdlops_nchw_kcyx_nkhw( - tmp[I0], - tmp[I1], - tmp[I2], - tmp[I3], - tmp[I4], - tmp[I5], - tmp[I6], - in, - wei, - out_device, - nrepeat); - } -#endif - -#if USE_CONV_FWD_V4R4R4_XDL_NHWC - if(algo == ConvForwardAlgo::V4R4R4XDLNHWC) - { - if(layout != ConvTensorLayout::NHWC) - { - throw std::runtime_error("wrong! layout"); - } - - const auto tmp = f_make_for_device_nhwc(); - - device_convolution_forward_implicit_gemm_v4r4r4_xdlops_nhwc_kyxc_nhwk( - tmp[I0], - tmp[I1], - tmp[I2], - tmp[I3], - tmp[I4], - tmp[I5], - tmp[I6], - in, - wei, - out_device, - nrepeat); - } -#endif - - if(do_verification) - { - host_convolution_forward(in, - wei, - out_host, - make_tuple(conv_stride_h, conv_stride_w), - make_tuple(conv_dilation_h, conv_dilation_w), - make_tuple(in_left_pad_h, in_left_pad_w), - make_tuple(in_right_pad_h, in_right_pad_w), - layout); - - ck::utils::check_err(out_device.mData, out_host.mData); - - if(do_log) - { - LogRangeAsType(std::cout << "in : ", in.mData, ",") << std::endl; - LogRangeAsType(std::cout << "wei: ", wei.mData, ",") << std::endl; - LogRangeAsType(std::cout << "out_host : ", out_host.mData, ",") << std::endl; - LogRangeAsType(std::cout << "out_device: ", out_device.mData, ",") << std::endl; - } - } -} diff --git a/library/src/obselete_driver_offline/conv_fwd_driver_offline_nchwc.cpp b/library/src/obselete_driver_offline/conv_fwd_driver_offline_nchwc.cpp deleted file mode 100644 index 6fb8b4c2aa..0000000000 --- a/library/src/obselete_driver_offline/conv_fwd_driver_offline_nchwc.cpp +++ /dev/null @@ -1,393 +0,0 @@ -#include -#include -#include -#include -#include -#include - -#include "check_err.hpp" -#include "config.hpp" -#include "debug.hpp" -#include "print.hpp" -#include "device.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" -#include "conv_common.hpp" -#include "device_tensor.hpp" -#include "device_convolution_forward_implicit_gemm_v5r1_dlops_nc0hwc1_kc0yxc1_nk0hwk1.hpp" - -#define USE_DYNAMIC_MODE 0 -#define USE_CONV_FWD_V5R1_NCHWC 1 - -enum ConvForwardAlgo -{ - V5R1NCHWC // 0 -}; - -template -void host_direct_convolution_nchwc(const Tensor& in, - const Tensor& wei, - const Tensor& bias, - Tensor& out, - const ConvStrides& conv_strides, - const ConvDilations& conv_dilations, - const InLeftPads& in_left_pads, - const InRightPads&, - const ck::ActivTypeEnum activ_type) -{ - using namespace ck; - - constexpr auto I0 = Number<0>{}; - constexpr auto I1 = Number<1>{}; - - auto f_nchw = [&](auto n, auto k0, auto ho, auto wo, auto k1) { - double v = 0; - const int k = k0 * out.mDesc.GetLengths()[4] + k1; - - for(int c0 = 0; c0 < wei.mDesc.GetLengths()[1]; ++c0) - { - for(int y = 0; y < wei.mDesc.GetLengths()[2]; ++y) - { - int hi = ho * conv_strides[I0] + y * conv_dilations[I0] - in_left_pads[I0]; - for(int x = 0; x < wei.mDesc.GetLengths()[3]; ++x) - { - int wi = wo * conv_strides[I1] + x * conv_dilations[I1] - in_left_pads[I1]; - if(hi >= 0 && hi < in.mDesc.GetLengths()[2] && wi >= 0 && - wi < in.mDesc.GetLengths()[3]) - { - for(int c1 = 0; c1 < wei.mDesc.GetLengths()[4]; ++c1) - { - v += static_cast(in(n, c0, hi, wi, c1)) * - static_cast(wei(k, c0, y, x, c1)); - } - } - } - } - } - v += bias(k0, k1); - out(n, k0, ho, wo, k1) = activ(v, activ_type); - }; - - make_ParallelTensorFunctor(f_nchw, - out.mDesc.GetLengths()[0], - out.mDesc.GetLengths()[1], - out.mDesc.GetLengths()[2], - out.mDesc.GetLengths()[3], - out.mDesc.GetLengths()[4])(std::thread::hardware_concurrency()); -} - -int main(int argc, char* argv[]) -{ - using namespace ck; - - constexpr auto I0 = Number<0>{}; - constexpr auto I1 = Number<1>{}; - constexpr auto I2 = Number<2>{}; - constexpr auto I3 = Number<3>{}; - constexpr auto I4 = Number<4>{}; - constexpr auto I5 = Number<5>{}; - constexpr auto I6 = Number<6>{}; - -#if USE_DYNAMIC_MODE - // dynamic mode - if(argc != 23) - { - printf("arg1 to 5: algo, do_verification, init_method, do_log, nrepeat\n"); - printf("rest: N, K0, K1, C0, C1, Y, X, Hi, Wi, Sy, Sx, Dy, Dx, LeftPy, LeftPx, RightPy, " - "RightPx\n"); - exit(1); - } - - constexpr ck::ActivTypeEnum activ_type = ActivTypeEnum::LeakyRelu; - - const ConvForwardAlgo algo = static_cast(std::stoi(argv[1])); - const bool do_verification = std::stoi(argv[2]); - const int init_method = std::stoi(argv[3]); - const bool do_log = std::stoi(argv[4]); - const int nrepeat = std::stoi(argv[5]); - - const index_t N = std::stoi(argv[6]); - const index_t K0 = std::stoi(argv[7]); - const index_t K1 = std::stoi(argv[8]); - const index_t C0 = std::stoi(argv[9]); - const index_t C1 = std::stoi(argv[10]); - const index_t Y = std::stoi(argv[11]); - const index_t X = std::stoi(argv[12]); - const index_t Hi = std::stoi(argv[13]); - const index_t Wi = std::stoi(argv[14]); - - const index_t conv_stride_h = std::stoi(argv[15]); - const index_t conv_stride_w = std::stoi(argv[16]); - const index_t conv_dilation_h = std::stoi(argv[17]); - const index_t conv_dilation_w = std::stoi(argv[18]); - const index_t in_left_pad_h = std::stoi(argv[19]); - const index_t in_left_pad_w = std::stoi(argv[20]); - const index_t in_right_pad_h = std::stoi(argv[21]); - const index_t in_right_pad_w = std::stoi(argv[22]); - - const index_t YEff = (Y - 1) * conv_dilation_h + 1; - const index_t XEff = (X - 1) * conv_dilation_w + 1; - - const index_t Ho = (Hi + in_left_pad_h + in_right_pad_h - YEff) / conv_stride_h + 1; - const index_t Wo = (Wi + in_left_pad_w + in_right_pad_w - XEff) / conv_stride_w + 1; -#else - // static mode - if(argc < 6) - { - printf("arg1 to 5: algo, do_verification, init_method, do_log, nrepeat\n"); - exit(1); - } - - const ConvForwardAlgo algo = static_cast(std::stoi(argv[1])); - - const bool do_verification = std::stoi(argv[2]); - const int init_method = std::stoi(argv[3]); - const bool do_log = std::stoi(argv[4]); - const int nrepeat = std::stoi(argv[5]); - - // constexpr ck::ActivTypeEnum activ_type = ActivTypeEnum::Sigmoid; - constexpr ck::ActivTypeEnum activ_type = ActivTypeEnum::LeakyRelu; - -#if 0 - constexpr auto N = Number<1>{}; - constexpr auto Hi = Number<1080>{}; - constexpr auto Wi = Number<1920>{}; - constexpr auto Y = Number<3>{}; - constexpr auto X = Number<3>{}; - constexpr auto C0 = Number<2>{}; - constexpr auto C1 = Number<8>{}; - constexpr auto K0 = Number<1>{}; - constexpr auto K1 = Number<4>{}; -#elif 1 - constexpr auto N = Number<1>{}; - constexpr auto Hi = Number<1080>{}; - constexpr auto Wi = Number<1920>{}; - constexpr auto Y = Number<3>{}; - constexpr auto X = Number<3>{}; - constexpr auto C0 = Number<2>{}; - constexpr auto C1 = Number<8>{}; - constexpr auto K0 = Number<2>{}; - constexpr auto K1 = Number<8>{}; -#elif 0 - constexpr auto N = Number<1>{}; - constexpr auto Hi = Number<1080>{}; - constexpr auto Wi = Number<1920>{}; - constexpr auto Y = Number<1>{}; - constexpr auto X = Number<1>{}; - constexpr auto C0 = Number<2>{}; - constexpr auto C1 = Number<8>{}; - constexpr auto K0 = Number<2>{}; - constexpr auto K1 = Number<8>{}; -#elif 0 - constexpr auto N = Number<1>{}; - constexpr auto Hi = Number<540>{}; - constexpr auto Wi = Number<960>{}; - constexpr auto Y = Number<1>{}; - constexpr auto X = Number<1>{}; - constexpr auto C0 = Number<2>{}; - constexpr auto C1 = Number<8>{}; - constexpr auto K0 = Number<2>{}; - constexpr auto K1 = Number<8>{}; -#elif 0 - constexpr auto N = Number<128>{}; - constexpr auto Hi = Number<270>{}; - constexpr auto Wi = Number<480>{}; - constexpr auto Y = Number<1>{}; - constexpr auto X = Number<1>{}; - constexpr auto C0 = Number<2>{}; - constexpr auto C1 = Number<8>{}; - constexpr auto K0 = Number<2>{}; - constexpr auto K1 = Number<8>{}; -#endif - - constexpr auto conv_stride_h = I1; - constexpr auto conv_stride_w = I1; - constexpr auto conv_dilation_h = I1; - constexpr auto conv_dilation_w = I1; - -#if 1 - constexpr auto in_left_pad_h = I1; - constexpr auto in_left_pad_w = I1; - constexpr auto in_right_pad_h = I1; - constexpr auto in_right_pad_w = I1; -#else - constexpr auto in_left_pad_h = I0; - constexpr auto in_left_pad_w = I0; - constexpr auto in_right_pad_h = I0; - constexpr auto in_right_pad_w = I0; -#endif - - constexpr auto YEff = (Y - I1) * conv_dilation_h + I1; - constexpr auto XEff = (X - I1) * conv_dilation_w + I1; - - constexpr auto Ho = (Hi + in_left_pad_h + in_right_pad_h - YEff) / conv_stride_h + I1; - constexpr auto Wo = (Wi + in_left_pad_w + in_right_pad_w - XEff) / conv_stride_w + I1; -#endif - -#if 0 - using in_data_t = float; - using acc_data_t = float; - using out_data_t = float; -#elif 1 - using in_data_t = half_t; - using acc_data_t = float; - using out_data_t = half_t; -#elif 1 - using in_data_t = int8_t; - using acc_data_t = int32_t; - using out_data_t = int8_t; -#endif - - std::vector in_lengths_host(5), wei_lengths_host(5), out_lengths_host(5), - bias_lengths_host(2); - - in_lengths_host[0] = static_cast(N); - in_lengths_host[1] = static_cast(C0); - in_lengths_host[2] = static_cast(Hi); - in_lengths_host[3] = static_cast(Wi); - in_lengths_host[4] = static_cast(C1); - - wei_lengths_host[0] = static_cast(K0 * K1); - wei_lengths_host[1] = static_cast(C0); - wei_lengths_host[2] = static_cast(Y); - wei_lengths_host[3] = static_cast(X); - wei_lengths_host[4] = static_cast(C1); - - out_lengths_host[0] = static_cast(N); - out_lengths_host[1] = static_cast(K0); - out_lengths_host[2] = static_cast(Ho); - out_lengths_host[3] = static_cast(Wo); - out_lengths_host[4] = static_cast(K1); - - bias_lengths_host[0] = static_cast(K0); - bias_lengths_host[1] = static_cast(K1); - - Tensor in(in_lengths_host); - Tensor wei(wei_lengths_host); - Tensor bias(bias_lengths_host); - Tensor out_host(out_lengths_host); - Tensor out_device(out_lengths_host); - - ostream_HostTensorDescriptor(in.mDesc, std::cout << "in: "); - ostream_HostTensorDescriptor(wei.mDesc, std::cout << "wei: "); - ostream_HostTensorDescriptor(bias.mDesc, std::cout << "bias: "); - ostream_HostTensorDescriptor(out_host.mDesc, std::cout << "out: "); - - print_array("InLeftPads", make_tuple(in_left_pad_h, in_left_pad_w)); - print_array("InRightPads", make_tuple(in_right_pad_h, in_right_pad_w)); - print_array("ConvStrides", make_tuple(conv_stride_h, conv_stride_w)); - print_array("ConvDilations", make_tuple(conv_dilation_h, conv_dilation_w)); - - std::size_t num_thread = 1; - - switch(init_method) - { - case 0: - // no initialization - break; - case 1: - in.GenerateTensorValue(GeneratorTensor_1{}, num_thread); - wei.GenerateTensorValue(GeneratorTensor_1{}, num_thread); - bias.GenerateTensorValue(GeneratorTensor_1{}, num_thread); - break; - case 2: - in.GenerateTensorValue(GeneratorTensor_1{}, num_thread); - wei.GenerateTensorValue(GeneratorTensor_2{-5, 5}, num_thread); - bias.GenerateTensorValue(GeneratorTensor_2{-5, 5}, num_thread); - break; - case 3: - in.GenerateTensorValue(GeneratorTensor_2{-5, 5}, num_thread); - wei.GenerateTensorValue(GeneratorTensor_1{}, num_thread); - bias.GenerateTensorValue(GeneratorTensor_1{}, num_thread); - break; - case 4: - in.GenerateTensorValue(GeneratorTensor_2{-5, 5}, num_thread); - wei.GenerateTensorValue(GeneratorTensor_2{-5, 5}, num_thread); - bias.GenerateTensorValue(GeneratorTensor_2{-5, 5}, num_thread); - break; - case 5: - in.GenerateTensorValue(GeneratorTensor_3{0.0, 1.0}, num_thread); - wei.GenerateTensorValue(GeneratorTensor_3{-0.5, 0.5}, num_thread); - bias.GenerateTensorValue(GeneratorTensor_3{-0.5, 0.5}, num_thread); - break; - default: - in.GenerateTensorValue(GeneratorTensor_2{1, 5}, num_thread); - - auto gen_wei = [](auto... is) { - return GeneratorTensor_2{1, 5}(is...) * GeneratorTensor_Checkboard{}(is...); - }; - wei.GenerateTensorValue(gen_wei, num_thread); - } - - auto f_make_for_device_nchwc = [&]() { - const auto in_lengths_dev = make_tuple(N, C0, Hi, Wi, C1); - const auto wei_lengths_dev = make_tuple(K0 * K1, C0, Y, X, C1); - const auto out_lengths_dev = make_tuple(N, K0, Ho, Wo, K1); - const auto conv_strides_dev = make_tuple(conv_stride_h, conv_stride_w); - const auto conv_dilations_dev = make_tuple(conv_dilation_h, conv_dilation_w); - const auto in_left_pads_dev = make_tuple(in_left_pad_h, in_left_pad_w); - const auto in_right_pads_dev = make_tuple(in_right_pad_h, in_right_pad_w); - - return make_tuple(in_lengths_dev, - wei_lengths_dev, - out_lengths_dev, - conv_strides_dev, - conv_dilations_dev, - in_left_pads_dev, - in_right_pads_dev); - }; - -#if USE_CONV_FWD_V5R1_NCHWC - if(algo == ConvForwardAlgo::V5R1NCHWC) - { - const auto tmp = f_make_for_device_nchwc(); - - device_convolution_forward_implicit_gemm_v5r1_dlops_nc0hwc1_kc0yxc1_nk0hwk1( - tmp[I0], - tmp[I1], - tmp[I2], - tmp[I3], - tmp[I4], - tmp[I5], - tmp[I6], - in, - wei, - bias, - out_device, - nrepeat); - } -#endif - - if(do_verification) - { - host_direct_convolution_nchwc(in, - wei, - bias, - out_host, - make_tuple(conv_stride_h, conv_stride_w), - make_tuple(conv_dilation_h, conv_dilation_w), - make_tuple(in_left_pad_h, in_left_pad_w), - make_tuple(in_right_pad_h, in_right_pad_w), - activ_type); - - ck::utils::check_err(out_device.mData, out_host.mData); - - if(do_log) - { - LogRangeAsType(std::cout << "in : ", in.mData, ",") << std::endl; - LogRangeAsType(std::cout << "wei: ", wei.mData, ",") << std::endl; - LogRangeAsType(std::cout << "bias: ", bias.mData, ",") << std::endl; - LogRangeAsType(std::cout << "out_host : ", out_host.mData, ",") << std::endl; - LogRangeAsType(std::cout << "out_device: ", out_device.mData, ",") << std::endl; - } - } -} diff --git a/library/src/obselete_driver_offline/conv_maxpool_fwd_driver_offline_nchwc.cpp b/library/src/obselete_driver_offline/conv_maxpool_fwd_driver_offline_nchwc.cpp deleted file mode 100644 index fb7e8e975b..0000000000 --- a/library/src/obselete_driver_offline/conv_maxpool_fwd_driver_offline_nchwc.cpp +++ /dev/null @@ -1,415 +0,0 @@ -#include -#include -#include -#include -#include -#include - -#include "check_err.hpp" -#include "config.hpp" -#include "debug.hpp" -#include "print.hpp" -#include "device.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" -#include "conv_common.hpp" -#include "device_tensor.hpp" -#include "device_convolution_maxpool_forward_implicit_gemm_v5r1_dlops_nc0hwc1_kc0yxc1_nk0hwk1.hpp" - -#define USE_DYNAMIC_MODE 0 -#define USE_CONV_FWD_V5R1_NCHWC 1 - -enum ConvForwardAlgo -{ - V5R1NCHWC // 0 -}; - -template -void host_direct_convolution_maxpool_nchwc(const Tensor& in, - const Tensor& wei, - const Tensor& bias, - Tensor& out_host, - Tensor& max_host, - const ConvStrides& conv_strides, - const ConvDilations& conv_dilations, - const InLeftPads& in_left_pads, - const InRightPads&, - const ck::ActivTypeEnum activ_type) -{ - using namespace ck; - - constexpr auto I0 = Number<0>{}; - constexpr auto I1 = Number<1>{}; - - auto f_nchw = [&](auto n, auto k0, auto ho, auto wo, auto k1) { - double v = 0; - auto k = k0 * out_host.mDesc.GetLengths()[4] + k1; - - for(int c0 = 0; c0 < wei.mDesc.GetLengths()[1]; ++c0) - { - for(int y = 0; y < wei.mDesc.GetLengths()[2]; ++y) - { - int hi = ho * conv_strides[I0] + y * conv_dilations[I0] - in_left_pads[I0]; - for(int x = 0; x < wei.mDesc.GetLengths()[3]; ++x) - { - int wi = wo * conv_strides[I1] + x * conv_dilations[I1] - in_left_pads[I1]; - if(hi >= 0 && hi < in.mDesc.GetLengths()[2] && wi >= 0 && - wi < in.mDesc.GetLengths()[3]) - { - for(int c1 = 0; c1 < wei.mDesc.GetLengths()[4]; ++c1) - { - v += static_cast(in(n, c0, hi, wi, c1)) * - static_cast(wei(k, c0, y, x, c1)); - } - } - } - } - } - - v += bias(k0, k1); - v = activ(v, activ_type); - - out_host(n, k0, ho, wo, k1) = v; - }; - - make_ParallelTensorFunctor(f_nchw, - out_host.mDesc.GetLengths()[0], - out_host.mDesc.GetLengths()[1], - out_host.mDesc.GetLengths()[2], - out_host.mDesc.GetLengths()[3], - out_host.mDesc.GetLengths()[4])(std::thread::hardware_concurrency()); - - auto maxpool_nchw = [&](auto n, auto k0, auto ho, auto wo, auto k1) { - auto hx = ho * 2; - auto wx = wo * 2; - - auto v0 = out_host(n, k0, hx, wx, k1); - auto v1 = out_host(n, k0, hx, wx + 1, k1); - auto v2 = out_host(n, k0, hx + 1, wx, k1); - auto v3 = out_host(n, k0, hx + 1, wx + 1, k1); - - max_host(n, k0, ho, wo, k1) = std::max({v0, v1, v2, v3}); - }; - - make_ParallelTensorFunctor(maxpool_nchw, - max_host.mDesc.GetLengths()[0], - max_host.mDesc.GetLengths()[1], - max_host.mDesc.GetLengths()[2], - max_host.mDesc.GetLengths()[3], - max_host.mDesc.GetLengths()[4])(std::thread::hardware_concurrency()); -} - -int main(int argc, char* argv[]) -{ - using namespace ck; - - constexpr auto I0 = Number<0>{}; - constexpr auto I1 = Number<1>{}; - constexpr auto I2 = Number<2>{}; - constexpr auto I3 = Number<3>{}; - constexpr auto I4 = Number<4>{}; - constexpr auto I5 = Number<5>{}; - constexpr auto I6 = Number<6>{}; - constexpr auto I7 = Number<7>{}; - -#if USE_DYNAMIC_MODE - // dynamic mode - if(argc != 23) - { - printf("arg1 to 5: algo, do_verification, init_method, do_log, nrepeat\n"); - printf("rest: N, K0, K1, C0, C1, Y, X, Hi, Wi, Sy, Sx, Dy, Dx, LeftPy, LeftPx, RightPy, " - "RightPx\n"); - exit(1); - } - - constexpr ck::ActivTypeEnum activ_type = ActivTypeEnum::LeakyRelu; - - const ConvForwardAlgo algo = static_cast(std::stoi(argv[1])); - const bool do_verification = std::stoi(argv[2]); - const int init_method = std::stoi(argv[3]); - const bool do_log = std::stoi(argv[4]); - const int nrepeat = std::stoi(argv[5]); - - const index_t N = std::stoi(argv[6]); - const index_t K0 = std::stoi(argv[7]); - const index_t K1 = std::stoi(argv[8]); - const index_t C0 = std::stoi(argv[9]); - const index_t C1 = std::stoi(argv[10]); - const index_t Y = std::stoi(argv[11]); - const index_t X = std::stoi(argv[12]); - const index_t Hi = std::stoi(argv[13]); - const index_t Wi = std::stoi(argv[14]); - - const index_t conv_stride_h = std::stoi(argv[15]); - const index_t conv_stride_w = std::stoi(argv[16]); - const index_t conv_dilation_h = std::stoi(argv[17]); - const index_t conv_dilation_w = std::stoi(argv[18]); - const index_t in_left_pad_h = std::stoi(argv[19]); - const index_t in_left_pad_w = std::stoi(argv[20]); - const index_t in_right_pad_h = std::stoi(argv[21]); - const index_t in_right_pad_w = std::stoi(argv[22]); - - const index_t YEff = (Y - 1) * conv_dilation_h + 1; - const index_t XEff = (X - 1) * conv_dilation_w + 1; - - const index_t Ho = (Hi + in_left_pad_h + in_right_pad_h - YEff) / conv_stride_h + 1; - const index_t Wo = (Wi + in_left_pad_w + in_right_pad_w - XEff) / conv_stride_w + 1; - - const index_t Ho_2 = Ho / 2; - const index_t Wo_2 = Wo / 2; -#else - // static mode - if(argc < 6) - { - printf("arg1 to 5: algo, do_verification, init_method, do_log, nrepeat\n"); - exit(1); - } - - const ConvForwardAlgo algo = static_cast(std::stoi(argv[1])); - - const bool do_verification = std::stoi(argv[2]); - const int init_method = std::stoi(argv[3]); - const bool do_log = std::stoi(argv[4]); - const int nrepeat = std::stoi(argv[5]); - - constexpr ck::ActivTypeEnum activ_type = ActivTypeEnum::LeakyRelu; - -#if 1 - constexpr auto N = Number<1>{}; - constexpr auto Hi = Number<1080>{}; - constexpr auto Wi = Number<1920>{}; - constexpr auto Y = Number<3>{}; - constexpr auto X = Number<3>{}; - constexpr auto C0 = Number<2>{}; - constexpr auto C1 = Number<8>{}; - constexpr auto K0 = Number<2>{}; - constexpr auto K1 = Number<8>{}; -#elif 0 - constexpr auto N = Number<1>{}; - constexpr auto Hi = Number<1080>{}; - constexpr auto Wi = Number<1920>{}; - constexpr auto Y = Number<3>{}; - constexpr auto X = Number<3>{}; - constexpr auto C0 = Number<3>{}; - constexpr auto C1 = Number<4>{}; - constexpr auto K0 = Number<2>{}; - constexpr auto K1 = Number<8>{}; -#elif 0 - constexpr auto N = Number<1>{}; - constexpr auto Hi = Number<540>{}; - constexpr auto Wi = Number<960>{}; - constexpr auto Y = Number<3>{}; - constexpr auto X = Number<3>{}; - constexpr auto C0 = Number<2>{}; - constexpr auto C1 = Number<8>{}; - constexpr auto K0 = Number<2>{}; - constexpr auto K1 = Number<8>{}; -#elif 0 - constexpr auto N = Number<128>{}; - constexpr auto Hi = Number<270>{}; - constexpr auto Wi = Number<480>{}; - constexpr auto Y = Number<3>{}; - constexpr auto X = Number<3>{}; - constexpr auto C0 = Number<2>{}; - constexpr auto C1 = Number<8>{}; - constexpr auto K0 = Number<2>{}; - constexpr auto K1 = Number<8>{}; -#endif - - constexpr auto conv_stride_h = I1; - constexpr auto conv_stride_w = I1; - constexpr auto conv_dilation_h = I1; - constexpr auto conv_dilation_w = I1; - constexpr auto in_left_pad_h = I1; - constexpr auto in_left_pad_w = I1; - constexpr auto in_right_pad_h = I1; - constexpr auto in_right_pad_w = I1; - - constexpr auto YEff = (Y - I1) * conv_dilation_h + I1; - constexpr auto XEff = (X - I1) * conv_dilation_w + I1; - - constexpr auto Ho = (Hi + in_left_pad_h + in_right_pad_h - YEff) / conv_stride_h + I1; - constexpr auto Wo = (Wi + in_left_pad_w + in_right_pad_w - XEff) / conv_stride_w + I1; - - constexpr auto Ho_2 = Number{}; - constexpr auto Wo_2 = Number{}; - -#endif - -#if 0 - using in_data_t = float; - using acc_data_t = float; - using out_data_t = float; -#elif 1 - using in_data_t = half_t; - using acc_data_t = float; - using out_data_t = half_t; -#elif 1 - using in_data_t = int8_t; - using acc_data_t = int32_t; - using out_data_t = int8_t; -#endif - - std::vector in_lengths_host(5), wei_lengths_host(5), out_lengths_host(5), - max_lengths_host(5), bias_lengths_host(2); - - in_lengths_host[0] = static_cast(N); - in_lengths_host[1] = static_cast(C0); - in_lengths_host[2] = static_cast(Hi); - in_lengths_host[3] = static_cast(Wi); - in_lengths_host[4] = static_cast(C1); - - wei_lengths_host[0] = static_cast(K0 * K1); - wei_lengths_host[1] = static_cast(C0); - wei_lengths_host[2] = static_cast(Y); - wei_lengths_host[3] = static_cast(X); - wei_lengths_host[4] = static_cast(C1); - - out_lengths_host[0] = static_cast(N); - out_lengths_host[1] = static_cast(K0); - out_lengths_host[2] = static_cast(Ho); - out_lengths_host[3] = static_cast(Wo); - out_lengths_host[4] = static_cast(K1); - - max_lengths_host[0] = static_cast(N); - max_lengths_host[1] = static_cast(K0); - max_lengths_host[2] = static_cast(Ho_2); - max_lengths_host[3] = static_cast(Wo_2); - max_lengths_host[4] = static_cast(K1); - - bias_lengths_host[0] = static_cast(K0); - bias_lengths_host[1] = static_cast(K1); - - Tensor in(in_lengths_host); - Tensor wei(wei_lengths_host); - Tensor bias(bias_lengths_host); - Tensor out_device(out_lengths_host); - Tensor out_host(out_lengths_host); - Tensor max_device(max_lengths_host); - Tensor max_host(max_lengths_host); - - ostream_HostTensorDescriptor(in.mDesc, std::cout << "in: "); - ostream_HostTensorDescriptor(wei.mDesc, std::cout << "wei: "); - - print_array("InLeftPads", make_tuple(in_left_pad_h, in_left_pad_w)); - print_array("InRightPads", make_tuple(in_right_pad_h, in_right_pad_w)); - print_array("ConvStrides", make_tuple(conv_stride_h, conv_stride_w)); - print_array("ConvDilations", make_tuple(conv_dilation_h, conv_dilation_w)); - - std::size_t num_thread = 1; - - switch(init_method) - { - case 0: - // no initialization - break; - case 1: - in.GenerateTensorValue(GeneratorTensor_1{}, num_thread); - wei.GenerateTensorValue(GeneratorTensor_1{}, num_thread); - break; - case 2: - in.GenerateTensorValue(GeneratorTensor_1{}, num_thread); - wei.GenerateTensorValue(GeneratorTensor_2{-5, 5}, num_thread); - break; - case 3: - in.GenerateTensorValue(GeneratorTensor_2{-5, 5}, num_thread); - wei.GenerateTensorValue(GeneratorTensor_1{}, num_thread); - break; - case 4: - in.GenerateTensorValue(GeneratorTensor_2{-5, 5}, num_thread); - wei.GenerateTensorValue(GeneratorTensor_2{-5, 5}, num_thread); - break; - case 5: - in.GenerateTensorValue(GeneratorTensor_3{0.0, 1.0}, num_thread); - wei.GenerateTensorValue(GeneratorTensor_3{-0.5, 0.5}, num_thread); - break; - default: - in.GenerateTensorValue(GeneratorTensor_2{1, 5}, num_thread); - - auto gen_wei = [](auto... is) { - return GeneratorTensor_2{1, 5}(is...) * GeneratorTensor_Checkboard{}(is...); - }; - wei.GenerateTensorValue(gen_wei, num_thread); - } - - bias.GenerateTensorValue(GeneratorTensor_1{}, num_thread); - - auto f_make_for_device_nchwc = [&]() { - const auto in_lengths_dev = make_tuple(N, C0, Hi, Wi, C1); - const auto wei_lengths_dev = make_tuple(K0 * K1, C0, Y, X, C1); - const auto max_lengths_dev = make_tuple(N, K0, Ho_2, Wo_2, K1); - const auto out_lengths_dev = make_tuple(N, K0, Ho, Wo, K1); - const auto conv_strides_dev = make_tuple(conv_stride_h, conv_stride_w); - const auto conv_dilations_dev = make_tuple(conv_dilation_h, conv_dilation_w); - const auto in_left_pads_dev = make_tuple(in_left_pad_h, in_left_pad_w); - const auto in_right_pads_dev = make_tuple(in_right_pad_h, in_right_pad_w); - - return make_tuple(in_lengths_dev, - wei_lengths_dev, - max_lengths_dev, - out_lengths_dev, - conv_strides_dev, - conv_dilations_dev, - in_left_pads_dev, - in_right_pads_dev); - }; - -#if USE_CONV_FWD_V5R1_NCHWC - if(algo == ConvForwardAlgo::V5R1NCHWC) - { - const auto tmp = f_make_for_device_nchwc(); - - device_convolution_maxpool_forward_implicit_gemm_v5r1_dlops_nc0hwc1_kc0yxc1_nk0hwk1< - in_data_t, - acc_data_t, - out_data_t, - activ_type>(tmp[I0], // in_lengths_dev - tmp[I1], // wei_lengths_dev - tmp[I2], // max_lengths_dev - tmp[I3], // out_lengths_dev - tmp[I4], // conv_strides_dev - tmp[I5], // conv_dilations_dev - tmp[I6], // in_left_pads_dev - tmp[I7], // in_right_pads_dev - in, - wei, - bias, - out_device, - max_device, - nrepeat); - } -#endif - - if(do_verification) - { - host_direct_convolution_maxpool_nchwc(in, - wei, - bias, - out_host, - max_host, - make_tuple(conv_stride_h, conv_stride_w), - make_tuple(conv_dilation_h, conv_dilation_w), - make_tuple(in_left_pad_h, in_left_pad_w), - make_tuple(in_right_pad_h, in_right_pad_w), - activ_type); - - ck::utils::check_err(out_device.mData, out_host.mData); - ck::utils::check_err(max_device.mData, max_host.mData); - - if(do_log) - { - // LogRangeAsType(std::cout << "in : ", in.mData, ",") << std::endl; - // LogRangeAsType(std::cout << "wei: ", wei.mData, ",") << std::endl; - // LogRangeAsType(std::cout << "out_device: ", out_device.mData, ",") << - // std::endl; - LogRangeAsType(std::cout << "max_host: ", max_host.mData, ",") << std::endl; - LogRangeAsType(std::cout << "max_device: ", max_device.mData, ",") << std::endl; - } - } -} diff --git a/library/src/obselete_driver_offline/conv_wrw_driver_offline.cpp b/library/src/obselete_driver_offline/conv_wrw_driver_offline.cpp deleted file mode 100644 index 1ac974202c..0000000000 --- a/library/src/obselete_driver_offline/conv_wrw_driver_offline.cpp +++ /dev/null @@ -1,532 +0,0 @@ -#include -#include -#include -#include -#include -#include - -#include "check_err.hpp" -#include "config.hpp" -#include "debug.hpp" -#include "print.hpp" -#include "device.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" -#include "conv_common.hpp" -#include "device_tensor.hpp" -#include "device_convolution_backward_weight_implicit_gemm_v4r4r2_xdlops_nchw_kcyx_nkhw.hpp" -#include "device_convolution_backward_weight_implicit_gemm_v4r4r4_xdlops_nhwc_kyxc_nhwk.hpp" -#include "device_convolution_backward_weight_implicit_gemm_v4r4r2_xdlops_atomic_nchw_kcyx_nkhw.hpp" -#include "device_convolution_backward_weight_implicit_gemm_v4r4r4_xdlops_atomic_nhwc_kyxc_nhwk.hpp" -#include "device_convolution_backward_weight_implicit_gemm_v4r4r5_xdlops_atomic_nhwc_kyxc_nhwk.hpp" - -enum ConvTensorLayout -{ - NCHW, - NHWC, - CHWN, - NCHWc, - NHWCc -}; - -#define USE_DYNAMIC_MODE 1 -#define USE_CONV_WRW_V4R4R2_XDL_NCHW 0 -#define USE_CONV_WRW_V4R4R4_XDL_NHWC 0 -#define USE_CONV_WRW_V4R4R2_XDL_ATOMIC_NCHW 0 -#define USE_CONV_WRW_V4R4R4_XDL_ATOMIC_NHWC 0 -#define USE_CONV_WRW_V4R4R5_XDL_ATOMIC_NHWC 1 - -enum ConvBackwardWeightAlgo -{ - V4R4R2XDLNCHW, // 0 - V4R4R4XDLNHWC, // 1 - V4R4R2XDLATOMICNCHW, // 2 - V4R4R4XDLATOMICNHWC, // 3 - V4R4R5XDLATOMICNHWC, // 4 -}; - -template -void host_convolution_backward_weight(const Tensor& out, - const Tensor& in, - Tensor& wei, - const ConvStrides& conv_strides, - const ConvDilations& conv_dilations, - const InLeftPads& in_left_pads, - const InRightPads&, - const ConvTensorLayout layout = ConvTensorLayout::NCHW) -{ - using namespace ck; - - constexpr auto I0 = Number<0>{}; - constexpr auto I1 = Number<1>{}; - auto f_kcyx = [&](auto k, auto c, auto y, auto x) { - double v = 0; - for(int n = 0; n < out.mDesc.GetLengths()[0]; ++n) - { - for(int ho = 0; ho < out.mDesc.GetLengths()[2]; ++ho) - { - int hi = ho * conv_strides[I0] + y * conv_dilations[I0] - in_left_pads[I0]; - for(int wo = 0; wo < out.mDesc.GetLengths()[3]; ++wo) - { - int wi = wo * conv_strides[I1] + x * conv_dilations[I1] - in_left_pads[I1]; - if(hi >= 0 && hi < in.mDesc.GetLengths()[2] && wi >= 0 && - wi < in.mDesc.GetLengths()[3]) - { - v += static_cast(in(n, c, hi, wi)) * - static_cast(out(n, k, ho, wo)); - } - } - } - } - wei(k, c, y, x) = v; - }; - - auto f_kyxc = [&](auto k, auto y, auto x, auto c) { - double v = 0; - for(int n = 0; n < out.mDesc.GetLengths()[0]; ++n) - { - for(int ho = 0; ho < out.mDesc.GetLengths()[1]; ++ho) - { - int hi = ho * conv_strides[I0] + y * conv_dilations[I0] - in_left_pads[I0]; - for(int wo = 0; wo < out.mDesc.GetLengths()[2]; ++wo) - { - int wi = wo * conv_strides[I1] + x * conv_dilations[I1] - in_left_pads[I1]; - if(hi >= 0 && hi < in.mDesc.GetLengths()[1] && wi >= 0 && - wi < in.mDesc.GetLengths()[2]) - { - v += static_cast(in(n, hi, wi, c)) * - static_cast(out(n, ho, wo, k)); - } - } - } - } - wei(k, y, x, c) = v; - }; - - if(layout == ConvTensorLayout::NCHW) - { - make_ParallelTensorFunctor(f_kcyx, - wei.mDesc.GetLengths()[0], - wei.mDesc.GetLengths()[1], - wei.mDesc.GetLengths()[2], - wei.mDesc.GetLengths()[3])(std::thread::hardware_concurrency()); - } - else if(layout == ConvTensorLayout::NHWC) - { - make_ParallelTensorFunctor(f_kyxc, - wei.mDesc.GetLengths()[0], - wei.mDesc.GetLengths()[1], - wei.mDesc.GetLengths()[2], - wei.mDesc.GetLengths()[3])(std::thread::hardware_concurrency()); - } - else - { - throw std::runtime_error("wrong! not supported layout"); - } -} - -int main(int argc, char* argv[]) -{ - using namespace ck; - - constexpr auto I0 = Number<0>{}; - constexpr auto I1 = Number<1>{}; - constexpr auto I2 = Number<2>{}; - constexpr auto I3 = Number<3>{}; - constexpr auto I4 = Number<4>{}; - constexpr auto I5 = Number<5>{}; - constexpr auto I6 = Number<6>{}; - -#if USE_DYNAMIC_MODE - // dynamic mode - if(argc != 23) - { - printf("arg1 to 6: layout, algo, do_verification, init_method, do_log, nrepeat\n"); - printf("rest: N, K, C, Y, X, Hi, Wi, Sy, Sx, Dy, Dx, LeftPy, LeftPx, RightPy, RightPx\n"); - printf("additional: desired_grid_size\n"); - exit(1); - } - - const ConvTensorLayout layout = static_cast(std::stoi(argv[1])); - const ConvBackwardWeightAlgo algo = static_cast(std::stoi(argv[2])); - const bool do_verification = std::stoi(argv[3]); - const int init_method = std::stoi(argv[4]); - const bool do_log = std::stoi(argv[5]); - const int nrepeat = std::stoi(argv[6]); - - const index_t N = std::stoi(argv[7]); - const index_t K = std::stoi(argv[8]); - const index_t C = std::stoi(argv[9]); - const index_t Y = std::stoi(argv[10]); - const index_t X = std::stoi(argv[11]); - const index_t Hi = std::stoi(argv[12]); - const index_t Wi = std::stoi(argv[13]); - - const index_t conv_stride_h = std::stoi(argv[14]); - const index_t conv_stride_w = std::stoi(argv[15]); - const index_t conv_dilation_h = std::stoi(argv[16]); - const index_t conv_dilation_w = std::stoi(argv[17]); - const index_t in_left_pad_h = std::stoi(argv[18]); - const index_t in_left_pad_w = std::stoi(argv[19]); - const index_t in_right_pad_h = std::stoi(argv[20]); - const index_t in_right_pad_w = std::stoi(argv[21]); - - const index_t desired_grid_size = std::stoi(argv[22]); - - const index_t YEff = (Y - 1) * conv_dilation_h + 1; - const index_t XEff = (X - 1) * conv_dilation_w + 1; - - const index_t Ho = (Hi + in_left_pad_h + in_right_pad_h - YEff) / conv_stride_h + 1; - const index_t Wo = (Wi + in_left_pad_w + in_right_pad_w - XEff) / conv_stride_w + 1; -#else - // static mode - if(argc < 7) - { - printf("arg1 to 6: layout, algo, do_verification, init_method, do_log, nrepeat\n"); - exit(1); - } - - const ConvTensorLayout layout = static_cast(std::stoi(argv[1])); - const ConvBackwardWeightAlgo algo = static_cast(std::stoi(argv[2])); - const bool do_verification = std::stoi(argv[3]); - const int init_method = std::stoi(argv[4]); - const bool do_log = std::stoi(argv[5]); - const int nrepeat = std::stoi(argv[6]); - - constexpr auto N = Number<128>{}; - constexpr auto C = Number<128>{}; - constexpr auto Hi = Number<14>{}; - constexpr auto Wi = Number<14>{}; - constexpr auto K = Number<256>{}; - constexpr auto Y = Number<3>{}; - constexpr auto X = Number<3>{}; - - constexpr auto conv_stride_h = I1; - constexpr auto conv_stride_w = I1; - constexpr auto conv_dilation_h = I1; - constexpr auto conv_dilation_w = I1; - constexpr auto in_left_pad_h = I1; - constexpr auto in_left_pad_w = I1; - constexpr auto in_right_pad_h = I1; - constexpr auto in_right_pad_w = I1; - - constexpr auto YEff = (Y - I1) * conv_dilation_h + I1; - constexpr auto XEff = (X - I1) * conv_dilation_w + I1; - - constexpr auto Ho = (Hi + in_left_pad_h + in_right_pad_h - YEff) / conv_stride_h + I1; - constexpr auto Wo = (Wi + in_left_pad_w + in_right_pad_w - XEff) / conv_stride_w + I1; -#endif - -#if 0 - using in_data_t = float; - using wei_data_t = float; - using acc_data_t = float; - using out_data_t = float; -#elif 1 - using in_data_t = half_t; - using out_data_t = half_t; - using acc_data_t = float; - using wei_data_t = float; -#elif 1 - using in_data_t = int8_t; - using out_data_t = int8_t; - using acc_data_t = int32_t; - using wei_data_t = int8_t; -#endif - - std::vector in_lengths_host(4), wei_lengths_host(4), out_lengths_host(4); - - if(layout == ConvTensorLayout::NCHW) - { - in_lengths_host[0] = static_cast(N); - in_lengths_host[1] = static_cast(C); - in_lengths_host[2] = static_cast(Hi); - in_lengths_host[3] = static_cast(Wi); - wei_lengths_host[0] = static_cast(K); - wei_lengths_host[1] = static_cast(C); - wei_lengths_host[2] = static_cast(Y); - wei_lengths_host[3] = static_cast(X); - out_lengths_host[0] = static_cast(N); - out_lengths_host[1] = static_cast(K); - out_lengths_host[2] = static_cast(Ho); - out_lengths_host[3] = static_cast(Wo); - } - else if(layout == ConvTensorLayout::NHWC) - { - in_lengths_host[0] = static_cast(N); - in_lengths_host[1] = static_cast(Hi); - in_lengths_host[2] = static_cast(Wi); - in_lengths_host[3] = static_cast(C); - wei_lengths_host[0] = static_cast(K); - wei_lengths_host[1] = static_cast(Y); - wei_lengths_host[2] = static_cast(X); - wei_lengths_host[3] = static_cast(C); - out_lengths_host[0] = static_cast(N); - out_lengths_host[1] = static_cast(Ho); - out_lengths_host[2] = static_cast(Wo); - out_lengths_host[3] = static_cast(K); - } - else - { - std::runtime_error("wrong! not implemented"); - } - - Tensor in(in_lengths_host); - Tensor wei_device(wei_lengths_host); - Tensor wei_host(wei_lengths_host); - Tensor out(out_lengths_host); - - std::cout << "layout: " << layout << std::endl; - ostream_HostTensorDescriptor(in.mDesc, std::cout << "in: "); - ostream_HostTensorDescriptor(wei_host.mDesc, std::cout << "wei: "); - ostream_HostTensorDescriptor(out.mDesc, std::cout << "out: "); - print_array("InLeftPads", make_tuple(in_left_pad_h, in_left_pad_w)); - print_array("InRightPads", make_tuple(in_right_pad_h, in_right_pad_w)); - print_array("ConvStrides", make_tuple(conv_stride_h, conv_stride_w)); - print_array("ConvDilations", make_tuple(conv_dilation_h, conv_dilation_w)); - - std::size_t num_thread = 1; - - switch(init_method) - { - case 0: - // no initialization - break; - case 1: - in.GenerateTensorValue(GeneratorTensor_1{}, num_thread); - out.GenerateTensorValue(GeneratorTensor_1{}, num_thread); - break; - case 2: - in.GenerateTensorValue(GeneratorTensor_1{}, num_thread); - out.GenerateTensorValue(GeneratorTensor_2{-5, 5}, num_thread); - break; - case 3: - in.GenerateTensorValue(GeneratorTensor_2{-5, 5}, num_thread); - out.GenerateTensorValue(GeneratorTensor_1{}, num_thread); - break; - case 4: - in.GenerateTensorValue(GeneratorTensor_2{-5, 5}, num_thread); - out.GenerateTensorValue(GeneratorTensor_2{-5, 5}, num_thread); - break; - case 5: - in.GenerateTensorValue(GeneratorTensor_3{-0.1, 0.1}, num_thread); - out.GenerateTensorValue(GeneratorTensor_3{-0.1, 0.1}, num_thread); - break; - default: - in.GenerateTensorValue(GeneratorTensor_2{1, 5}, num_thread); - - auto gen_out = [](auto... is) { - return GeneratorTensor_2{1, 5}(is...) * GeneratorTensor_Checkboard{}(is...); - }; - out.GenerateTensorValue(gen_out, num_thread); - } - - auto f_make_for_device_nchw = [&]() { - const auto in_lengths_dev = make_tuple(N, C, Hi, Wi); - const auto wei_lengths_dev = make_tuple(K, C, Y, X); - const auto out_lengths_dev = make_tuple(N, K, Ho, Wo); - const auto conv_strides_dev = make_tuple(conv_stride_h, conv_stride_w); - const auto conv_dilations_dev = make_tuple(conv_dilation_h, conv_dilation_w); - const auto in_left_pads_dev = make_tuple(in_left_pad_h, in_left_pad_w); - const auto in_right_pads_dev = make_tuple(in_right_pad_h, in_right_pad_w); - - return make_tuple(in_lengths_dev, - wei_lengths_dev, - out_lengths_dev, - conv_strides_dev, - conv_dilations_dev, - in_left_pads_dev, - in_right_pads_dev); - }; - - auto f_make_for_device_nhwc = [&]() { - const auto in_lengths_dev = make_tuple(N, Hi, Wi, C); - const auto wei_lengths_dev = make_tuple(K, Y, X, C); - const auto out_lengths_dev = make_tuple(N, Ho, Wo, K); - const auto conv_strides_dev = make_tuple(conv_stride_h, conv_stride_w); - const auto conv_dilations_dev = make_tuple(conv_dilation_h, conv_dilation_w); - const auto in_left_pads_dev = make_tuple(in_left_pad_h, in_left_pad_w); - const auto in_right_pads_dev = make_tuple(in_right_pad_h, in_right_pad_w); - - return make_tuple(in_lengths_dev, - wei_lengths_dev, - out_lengths_dev, - conv_strides_dev, - conv_dilations_dev, - in_left_pads_dev, - in_right_pads_dev); - }; - - // set zero to wei_device - wei_device.GenerateTensorValue(GeneratorTensor_0{}, num_thread); -#if USE_CONV_WRW_V4R4R2_XDL_NCHW - if(algo == ConvBackwardWeightAlgo::V4R4R2XDLNCHW) - { - if(layout != ConvTensorLayout::NCHW) - { - throw std::runtime_error("wrong! layout"); - } - - const auto tmp = f_make_for_device_nchw(); - - device_convolution_backward_weight_implicit_gemm_v4r4r2_xdlops_nchw_kcyx_nkhw( - tmp[I0], - tmp[I1], - tmp[I2], - tmp[I3], - tmp[I4], - tmp[I5], - tmp[I6], - in, - wei_device, - out, - nrepeat); - } -#endif - -#if USE_CONV_WRW_V4R4R4_XDL_NHWC - if(algo == ConvBackwardWeightAlgo::V4R4R4XDLNHWC) - { - if(layout != ConvTensorLayout::NHWC) - { - throw std::runtime_error("wrong! layout"); - } - - const auto tmp = f_make_for_device_nhwc(); - - device_convolution_backward_weight_implicit_gemm_v4r4r4_xdlops_nhwc_kyxc_nhwk( - tmp[I0], - tmp[I1], - tmp[I2], - tmp[I3], - tmp[I4], - tmp[I5], - tmp[I6], - in, - wei_device, - out, - nrepeat); - } -#endif - -#if USE_CONV_WRW_V4R4R2_XDL_ATOMIC_NCHW - if(algo == ConvBackwardWeightAlgo::V4R4R2XDLATOMICNCHW) - { - if(layout != ConvTensorLayout::NCHW) - { - throw std::runtime_error("wrong! layout"); - } - - const auto tmp = f_make_for_device_nchw(); - - device_convolution_backward_weight_implicit_gemm_v4r4r2_xdlops_atomic_nchw_kcyx_nkhw< - in_data_t, - wei_data_t, - acc_data_t, - out_data_t>(tmp[I0], - tmp[I1], - tmp[I2], - tmp[I3], - tmp[I4], - tmp[I5], - tmp[I6], - in, - wei_device, - out, - desired_grid_size, - nrepeat); - } -#endif - -#if USE_CONV_WRW_V4R4R4_XDL_ATOMIC_NHWC - if(algo == ConvBackwardWeightAlgo::V4R4R4XDLATOMICNHWC) - { - if(layout != ConvTensorLayout::NHWC) - { - throw std::runtime_error("wrong! layout"); - } - - const auto tmp = f_make_for_device_nhwc(); - - device_convolution_backward_weight_implicit_gemm_v4r4r4_xdlops_atomic_nhwc_kyxc_nhwk< - in_data_t, - wei_data_t, - acc_data_t, - out_data_t>(tmp[I0], - tmp[I1], - tmp[I2], - tmp[I3], - tmp[I4], - tmp[I5], - tmp[I6], - in, - wei_device, - out, - desired_grid_size, - nrepeat); - } -#endif - -#if USE_CONV_WRW_V4R4R5_XDL_ATOMIC_NHWC - if(algo == ConvBackwardWeightAlgo::V4R4R5XDLATOMICNHWC) - { - if(layout != ConvTensorLayout::NHWC) - { - throw std::runtime_error("wrong! layout"); - } - - const auto tmp = f_make_for_device_nhwc(); - - device_convolution_backward_weight_implicit_gemm_v4r4r5_xdlops_atomic_nhwc_kyxc_nhwk< - in_data_t, - wei_data_t, - acc_data_t, - out_data_t>(tmp[I0], - tmp[I1], - tmp[I2], - tmp[I3], - tmp[I4], - tmp[I5], - tmp[I6], - in, - wei_device, - out, - desired_grid_size, - nrepeat); - } -#endif - - if(do_verification) - { - host_convolution_backward_weight(out, - in, - wei_host, - make_tuple(conv_stride_h, conv_stride_w), - make_tuple(conv_dilation_h, conv_dilation_w), - make_tuple(in_left_pad_h, in_left_pad_w), - make_tuple(in_right_pad_h, in_right_pad_w), - layout); - - ck::utils::check_err(wei_device.mData, wei_host.mData); - - if(do_log) - { - LogRangeAsType(std::cout << "out: ", out.mData, ",") << std::endl; - LogRangeAsType(std::cout << "in : ", in.mData, ",") << std::endl; - LogRangeAsType(std::cout << "wei_device: ", wei_device.mData, ",") << std::endl; - LogRangeAsType(std::cout << "wei_host : ", wei_host.mData, ",") << std::endl; - } - } -} diff --git a/library/src/obselete_driver_offline/gemm_driver_offline.cpp b/library/src/obselete_driver_offline/gemm_driver_offline.cpp deleted file mode 100644 index a09cb932d6..0000000000 --- a/library/src/obselete_driver_offline/gemm_driver_offline.cpp +++ /dev/null @@ -1,456 +0,0 @@ -#include -#include -#include -#include -#include -#include - -#include "check_err.hpp" -#include "config.hpp" -#include "debug.hpp" -#include "print.hpp" -#include "device.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" -#include "host_gemm.hpp" -#include "device_tensor.hpp" -#include "device_gemm_xdlops_mk_kn_mn.hpp" -#include "device_gemm_xdlops_mk_nk_mn.hpp" -#include "device_gemm_xdlops_km_kn_mn.hpp" -#include "device_gemm_xdlops_km_nk_mn.hpp" -#include "device_gemm_xdlops_mk_kn_nm.hpp" -#include "device_gemm_xdlops_mk_nk_nm.hpp" -#include "device_gemm_xdlops_km_kn_nm.hpp" -#include "device_gemm_xdlops_km_nk_nm.hpp" - -#define USE_GEMM_XDL_MK_KN_MN 1 -#define USE_GEMM_XDL_MK_NK_MN 1 -#define USE_GEMM_XDL_KM_KN_MN 1 -#define USE_GEMM_XDL_KM_NK_MN 1 -#define USE_GEMM_XDL_MK_KN_NM 0 -#define USE_GEMM_XDL_MK_NK_NM 0 -#define USE_GEMM_XDL_KM_KN_NM 0 -#define USE_GEMM_XDL_KM_NK_NM 0 - -enum struct GemmMatrixLayout -{ - MK_KN_MN, // 0 - MK_NK_MN, // 1 - KM_KN_MN, // 2 - KM_NK_MN, // 3 - MK_KN_NM, // 4 - MK_NK_NM, // 5 - KM_KN_NM, // 6 - KM_NK_NM // 7 -}; - -enum struct GemmAlgo -{ - Xdl_MK_KN_MN, // 0 - Xdl_MK_NK_MN, // 1 - Xdl_KM_KN_MN, // 2 - Xdl_KM_NK_MN, // 3 - Xdl_MK_KN_NM, // 4 - Xdl_MK_NK_NM, // 5 - Xdl_KM_KN_NM, // 6 - Xdl_KM_NK_NM, // 7 -}; - -template -void host_gemm(const Tensor& a, - const Tensor& b, - Tensor& c, - const GemmMatrixLayout layout) -{ - if(layout == GemmMatrixLayout::MK_KN_MN) - { - auto f_mk_kn_mn = [&](auto m, auto n) { - const int K = a.mDesc.GetLengths()[1]; - - double v = 0; - - for(int k = 0; k < K; ++k) - { - v += static_cast(a(m, k)) * static_cast(b(k, n)); - } - - c(m, n) = v; - }; - - make_ParallelTensorFunctor(f_mk_kn_mn, c.mDesc.GetLengths()[0], c.mDesc.GetLengths()[1])( - std::thread::hardware_concurrency()); - } - else if(layout == GemmMatrixLayout::MK_NK_MN) - { - auto f_mk_nk_mn = [&](auto m, auto n) { - const int K = a.mDesc.GetLengths()[1]; - - double v = 0; - - for(int k = 0; k < K; ++k) - { - v += static_cast(a(m, k)) * static_cast(b(n, k)); - } - - c(m, n) = v; - }; - - make_ParallelTensorFunctor(f_mk_nk_mn, c.mDesc.GetLengths()[0], c.mDesc.GetLengths()[1])( - std::thread::hardware_concurrency()); - } - else if(layout == GemmMatrixLayout::KM_KN_MN) - { - auto f_km_kn_mn = [&](auto m, auto n) { - const int K = a.mDesc.GetLengths()[0]; - - double v = 0; - - for(int k = 0; k < K; ++k) - { - v += static_cast(a(k, m)) * static_cast(b(k, n)); - } - - c(m, n) = v; - }; - - make_ParallelTensorFunctor(f_km_kn_mn, c.mDesc.GetLengths()[0], c.mDesc.GetLengths()[1])( - std::thread::hardware_concurrency()); - } - else if(layout == GemmMatrixLayout::KM_NK_MN) - { - auto f_km_nk_mn = [&](auto m, auto n) { - const int K = a.mDesc.GetLengths()[0]; - - double v = 0; - - for(int k = 0; k < K; ++k) - { - v += static_cast(a(k, m)) * static_cast(b(n, k)); - } - - c(m, n) = v; - }; - - make_ParallelTensorFunctor(f_km_nk_mn, c.mDesc.GetLengths()[0], c.mDesc.GetLengths()[1])( - std::thread::hardware_concurrency()); - } - else if(layout == GemmMatrixLayout::MK_KN_NM) - { - auto f_mk_kn_nm = [&](auto n, auto m) { - const int K = a.mDesc.GetLengths()[1]; - - double v = 0; - - for(int k = 0; k < K; ++k) - { - v += static_cast(a(m, k)) * static_cast(b(k, n)); - } - - c(n, m) = v; - }; - - make_ParallelTensorFunctor(f_mk_kn_nm, c.mDesc.GetLengths()[0], c.mDesc.GetLengths()[1])( - std::thread::hardware_concurrency()); - } - else if(layout == GemmMatrixLayout::MK_NK_NM) - { - auto f_mk_nk_nm = [&](auto n, auto m) { - const int K = a.mDesc.GetLengths()[1]; - - double v = 0; - - for(int k = 0; k < K; ++k) - { - v += static_cast(a(m, k)) * static_cast(b(n, k)); - } - - c(n, m) = v; - }; - - make_ParallelTensorFunctor(f_mk_nk_nm, c.mDesc.GetLengths()[0], c.mDesc.GetLengths()[1])( - std::thread::hardware_concurrency()); - } - else if(layout == GemmMatrixLayout::KM_KN_NM) - { - auto f_km_kn_nm = [&](auto n, auto m) { - const int K = a.mDesc.GetLengths()[0]; - - double v = 0; - - for(int k = 0; k < K; ++k) - { - v += static_cast(a(k, m)) * static_cast(b(k, n)); - } - - c(n, m) = v; - }; - - make_ParallelTensorFunctor(f_km_kn_nm, c.mDesc.GetLengths()[0], c.mDesc.GetLengths()[1])( - std::thread::hardware_concurrency()); - } - else if(layout == GemmMatrixLayout::KM_NK_NM) - { - auto f_km_nk_nm = [&](auto n, auto m) { - const int K = a.mDesc.GetLengths()[0]; - - double v = 0; - - for(int k = 0; k < K; ++k) - { - v += static_cast(a(k, m)) * static_cast(b(n, k)); - } - - c(n, m) = v; - }; - - make_ParallelTensorFunctor(f_km_nk_nm, c.mDesc.GetLengths()[0], c.mDesc.GetLengths()[1])( - std::thread::hardware_concurrency()); - } - else - { - throw std::runtime_error("wrong! not supported layout"); - } -} -int main(int argc, char* argv[]) -{ - using namespace ck; - - if(argc != 12) - { - printf("arg1 to 6: layout, algo, do_verification, init_method, do_log, nrepeat\n"); - printf("rest: M, N, K\n"); - printf("debug_driver_gemm_xdlops_v2r3::M01, debug_driver_gemm_xdlops_v2r3::N01\n"); - exit(1); - } - - const auto layout = static_cast(std::stoi(argv[1])); - const auto algo = static_cast(std::stoi(argv[2])); - const bool do_verification = std::stoi(argv[3]); - const int init_method = std::stoi(argv[4]); - const bool do_log = std::stoi(argv[5]); - const int nrepeat = std::stoi(argv[6]); - - const index_t M = std::stoi(argv[7]); - const index_t N = std::stoi(argv[8]); - const index_t K = std::stoi(argv[9]); - - debug::debug_driver_gemm_xdlops_v2r3::M01 = std::stoi(argv[10]); - debug::debug_driver_gemm_xdlops_v2r3::N01 = std::stoi(argv[11]); - -#if 0 - using ab_data_t = float; - using acc_data_t = float; - using c_data_t = float; -#elif 1 - using ab_data_t = half_t; - using acc_data_t = float; - using c_data_t = half_t; -#elif 1 - using ab_data_t = int8_t; - using acc_data_t = int32_t; - using c_data_t = int8_t; -#endif - - std::vector a_lengths_host(2), b_lengths_host(2), c_lengths_host(2); - std::vector a_strides_host(2), b_strides_host(2), c_strides_host(2); - - // A - if(layout == GemmMatrixLayout::MK_KN_MN || layout == GemmMatrixLayout::MK_NK_MN || - layout == GemmMatrixLayout::MK_KN_NM || layout == GemmMatrixLayout::MK_NK_NM) - { - a_lengths_host[0] = static_cast(M); - a_lengths_host[1] = static_cast(K); - a_strides_host[0] = static_cast(K); - a_strides_host[1] = static_cast(1); - } - else - { - a_lengths_host[0] = static_cast(K); - a_lengths_host[1] = static_cast(M); - a_strides_host[0] = static_cast(M); - a_strides_host[1] = static_cast(1); - } - - // B - if(layout == GemmMatrixLayout::MK_NK_MN || layout == GemmMatrixLayout::KM_NK_MN || - layout == GemmMatrixLayout::MK_NK_NM || layout == GemmMatrixLayout::KM_NK_NM) - { - b_lengths_host[0] = static_cast(N); - b_lengths_host[1] = static_cast(K); - b_strides_host[0] = static_cast(K); - b_strides_host[1] = static_cast(1); - } - else - { - b_lengths_host[0] = static_cast(K); - b_lengths_host[1] = static_cast(N); - b_strides_host[0] = static_cast(N); - b_strides_host[1] = static_cast(1); - } - - // C - if(layout == GemmMatrixLayout::MK_KN_MN || layout == GemmMatrixLayout::KM_KN_MN || - layout == GemmMatrixLayout::MK_NK_MN || layout == GemmMatrixLayout::KM_NK_MN) - { - c_lengths_host[0] = static_cast(M); - c_lengths_host[1] = static_cast(N); - c_strides_host[0] = static_cast(N); - c_strides_host[1] = static_cast(1); - } - else - { - c_lengths_host[0] = static_cast(N); - c_lengths_host[1] = static_cast(M); - c_strides_host[0] = static_cast(M); - c_strides_host[1] = static_cast(1); - } - - Tensor a(a_lengths_host, a_strides_host); - Tensor b(b_lengths_host, b_strides_host); - Tensor c_host(c_lengths_host, c_strides_host); - Tensor c_device(c_lengths_host, c_strides_host); - - std::cout << "layout: " << layout << std::endl; - ostream_HostTensorDescriptor(a.mDesc, std::cout << "a: "); - ostream_HostTensorDescriptor(b.mDesc, std::cout << "b: "); - ostream_HostTensorDescriptor(c_host.mDesc, std::cout << "c: "); - - std::size_t num_thread = 1; - - switch(init_method) - { - case 0: - // no initialization - break; - case 1: - a.GenerateTensorValue(GeneratorTensor_1{}, num_thread); - b.GenerateTensorValue(GeneratorTensor_1{}, num_thread); - break; - case 2: - a.GenerateTensorValue(GeneratorTensor_1{}, num_thread); - b.GenerateTensorValue(GeneratorTensor_2{-5, 5}, num_thread); - break; - case 3: - a.GenerateTensorValue(GeneratorTensor_2{-5, 5}, num_thread); - b.GenerateTensorValue(GeneratorTensor_1{}, num_thread); - break; - case 4: - a.GenerateTensorValue(GeneratorTensor_2{-5, 5}, num_thread); - b.GenerateTensorValue(GeneratorTensor_2{-5, 5}, num_thread); - break; - default: - a.GenerateTensorValue(GeneratorTensor_3{0.0, 1.0}, num_thread); - b.GenerateTensorValue(GeneratorTensor_3{-0.5, 0.5}, num_thread); - } - -#if USE_GEMM_XDL_MK_KN_MN - if(algo == GemmAlgo::Xdl_MK_KN_MN) - { - if(layout != GemmMatrixLayout::MK_KN_MN) - { - throw std::runtime_error("wrong! layout"); - } - - device_gemm_xdlops_mk_kn_mn(a, b, c_device, nrepeat); - } -#endif - -#if USE_GEMM_XDL_MK_NK_MN - if(algo == GemmAlgo::Xdl_MK_NK_MN) - { - if(layout != GemmMatrixLayout::MK_NK_MN) - { - throw std::runtime_error("wrong! layout"); - } - - device_gemm_xdlops_mk_nk_mn(a, b, c_device, nrepeat); - } -#endif - -#if USE_GEMM_XDL_KM_KN_MN - if(algo == GemmAlgo::Xdl_KM_KN_MN) - { - if(layout != GemmMatrixLayout::KM_KN_MN) - { - throw std::runtime_error("wrong! layout"); - } - - device_gemm_xdlops_km_kn_mn(a, b, c_device, nrepeat); - } -#endif - -#if USE_GEMM_XDL_KM_NK_MN - if(algo == GemmAlgo::Xdl_KM_NK_MN) - { - if(layout != GemmMatrixLayout::KM_NK_MN) - { - throw std::runtime_error("wrong! layout"); - } - - device_gemm_xdlops_km_nk_mn(a, b, c_device, nrepeat); - } -#endif - -#if USE_GEMM_XDL_MK_KN_NM - if(algo == GemmAlgo::Xdl_MK_KN_NM) - { - if(layout != GemmMatrixLayout::MK_KN_NM) - { - throw std::runtime_error("wrong! layout"); - } - - device_gemm_xdlops_mk_kn_nm(a, b, c_device, nrepeat); - } -#endif - -#if USE_GEMM_XDL_MK_NK_NM - if(algo == GemmAlgo::Xdl_MK_NK_NM) - { - if(layout != GemmMatrixLayout::MK_NK_NM) - { - throw std::runtime_error("wrong! layout"); - } - - device_gemm_xdlops_mk_nk_nm(a, b, c_device, nrepeat); - } -#endif - -#if USE_GEMM_XDL_KM_KN_NM - if(algo == GemmAlgo::Xdl_KM_KN_NM) - { - if(layout != GemmMatrixLayout::KM_KN_NM) - { - throw std::runtime_error("wrong! layout"); - } - - device_gemm_xdlops_km_kn_nm(a, b, c_device, nrepeat); - } -#endif - -#if USE_GEMM_XDL_KM_NK_NM - if(algo == GemmAlgo::Xdl_KM_NK_NM) - { - if(layout != GemmMatrixLayout::KM_NK_NM) - { - throw std::runtime_error("wrong! layout"); - } - - device_gemm_xdlops_km_nk_nm(a, b, c_device, nrepeat); - } -#endif - - if(do_verification) - { - host_gemm(a, b, c_host, layout); - - ck::utils::check_err(c_device.mData, c_host.mData); - - if(do_log) - { - LogRangeAsType(std::cout << "a : ", a.mData, ",") << std::endl; - LogRangeAsType(std::cout << "b: ", b.mData, ",") << std::endl; - LogRangeAsType(std::cout << "c_host : ", c_host.mData, ",") << std::endl; - LogRangeAsType(std::cout << "c_device: ", c_device.mData, ",") << std::endl; - } - } -} diff --git a/library/src/tensor_operation_instance/gpu/CMakeLists.txt b/library/src/tensor_operation_instance/gpu/CMakeLists.txt index 128aea334a..c50b3ef649 100644 --- a/library/src/tensor_operation_instance/gpu/CMakeLists.txt +++ b/library/src/tensor_operation_instance/gpu/CMakeLists.txt @@ -1,23 +1,3 @@ -include_directories(BEFORE - ${PROJECT_SOURCE_DIR}/include/ck - ${PROJECT_SOURCE_DIR}/include/ck/utility - ${PROJECT_SOURCE_DIR}/include/ck/host_utility - ${PROJECT_SOURCE_DIR}/include/ck/tensor_description - ${PROJECT_SOURCE_DIR}/include/ck/tensor - ${PROJECT_SOURCE_DIR}/include/ck/problem_transform - ${PROJECT_SOURCE_DIR}/include/ck/tensor_operation/gpu/device - ${PROJECT_SOURCE_DIR}/include/ck/tensor_operation/gpu/grid - ${PROJECT_SOURCE_DIR}/include/ck/tensor_operation/gpu/block - ${PROJECT_SOURCE_DIR}/include/ck/tensor_operation/gpu/warp - ${PROJECT_SOURCE_DIR}/include/ck/tensor_operation/gpu/thread - ${PROJECT_SOURCE_DIR}/include/ck/tensor_operation/gpu/element - ${PROJECT_SOURCE_DIR}/library/include/ck/library/host_tensor - ${PROJECT_SOURCE_DIR}/library/include/ck/library/host - ${PROJECT_SOURCE_DIR}/library/include/ck/library/tensor_operation_instance - ${PROJECT_SOURCE_DIR}/library/include/ck/library/tensor_operation_instance/gpu/reduce - ${PROJECT_SOURCE_DIR}/external/include/half -) - function(add_instance_library INSTANCE_NAME) message("adding instance ${INSTANCE_NAME}") add_library(${INSTANCE_NAME} OBJECT ${ARGN}) @@ -37,7 +17,6 @@ add_subdirectory(conv2d_fwd) add_subdirectory(conv3d_fwd) add_subdirectory(conv2d_fwd_bias_relu) add_subdirectory(conv2d_fwd_bias_relu_add) -add_subdirectory(conv2d_fwd_bias_relu_atomic_add) add_subdirectory(conv2d_bwd_data) add_subdirectory(reduce) add_subdirectory(convnd_bwd_data) @@ -53,7 +32,6 @@ add_library(device_operations STATIC $ $ $ - $ $ $ $ @@ -65,7 +43,6 @@ add_library(device_operations STATIC $ $ $ - device_conv2d.cpp ) add_library(composablekernels::device_operations ALIAS device_operations) @@ -73,8 +50,8 @@ add_library(composablekernels::device_operations ALIAS device_operations) set(DEV_OPS_INC_DIRS ${PROJECT_SOURCE_DIR}/include/ck/ ${PROJECT_SOURCE_DIR}/library/include/ck/ - ${PROJECT_SOURCE_DIR}/external/include/ ) + target_compile_features(device_operations PUBLIC) set_target_properties(device_operations PROPERTIES POSITION_INDEPENDENT_CODE ON) target_include_directories(device_operations PUBLIC @@ -93,7 +70,6 @@ target_include_directories(device_operations PUBLIC $ $ $ - $ ) #once new arches are enabled make this an option on the main cmake file diff --git a/library/src/tensor_operation_instance/gpu/batched_gemm/device_batched_gemm_xdl_bf16_bf16_bf16_gkm_gkn_gmn_instance.cpp b/library/src/tensor_operation_instance/gpu/batched_gemm/device_batched_gemm_xdl_bf16_bf16_bf16_gkm_gkn_gmn_instance.cpp index 9641e3cf72..0eadcab903 100644 --- a/library/src/tensor_operation_instance/gpu/batched_gemm/device_batched_gemm_xdl_bf16_bf16_bf16_gkm_gkn_gmn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/batched_gemm/device_batched_gemm_xdl_bf16_bf16_bf16_gkm_gkn_gmn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_batched_gemm_xdl.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_batched_gemm_xdl.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/batched_gemm/device_batched_gemm_xdl_bf16_bf16_bf16_gkm_gnk_gmn_instance.cpp b/library/src/tensor_operation_instance/gpu/batched_gemm/device_batched_gemm_xdl_bf16_bf16_bf16_gkm_gnk_gmn_instance.cpp index c93c77dccc..3dbda7c706 100644 --- a/library/src/tensor_operation_instance/gpu/batched_gemm/device_batched_gemm_xdl_bf16_bf16_bf16_gkm_gnk_gmn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/batched_gemm/device_batched_gemm_xdl_bf16_bf16_bf16_gkm_gnk_gmn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_batched_gemm_xdl.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_batched_gemm_xdl.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/batched_gemm/device_batched_gemm_xdl_bf16_bf16_bf16_gmk_gkn_gmn_instance.cpp b/library/src/tensor_operation_instance/gpu/batched_gemm/device_batched_gemm_xdl_bf16_bf16_bf16_gmk_gkn_gmn_instance.cpp index 8da334071a..b806701ad2 100644 --- a/library/src/tensor_operation_instance/gpu/batched_gemm/device_batched_gemm_xdl_bf16_bf16_bf16_gmk_gkn_gmn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/batched_gemm/device_batched_gemm_xdl_bf16_bf16_bf16_gmk_gkn_gmn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_batched_gemm_xdl.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_batched_gemm_xdl.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/batched_gemm/device_batched_gemm_xdl_bf16_bf16_bf16_gmk_gnk_gmn_instance.cpp b/library/src/tensor_operation_instance/gpu/batched_gemm/device_batched_gemm_xdl_bf16_bf16_bf16_gmk_gnk_gmn_instance.cpp index 9566d5ecd4..079555e216 100644 --- a/library/src/tensor_operation_instance/gpu/batched_gemm/device_batched_gemm_xdl_bf16_bf16_bf16_gmk_gnk_gmn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/batched_gemm/device_batched_gemm_xdl_bf16_bf16_bf16_gmk_gnk_gmn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_batched_gemm_xdl.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_batched_gemm_xdl.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/batched_gemm/device_batched_gemm_xdl_f16_f16_f16_gkm_gkn_gmn_instance.cpp b/library/src/tensor_operation_instance/gpu/batched_gemm/device_batched_gemm_xdl_f16_f16_f16_gkm_gkn_gmn_instance.cpp index 3be8083713..03fa8361c8 100644 --- a/library/src/tensor_operation_instance/gpu/batched_gemm/device_batched_gemm_xdl_f16_f16_f16_gkm_gkn_gmn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/batched_gemm/device_batched_gemm_xdl_f16_f16_f16_gkm_gkn_gmn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_batched_gemm_xdl.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_batched_gemm_xdl.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/batched_gemm/device_batched_gemm_xdl_f16_f16_f16_gkm_gnk_gmn_instance.cpp b/library/src/tensor_operation_instance/gpu/batched_gemm/device_batched_gemm_xdl_f16_f16_f16_gkm_gnk_gmn_instance.cpp index 21daf0b193..a3f932737c 100644 --- a/library/src/tensor_operation_instance/gpu/batched_gemm/device_batched_gemm_xdl_f16_f16_f16_gkm_gnk_gmn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/batched_gemm/device_batched_gemm_xdl_f16_f16_f16_gkm_gnk_gmn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_batched_gemm_xdl.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_batched_gemm_xdl.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/batched_gemm/device_batched_gemm_xdl_f16_f16_f16_gmk_gkn_gmn_instance.cpp b/library/src/tensor_operation_instance/gpu/batched_gemm/device_batched_gemm_xdl_f16_f16_f16_gmk_gkn_gmn_instance.cpp index 9606b1f0cc..d29b68fdf1 100644 --- a/library/src/tensor_operation_instance/gpu/batched_gemm/device_batched_gemm_xdl_f16_f16_f16_gmk_gkn_gmn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/batched_gemm/device_batched_gemm_xdl_f16_f16_f16_gmk_gkn_gmn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_batched_gemm_xdl.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_batched_gemm_xdl.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/batched_gemm/device_batched_gemm_xdl_f16_f16_f16_gmk_gnk_gmn_instance.cpp b/library/src/tensor_operation_instance/gpu/batched_gemm/device_batched_gemm_xdl_f16_f16_f16_gmk_gnk_gmn_instance.cpp index 3d3e35e8e4..c821ab9bf0 100644 --- a/library/src/tensor_operation_instance/gpu/batched_gemm/device_batched_gemm_xdl_f16_f16_f16_gmk_gnk_gmn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/batched_gemm/device_batched_gemm_xdl_f16_f16_f16_gmk_gnk_gmn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_batched_gemm_xdl.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_batched_gemm_xdl.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/batched_gemm/device_batched_gemm_xdl_f32_f32_f32_gkm_gkn_gmn_instance.cpp b/library/src/tensor_operation_instance/gpu/batched_gemm/device_batched_gemm_xdl_f32_f32_f32_gkm_gkn_gmn_instance.cpp index c6d6a1ba6a..cf939d5b45 100644 --- a/library/src/tensor_operation_instance/gpu/batched_gemm/device_batched_gemm_xdl_f32_f32_f32_gkm_gkn_gmn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/batched_gemm/device_batched_gemm_xdl_f32_f32_f32_gkm_gkn_gmn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_batched_gemm_xdl.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_batched_gemm_xdl.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/batched_gemm/device_batched_gemm_xdl_f32_f32_f32_gkm_gnk_gmn_instance.cpp b/library/src/tensor_operation_instance/gpu/batched_gemm/device_batched_gemm_xdl_f32_f32_f32_gkm_gnk_gmn_instance.cpp index 157bf413ac..acf9d61765 100644 --- a/library/src/tensor_operation_instance/gpu/batched_gemm/device_batched_gemm_xdl_f32_f32_f32_gkm_gnk_gmn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/batched_gemm/device_batched_gemm_xdl_f32_f32_f32_gkm_gnk_gmn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_batched_gemm_xdl.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_batched_gemm_xdl.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/batched_gemm/device_batched_gemm_xdl_f32_f32_f32_gmk_gkn_gmn_instance.cpp b/library/src/tensor_operation_instance/gpu/batched_gemm/device_batched_gemm_xdl_f32_f32_f32_gmk_gkn_gmn_instance.cpp index 5a8988722e..836f0a4652 100644 --- a/library/src/tensor_operation_instance/gpu/batched_gemm/device_batched_gemm_xdl_f32_f32_f32_gmk_gkn_gmn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/batched_gemm/device_batched_gemm_xdl_f32_f32_f32_gmk_gkn_gmn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_batched_gemm_xdl.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_batched_gemm_xdl.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/batched_gemm/device_batched_gemm_xdl_f32_f32_f32_gmk_gnk_gmn_instance.cpp b/library/src/tensor_operation_instance/gpu/batched_gemm/device_batched_gemm_xdl_f32_f32_f32_gmk_gnk_gmn_instance.cpp index 2e892d97f5..4bb16a4eed 100644 --- a/library/src/tensor_operation_instance/gpu/batched_gemm/device_batched_gemm_xdl_f32_f32_f32_gmk_gnk_gmn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/batched_gemm/device_batched_gemm_xdl_f32_f32_f32_gmk_gnk_gmn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_batched_gemm_xdl.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_batched_gemm_xdl.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/batched_gemm/device_batched_gemm_xdl_int8_int8_int8_gkm_gkn_gmn_instance.cpp b/library/src/tensor_operation_instance/gpu/batched_gemm/device_batched_gemm_xdl_int8_int8_int8_gkm_gkn_gmn_instance.cpp index 1f3951c938..5b438c6c76 100644 --- a/library/src/tensor_operation_instance/gpu/batched_gemm/device_batched_gemm_xdl_int8_int8_int8_gkm_gkn_gmn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/batched_gemm/device_batched_gemm_xdl_int8_int8_int8_gkm_gkn_gmn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_batched_gemm_xdl.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_batched_gemm_xdl.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/batched_gemm/device_batched_gemm_xdl_int8_int8_int8_gkm_gnk_gmn_instance.cpp b/library/src/tensor_operation_instance/gpu/batched_gemm/device_batched_gemm_xdl_int8_int8_int8_gkm_gnk_gmn_instance.cpp index d6faa5a9cb..707bdde582 100644 --- a/library/src/tensor_operation_instance/gpu/batched_gemm/device_batched_gemm_xdl_int8_int8_int8_gkm_gnk_gmn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/batched_gemm/device_batched_gemm_xdl_int8_int8_int8_gkm_gnk_gmn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_batched_gemm_xdl.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_batched_gemm_xdl.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/batched_gemm/device_batched_gemm_xdl_int8_int8_int8_gmk_gkn_gmn_instance.cpp b/library/src/tensor_operation_instance/gpu/batched_gemm/device_batched_gemm_xdl_int8_int8_int8_gmk_gkn_gmn_instance.cpp index b5bc2786f2..ebb067b69a 100644 --- a/library/src/tensor_operation_instance/gpu/batched_gemm/device_batched_gemm_xdl_int8_int8_int8_gmk_gkn_gmn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/batched_gemm/device_batched_gemm_xdl_int8_int8_int8_gmk_gkn_gmn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_batched_gemm_xdl.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_batched_gemm_xdl.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/batched_gemm/device_batched_gemm_xdl_int8_int8_int8_gmk_gnk_gmn_instance.cpp b/library/src/tensor_operation_instance/gpu/batched_gemm/device_batched_gemm_xdl_int8_int8_int8_gmk_gnk_gmn_instance.cpp index 6858903ff4..1be64130ab 100644 --- a/library/src/tensor_operation_instance/gpu/batched_gemm/device_batched_gemm_xdl_int8_int8_int8_gmk_gnk_gmn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/batched_gemm/device_batched_gemm_xdl_int8_int8_int8_gmk_gnk_gmn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_batched_gemm_xdl.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_batched_gemm_xdl.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/batched_gemm_reduce/device_batched_gemm_reduce_xdl_cshuffle_f16_f16_f16_f32_f32_gkm_gkn_gmn_instance.cpp b/library/src/tensor_operation_instance/gpu/batched_gemm_reduce/device_batched_gemm_reduce_xdl_cshuffle_f16_f16_f16_f32_f32_gkm_gkn_gmn_instance.cpp index 886863c73b..3b7ac78042 100644 --- a/library/src/tensor_operation_instance/gpu/batched_gemm_reduce/device_batched_gemm_reduce_xdl_cshuffle_f16_f16_f16_f32_f32_gkm_gkn_gmn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/batched_gemm_reduce/device_batched_gemm_reduce_xdl_cshuffle_f16_f16_f16_f32_f32_gkm_gkn_gmn_instance.cpp @@ -1,9 +1,11 @@ -#include -#include "config.hpp" -#include "device_batched_gemm_reduce_xdl_cshuffle.hpp" -#include "element_wise_operation.hpp" -#include "reduction_operator.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/utility/reduction_operator.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_batched_gemm_reduce_xdl_cshuffle.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/batched_gemm_reduce/device_batched_gemm_reduce_xdl_cshuffle_f16_f16_f16_f32_f32_gkm_gnk_gmn_instance.cpp b/library/src/tensor_operation_instance/gpu/batched_gemm_reduce/device_batched_gemm_reduce_xdl_cshuffle_f16_f16_f16_f32_f32_gkm_gnk_gmn_instance.cpp index b5ddc43838..abc5bd1c3a 100644 --- a/library/src/tensor_operation_instance/gpu/batched_gemm_reduce/device_batched_gemm_reduce_xdl_cshuffle_f16_f16_f16_f32_f32_gkm_gnk_gmn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/batched_gemm_reduce/device_batched_gemm_reduce_xdl_cshuffle_f16_f16_f16_f32_f32_gkm_gnk_gmn_instance.cpp @@ -1,9 +1,11 @@ -#include -#include "config.hpp" -#include "device_batched_gemm_reduce_xdl_cshuffle.hpp" -#include "element_wise_operation.hpp" -#include "reduction_operator.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/utility/reduction_operator.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_batched_gemm_reduce_xdl_cshuffle.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/batched_gemm_reduce/device_batched_gemm_reduce_xdl_cshuffle_f16_f16_f16_f32_f32_gmk_gkn_gmn_instance.cpp b/library/src/tensor_operation_instance/gpu/batched_gemm_reduce/device_batched_gemm_reduce_xdl_cshuffle_f16_f16_f16_f32_f32_gmk_gkn_gmn_instance.cpp index 8426ab79c9..ca5d2844fc 100644 --- a/library/src/tensor_operation_instance/gpu/batched_gemm_reduce/device_batched_gemm_reduce_xdl_cshuffle_f16_f16_f16_f32_f32_gmk_gkn_gmn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/batched_gemm_reduce/device_batched_gemm_reduce_xdl_cshuffle_f16_f16_f16_f32_f32_gmk_gkn_gmn_instance.cpp @@ -1,9 +1,11 @@ -#include -#include "config.hpp" -#include "device_batched_gemm_reduce_xdl_cshuffle.hpp" -#include "element_wise_operation.hpp" -#include "reduction_operator.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/utility/reduction_operator.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_batched_gemm_reduce_xdl_cshuffle.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/batched_gemm_reduce/device_batched_gemm_reduce_xdl_cshuffle_f16_f16_f16_f32_f32_gmk_gnk_gmn_instance.cpp b/library/src/tensor_operation_instance/gpu/batched_gemm_reduce/device_batched_gemm_reduce_xdl_cshuffle_f16_f16_f16_f32_f32_gmk_gnk_gmn_instance.cpp index 7cd1908803..6f894d3571 100644 --- a/library/src/tensor_operation_instance/gpu/batched_gemm_reduce/device_batched_gemm_reduce_xdl_cshuffle_f16_f16_f16_f32_f32_gmk_gnk_gmn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/batched_gemm_reduce/device_batched_gemm_reduce_xdl_cshuffle_f16_f16_f16_f32_f32_gmk_gnk_gmn_instance.cpp @@ -1,9 +1,11 @@ -#include -#include "config.hpp" -#include "device_batched_gemm_reduce_xdl_cshuffle.hpp" -#include "element_wise_operation.hpp" -#include "reduction_operator.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/utility/reduction_operator.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_batched_gemm_reduce_xdl_cshuffle.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/conv1d_fwd/device_conv1d_fwd_xdl_nwc_kxc_nwk_bf16_instance.cpp b/library/src/tensor_operation_instance/gpu/conv1d_fwd/device_conv1d_fwd_xdl_nwc_kxc_nwk_bf16_instance.cpp index a133300f73..d19c9a4644 100644 --- a/library/src/tensor_operation_instance/gpu/conv1d_fwd/device_conv1d_fwd_xdl_nwc_kxc_nwk_bf16_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/conv1d_fwd/device_conv1d_fwd_xdl_nwc_kxc_nwk_bf16_instance.cpp @@ -1,8 +1,11 @@ -#include -#include "config.hpp" -#include "device_convnd_fwd_xdl_nhwc_kyxc_nhwk.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_convnd_fwd_xdl_nhwc_kyxc_nhwk.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/conv1d_fwd/device_conv1d_fwd_xdl_nwc_kxc_nwk_f16_instance.cpp b/library/src/tensor_operation_instance/gpu/conv1d_fwd/device_conv1d_fwd_xdl_nwc_kxc_nwk_f16_instance.cpp index 669dca617a..375c364a80 100644 --- a/library/src/tensor_operation_instance/gpu/conv1d_fwd/device_conv1d_fwd_xdl_nwc_kxc_nwk_f16_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/conv1d_fwd/device_conv1d_fwd_xdl_nwc_kxc_nwk_f16_instance.cpp @@ -1,8 +1,11 @@ -#include -#include "config.hpp" -#include "device_convnd_fwd_xdl_nhwc_kyxc_nhwk.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_convnd_fwd_xdl_nhwc_kyxc_nhwk.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/conv1d_fwd/device_conv1d_fwd_xdl_nwc_kxc_nwk_f32_instance.cpp b/library/src/tensor_operation_instance/gpu/conv1d_fwd/device_conv1d_fwd_xdl_nwc_kxc_nwk_f32_instance.cpp index 0abd47142b..88e2f68e0c 100644 --- a/library/src/tensor_operation_instance/gpu/conv1d_fwd/device_conv1d_fwd_xdl_nwc_kxc_nwk_f32_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/conv1d_fwd/device_conv1d_fwd_xdl_nwc_kxc_nwk_f32_instance.cpp @@ -1,8 +1,11 @@ -#include -#include "config.hpp" -#include "device_convnd_fwd_xdl_nhwc_kyxc_nhwk.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_convnd_fwd_xdl_nhwc_kyxc_nhwk.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/conv1d_fwd/device_conv1d_fwd_xdl_nwc_kxc_nwk_int8_instance.cpp b/library/src/tensor_operation_instance/gpu/conv1d_fwd/device_conv1d_fwd_xdl_nwc_kxc_nwk_int8_instance.cpp index 53e0f77550..714de16ba7 100644 --- a/library/src/tensor_operation_instance/gpu/conv1d_fwd/device_conv1d_fwd_xdl_nwc_kxc_nwk_int8_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/conv1d_fwd/device_conv1d_fwd_xdl_nwc_kxc_nwk_int8_instance.cpp @@ -1,8 +1,11 @@ -#include -#include "config.hpp" -#include "device_convnd_fwd_xdl_nhwc_kyxc_nhwk.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_convnd_fwd_xdl_nhwc_kyxc_nhwk.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/conv2d_bwd_data/device_conv2d_bwd_data_xdl_nhwc_kyxc_nhwk_bf16_instance.cpp b/library/src/tensor_operation_instance/gpu/conv2d_bwd_data/device_conv2d_bwd_data_xdl_nhwc_kyxc_nhwk_bf16_instance.cpp index b5814aa17f..248c3e33e8 100644 --- a/library/src/tensor_operation_instance/gpu/conv2d_bwd_data/device_conv2d_bwd_data_xdl_nhwc_kyxc_nhwk_bf16_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/conv2d_bwd_data/device_conv2d_bwd_data_xdl_nhwc_kyxc_nhwk_bf16_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_conv2d_bwd_data_xdl_nhwc_kyxc_nhwk.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_conv2d_bwd_data_xdl_nhwc_kyxc_nhwk.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/conv2d_bwd_data/device_conv2d_bwd_data_xdl_nhwc_kyxc_nhwk_f16_instance.cpp b/library/src/tensor_operation_instance/gpu/conv2d_bwd_data/device_conv2d_bwd_data_xdl_nhwc_kyxc_nhwk_f16_instance.cpp index 53498aff34..8846373ca7 100644 --- a/library/src/tensor_operation_instance/gpu/conv2d_bwd_data/device_conv2d_bwd_data_xdl_nhwc_kyxc_nhwk_f16_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/conv2d_bwd_data/device_conv2d_bwd_data_xdl_nhwc_kyxc_nhwk_f16_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_conv2d_bwd_data_xdl_nhwc_kyxc_nhwk.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_conv2d_bwd_data_xdl_nhwc_kyxc_nhwk.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/conv2d_bwd_data/device_conv2d_bwd_data_xdl_nhwc_kyxc_nhwk_f32_instance.cpp b/library/src/tensor_operation_instance/gpu/conv2d_bwd_data/device_conv2d_bwd_data_xdl_nhwc_kyxc_nhwk_f32_instance.cpp index fbe279e033..5d31a3ab5e 100644 --- a/library/src/tensor_operation_instance/gpu/conv2d_bwd_data/device_conv2d_bwd_data_xdl_nhwc_kyxc_nhwk_f32_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/conv2d_bwd_data/device_conv2d_bwd_data_xdl_nhwc_kyxc_nhwk_f32_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_conv2d_bwd_data_xdl_nhwc_kyxc_nhwk.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_conv2d_bwd_data_xdl_nhwc_kyxc_nhwk.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/conv2d_bwd_data/device_conv2d_bwd_data_xdl_nhwc_kyxc_nhwk_int8_instance.cpp b/library/src/tensor_operation_instance/gpu/conv2d_bwd_data/device_conv2d_bwd_data_xdl_nhwc_kyxc_nhwk_int8_instance.cpp index 7fd51bbfbf..590f62fdb6 100644 --- a/library/src/tensor_operation_instance/gpu/conv2d_bwd_data/device_conv2d_bwd_data_xdl_nhwc_kyxc_nhwk_int8_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/conv2d_bwd_data/device_conv2d_bwd_data_xdl_nhwc_kyxc_nhwk_int8_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_conv2d_bwd_data_xdl_nhwc_kyxc_nhwk.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_conv2d_bwd_data_xdl_nhwc_kyxc_nhwk.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/conv2d_bwd_weight/device_conv2d_bwd_weight_xdl_nhwc_kyxc_nhwk_f16_instance.cpp b/library/src/tensor_operation_instance/gpu/conv2d_bwd_weight/device_conv2d_bwd_weight_xdl_nhwc_kyxc_nhwk_f16_instance.cpp index d915db6758..76aef456ac 100644 --- a/library/src/tensor_operation_instance/gpu/conv2d_bwd_weight/device_conv2d_bwd_weight_xdl_nhwc_kyxc_nhwk_f16_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/conv2d_bwd_weight/device_conv2d_bwd_weight_xdl_nhwc_kyxc_nhwk_f16_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_conv2d_backward_weight_xdl_c_shuffle_nhwc_kyxc_nhwk.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_conv2d_backward_weight_xdl_c_shuffle_nhwc_kyxc_nhwk.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/conv2d_bwd_weight/device_conv2d_bwd_weight_xdl_nhwc_kyxc_nhwk_f32_instance.cpp b/library/src/tensor_operation_instance/gpu/conv2d_bwd_weight/device_conv2d_bwd_weight_xdl_nhwc_kyxc_nhwk_f32_instance.cpp index e9f6636518..c7b7657c63 100644 --- a/library/src/tensor_operation_instance/gpu/conv2d_bwd_weight/device_conv2d_bwd_weight_xdl_nhwc_kyxc_nhwk_f32_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/conv2d_bwd_weight/device_conv2d_bwd_weight_xdl_nhwc_kyxc_nhwk_f32_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_conv2d_backward_weight_xdl_c_shuffle_nhwc_kyxc_nhwk.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_conv2d_backward_weight_xdl_c_shuffle_nhwc_kyxc_nhwk.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/conv2d_fwd/device_conv2d_fwd_xdl_c_shuffle_nhwc_kyxc_nhwk_f16_instance.cpp b/library/src/tensor_operation_instance/gpu/conv2d_fwd/device_conv2d_fwd_xdl_c_shuffle_nhwc_kyxc_nhwk_f16_instance.cpp index b2f6f9335e..3b38b3129b 100644 --- a/library/src/tensor_operation_instance/gpu/conv2d_fwd/device_conv2d_fwd_xdl_c_shuffle_nhwc_kyxc_nhwk_f16_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/conv2d_fwd/device_conv2d_fwd_xdl_c_shuffle_nhwc_kyxc_nhwk_f16_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_conv2d_fwd_xdl_c_shuffle_nhwc_kyxc_nhwk.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_conv2d_fwd_xdl_c_shuffle_nhwc_kyxc_nhwk.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/conv2d_fwd/device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_bf16_instance.cpp b/library/src/tensor_operation_instance/gpu/conv2d_fwd/device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_bf16_instance.cpp index 47405ea1bf..33c9bf80e2 100644 --- a/library/src/tensor_operation_instance/gpu/conv2d_fwd/device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_bf16_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/conv2d_fwd/device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_bf16_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_conv2d_fwd_xdl_nhwc_kyxc_nhwk.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_conv2d_fwd_xdl_nhwc_kyxc_nhwk.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/conv2d_fwd/device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_f16_instance.cpp b/library/src/tensor_operation_instance/gpu/conv2d_fwd/device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_f16_instance.cpp index a4060f8bf2..8351d227b3 100644 --- a/library/src/tensor_operation_instance/gpu/conv2d_fwd/device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_f16_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/conv2d_fwd/device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_f16_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_conv2d_fwd_xdl_nhwc_kyxc_nhwk.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_conv2d_fwd_xdl_nhwc_kyxc_nhwk.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/conv2d_fwd/device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_f32_instance.cpp b/library/src/tensor_operation_instance/gpu/conv2d_fwd/device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_f32_instance.cpp index 3c46c2f7e9..00ad47578d 100644 --- a/library/src/tensor_operation_instance/gpu/conv2d_fwd/device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_f32_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/conv2d_fwd/device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_f32_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_conv2d_fwd_xdl_nhwc_kyxc_nhwk.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_conv2d_fwd_xdl_nhwc_kyxc_nhwk.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/conv2d_fwd/device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_int8_instance.cpp b/library/src/tensor_operation_instance/gpu/conv2d_fwd/device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_int8_instance.cpp index 0db59ca394..2804a3314c 100644 --- a/library/src/tensor_operation_instance/gpu/conv2d_fwd/device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_int8_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/conv2d_fwd/device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_int8_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_conv2d_fwd_xdl_nhwc_kyxc_nhwk.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_conv2d_fwd_xdl_nhwc_kyxc_nhwk.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/conv2d_fwd/device_convnd_2d_fwd_xdl_nhwc_kyxc_nhwk_bf16_instance.cpp b/library/src/tensor_operation_instance/gpu/conv2d_fwd/device_convnd_2d_fwd_xdl_nhwc_kyxc_nhwk_bf16_instance.cpp index de98151ef8..6768bfbd86 100644 --- a/library/src/tensor_operation_instance/gpu/conv2d_fwd/device_convnd_2d_fwd_xdl_nhwc_kyxc_nhwk_bf16_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/conv2d_fwd/device_convnd_2d_fwd_xdl_nhwc_kyxc_nhwk_bf16_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_convnd_fwd_xdl_nhwc_kyxc_nhwk.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_convnd_fwd_xdl_nhwc_kyxc_nhwk.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/conv2d_fwd/device_convnd_2d_fwd_xdl_nhwc_kyxc_nhwk_f16_instance.cpp b/library/src/tensor_operation_instance/gpu/conv2d_fwd/device_convnd_2d_fwd_xdl_nhwc_kyxc_nhwk_f16_instance.cpp index 4b4a0fc25a..dfa7ee4691 100644 --- a/library/src/tensor_operation_instance/gpu/conv2d_fwd/device_convnd_2d_fwd_xdl_nhwc_kyxc_nhwk_f16_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/conv2d_fwd/device_convnd_2d_fwd_xdl_nhwc_kyxc_nhwk_f16_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_convnd_fwd_xdl_nhwc_kyxc_nhwk.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_convnd_fwd_xdl_nhwc_kyxc_nhwk.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/conv2d_fwd/device_convnd_2d_fwd_xdl_nhwc_kyxc_nhwk_f32_instance.cpp b/library/src/tensor_operation_instance/gpu/conv2d_fwd/device_convnd_2d_fwd_xdl_nhwc_kyxc_nhwk_f32_instance.cpp index 5603fc5d06..53d53ebd34 100644 --- a/library/src/tensor_operation_instance/gpu/conv2d_fwd/device_convnd_2d_fwd_xdl_nhwc_kyxc_nhwk_f32_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/conv2d_fwd/device_convnd_2d_fwd_xdl_nhwc_kyxc_nhwk_f32_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_convnd_fwd_xdl_nhwc_kyxc_nhwk.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_convnd_fwd_xdl_nhwc_kyxc_nhwk.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/conv2d_fwd/device_convnd_2d_fwd_xdl_nhwc_kyxc_nhwk_int8_instance.cpp b/library/src/tensor_operation_instance/gpu/conv2d_fwd/device_convnd_2d_fwd_xdl_nhwc_kyxc_nhwk_int8_instance.cpp index b4447bcb82..12652f5312 100644 --- a/library/src/tensor_operation_instance/gpu/conv2d_fwd/device_convnd_2d_fwd_xdl_nhwc_kyxc_nhwk_int8_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/conv2d_fwd/device_convnd_2d_fwd_xdl_nhwc_kyxc_nhwk_int8_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_convnd_fwd_xdl_nhwc_kyxc_nhwk.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_convnd_fwd_xdl_nhwc_kyxc_nhwk.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/conv2d_fwd_bias_relu/device_conv2d_fwd_xdl_c_shuffle_bias_relu_nhwc_kyxc_nhwk_f16_instance.cpp b/library/src/tensor_operation_instance/gpu/conv2d_fwd_bias_relu/device_conv2d_fwd_xdl_c_shuffle_bias_relu_nhwc_kyxc_nhwk_f16_instance.cpp index 9c3f0a4b96..75701a7ec6 100644 --- a/library/src/tensor_operation_instance/gpu/conv2d_fwd_bias_relu/device_conv2d_fwd_xdl_c_shuffle_bias_relu_nhwc_kyxc_nhwk_f16_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/conv2d_fwd_bias_relu/device_conv2d_fwd_xdl_c_shuffle_bias_relu_nhwc_kyxc_nhwk_f16_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_conv2d_fwd_xdl_c_shuffle_bias_activation_nhwc_kyxc_nhwk.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_conv2d_fwd_xdl_c_shuffle_bias_activation_nhwc_kyxc_nhwk.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/conv2d_fwd_bias_relu_add/device_conv2d_fwd_xdl_c_shuffle_bias_relu_add_nhwc_kyxc_nhwk_f16_instance.cpp b/library/src/tensor_operation_instance/gpu/conv2d_fwd_bias_relu_add/device_conv2d_fwd_xdl_c_shuffle_bias_relu_add_nhwc_kyxc_nhwk_f16_instance.cpp index b9f46e2611..855630cd9a 100644 --- a/library/src/tensor_operation_instance/gpu/conv2d_fwd_bias_relu_add/device_conv2d_fwd_xdl_c_shuffle_bias_relu_add_nhwc_kyxc_nhwk_f16_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/conv2d_fwd_bias_relu_add/device_conv2d_fwd_xdl_c_shuffle_bias_relu_add_nhwc_kyxc_nhwk_f16_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_conv2d_fwd_xdl_c_shuffle_bias_activation_add_nhwc_kyxc_nhwk.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_conv2d_fwd_xdl_c_shuffle_bias_activation_add_nhwc_kyxc_nhwk.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/conv2d_fwd_bias_relu_atomic_add/CMakeLists.txt b/library/src/tensor_operation_instance/gpu/conv2d_fwd_bias_relu_atomic_add/CMakeLists.txt deleted file mode 100644 index 5906c7c5ac..0000000000 --- a/library/src/tensor_operation_instance/gpu/conv2d_fwd_bias_relu_atomic_add/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -# device_conv2d_fwd_bias_relu_atomic_add_instance -set(DEVICE_CONV2D_FWD_BIAS_RELU_ATOMIC_ADD_INSTANCE_SOURCE - device_conv2d_fwd_xdl_c_shuffle_bias_relu_atomic_add_nhwc_kyxc_nhwk_f16_instance.cpp; -) - -add_library(device_conv2d_fwd_bias_relu_atomic_add_instance OBJECT ${DEVICE_CONV2D_FWD_BIAS_RELU_ATOMIC_ADD_INSTANCE_SOURCE}) -set_target_properties(device_conv2d_fwd_bias_relu_atomic_add_instance PROPERTIES POSITION_INDEPENDENT_CODE ON) - -clang_tidy_check(device_conv2d_fwd_bias_relu_atomic_add_instance) diff --git a/library/src/tensor_operation_instance/gpu/conv2d_fwd_bias_relu_atomic_add/device_conv2d_fwd_xdl_c_shuffle_bias_relu_atomic_add_nhwc_kyxc_nhwk_f16_instance.cpp b/library/src/tensor_operation_instance/gpu/conv2d_fwd_bias_relu_atomic_add/device_conv2d_fwd_xdl_c_shuffle_bias_relu_atomic_add_nhwc_kyxc_nhwk_f16_instance.cpp deleted file mode 100644 index c56ad270aa..0000000000 --- a/library/src/tensor_operation_instance/gpu/conv2d_fwd_bias_relu_atomic_add/device_conv2d_fwd_xdl_c_shuffle_bias_relu_atomic_add_nhwc_kyxc_nhwk_f16_instance.cpp +++ /dev/null @@ -1,69 +0,0 @@ -#include -#include "config.hpp" -#include "device_conv2d_fwd_xdl_c_shuffle_bias_activation_nhwc_kyxc_nhwk.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace device_conv2d_fwd_bias_activation_atomic_add_instance { - -using F16 = ck::half_t; -using F32 = float; - -template -using S = ck::Sequence; - -using PassThrough = ck::tensor_operation::element_wise::PassThrough; -using AddRelu = ck::tensor_operation::element_wise::AddRelu; - -static constexpr auto InMemoryAtomicAdd = ck::InMemoryDataOperationEnum::AtomicAdd; - -static constexpr auto ConvFwdDefault = - ck::tensor_operation::device::ConvolutionForwardSpecialization::Default; - -using device_conv2d_fwd_xdl_c_shuffle_bias_relu_atomic_add_nhwc_kyxc_nhwk_f16_instances = std::tuple< - // clang-format off - //##########################################################################################| InData| WeiData| OutData| AccData| In| Wei| Out| Out| ConvForward| Block| MPer| NPer| K0Per| K1| MPer| NPer| MXdl| NXdl| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockLds| BBlockTransfer| BBlockTransfer| BBlockTransfer| BlockTransfer| BBlockTransfer| BBlockTransfer| BBlockLds| CShuffle| CShuffle| CBlockTransferClusterLengths| CBlockTransfer| - //##########################################################################################| Type| Type| Type| Type| Elementwise| Elementwise| Elementwise| GlobalMemory| Specialization| Size| Block| Block| Block| | XDL| XDL| Per| Per| ThreadCluster| ThreadCluster| SrcAccessOrder| SrcVectorDim| SrcScalar| DstScalar| AddExtraM| ThreadCluster| ThreadCluster| SrcAccessOrder| SrcVectorDim| SrcScalar| DstScalar| AddExtraN| MXdlPerWave| NXdlPerWave| _MBlock_MXdlPerWave_MWaveMPerXdl| ScalarPerVector| - //##########################################################################################| | | | | Operation| Operation| Operation| DataOperation| | | | | | | | | Wave| Wave| Lengths_K0_M_K1| ArrangeOrder| | | PerVector| PerVector_K1| | Lengths_K0_N_K1| ArrangeOrder| | | PerVector| PerVector_K1| | PerShuffle| PerShuffle| _NBlock_NXdlPerWave_NWaveNPerXdl| _NWaveNPerXdl| - //##########################################################################################| | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | - DeviceConv2dFwdXdl_C_Shuffle_Bias_Activation_Input_N_Hi_Wi_C_Weight_K_Y_X_C_Output_N_Ho_Wo_K< F16, F16, F16, F32, PassThrough, PassThrough, AddRelu, InMemoryAtomicAdd, ConvFwdDefault, 256, 256, 128, 4, 8, 32, 32, 4, 2, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, true, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, true, 1, 1, S<1, 1, 8, 1, 1, 32>, 2>, - DeviceConv2dFwdXdl_C_Shuffle_Bias_Activation_Input_N_Hi_Wi_C_Weight_K_Y_X_C_Output_N_Ho_Wo_K< F16, F16, F16, F32, PassThrough, PassThrough, AddRelu, InMemoryAtomicAdd, ConvFwdDefault, 256, 128, 256, 4, 8, 32, 32, 2, 4, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, true, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, true, 1, 1, S<1, 1, 8, 1, 1, 32>, 2>, - DeviceConv2dFwdXdl_C_Shuffle_Bias_Activation_Input_N_Hi_Wi_C_Weight_K_Y_X_C_Output_N_Ho_Wo_K< F16, F16, F16, F32, PassThrough, PassThrough, AddRelu, InMemoryAtomicAdd, ConvFwdDefault, 128, 128, 128, 4, 8, 32, 32, 4, 2, S<4, 32, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, true, S<4, 32, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, true, 1, 1, S<1, 1, 4, 1, 1, 32>, 2>, - DeviceConv2dFwdXdl_C_Shuffle_Bias_Activation_Input_N_Hi_Wi_C_Weight_K_Y_X_C_Output_N_Ho_Wo_K< F16, F16, F16, F32, PassThrough, PassThrough, AddRelu, InMemoryAtomicAdd, ConvFwdDefault, 256, 128, 128, 4, 8, 32, 32, 2, 2, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, true, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, true, 1, 1, S<1, 1, 8, 1, 1, 32>, 2>, - DeviceConv2dFwdXdl_C_Shuffle_Bias_Activation_Input_N_Hi_Wi_C_Weight_K_Y_X_C_Output_N_Ho_Wo_K< F16, F16, F16, F32, PassThrough, PassThrough, AddRelu, InMemoryAtomicAdd, ConvFwdDefault, 128, 128, 64, 4, 8, 32, 32, 2, 2, S<4, 32, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, true, S<4, 32, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, true, 1, 1, S<1, 1, 8, 1, 1, 16>, 2>, - DeviceConv2dFwdXdl_C_Shuffle_Bias_Activation_Input_N_Hi_Wi_C_Weight_K_Y_X_C_Output_N_Ho_Wo_K< F16, F16, F16, F32, PassThrough, PassThrough, AddRelu, InMemoryAtomicAdd, ConvFwdDefault, 128, 64, 128, 4, 8, 32, 32, 2, 2, S<4, 32, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, true, S<4, 32, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, true, 1, 1, S<1, 1, 4, 1, 1, 32>, 2>, - DeviceConv2dFwdXdl_C_Shuffle_Bias_Activation_Input_N_Hi_Wi_C_Weight_K_Y_X_C_Output_N_Ho_Wo_K< F16, F16, F16, F32, PassThrough, PassThrough, AddRelu, InMemoryAtomicAdd, ConvFwdDefault, 64, 64, 64, 4, 8, 32, 32, 2, 2, S<4, 16, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, true, S<4, 16, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, true, 1, 1, S<1, 1, 4, 1, 1, 16>, 2>, - DeviceConv2dFwdXdl_C_Shuffle_Bias_Activation_Input_N_Hi_Wi_C_Weight_K_Y_X_C_Output_N_Ho_Wo_K< F16, F16, F16, F32, PassThrough, PassThrough, AddRelu, InMemoryAtomicAdd, ConvFwdDefault, 256, 128, 64, 4, 8, 32, 32, 2, 1, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, true, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, true, 1, 1, S<1, 1, 8, 1, 1, 32>, 2>, - DeviceConv2dFwdXdl_C_Shuffle_Bias_Activation_Input_N_Hi_Wi_C_Weight_K_Y_X_C_Output_N_Ho_Wo_K< F16, F16, F16, F32, PassThrough, PassThrough, AddRelu, InMemoryAtomicAdd, ConvFwdDefault, 256, 64, 128, 4, 8, 32, 32, 1, 2, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, true, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, true, 1, 1, S<1, 1, 8, 1, 1, 32>, 2>, - DeviceConv2dFwdXdl_C_Shuffle_Bias_Activation_Input_N_Hi_Wi_C_Weight_K_Y_X_C_Output_N_Ho_Wo_K< F16, F16, F16, F32, PassThrough, PassThrough, AddRelu, InMemoryAtomicAdd, ConvFwdDefault, 128, 128, 32, 4, 8, 32, 32, 2, 1, S<4, 32, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, true, S<4, 32, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, true, 1, 1, S<1, 1, 8, 1, 1, 16>, 2>, - DeviceConv2dFwdXdl_C_Shuffle_Bias_Activation_Input_N_Hi_Wi_C_Weight_K_Y_X_C_Output_N_Ho_Wo_K< F16, F16, F16, F32, PassThrough, PassThrough, AddRelu, InMemoryAtomicAdd, ConvFwdDefault, 128, 32, 128, 4, 8, 32, 32, 1, 2, S<4, 32, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, true, S<4, 32, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, true, 1, 1, S<1, 1, 4, 1, 1, 32>, 2>, - DeviceConv2dFwdXdl_C_Shuffle_Bias_Activation_Input_N_Hi_Wi_C_Weight_K_Y_X_C_Output_N_Ho_Wo_K< F16, F16, F16, F32, PassThrough, PassThrough, AddRelu, InMemoryAtomicAdd, ConvFwdDefault, 64, 64, 32, 4, 8, 32, 32, 2, 1, S<4, 16, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, true, S<4, 16, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, true, 1, 1, S<1, 1, 4, 1, 1, 16>, 2>, - DeviceConv2dFwdXdl_C_Shuffle_Bias_Activation_Input_N_Hi_Wi_C_Weight_K_Y_X_C_Output_N_Ho_Wo_K< F16, F16, F16, F32, PassThrough, PassThrough, AddRelu, InMemoryAtomicAdd, ConvFwdDefault, 64, 32, 64, 4, 8, 32, 32, 1, 2, S<4, 16, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, true, S<4, 16, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, true, 1, 1, S<1, 1, 4, 1, 1, 16>, 2> - // clang-format on - >; - -void add_device_conv2d_fwd_xdl_c_shuffle_bias_relu_atomic_add_nhwc_kyxc_nhwk_f16_instances( - std::vector>& - instance_container) -{ - using Instances = - device_conv2d_fwd_xdl_c_shuffle_bias_relu_atomic_add_nhwc_kyxc_nhwk_f16_instances; - - const auto instances = Instances{}; - - ck::static_for<0, std::tuple_size_v, 1>{}([&](auto i) { - using Instance = remove_cvref_t(instances))>; - - auto instance = Instance{}; - - instance_container.push_back(std::make_unique(instance)); - }); -} - -} // namespace device_conv2d_fwd_bias_activation_atomic_add_instance -} // namespace device -} // namespace tensor_operation -} // namespace ck diff --git a/library/src/tensor_operation_instance/gpu/conv3d_fwd/device_conv3d_fwd_xdl_ndhwc_kzyxc_ndhwk_bf16_instance.cpp b/library/src/tensor_operation_instance/gpu/conv3d_fwd/device_conv3d_fwd_xdl_ndhwc_kzyxc_ndhwk_bf16_instance.cpp index bff51affd1..b4503271bf 100644 --- a/library/src/tensor_operation_instance/gpu/conv3d_fwd/device_conv3d_fwd_xdl_ndhwc_kzyxc_ndhwk_bf16_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/conv3d_fwd/device_conv3d_fwd_xdl_ndhwc_kzyxc_ndhwk_bf16_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_convnd_fwd_xdl_nhwc_kyxc_nhwk.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_convnd_fwd_xdl_nhwc_kyxc_nhwk.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/conv3d_fwd/device_conv3d_fwd_xdl_ndhwc_kzyxc_ndhwk_f16_instance.cpp b/library/src/tensor_operation_instance/gpu/conv3d_fwd/device_conv3d_fwd_xdl_ndhwc_kzyxc_ndhwk_f16_instance.cpp index 4d51180e72..713fd94086 100644 --- a/library/src/tensor_operation_instance/gpu/conv3d_fwd/device_conv3d_fwd_xdl_ndhwc_kzyxc_ndhwk_f16_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/conv3d_fwd/device_conv3d_fwd_xdl_ndhwc_kzyxc_ndhwk_f16_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_convnd_fwd_xdl_nhwc_kyxc_nhwk.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_convnd_fwd_xdl_nhwc_kyxc_nhwk.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/conv3d_fwd/device_conv3d_fwd_xdl_ndhwc_kzyxc_ndhwk_f32_instance.cpp b/library/src/tensor_operation_instance/gpu/conv3d_fwd/device_conv3d_fwd_xdl_ndhwc_kzyxc_ndhwk_f32_instance.cpp index 9a8ff8d714..9fc692eba9 100644 --- a/library/src/tensor_operation_instance/gpu/conv3d_fwd/device_conv3d_fwd_xdl_ndhwc_kzyxc_ndhwk_f32_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/conv3d_fwd/device_conv3d_fwd_xdl_ndhwc_kzyxc_ndhwk_f32_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_convnd_fwd_xdl_nhwc_kyxc_nhwk.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_convnd_fwd_xdl_nhwc_kyxc_nhwk.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/conv3d_fwd/device_conv3d_fwd_xdl_ndhwc_kzyxc_ndhwk_int8_instance.cpp b/library/src/tensor_operation_instance/gpu/conv3d_fwd/device_conv3d_fwd_xdl_ndhwc_kzyxc_ndhwk_int8_instance.cpp index 7f54b66f9b..d3faf90f99 100644 --- a/library/src/tensor_operation_instance/gpu/conv3d_fwd/device_conv3d_fwd_xdl_ndhwc_kzyxc_ndhwk_int8_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/conv3d_fwd/device_conv3d_fwd_xdl_ndhwc_kzyxc_ndhwk_int8_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_convnd_fwd_xdl_nhwc_kyxc_nhwk.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_convnd_fwd_xdl_nhwc_kyxc_nhwk.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/convnd_bwd_data/device_conv1d_bwd_data_xdl_nwc_kxc_nwk_bf16_instance.cpp b/library/src/tensor_operation_instance/gpu/convnd_bwd_data/device_conv1d_bwd_data_xdl_nwc_kxc_nwk_bf16_instance.cpp index 5c915dcc42..01c52fea81 100644 --- a/library/src/tensor_operation_instance/gpu/convnd_bwd_data/device_conv1d_bwd_data_xdl_nwc_kxc_nwk_bf16_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/convnd_bwd_data/device_conv1d_bwd_data_xdl_nwc_kxc_nwk_bf16_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_convnd_bwd_data_xdl_ndhwc_kzyxc_ndhwk.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_convnd_bwd_data_xdl_ndhwc_kzyxc_ndhwk.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/convnd_bwd_data/device_conv1d_bwd_data_xdl_nwc_kxc_nwk_f16_instance.cpp b/library/src/tensor_operation_instance/gpu/convnd_bwd_data/device_conv1d_bwd_data_xdl_nwc_kxc_nwk_f16_instance.cpp index e8f7d4f11a..f2dabd1482 100644 --- a/library/src/tensor_operation_instance/gpu/convnd_bwd_data/device_conv1d_bwd_data_xdl_nwc_kxc_nwk_f16_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/convnd_bwd_data/device_conv1d_bwd_data_xdl_nwc_kxc_nwk_f16_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_convnd_bwd_data_xdl_ndhwc_kzyxc_ndhwk.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_convnd_bwd_data_xdl_ndhwc_kzyxc_ndhwk.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/convnd_bwd_data/device_conv1d_bwd_data_xdl_nwc_kxc_nwk_f32_instance.cpp b/library/src/tensor_operation_instance/gpu/convnd_bwd_data/device_conv1d_bwd_data_xdl_nwc_kxc_nwk_f32_instance.cpp index b4c65ab66a..a019e3ac86 100644 --- a/library/src/tensor_operation_instance/gpu/convnd_bwd_data/device_conv1d_bwd_data_xdl_nwc_kxc_nwk_f32_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/convnd_bwd_data/device_conv1d_bwd_data_xdl_nwc_kxc_nwk_f32_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_convnd_bwd_data_xdl_ndhwc_kzyxc_ndhwk.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_convnd_bwd_data_xdl_ndhwc_kzyxc_ndhwk.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/convnd_bwd_data/device_conv1d_bwd_data_xdl_nwc_kxc_nwk_int8_instance.cpp b/library/src/tensor_operation_instance/gpu/convnd_bwd_data/device_conv1d_bwd_data_xdl_nwc_kxc_nwk_int8_instance.cpp index e3958ef689..0a8b10f200 100644 --- a/library/src/tensor_operation_instance/gpu/convnd_bwd_data/device_conv1d_bwd_data_xdl_nwc_kxc_nwk_int8_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/convnd_bwd_data/device_conv1d_bwd_data_xdl_nwc_kxc_nwk_int8_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_convnd_bwd_data_xdl_ndhwc_kzyxc_ndhwk.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_convnd_bwd_data_xdl_ndhwc_kzyxc_ndhwk.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/convnd_bwd_data/device_conv2d_bwd_data_xdl_nhwc_kyxc_nhwk_bf16_instance.cpp b/library/src/tensor_operation_instance/gpu/convnd_bwd_data/device_conv2d_bwd_data_xdl_nhwc_kyxc_nhwk_bf16_instance.cpp index 2e4cd5cf31..a34d8de610 100644 --- a/library/src/tensor_operation_instance/gpu/convnd_bwd_data/device_conv2d_bwd_data_xdl_nhwc_kyxc_nhwk_bf16_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/convnd_bwd_data/device_conv2d_bwd_data_xdl_nhwc_kyxc_nhwk_bf16_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_convnd_bwd_data_xdl_ndhwc_kzyxc_ndhwk.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_convnd_bwd_data_xdl_ndhwc_kzyxc_ndhwk.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/convnd_bwd_data/device_conv2d_bwd_data_xdl_nhwc_kyxc_nhwk_f16_instance.cpp b/library/src/tensor_operation_instance/gpu/convnd_bwd_data/device_conv2d_bwd_data_xdl_nhwc_kyxc_nhwk_f16_instance.cpp index 7170decc43..ed467947e4 100644 --- a/library/src/tensor_operation_instance/gpu/convnd_bwd_data/device_conv2d_bwd_data_xdl_nhwc_kyxc_nhwk_f16_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/convnd_bwd_data/device_conv2d_bwd_data_xdl_nhwc_kyxc_nhwk_f16_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_convnd_bwd_data_xdl_ndhwc_kzyxc_ndhwk.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_convnd_bwd_data_xdl_ndhwc_kzyxc_ndhwk.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { @@ -33,13 +35,11 @@ using device_conv2d_bwd_data_xdl_nhwc_kyxc_nhwk_f16_instances = DeviceConvndBwdDataXdl_Input_N_Di_Hi_Wi_C_Weight_K_Z_Y_X_C_Output_N_Do_Ho_Wo_K< F16, F16, F16, F32, PassThrough, PassThrough, PassThrough, ConvBwdDataDefault, 2, 256, 256, 128, 4, 8, 32, 32, 4, 2, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, true, S<4, 64, 1>, S<2, 0, 1>, S<0, 2, 1>, 1, 2, 8, true, 7, 1>, DeviceConvndBwdDataXdl_Input_N_Di_Hi_Wi_C_Weight_K_Z_Y_X_C_Output_N_Do_Ho_Wo_K< F16, F16, F16, F32, PassThrough, PassThrough, PassThrough, ConvBwdDataDefault, 2, 256, 128, 256, 4, 8, 32, 32, 2, 4, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, true, S<4, 64, 1>, S<2, 0, 1>, S<0, 2, 1>, 1, 4, 8, true, 7, 1>, DeviceConvndBwdDataXdl_Input_N_Di_Hi_Wi_C_Weight_K_Z_Y_X_C_Output_N_Do_Ho_Wo_K< F16, F16, F16, F32, PassThrough, PassThrough, PassThrough, ConvBwdDataDefault, 2, 128, 128, 128, 4, 8, 32, 32, 4, 2, S<4, 32, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, true, S<4, 32, 1>, S<2, 0, 1>, S<0, 2, 1>, 1, 4, 8, true, 7, 1>, -#if 1 DeviceConvndBwdDataXdl_Input_N_Di_Hi_Wi_C_Weight_K_Z_Y_X_C_Output_N_Do_Ho_Wo_K< F16, F16, F16, F32, PassThrough, PassThrough, PassThrough, ConvBwdDataDefault, 2, 256, 128, 128, 4, 8, 32, 32, 2, 2, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, true, S<4, 64, 1>, S<2, 0, 1>, S<0, 2, 1>, 1, 2, 8, true, 7, 1>, DeviceConvndBwdDataXdl_Input_N_Di_Hi_Wi_C_Weight_K_Z_Y_X_C_Output_N_Do_Ho_Wo_K< F16, F16, F16, F32, PassThrough, PassThrough, PassThrough, ConvBwdDataDefault, 2, 256, 64, 128, 4, 8, 32, 32, 1, 2, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, true, S<4, 64, 1>, S<2, 0, 1>, S<0, 2, 1>, 1, 2, 8, true, 7, 1>, DeviceConvndBwdDataXdl_Input_N_Di_Hi_Wi_C_Weight_K_Z_Y_X_C_Output_N_Do_Ho_Wo_K< F16, F16, F16, F32, PassThrough, PassThrough, PassThrough, ConvBwdDataDefault, 2, 128, 32, 128, 4, 8, 32, 32, 1, 2, S<4, 32, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, true, S<4, 32, 1>, S<2, 0, 1>, S<0, 2, 1>, 1, 4, 8, true, 7, 1>, DeviceConvndBwdDataXdl_Input_N_Di_Hi_Wi_C_Weight_K_Z_Y_X_C_Output_N_Do_Ho_Wo_K< F16, F16, F16, F32, PassThrough, PassThrough, PassThrough, ConvBwdDataDefault, 2, 128, 64, 128, 4, 8, 32, 32, 2, 2, S<4, 32, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, true, S<4, 32, 1>, S<2, 0, 1>, S<0, 2, 1>, 1, 4, 8, true, 7, 1>, DeviceConvndBwdDataXdl_Input_N_Di_Hi_Wi_C_Weight_K_Z_Y_X_C_Output_N_Do_Ho_Wo_K< F16, F16, F16, F32, PassThrough, PassThrough, PassThrough, ConvBwdDataDefault, 2, 256, 128, 64, 4, 8, 32, 32, 2, 1, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, true, S<4, 64, 1>, S<2, 0, 1>, S<0, 2, 1>, 1, 1, 8, true, 7, 1>, -#endif DeviceConvndBwdDataXdl_Input_N_Di_Hi_Wi_C_Weight_K_Z_Y_X_C_Output_N_Do_Ho_Wo_K< F16, F16, F16, F32, PassThrough, PassThrough, PassThrough, ConvBwdDataDefault, 2, 128, 128, 64, 4, 8, 32, 32, 2, 2, S<4, 32, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, true, S<4, 32, 1>, S<2, 0, 1>, S<0, 2, 1>, 1, 2, 8, true, 7, 1>, DeviceConvndBwdDataXdl_Input_N_Di_Hi_Wi_C_Weight_K_Z_Y_X_C_Output_N_Do_Ho_Wo_K< F16, F16, F16, F32, PassThrough, PassThrough, PassThrough, ConvBwdDataDefault, 2, 64, 64, 64, 4, 8, 32, 32, 2, 2, S<4, 16, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, true, S<4, 16, 1>, S<2, 0, 1>, S<0, 2, 1>, 1, 4, 8, true, 7, 1>, DeviceConvndBwdDataXdl_Input_N_Di_Hi_Wi_C_Weight_K_Z_Y_X_C_Output_N_Do_Ho_Wo_K< F16, F16, F16, F32, PassThrough, PassThrough, PassThrough, ConvBwdDataDefault, 2, 128, 128, 32, 4, 8, 32, 32, 2, 1, S<4, 32, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, true, S<4, 32, 1>, S<2, 0, 1>, S<0, 2, 1>, 1, 1, 8, true, 7, 1>, diff --git a/library/src/tensor_operation_instance/gpu/convnd_bwd_data/device_conv2d_bwd_data_xdl_nhwc_kyxc_nhwk_f32_instance.cpp b/library/src/tensor_operation_instance/gpu/convnd_bwd_data/device_conv2d_bwd_data_xdl_nhwc_kyxc_nhwk_f32_instance.cpp index 5a727b1113..046e6d07e7 100644 --- a/library/src/tensor_operation_instance/gpu/convnd_bwd_data/device_conv2d_bwd_data_xdl_nhwc_kyxc_nhwk_f32_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/convnd_bwd_data/device_conv2d_bwd_data_xdl_nhwc_kyxc_nhwk_f32_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_convnd_bwd_data_xdl_ndhwc_kzyxc_ndhwk.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_convnd_bwd_data_xdl_ndhwc_kzyxc_ndhwk.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/convnd_bwd_data/device_conv2d_bwd_data_xdl_nhwc_kyxc_nhwk_int8_instance.cpp b/library/src/tensor_operation_instance/gpu/convnd_bwd_data/device_conv2d_bwd_data_xdl_nhwc_kyxc_nhwk_int8_instance.cpp index 3c53644ddc..9ae158c96d 100644 --- a/library/src/tensor_operation_instance/gpu/convnd_bwd_data/device_conv2d_bwd_data_xdl_nhwc_kyxc_nhwk_int8_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/convnd_bwd_data/device_conv2d_bwd_data_xdl_nhwc_kyxc_nhwk_int8_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_convnd_bwd_data_xdl_ndhwc_kzyxc_ndhwk.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_convnd_bwd_data_xdl_ndhwc_kzyxc_ndhwk.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { @@ -32,10 +34,8 @@ using device_conv2d_bwd_data_xdl_nhwc_kyxc_nhwk_int8_instances = //#############################################################################| | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | DeviceConvndBwdDataXdl_Input_N_Di_Hi_Wi_C_Weight_K_Z_Y_X_C_Output_N_Do_Ho_Wo_K< DataType, DataType, DataType, AccType, PassThrough, PassThrough, PassThrough, ConvBwdDataDefault, 2, 256, 128, 256, 4, 16, 32, 32, 2, 4, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 16, 16, true, S<4, 64, 1>, S<2, 0, 1>, S<0, 2, 1>, 1, 4, 16, true, 7, 1>, DeviceConvndBwdDataXdl_Input_N_Di_Hi_Wi_C_Weight_K_Z_Y_X_C_Output_N_Do_Ho_Wo_K< DataType, DataType, DataType, AccType, PassThrough, PassThrough, PassThrough, ConvBwdDataDefault, 2, 128, 128, 128, 4, 16, 32, 32, 4, 2, S<4, 32, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 16, 16, true, S<4, 32, 1>, S<2, 0, 1>, S<0, 2, 1>, 1, 2, 16, true, 7, 1>, - #if 1 DeviceConvndBwdDataXdl_Input_N_Di_Hi_Wi_C_Weight_K_Z_Y_X_C_Output_N_Do_Ho_Wo_K< DataType, DataType, DataType, AccType, PassThrough, PassThrough, PassThrough, ConvBwdDataDefault, 2, 256, 256, 128, 4, 16, 32, 32, 4, 2, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 16, 16, true, S<4, 64, 1>, S<2, 0, 1>, S<0, 2, 1>, 1, 2, 16, true, 7, 1>, DeviceConvndBwdDataXdl_Input_N_Di_Hi_Wi_C_Weight_K_Z_Y_X_C_Output_N_Do_Ho_Wo_K< DataType, DataType, DataType, AccType, PassThrough, PassThrough, PassThrough, ConvBwdDataDefault, 2, 256, 128, 128, 4, 16, 32, 32, 2, 2, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 16, 16, true, S<4, 64, 1>, S<2, 0, 1>, S<0, 2, 1>, 1, 2, 16, true, 7, 1>, - #endif DeviceConvndBwdDataXdl_Input_N_Di_Hi_Wi_C_Weight_K_Z_Y_X_C_Output_N_Do_Ho_Wo_K< DataType, DataType, DataType, AccType, PassThrough, PassThrough, PassThrough, ConvBwdDataDefault, 2, 128, 128, 64, 4, 16, 32, 32, 2, 2, S<4, 32, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 16, 16, true, S<4, 32, 1>, S<2, 0, 1>, S<0, 2, 1>, 1, 2, 16, true, 7, 1>, DeviceConvndBwdDataXdl_Input_N_Di_Hi_Wi_C_Weight_K_Z_Y_X_C_Output_N_Do_Ho_Wo_K< DataType, DataType, DataType, AccType, PassThrough, PassThrough, PassThrough, ConvBwdDataDefault, 2, 128, 64, 128, 4, 16, 32, 32, 2, 2, S<4, 32, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 16, 16, true, S<4, 32, 1>, S<2, 0, 1>, S<0, 2, 1>, 1, 4, 16, true, 7, 1>, DeviceConvndBwdDataXdl_Input_N_Di_Hi_Wi_C_Weight_K_Z_Y_X_C_Output_N_Do_Ho_Wo_K< DataType, DataType, DataType, AccType, PassThrough, PassThrough, PassThrough, ConvBwdDataDefault, 2, 64, 64, 64, 4, 16, 32, 32, 2, 2, S<4, 16, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 16, 16, true, S<4, 16, 1>, S<2, 0, 1>, S<0, 2, 1>, 1, 4, 16, true, 7, 1>, @@ -58,9 +58,7 @@ using device_conv2d_bwd_data_xdl_nhwc_kyxc_nhwk_1x1_s1_p0_int8_instances = DeviceConvndBwdDataXdl_Input_N_Di_Hi_Wi_C_Weight_K_Z_Y_X_C_Output_N_Do_Ho_Wo_K< DataType, DataType, DataType, AccType, PassThrough, PassThrough, PassThrough, ConvBwdDataFilter1x1Stride1Pad0, 2, 256, 256, 128, 4, 16, 32, 32, 4, 2, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 16, 16, true, S<4, 64, 1>, S<2, 0, 1>, S<0, 2, 1>, 1, 2, 16, true, 7, 1>, DeviceConvndBwdDataXdl_Input_N_Di_Hi_Wi_C_Weight_K_Z_Y_X_C_Output_N_Do_Ho_Wo_K< DataType, DataType, DataType, AccType, PassThrough, PassThrough, PassThrough, ConvBwdDataFilter1x1Stride1Pad0, 2, 256, 128, 256, 4, 16, 32, 32, 2, 4, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 16, 16, true, S<4, 64, 1>, S<2, 0, 1>, S<0, 2, 1>, 1, 4, 16, true, 7, 1>, DeviceConvndBwdDataXdl_Input_N_Di_Hi_Wi_C_Weight_K_Z_Y_X_C_Output_N_Do_Ho_Wo_K< DataType, DataType, DataType, AccType, PassThrough, PassThrough, PassThrough, ConvBwdDataFilter1x1Stride1Pad0, 2, 128, 128, 128, 4, 16, 32, 32, 4, 2, S<4, 32, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 16, 16, true, S<4, 32, 1>, S<2, 0, 1>, S<0, 2, 1>, 1, 2, 16, true, 7, 1>, - #if 1 DeviceConvndBwdDataXdl_Input_N_Di_Hi_Wi_C_Weight_K_Z_Y_X_C_Output_N_Do_Ho_Wo_K< DataType, DataType, DataType, AccType, PassThrough, PassThrough, PassThrough, ConvBwdDataFilter1x1Stride1Pad0, 2, 256, 128, 128, 4, 16, 32, 32, 2, 2, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 16, 16, true, S<4, 64, 1>, S<2, 0, 1>, S<0, 2, 1>, 1, 2, 16, true, 7, 1>, - #endif DeviceConvndBwdDataXdl_Input_N_Di_Hi_Wi_C_Weight_K_Z_Y_X_C_Output_N_Do_Ho_Wo_K< DataType, DataType, DataType, AccType, PassThrough, PassThrough, PassThrough, ConvBwdDataFilter1x1Stride1Pad0, 2, 128, 128, 64, 4, 16, 32, 32, 2, 2, S<4, 32, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 16, 16, true, S<4, 32, 1>, S<2, 0, 1>, S<0, 2, 1>, 1, 2, 16, true, 7, 1>, DeviceConvndBwdDataXdl_Input_N_Di_Hi_Wi_C_Weight_K_Z_Y_X_C_Output_N_Do_Ho_Wo_K< DataType, DataType, DataType, AccType, PassThrough, PassThrough, PassThrough, ConvBwdDataFilter1x1Stride1Pad0, 2, 128, 64, 128, 4, 16, 32, 32, 2, 2, S<4, 32, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 16, 16, true, S<4, 32, 1>, S<2, 0, 1>, S<0, 2, 1>, 1, 4, 16, true, 7, 1>, DeviceConvndBwdDataXdl_Input_N_Di_Hi_Wi_C_Weight_K_Z_Y_X_C_Output_N_Do_Ho_Wo_K< DataType, DataType, DataType, AccType, PassThrough, PassThrough, PassThrough, ConvBwdDataFilter1x1Stride1Pad0, 2, 64, 64, 64, 4, 16, 32, 32, 2, 2, S<4, 16, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 16, 16, true, S<4, 16, 1>, S<2, 0, 1>, S<0, 2, 1>, 1, 4, 16, true, 7, 1>, diff --git a/library/src/tensor_operation_instance/gpu/convnd_bwd_data/device_conv3d_bwd_data_xdl_ndhwc_kzyxc_ndhwk_bf16_instance.cpp b/library/src/tensor_operation_instance/gpu/convnd_bwd_data/device_conv3d_bwd_data_xdl_ndhwc_kzyxc_ndhwk_bf16_instance.cpp index edbb7a14d9..765897fb23 100644 --- a/library/src/tensor_operation_instance/gpu/convnd_bwd_data/device_conv3d_bwd_data_xdl_ndhwc_kzyxc_ndhwk_bf16_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/convnd_bwd_data/device_conv3d_bwd_data_xdl_ndhwc_kzyxc_ndhwk_bf16_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_convnd_bwd_data_xdl_ndhwc_kzyxc_ndhwk.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_convnd_bwd_data_xdl_ndhwc_kzyxc_ndhwk.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/convnd_bwd_data/device_conv3d_bwd_data_xdl_ndhwc_kzyxc_ndhwk_f16_instance.cpp b/library/src/tensor_operation_instance/gpu/convnd_bwd_data/device_conv3d_bwd_data_xdl_ndhwc_kzyxc_ndhwk_f16_instance.cpp index 5d00fa8f08..893d055e79 100644 --- a/library/src/tensor_operation_instance/gpu/convnd_bwd_data/device_conv3d_bwd_data_xdl_ndhwc_kzyxc_ndhwk_f16_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/convnd_bwd_data/device_conv3d_bwd_data_xdl_ndhwc_kzyxc_ndhwk_f16_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_convnd_bwd_data_xdl_ndhwc_kzyxc_ndhwk.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_convnd_bwd_data_xdl_ndhwc_kzyxc_ndhwk.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { @@ -32,7 +34,6 @@ using device_conv3d_bwd_data_xdl_ndhwc_kzyxc_ndhwk_f16_instances = //#############################################################################| | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | DeviceConvndBwdDataXdl_Input_N_Di_Hi_Wi_C_Weight_K_Z_Y_X_C_Output_N_Do_Ho_Wo_K< F16, F16, F16, F32, PassThrough, PassThrough, PassThrough, ConvBwdDataDefault, 3, 256, 256, 128, 4, 8, 32, 32, 4, 2, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, true, S<4, 64, 1>, S<2, 0, 1>, S<0, 2, 1>, 1, 2, 8, true, 7, 1>, DeviceConvndBwdDataXdl_Input_N_Di_Hi_Wi_C_Weight_K_Z_Y_X_C_Output_N_Do_Ho_Wo_K< F16, F16, F16, F32, PassThrough, PassThrough, PassThrough, ConvBwdDataDefault, 3, 128, 128, 128, 4, 8, 32, 32, 4, 2, S<4, 32, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, true, S<4, 32, 1>, S<2, 0, 1>, S<0, 2, 1>, 1, 4, 8, true, 7, 1>, -#if 1 DeviceConvndBwdDataXdl_Input_N_Di_Hi_Wi_C_Weight_K_Z_Y_X_C_Output_N_Do_Ho_Wo_K< F16, F16, F16, F32, PassThrough, PassThrough, PassThrough, ConvBwdDataDefault, 3, 256, 128, 256, 4, 8, 32, 32, 2, 4, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, true, S<4, 64, 1>, S<2, 0, 1>, S<0, 2, 1>, 1, 4, 8, true, 7, 1>, DeviceConvndBwdDataXdl_Input_N_Di_Hi_Wi_C_Weight_K_Z_Y_X_C_Output_N_Do_Ho_Wo_K< F16, F16, F16, F32, PassThrough, PassThrough, PassThrough, ConvBwdDataDefault, 3, 256, 128, 128, 4, 8, 32, 32, 2, 2, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, true, S<4, 64, 1>, S<2, 0, 1>, S<0, 2, 1>, 1, 2, 8, true, 7, 1>, DeviceConvndBwdDataXdl_Input_N_Di_Hi_Wi_C_Weight_K_Z_Y_X_C_Output_N_Do_Ho_Wo_K< F16, F16, F16, F32, PassThrough, PassThrough, PassThrough, ConvBwdDataDefault, 3, 256, 64, 128, 4, 8, 32, 32, 1, 2, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, true, S<4, 64, 1>, S<2, 0, 1>, S<0, 2, 1>, 1, 2, 8, true, 7, 1>, @@ -40,7 +41,6 @@ using device_conv3d_bwd_data_xdl_ndhwc_kzyxc_ndhwk_f16_instances = DeviceConvndBwdDataXdl_Input_N_Di_Hi_Wi_C_Weight_K_Z_Y_X_C_Output_N_Do_Ho_Wo_K< F16, F16, F16, F32, PassThrough, PassThrough, PassThrough, ConvBwdDataDefault, 3, 128, 64, 128, 4, 8, 32, 32, 2, 2, S<4, 32, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, true, S<4, 32, 1>, S<2, 0, 1>, S<0, 2, 1>, 1, 4, 8, true, 7, 1>, DeviceConvndBwdDataXdl_Input_N_Di_Hi_Wi_C_Weight_K_Z_Y_X_C_Output_N_Do_Ho_Wo_K< F16, F16, F16, F32, PassThrough, PassThrough, PassThrough, ConvBwdDataDefault, 3, 256, 128, 64, 4, 8, 32, 32, 2, 1, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, true, S<4, 64, 1>, S<2, 0, 1>, S<0, 2, 1>, 1, 1, 8, true, 7, 1>, DeviceConvndBwdDataXdl_Input_N_Di_Hi_Wi_C_Weight_K_Z_Y_X_C_Output_N_Do_Ho_Wo_K< F16, F16, F16, F32, PassThrough, PassThrough, PassThrough, ConvBwdDataDefault, 3, 64, 32, 64, 4, 8, 32, 32, 1, 2, S<4, 16, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, true, S<4, 16, 1>, S<2, 0, 1>, S<0, 2, 1>, 1, 2, 8, true, 7, 1>, -#endif DeviceConvndBwdDataXdl_Input_N_Di_Hi_Wi_C_Weight_K_Z_Y_X_C_Output_N_Do_Ho_Wo_K< F16, F16, F16, F32, PassThrough, PassThrough, PassThrough, ConvBwdDataDefault, 3, 128, 128, 64, 4, 8, 32, 32, 2, 2, S<4, 32, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, true, S<4, 32, 1>, S<2, 0, 1>, S<0, 2, 1>, 1, 2, 8, true, 7, 1>, DeviceConvndBwdDataXdl_Input_N_Di_Hi_Wi_C_Weight_K_Z_Y_X_C_Output_N_Do_Ho_Wo_K< F16, F16, F16, F32, PassThrough, PassThrough, PassThrough, ConvBwdDataDefault, 3, 64, 64, 64, 4, 8, 32, 32, 2, 2, S<4, 16, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, true, S<4, 16, 1>, S<2, 0, 1>, S<0, 2, 1>, 1, 4, 8, true, 7, 1>, DeviceConvndBwdDataXdl_Input_N_Di_Hi_Wi_C_Weight_K_Z_Y_X_C_Output_N_Do_Ho_Wo_K< F16, F16, F16, F32, PassThrough, PassThrough, PassThrough, ConvBwdDataDefault, 3, 128, 128, 32, 4, 8, 32, 32, 2, 1, S<4, 32, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, true, S<4, 32, 1>, S<2, 0, 1>, S<0, 2, 1>, 1, 1, 8, true, 7, 1>, diff --git a/library/src/tensor_operation_instance/gpu/convnd_bwd_data/device_conv3d_bwd_data_xdl_ndhwc_kzyxc_ndhwk_f32_instance.cpp b/library/src/tensor_operation_instance/gpu/convnd_bwd_data/device_conv3d_bwd_data_xdl_ndhwc_kzyxc_ndhwk_f32_instance.cpp index d5cd04de6b..ce4eec79a7 100644 --- a/library/src/tensor_operation_instance/gpu/convnd_bwd_data/device_conv3d_bwd_data_xdl_ndhwc_kzyxc_ndhwk_f32_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/convnd_bwd_data/device_conv3d_bwd_data_xdl_ndhwc_kzyxc_ndhwk_f32_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_convnd_bwd_data_xdl_ndhwc_kzyxc_ndhwk.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_convnd_bwd_data_xdl_ndhwc_kzyxc_ndhwk.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/convnd_bwd_data/device_conv3d_bwd_data_xdl_ndhwc_kzyxc_ndhwk_int8_instance.cpp b/library/src/tensor_operation_instance/gpu/convnd_bwd_data/device_conv3d_bwd_data_xdl_ndhwc_kzyxc_ndhwk_int8_instance.cpp index d551970606..6242351733 100644 --- a/library/src/tensor_operation_instance/gpu/convnd_bwd_data/device_conv3d_bwd_data_xdl_ndhwc_kzyxc_ndhwk_int8_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/convnd_bwd_data/device_conv3d_bwd_data_xdl_ndhwc_kzyxc_ndhwk_int8_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_convnd_bwd_data_xdl_ndhwc_kzyxc_ndhwk.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_convnd_bwd_data_xdl_ndhwc_kzyxc_ndhwk.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { @@ -33,13 +35,11 @@ using device_conv3d_bwd_data_xdl_ndhwc_kzyxc_ndhwk_int8_instances = DeviceConvndBwdDataXdl_Input_N_Di_Hi_Wi_C_Weight_K_Z_Y_X_C_Output_N_Do_Ho_Wo_K< DataType, DataType, DataType, AccType, PassThrough, PassThrough, PassThrough, ConvBwdDataDefault, 3, 256, 256, 128, 4, 16, 32, 32, 4, 2, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 16, 16, true, S<4, 64, 1>, S<2, 0, 1>, S<0, 2, 1>, 1, 2, 16, true, 7, 1>, DeviceConvndBwdDataXdl_Input_N_Di_Hi_Wi_C_Weight_K_Z_Y_X_C_Output_N_Do_Ho_Wo_K< DataType, DataType, DataType, AccType, PassThrough, PassThrough, PassThrough, ConvBwdDataDefault, 3, 256, 128, 256, 4, 16, 32, 32, 2, 4, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 16, 16, true, S<4, 64, 1>, S<2, 0, 1>, S<0, 2, 1>, 1, 4, 16, true, 7, 1>, DeviceConvndBwdDataXdl_Input_N_Di_Hi_Wi_C_Weight_K_Z_Y_X_C_Output_N_Do_Ho_Wo_K< DataType, DataType, DataType, AccType, PassThrough, PassThrough, PassThrough, ConvBwdDataDefault, 3, 128, 128, 128, 4, 16, 32, 32, 4, 2, S<4, 32, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 16, 16, true, S<4, 32, 1>, S<2, 0, 1>, S<0, 2, 1>, 1, 2, 16, true, 7, 1>, -#if 1 DeviceConvndBwdDataXdl_Input_N_Di_Hi_Wi_C_Weight_K_Z_Y_X_C_Output_N_Do_Ho_Wo_K< DataType, DataType, DataType, AccType, PassThrough, PassThrough, PassThrough, ConvBwdDataDefault, 3, 256, 128, 128, 4, 16, 32, 32, 2, 2, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 16, 16, true, S<4, 64, 1>, S<2, 0, 1>, S<0, 2, 1>, 1, 2, 16, true, 7, 1>, DeviceConvndBwdDataXdl_Input_N_Di_Hi_Wi_C_Weight_K_Z_Y_X_C_Output_N_Do_Ho_Wo_K< DataType, DataType, DataType, AccType, PassThrough, PassThrough, PassThrough, ConvBwdDataDefault, 3, 128, 64, 128, 4, 16, 32, 32, 2, 2, S<4, 32, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 16, 16, true, S<4, 32, 1>, S<2, 0, 1>, S<0, 2, 1>, 1, 4, 16, true, 7, 1>, DeviceConvndBwdDataXdl_Input_N_Di_Hi_Wi_C_Weight_K_Z_Y_X_C_Output_N_Do_Ho_Wo_K< DataType, DataType, DataType, AccType, PassThrough, PassThrough, PassThrough, ConvBwdDataDefault, 3, 256, 64, 128, 4, 16, 32, 32, 1, 2, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 16, 16, true, S<4, 64, 1>, S<2, 0, 1>, S<0, 2, 1>, 1, 2, 16, true, 7, 1>, DeviceConvndBwdDataXdl_Input_N_Di_Hi_Wi_C_Weight_K_Z_Y_X_C_Output_N_Do_Ho_Wo_K< DataType, DataType, DataType, AccType, PassThrough, PassThrough, PassThrough, ConvBwdDataDefault, 3, 128, 32, 128, 4, 16, 32, 32, 1, 2, S<4, 32, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 16, 16, true, S<4, 32, 1>, S<2, 0, 1>, S<0, 2, 1>, 1, 4, 16, true, 7, 1>, DeviceConvndBwdDataXdl_Input_N_Di_Hi_Wi_C_Weight_K_Z_Y_X_C_Output_N_Do_Ho_Wo_K< DataType, DataType, DataType, AccType, PassThrough, PassThrough, PassThrough, ConvBwdDataDefault, 3, 64, 32, 64, 4, 16, 32, 32, 1, 2, S<4, 16, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 16, 16, true, S<4, 16, 1>, S<2, 0, 1>, S<0, 2, 1>, 1, 4, 16, true, 7, 1>, -#endif DeviceConvndBwdDataXdl_Input_N_Di_Hi_Wi_C_Weight_K_Z_Y_X_C_Output_N_Do_Ho_Wo_K< DataType, DataType, DataType, AccType, PassThrough, PassThrough, PassThrough, ConvBwdDataDefault, 3, 128, 128, 64, 4, 16, 32, 32, 2, 2, S<4, 32, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 16, 16, true, S<4, 32, 1>, S<2, 0, 1>, S<0, 2, 1>, 1, 2, 16, true, 7, 1>, DeviceConvndBwdDataXdl_Input_N_Di_Hi_Wi_C_Weight_K_Z_Y_X_C_Output_N_Do_Ho_Wo_K< DataType, DataType, DataType, AccType, PassThrough, PassThrough, PassThrough, ConvBwdDataDefault, 3, 64, 64, 64, 4, 16, 32, 32, 2, 2, S<4, 16, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 16, 16, true, S<4, 16, 1>, S<2, 0, 1>, S<0, 2, 1>, 1, 4, 16, true, 7, 1>, DeviceConvndBwdDataXdl_Input_N_Di_Hi_Wi_C_Weight_K_Z_Y_X_C_Output_N_Do_Ho_Wo_K< DataType, DataType, DataType, AccType, PassThrough, PassThrough, PassThrough, ConvBwdDataDefault, 3, 256, 128, 64, 4, 16, 32, 32, 2, 1, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 16, 16, true, S<4, 64, 1>, S<2, 0, 1>, S<0, 2, 1>, 1, 1, 16, true, 7, 1>, diff --git a/library/src/tensor_operation_instance/gpu/device_conv2d.cpp b/library/src/tensor_operation_instance/gpu/device_conv2d.cpp deleted file mode 100644 index 6b99433ffa..0000000000 --- a/library/src/tensor_operation_instance/gpu/device_conv2d.cpp +++ /dev/null @@ -1,201 +0,0 @@ -#include -#include "config.hpp" -#include "device_conv2d_fwd_xdl_c_shuffle_nhwc_kyxc_nhwk.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" -#include "host_interface.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace device_conv2d_fwd_instance { -using PassThrough = ck::tensor_operation::element_wise::PassThrough; -void add_device_conv2d_fwd_xdl_c_shuffle_nhwc_kyxc_nhwk_f16_instances( - std::vector>& instances); -void add_device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_f32_instances( - std::vector>& instances); -void add_device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_bf16_instances( - std::vector>& instances); -void add_device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_f16_instances( - std::vector>& instances); -void add_device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_int8_instances( - std::vector>& instances); - -} // namespace device_conv2d_fwd_instance -} // namespace device -} // namespace tensor_operation -} // namespace ck - -using PassThrough = ck::tensor_operation::element_wise::PassThrough; -struct DeviceConvFwdPtr_t::DeviceConvFwdPtrImpl -{ - std::unique_ptr - MakeArgumentPointer(void* in_ptr, - void* wei_ptr, - void* out_ptr, - size_t N, - size_t K, - size_t C, - std::vector input_spatial_lengths, - std::vector filter_spatial_lengths, - std::vector output_spatial_lengths, - std::vector conv_filter_strides, - std::vector conv_filter_dilations, - std::vector input_left_pads, - std::vector input_right_pads) const - { - return el->MakeArgumentPointer(in_ptr, - wei_ptr, - out_ptr, - N, - K, - C, - input_spatial_lengths, - filter_spatial_lengths, - output_spatial_lengths, - conv_filter_strides, - conv_filter_dilations, - input_left_pads, - input_right_pads, - PassThrough{}, - PassThrough{}, - PassThrough{}); - } - std::unique_ptr MakeInvokerPointer() const - { - return el->MakeInvokerPointer(); - } - - std::string GetTypeString() { return el->GetTypeString(); } - bool IsSupportedArgument(const DeviceConvFwdPtr_t::BaseArgument* arg) - { - return el->IsSupportedArgument(arg); - } - - ck::tensor_operation::device::DeviceConvFwdPtr el; -}; - -DeviceConvFwdPtr_t::DeviceConvFwdPtr_t() : pImpl(nullptr) {} -DeviceConvFwdPtr_t::~DeviceConvFwdPtr_t() = default; -DeviceConvFwdPtr_t::DeviceConvFwdPtr_t(DeviceConvFwdPtr_t&&) = default; -DeviceConvFwdPtr_t::DeviceConvFwdPtr_t(DeviceConvFwdPtr_t::DeviceConvFwdPtrImpl& other) - : pImpl(std::make_unique(std::move(other))) -{ -} - -std::unique_ptr -DeviceConvFwdPtr_t::MakeArgumentPointer(void* in_ptr, - void* wei_ptr, - void* out_ptr, - size_t N, - size_t K, - size_t C, - std::vector input_spatial_lengths, - std::vector filter_spatial_lengths, - std::vector output_spatial_lengths, - std::vector conv_filter_strides, - std::vector conv_filter_dilations, - std::vector input_left_pads, - std::vector input_right_pads) const -{ - return pImpl->MakeArgumentPointer(in_ptr, - wei_ptr, - out_ptr, - N, - K, - C, - input_spatial_lengths, - filter_spatial_lengths, - output_spatial_lengths, - conv_filter_strides, - conv_filter_dilations, - input_left_pads, - input_right_pads); -} - -std::unique_ptr DeviceConvFwdPtr_t::MakeInvokerPointer() const -{ - return pImpl->MakeInvokerPointer(); -} - -std::string DeviceConvFwdPtr_t::GetTypeString() { return pImpl->GetTypeString(); } -bool DeviceConvFwdPtr_t::IsSupportedArgument(const DeviceConvFwdPtr_t::BaseArgument* arg_ptr) -{ - return pImpl->IsSupportedArgument(arg_ptr); -} - -using namespace ck::tensor_operation::device::device_conv2d_fwd_instance; -void add_device_conv2d_fwd_xdl_c_shuffle_nhwc_kyxc_nhwk_f16_instances_t( - std::vector& instances) -{ - std::vector< - ck::tensor_operation::device::DeviceConvFwdPtr> - local_instances; - add_device_conv2d_fwd_xdl_c_shuffle_nhwc_kyxc_nhwk_f16_instances(local_instances); - for(auto& kinder : local_instances) - { - DeviceConvFwdPtr_t::DeviceConvFwdPtrImpl tmp{std::move(kinder)}; - instances.emplace_back(tmp); - } - return; -} - -void add_device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_f32_instances_t( - std::vector& instances) -{ - std::vector< - ck::tensor_operation::device::DeviceConvFwdPtr> - local_instances; - add_device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_f32_instances(local_instances); - for(auto& kinder : local_instances) - { - DeviceConvFwdPtr_t::DeviceConvFwdPtrImpl tmp{std::move(kinder)}; - instances.emplace_back(tmp); // Perhaps we can do better - } - return; -} - -void add_device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_bf16_instances_t( - std::vector& instances) -{ - std::vector< - ck::tensor_operation::device::DeviceConvFwdPtr> - local_instances; - add_device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_bf16_instances(local_instances); - for(auto& kinder : local_instances) - { - DeviceConvFwdPtr_t::DeviceConvFwdPtrImpl tmp{std::move(kinder)}; - instances.emplace_back(tmp); // Perhaps we can do better - } - return; -} - -void add_device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_f16_instances_t( - std::vector& instances) -{ - std::vector< - ck::tensor_operation::device::DeviceConvFwdPtr> - local_instances; - add_device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_f16_instances(local_instances); - for(auto& kinder : local_instances) - { - DeviceConvFwdPtr_t::DeviceConvFwdPtrImpl tmp{std::move(kinder)}; - instances.emplace_back(tmp); // Perhaps we can do better - } - return; -} - -void add_device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_int8_instances_t( - std::vector& instances) -{ - std::vector< - ck::tensor_operation::device::DeviceConvFwdPtr> - local_instances; - add_device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_int8_instances(local_instances); - for(auto& kinder : local_instances) - { - DeviceConvFwdPtr_t::DeviceConvFwdPtrImpl tmp{std::move(kinder)}; - instances.emplace_back(tmp); - } - return; -} diff --git a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_dl_f16_f16_f16_km_kn_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_dl_f16_f16_f16_km_kn_mn_instance.cpp index db7f6af04b..65222a9df7 100644 --- a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_dl_f16_f16_f16_km_kn_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_dl_f16_f16_f16_km_kn_mn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_gemm_dl.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_dl.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_dl_f16_f16_f16_km_nk_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_dl_f16_f16_f16_km_nk_mn_instance.cpp index c4253bcc4c..9d6437962b 100644 --- a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_dl_f16_f16_f16_km_nk_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_dl_f16_f16_f16_km_nk_mn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_gemm_dl.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_dl.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_dl_f16_f16_f16_mk_kn_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_dl_f16_f16_f16_mk_kn_mn_instance.cpp index d19d11f1f8..2b34196056 100644 --- a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_dl_f16_f16_f16_mk_kn_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_dl_f16_f16_f16_mk_kn_mn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_gemm_dl.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_dl.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_dl_f16_f16_f16_mk_nk_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_dl_f16_f16_f16_mk_nk_mn_instance.cpp index cd86e5ceae..67f178609b 100644 --- a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_dl_f16_f16_f16_mk_nk_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_dl_f16_f16_f16_mk_nk_mn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_gemm_dl.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_dl.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_dl_f32_f32_f32_km_kn_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_dl_f32_f32_f32_km_kn_mn_instance.cpp index 3fcc5fdfdc..8816cd0189 100644 --- a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_dl_f32_f32_f32_km_kn_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_dl_f32_f32_f32_km_kn_mn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_gemm_dl.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_dl.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_dl_f32_f32_f32_km_nk_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_dl_f32_f32_f32_km_nk_mn_instance.cpp index 8cd32128b5..11ae9ce41f 100644 --- a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_dl_f32_f32_f32_km_nk_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_dl_f32_f32_f32_km_nk_mn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_gemm_dl.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_dl.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_dl_f32_f32_f32_mk_kn_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_dl_f32_f32_f32_mk_kn_mn_instance.cpp index 4c4bfc440d..9b52d681d5 100644 --- a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_dl_f32_f32_f32_mk_kn_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_dl_f32_f32_f32_mk_kn_mn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_gemm_dl.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_dl.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_dl_f32_f32_f32_mk_nk_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_dl_f32_f32_f32_mk_nk_mn_instance.cpp index c6077341b1..2975e95d03 100644 --- a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_dl_f32_f32_f32_mk_nk_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_dl_f32_f32_f32_mk_nk_mn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_gemm_dl.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_dl.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_dl_i8_i8_i8_km_kn_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_dl_i8_i8_i8_km_kn_mn_instance.cpp index 91b68d4bf2..74cde7ee10 100644 --- a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_dl_i8_i8_i8_km_kn_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_dl_i8_i8_i8_km_kn_mn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_gemm_dl.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_dl.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_dl_i8_i8_i8_km_nk_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_dl_i8_i8_i8_km_nk_mn_instance.cpp index 13b185fd93..6d30ff9e51 100644 --- a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_dl_i8_i8_i8_km_nk_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_dl_i8_i8_i8_km_nk_mn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_gemm_dl.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_dl.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_dl_i8_i8_i8_mk_kn_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_dl_i8_i8_i8_mk_kn_mn_instance.cpp index ff4a89beb4..cea6f0faa2 100644 --- a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_dl_i8_i8_i8_mk_kn_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_dl_i8_i8_i8_mk_kn_mn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_gemm_dl.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_dl.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_dl_i8_i8_i8_mk_nk_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_dl_i8_i8_i8_mk_nk_mn_instance.cpp index e32158a292..cdab613a60 100644 --- a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_dl_i8_i8_i8_mk_nk_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_dl_i8_i8_i8_mk_nk_mn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_gemm_dl.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_dl.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_2_stage_f16_f16_f16_mk_nk_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_2_stage_f16_f16_f16_mk_nk_mn_instance.cpp index de97b60a62..6ddf31005f 100644 --- a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_2_stage_f16_f16_f16_mk_nk_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_2_stage_f16_f16_f16_mk_nk_mn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_gemm_xdl_cshuffle.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_xdl_cshuffle.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_bf16_bf16_bf16_km_kn_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_bf16_bf16_bf16_km_kn_mn_instance.cpp index 5e99c67b3f..ea08c76eb0 100644 --- a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_bf16_bf16_bf16_km_kn_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_bf16_bf16_bf16_km_kn_mn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_gemm_xdl_cshuffle.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_xdl_cshuffle.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_bf16_bf16_bf16_km_nk_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_bf16_bf16_bf16_km_nk_mn_instance.cpp index 321b97fd30..3c25cdd1a4 100644 --- a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_bf16_bf16_bf16_km_nk_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_bf16_bf16_bf16_km_nk_mn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_gemm_xdl_cshuffle.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_xdl_cshuffle.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_bf16_bf16_bf16_mk_kn_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_bf16_bf16_bf16_mk_kn_mn_instance.cpp index 1d69a23dd7..bff8327707 100644 --- a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_bf16_bf16_bf16_mk_kn_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_bf16_bf16_bf16_mk_kn_mn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_gemm_xdl_cshuffle.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_xdl_cshuffle.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_bf16_bf16_bf16_mk_nk_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_bf16_bf16_bf16_mk_nk_mn_instance.cpp index 8ffa2b8b86..93b20f5634 100644 --- a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_bf16_bf16_bf16_mk_nk_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_bf16_bf16_bf16_mk_nk_mn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_gemm_xdl_cshuffle.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_xdl_cshuffle.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_f16_f16_f16_km_kn_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_f16_f16_f16_km_kn_mn_instance.cpp index 09adf1678d..7788b4570e 100644 --- a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_f16_f16_f16_km_kn_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_f16_f16_f16_km_kn_mn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_gemm_xdl_cshuffle.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_xdl_cshuffle.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_f16_f16_f16_km_nk_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_f16_f16_f16_km_nk_mn_instance.cpp index 121b5857b2..35af7c3e16 100644 --- a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_f16_f16_f16_km_nk_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_f16_f16_f16_km_nk_mn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_gemm_xdl_cshuffle.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_xdl_cshuffle.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_f16_f16_f16_mk_kn_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_f16_f16_f16_mk_kn_mn_instance.cpp index 2073d5f50e..efc8ba715a 100644 --- a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_f16_f16_f16_mk_kn_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_f16_f16_f16_mk_kn_mn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_gemm_xdl_cshuffle.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_xdl_cshuffle.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_f16_f16_f16_mk_nk_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_f16_f16_f16_mk_nk_mn_instance.cpp index e177ee60ec..e37402157d 100644 --- a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_f16_f16_f16_mk_nk_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_f16_f16_f16_mk_nk_mn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_gemm_xdl_cshuffle.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_xdl_cshuffle.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_f32_f32_f32_km_kn_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_f32_f32_f32_km_kn_mn_instance.cpp index ff830d4161..6c82745c28 100644 --- a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_f32_f32_f32_km_kn_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_f32_f32_f32_km_kn_mn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_gemm_xdl_cshuffle.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_xdl_cshuffle.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_f32_f32_f32_km_nk_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_f32_f32_f32_km_nk_mn_instance.cpp index 79bca77aad..006998d682 100644 --- a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_f32_f32_f32_km_nk_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_f32_f32_f32_km_nk_mn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_gemm_xdl_cshuffle.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_xdl_cshuffle.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_f32_f32_f32_mk_kn_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_f32_f32_f32_mk_kn_mn_instance.cpp index fac4e8d96e..69b77ace18 100644 --- a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_f32_f32_f32_mk_kn_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_f32_f32_f32_mk_kn_mn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_gemm_xdl_cshuffle.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_xdl_cshuffle.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_f32_f32_f32_mk_nk_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_f32_f32_f32_mk_nk_mn_instance.cpp index ffcd957913..7f45690832 100644 --- a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_f32_f32_f32_mk_nk_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_f32_f32_f32_mk_nk_mn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_gemm_xdl_cshuffle.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_xdl_cshuffle.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_i8_i8_i8_km_kn_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_i8_i8_i8_km_kn_mn_instance.cpp index 2185b55aac..02fda79f8b 100644 --- a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_i8_i8_i8_km_kn_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_i8_i8_i8_km_kn_mn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_gemm_xdl_cshuffle.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_xdl_cshuffle.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_i8_i8_i8_km_nk_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_i8_i8_i8_km_nk_mn_instance.cpp index 90966349b2..2918c95763 100644 --- a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_i8_i8_i8_km_nk_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_i8_i8_i8_km_nk_mn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_gemm_xdl_cshuffle.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_xdl_cshuffle.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_i8_i8_i8_mk_kn_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_i8_i8_i8_mk_kn_mn_instance.cpp index aa5a13001c..af54e4c3da 100644 --- a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_i8_i8_i8_mk_kn_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_i8_i8_i8_mk_kn_mn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_gemm_xdl_cshuffle.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_xdl_cshuffle.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_i8_i8_i8_mk_nk_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_i8_i8_i8_mk_nk_mn_instance.cpp index 82eec1164a..1fcadcc33d 100644 --- a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_i8_i8_i8_mk_nk_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_c_shuffle_i8_i8_i8_mk_nk_mn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_gemm_xdl_cshuffle.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_xdl_cshuffle.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_f16_f16_f16_km_kn_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_f16_f16_f16_km_kn_mn_instance.cpp index 08047c7e52..40e895d16d 100644 --- a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_f16_f16_f16_km_kn_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_f16_f16_f16_km_kn_mn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_gemm_xdl.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_xdl.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_f16_f16_f16_km_nk_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_f16_f16_f16_km_nk_mn_instance.cpp index 05cb080cbf..3efc94ecec 100644 --- a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_f16_f16_f16_km_nk_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_f16_f16_f16_km_nk_mn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_gemm_xdl.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_xdl.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_f16_f16_f16_mk_kn_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_f16_f16_f16_mk_kn_mn_instance.cpp index 4de989caf0..5e8716e6ed 100644 --- a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_f16_f16_f16_mk_kn_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_f16_f16_f16_mk_kn_mn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_gemm_xdl.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_xdl.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_f16_f16_f16_mk_nk_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_f16_f16_f16_mk_nk_mn_instance.cpp index 633e2aac2e..b03265b954 100644 --- a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_f16_f16_f16_mk_nk_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_f16_f16_f16_mk_nk_mn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_gemm_xdl.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_xdl.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_f32_f32_f32_km_kn_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_f32_f32_f32_km_kn_mn_instance.cpp index 8284311102..ce2da9889c 100644 --- a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_f32_f32_f32_km_kn_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_f32_f32_f32_km_kn_mn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_gemm_xdl.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_xdl.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_f32_f32_f32_km_nk_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_f32_f32_f32_km_nk_mn_instance.cpp index 235c4771f9..299f364028 100644 --- a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_f32_f32_f32_km_nk_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_f32_f32_f32_km_nk_mn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_gemm_xdl.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_xdl.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_f32_f32_f32_mk_kn_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_f32_f32_f32_mk_kn_mn_instance.cpp index b7000bddf8..92270bf9ad 100644 --- a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_f32_f32_f32_mk_kn_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_f32_f32_f32_mk_kn_mn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_gemm_xdl.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_xdl.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_f32_f32_f32_mk_nk_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_f32_f32_f32_mk_nk_mn_instance.cpp index 1b4f23141b..1b254b11d3 100644 --- a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_f32_f32_f32_mk_nk_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_f32_f32_f32_mk_nk_mn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_gemm_xdl.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_xdl.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_f64_f64_f64_km_kn_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_f64_f64_f64_km_kn_mn_instance.cpp index fdc85dfc71..d4022c0cf3 100644 --- a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_f64_f64_f64_km_kn_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_f64_f64_f64_km_kn_mn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_gemm_xdl.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_xdl.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_f64_f64_f64_km_nk_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_f64_f64_f64_km_nk_mn_instance.cpp index e400cd9bbb..456bfc4c68 100644 --- a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_f64_f64_f64_km_nk_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_f64_f64_f64_km_nk_mn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_gemm_xdl.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_xdl.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_f64_f64_f64_mk_kn_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_f64_f64_f64_mk_kn_mn_instance.cpp index 2f9241b93b..4e3ef7f587 100644 --- a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_f64_f64_f64_mk_kn_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_f64_f64_f64_mk_kn_mn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_gemm_xdl.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_xdl.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_f64_f64_f64_mk_nk_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_f64_f64_f64_mk_nk_mn_instance.cpp index 537fe2bdae..ca40376ba6 100644 --- a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_f64_f64_f64_mk_nk_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_f64_f64_f64_mk_nk_mn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_gemm_xdl.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_xdl.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_splitk_f16_f16_f16_km_kn_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_splitk_f16_f16_f16_km_kn_mn_instance.cpp index 26ec965bb5..59c2577a06 100644 --- a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_splitk_f16_f16_f16_km_kn_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_splitk_f16_f16_f16_km_kn_mn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_gemm_xdl_splitk_c_shuffle.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_xdl_splitk_c_shuffle.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_splitk_f16_f16_f16_km_nk_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_splitk_f16_f16_f16_km_nk_mn_instance.cpp index 45e3f9f940..f357ed553d 100644 --- a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_splitk_f16_f16_f16_km_nk_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_splitk_f16_f16_f16_km_nk_mn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_gemm_xdl_splitk_c_shuffle.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_xdl_splitk_c_shuffle.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_splitk_f16_f16_f16_mk_kn_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_splitk_f16_f16_f16_mk_kn_mn_instance.cpp index 042ac2b8ca..f247e7c7ca 100644 --- a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_splitk_f16_f16_f16_mk_kn_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_splitk_f16_f16_f16_mk_kn_mn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_gemm_xdl_splitk_c_shuffle.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_xdl_splitk_c_shuffle.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_splitk_f16_f16_f16_mk_nk_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_splitk_f16_f16_f16_mk_nk_mn_instance.cpp index 21fdb7cd9d..defb97f9bf 100644 --- a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_splitk_f16_f16_f16_mk_nk_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_splitk_f16_f16_f16_mk_nk_mn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_gemm_xdl_splitk_c_shuffle.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_xdl_splitk_c_shuffle.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_splitk_f32_f32_f32_km_kn_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_splitk_f32_f32_f32_km_kn_mn_instance.cpp index 971bdcad58..f664ce9ccd 100644 --- a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_splitk_f32_f32_f32_km_kn_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_splitk_f32_f32_f32_km_kn_mn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_gemm_xdl_splitk.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_xdl_splitk.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_splitk_f32_f32_f32_km_nk_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_splitk_f32_f32_f32_km_nk_mn_instance.cpp index 3b7bdb87be..fb6e453dd8 100644 --- a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_splitk_f32_f32_f32_km_nk_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_splitk_f32_f32_f32_km_nk_mn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_gemm_xdl_splitk.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_xdl_splitk.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_splitk_f32_f32_f32_mk_kn_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_splitk_f32_f32_f32_mk_kn_mn_instance.cpp index 8366616246..44ec005308 100644 --- a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_splitk_f32_f32_f32_mk_kn_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_splitk_f32_f32_f32_mk_kn_mn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_gemm_xdl_splitk.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_xdl_splitk.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_splitk_f32_f32_f32_mk_nk_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_splitk_f32_f32_f32_mk_nk_mn_instance.cpp index 396de62cfb..dd2f6aec83 100644 --- a/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_splitk_f32_f32_f32_mk_nk_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm/device_gemm_xdl_splitk_f32_f32_f32_mk_nk_mn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_gemm_xdl_splitk.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_xdl_splitk.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm_add_add_fastgelu/device_gemm_add_add_fastgelu_xdl_c_shuffle_f16_f16_f16_km_kn_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm_add_add_fastgelu/device_gemm_add_add_fastgelu_xdl_c_shuffle_f16_f16_f16_km_kn_mn_instance.cpp index 15ef0f00e8..8ba6bce33f 100644 --- a/library/src/tensor_operation_instance/gpu/gemm_add_add_fastgelu/device_gemm_add_add_fastgelu_xdl_c_shuffle_f16_f16_f16_km_kn_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm_add_add_fastgelu/device_gemm_add_add_fastgelu_xdl_c_shuffle_f16_f16_f16_km_kn_mn_instance.cpp @@ -1,9 +1,9 @@ -#include +#include -#include "config.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" -#include "device_gemm_multiple_d_xdl_cshuffle.hpp" +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_multiple_d_xdl_cshuffle.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { @@ -25,9 +25,9 @@ using AddAddFastGelu = ck::tensor_operation::element_wise::AddAddFastGelu; static constexpr auto GemmDefault = ck::tensor_operation::device::GemmSpecialization::Default; -// e = elementwise((a * b), d) +// e = elementwise((a * b), d0, d1) // outout: e[m, n] -// input: a[k, m], b[k, n], d[m, n] +// input: a[k, m], b[k, n], d0[m, n], d1[m, n] using device_gemm_add_add_fastgelu_xdl_c_shuffle_f16_f16_f16_km_kn_mn_instances = std::tuple< // clang-format off //##############################| ALayout| BLayout| ELayout| AData| BData| AccData| CShuffle| DsData| EData| A| B| CDE| GEMM| NumGemmK| Block| MPer| NPer| KPer| AK1| BK1| MPer| NPer| MXdl| NXdl| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockLds| BBlockTransfer| BBlockTransfer| BBlockTransfer| BlockTransfer| BBlockTransfer| BBlockTransfer| BBlockLds| CShuffle| CShuffle| CBlockTransferClusterLengths| CBlockTransfer| diff --git a/library/src/tensor_operation_instance/gpu/gemm_add_add_fastgelu/device_gemm_add_add_fastgelu_xdl_c_shuffle_f16_f16_f16_km_nk_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm_add_add_fastgelu/device_gemm_add_add_fastgelu_xdl_c_shuffle_f16_f16_f16_km_nk_mn_instance.cpp index 54386e8a8a..3429b41e25 100644 --- a/library/src/tensor_operation_instance/gpu/gemm_add_add_fastgelu/device_gemm_add_add_fastgelu_xdl_c_shuffle_f16_f16_f16_km_nk_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm_add_add_fastgelu/device_gemm_add_add_fastgelu_xdl_c_shuffle_f16_f16_f16_km_nk_mn_instance.cpp @@ -1,9 +1,9 @@ -#include +#include -#include "config.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" -#include "device_gemm_multiple_d_xdl_cshuffle.hpp" +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_multiple_d_xdl_cshuffle.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { @@ -25,9 +25,9 @@ using AddAddFastGelu = ck::tensor_operation::element_wise::AddAddFastGelu; static constexpr auto GemmDefault = ck::tensor_operation::device::GemmSpecialization::Default; -// e = elementwise((a * b), d) +// e = elementwise((a * b), d0, d1) // outout: e[m, n] -// input: a[k, m], b[n, k], d[m, n] +// input: a[k, m], b[n, k], d0[m, n], d1[m, n] using device_gemm_add_add_fastgelu_xdl_c_shuffle_f16_f16_f16_km_nk_mn_instances = std::tuple< // clang-format off //##############################| ALayout| BLayout| ELayout| AData| BData| AccData| CShuffle| DsData| EData| A| B| CDE| GEMM| NumGemmK| Block| MPer| NPer| KPer| AK1| BK1| MPer| NPer| MXdl| NXdl| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockLds| BBlockTransfer| BBlockTransfer| BBlockTransfer| BlockTransfer| BBlockTransfer| BBlockTransfer| BBlockLds| CShuffle| CShuffle| CBlockTransferClusterLengths| CBlockTransfer| diff --git a/library/src/tensor_operation_instance/gpu/gemm_add_add_fastgelu/device_gemm_add_add_fastgelu_xdl_c_shuffle_f16_f16_f16_mk_kn_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm_add_add_fastgelu/device_gemm_add_add_fastgelu_xdl_c_shuffle_f16_f16_f16_mk_kn_mn_instance.cpp index b78fd155fa..a066fefa60 100644 --- a/library/src/tensor_operation_instance/gpu/gemm_add_add_fastgelu/device_gemm_add_add_fastgelu_xdl_c_shuffle_f16_f16_f16_mk_kn_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm_add_add_fastgelu/device_gemm_add_add_fastgelu_xdl_c_shuffle_f16_f16_f16_mk_kn_mn_instance.cpp @@ -1,9 +1,9 @@ -#include +#include -#include "config.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" -#include "device_gemm_multiple_d_xdl_cshuffle.hpp" +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_multiple_d_xdl_cshuffle.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { @@ -25,9 +25,9 @@ using AddAddFastGelu = ck::tensor_operation::element_wise::AddAddFastGelu; static constexpr auto GemmDefault = ck::tensor_operation::device::GemmSpecialization::Default; -// e = elementwise((a * b), d) +// e = elementwise((a * b), d0, d1) // outout: e[m, n] -// input: a[m, k], b[k, n], d[m, n] +// input: a[m, k], b[k, n], d0[m, n], d1[m, n] using device_gemm_add_add_fastgelu_xdl_c_shuffle_f16_f16_f16_mk_kn_mn_instances = std::tuple< // clang-format off //##############################| ALayout| BLayout| CLayout| AData| BData| AccData| CShuffle| DsData| EData| A| B| CDE| GEMM| NumGemmK| Block| MPer| NPer| KPer| AK1| BK1| MPer| NPer| MXdl| NXdl| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockLds| BBlockTransfer| BBlockTransfer| BBlockTransfer| BlockTransfer| BBlockTransfer| BBlockTransfer| BBlockLds| CShuffle| CShuffle| CBlockTransferClusterLengths| CBlockTransfer| diff --git a/library/src/tensor_operation_instance/gpu/gemm_add_add_fastgelu/device_gemm_add_add_fastgelu_xdl_c_shuffle_f16_f16_f16_mk_nk_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm_add_add_fastgelu/device_gemm_add_add_fastgelu_xdl_c_shuffle_f16_f16_f16_mk_nk_mn_instance.cpp index 4641cb40e0..221d9b4360 100644 --- a/library/src/tensor_operation_instance/gpu/gemm_add_add_fastgelu/device_gemm_add_add_fastgelu_xdl_c_shuffle_f16_f16_f16_mk_nk_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm_add_add_fastgelu/device_gemm_add_add_fastgelu_xdl_c_shuffle_f16_f16_f16_mk_nk_mn_instance.cpp @@ -1,9 +1,9 @@ -#include +#include -#include "config.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" -#include "device_gemm_multiple_d_xdl_cshuffle.hpp" +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_multiple_d_xdl_cshuffle.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { @@ -25,9 +25,9 @@ using AddAddFastGelu = ck::tensor_operation::element_wise::AddAddFastGelu; static constexpr auto GemmDefault = ck::tensor_operation::device::GemmSpecialization::Default; -// e = elementwise((a * b), d) +// e = elementwise((a * b), d0, d1) // outout: e[m, n] -// input: a[m, k], b[n, k], d[m, n] +// input: a[m, k], b[n, k], d0[m, n], d1[m ,n] using device_gemm_add_add_fastgelu_xdl_c_shuffle_f16_f16_f16_mk_nk_mn_instances = std::tuple< // clang-format off //##############################| ALayout| BLayout| CLayout| AData| BData| AccData| CShuffle| DsData| EData| A| B| CDE| GEMM| NumGemmK| Block| MPer| NPer| KPer| AK1| BK1| MPer| NPer| MXdl| NXdl| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockLds| BBlockTransfer| BBlockTransfer| BBlockTransfer| BlockTransfer| BBlockTransfer| BBlockTransfer| BBlockLds| CShuffle| CShuffle| CBlockTransferClusterLengths| CBlockTransfer| diff --git a/library/src/tensor_operation_instance/gpu/gemm_bias2d/device_gemm_xdl_c_shuffle_bias_2d_f16_f16_f16_km_kn_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm_bias2d/device_gemm_xdl_c_shuffle_bias_2d_f16_f16_f16_km_kn_mn_instance.cpp index bd16850ee4..e86511f10c 100644 --- a/library/src/tensor_operation_instance/gpu/gemm_bias2d/device_gemm_xdl_c_shuffle_bias_2d_f16_f16_f16_km_kn_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm_bias2d/device_gemm_xdl_c_shuffle_bias_2d_f16_f16_f16_km_kn_mn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_gemm_xdl_c_shuffle_bias_2d.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_xdl_c_shuffle_bias_2d.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm_bias2d/device_gemm_xdl_c_shuffle_bias_2d_f16_f16_f16_km_nk_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm_bias2d/device_gemm_xdl_c_shuffle_bias_2d_f16_f16_f16_km_nk_mn_instance.cpp index 12740ce256..d8f6eb46fa 100644 --- a/library/src/tensor_operation_instance/gpu/gemm_bias2d/device_gemm_xdl_c_shuffle_bias_2d_f16_f16_f16_km_nk_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm_bias2d/device_gemm_xdl_c_shuffle_bias_2d_f16_f16_f16_km_nk_mn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_gemm_xdl_c_shuffle_bias_2d.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_xdl_c_shuffle_bias_2d.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm_bias2d/device_gemm_xdl_c_shuffle_bias_2d_f16_f16_f16_mk_kn_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm_bias2d/device_gemm_xdl_c_shuffle_bias_2d_f16_f16_f16_mk_kn_mn_instance.cpp index 56db0475ef..169f105381 100644 --- a/library/src/tensor_operation_instance/gpu/gemm_bias2d/device_gemm_xdl_c_shuffle_bias_2d_f16_f16_f16_mk_kn_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm_bias2d/device_gemm_xdl_c_shuffle_bias_2d_f16_f16_f16_mk_kn_mn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_gemm_xdl_c_shuffle_bias_2d.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_xdl_c_shuffle_bias_2d.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm_bias2d/device_gemm_xdl_c_shuffle_bias_2d_f16_f16_f16_mk_nk_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm_bias2d/device_gemm_xdl_c_shuffle_bias_2d_f16_f16_f16_mk_nk_mn_instance.cpp index b20ee8db69..ab137b57d4 100644 --- a/library/src/tensor_operation_instance/gpu/gemm_bias2d/device_gemm_xdl_c_shuffle_bias_2d_f16_f16_f16_mk_nk_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm_bias2d/device_gemm_xdl_c_shuffle_bias_2d_f16_f16_f16_mk_nk_mn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_gemm_xdl_c_shuffle_bias_2d.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_xdl_c_shuffle_bias_2d.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm_bias2d/device_gemm_xdl_c_shuffle_bias_2d_f32_f32_f32_km_kn_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm_bias2d/device_gemm_xdl_c_shuffle_bias_2d_f32_f32_f32_km_kn_mn_instance.cpp index 11984c36db..ac2bdab844 100644 --- a/library/src/tensor_operation_instance/gpu/gemm_bias2d/device_gemm_xdl_c_shuffle_bias_2d_f32_f32_f32_km_kn_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm_bias2d/device_gemm_xdl_c_shuffle_bias_2d_f32_f32_f32_km_kn_mn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_gemm_xdl_c_shuffle_bias_2d.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_xdl_c_shuffle_bias_2d.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm_bias2d/device_gemm_xdl_c_shuffle_bias_2d_f32_f32_f32_km_nk_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm_bias2d/device_gemm_xdl_c_shuffle_bias_2d_f32_f32_f32_km_nk_mn_instance.cpp index bd0a988059..82ad1fe00c 100644 --- a/library/src/tensor_operation_instance/gpu/gemm_bias2d/device_gemm_xdl_c_shuffle_bias_2d_f32_f32_f32_km_nk_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm_bias2d/device_gemm_xdl_c_shuffle_bias_2d_f32_f32_f32_km_nk_mn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_gemm_xdl_c_shuffle_bias_2d.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_xdl_c_shuffle_bias_2d.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm_bias2d/device_gemm_xdl_c_shuffle_bias_2d_f32_f32_f32_mk_kn_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm_bias2d/device_gemm_xdl_c_shuffle_bias_2d_f32_f32_f32_mk_kn_mn_instance.cpp index 440ea1582e..0bd6a77855 100644 --- a/library/src/tensor_operation_instance/gpu/gemm_bias2d/device_gemm_xdl_c_shuffle_bias_2d_f32_f32_f32_mk_kn_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm_bias2d/device_gemm_xdl_c_shuffle_bias_2d_f32_f32_f32_mk_kn_mn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_gemm_xdl_c_shuffle_bias_2d.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_xdl_c_shuffle_bias_2d.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm_bias2d/device_gemm_xdl_c_shuffle_bias_2d_f32_f32_f32_mk_nk_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm_bias2d/device_gemm_xdl_c_shuffle_bias_2d_f32_f32_f32_mk_nk_mn_instance.cpp index fab885969f..e8a74dc159 100644 --- a/library/src/tensor_operation_instance/gpu/gemm_bias2d/device_gemm_xdl_c_shuffle_bias_2d_f32_f32_f32_mk_nk_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm_bias2d/device_gemm_xdl_c_shuffle_bias_2d_f32_f32_f32_mk_nk_mn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_gemm_xdl_c_shuffle_bias_2d.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_xdl_c_shuffle_bias_2d.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm_bias_add_reduce/device_gemm_bias_add_reduce_xdl_cshuffle_f16_f16_f16_f32_f32_km_kn_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm_bias_add_reduce/device_gemm_bias_add_reduce_xdl_cshuffle_f16_f16_f16_f32_f32_km_kn_mn_instance.cpp index 2e1a7f531c..e42afa0cf4 100644 --- a/library/src/tensor_operation_instance/gpu/gemm_bias_add_reduce/device_gemm_bias_add_reduce_xdl_cshuffle_f16_f16_f16_f32_f32_km_kn_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm_bias_add_reduce/device_gemm_bias_add_reduce_xdl_cshuffle_f16_f16_f16_f32_f32_km_kn_mn_instance.cpp @@ -1,9 +1,12 @@ -#include -#include "config.hpp" -#include "device_gemm_bias_add_reduce_xdl_cshuffle.hpp" -#include "element_wise_operation.hpp" -#include "reduction_operator.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/utility/reduction_operator.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_bias_add_reduce_xdl_cshuffle.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm_bias_add_reduce/device_gemm_bias_add_reduce_xdl_cshuffle_f16_f16_f16_f32_f32_km_nk_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm_bias_add_reduce/device_gemm_bias_add_reduce_xdl_cshuffle_f16_f16_f16_f32_f32_km_nk_mn_instance.cpp index db6140ea61..97aa910aef 100644 --- a/library/src/tensor_operation_instance/gpu/gemm_bias_add_reduce/device_gemm_bias_add_reduce_xdl_cshuffle_f16_f16_f16_f32_f32_km_nk_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm_bias_add_reduce/device_gemm_bias_add_reduce_xdl_cshuffle_f16_f16_f16_f32_f32_km_nk_mn_instance.cpp @@ -1,9 +1,12 @@ -#include -#include "config.hpp" -#include "device_gemm_bias_add_reduce_xdl_cshuffle.hpp" -#include "element_wise_operation.hpp" -#include "reduction_operator.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/utility/reduction_operator.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_bias_add_reduce_xdl_cshuffle.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm_bias_add_reduce/device_gemm_bias_add_reduce_xdl_cshuffle_f16_f16_f16_f32_f32_mk_kn_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm_bias_add_reduce/device_gemm_bias_add_reduce_xdl_cshuffle_f16_f16_f16_f32_f32_mk_kn_mn_instance.cpp index 050473886f..3cc40eae7f 100644 --- a/library/src/tensor_operation_instance/gpu/gemm_bias_add_reduce/device_gemm_bias_add_reduce_xdl_cshuffle_f16_f16_f16_f32_f32_mk_kn_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm_bias_add_reduce/device_gemm_bias_add_reduce_xdl_cshuffle_f16_f16_f16_f32_f32_mk_kn_mn_instance.cpp @@ -1,9 +1,12 @@ -#include -#include "config.hpp" -#include "device_gemm_bias_add_reduce_xdl_cshuffle.hpp" -#include "element_wise_operation.hpp" -#include "reduction_operator.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/utility/reduction_operator.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_bias_add_reduce_xdl_cshuffle.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm_bias_add_reduce/device_gemm_bias_add_reduce_xdl_cshuffle_f16_f16_f16_f32_f32_mk_nk_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm_bias_add_reduce/device_gemm_bias_add_reduce_xdl_cshuffle_f16_f16_f16_f32_f32_mk_nk_mn_instance.cpp index c50e6cf83d..b1eeacb564 100644 --- a/library/src/tensor_operation_instance/gpu/gemm_bias_add_reduce/device_gemm_bias_add_reduce_xdl_cshuffle_f16_f16_f16_f32_f32_mk_nk_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm_bias_add_reduce/device_gemm_bias_add_reduce_xdl_cshuffle_f16_f16_f16_f32_f32_mk_nk_mn_instance.cpp @@ -1,9 +1,12 @@ -#include -#include "config.hpp" -#include "device_gemm_bias_add_reduce_xdl_cshuffle.hpp" -#include "element_wise_operation.hpp" -#include "reduction_operator.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/utility/reduction_operator.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_bias_add_reduce_xdl_cshuffle.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm_bias_relu/device_gemm_xdl_c_shuffle_bias_relu_f16_f16_f16_km_kn_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm_bias_relu/device_gemm_xdl_c_shuffle_bias_relu_f16_f16_f16_km_kn_mn_instance.cpp index 4927a05ca4..79c2fa403c 100644 --- a/library/src/tensor_operation_instance/gpu/gemm_bias_relu/device_gemm_xdl_c_shuffle_bias_relu_f16_f16_f16_km_kn_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm_bias_relu/device_gemm_xdl_c_shuffle_bias_relu_f16_f16_f16_km_kn_mn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_gemm_xdl_c_shuffle_bias_activation.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_xdl_c_shuffle_bias_activation.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm_bias_relu/device_gemm_xdl_c_shuffle_bias_relu_f16_f16_f16_km_nk_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm_bias_relu/device_gemm_xdl_c_shuffle_bias_relu_f16_f16_f16_km_nk_mn_instance.cpp index f712f9de11..0a019c982e 100644 --- a/library/src/tensor_operation_instance/gpu/gemm_bias_relu/device_gemm_xdl_c_shuffle_bias_relu_f16_f16_f16_km_nk_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm_bias_relu/device_gemm_xdl_c_shuffle_bias_relu_f16_f16_f16_km_nk_mn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_gemm_xdl_c_shuffle_bias_activation.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_xdl_c_shuffle_bias_activation.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm_bias_relu/device_gemm_xdl_c_shuffle_bias_relu_f16_f16_f16_mk_kn_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm_bias_relu/device_gemm_xdl_c_shuffle_bias_relu_f16_f16_f16_mk_kn_mn_instance.cpp index 26af05bbde..baa54c3320 100644 --- a/library/src/tensor_operation_instance/gpu/gemm_bias_relu/device_gemm_xdl_c_shuffle_bias_relu_f16_f16_f16_mk_kn_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm_bias_relu/device_gemm_xdl_c_shuffle_bias_relu_f16_f16_f16_mk_kn_mn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_gemm_xdl_c_shuffle_bias_activation.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_xdl_c_shuffle_bias_activation.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm_bias_relu/device_gemm_xdl_c_shuffle_bias_relu_f16_f16_f16_mk_nk_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm_bias_relu/device_gemm_xdl_c_shuffle_bias_relu_f16_f16_f16_mk_nk_mn_instance.cpp index 901b7a5d64..159ebdc572 100644 --- a/library/src/tensor_operation_instance/gpu/gemm_bias_relu/device_gemm_xdl_c_shuffle_bias_relu_f16_f16_f16_mk_nk_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm_bias_relu/device_gemm_xdl_c_shuffle_bias_relu_f16_f16_f16_mk_nk_mn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_gemm_xdl_c_shuffle_bias_activation.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_xdl_c_shuffle_bias_activation.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm_bias_relu_add/device_gemm_xdl_c_shuffle_bias_relu_add_f16_f16_f16_km_kn_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm_bias_relu_add/device_gemm_xdl_c_shuffle_bias_relu_add_f16_f16_f16_km_kn_mn_instance.cpp index c26f66a9ed..0281436928 100644 --- a/library/src/tensor_operation_instance/gpu/gemm_bias_relu_add/device_gemm_xdl_c_shuffle_bias_relu_add_f16_f16_f16_km_kn_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm_bias_relu_add/device_gemm_xdl_c_shuffle_bias_relu_add_f16_f16_f16_km_kn_mn_instance.cpp @@ -1,8 +1,12 @@ -#include -#include "config.hpp" -#include "device_gemm_xdl_c_shuffle_bias_activation_add.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/utility/reduction_operator.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_xdl_c_shuffle_bias_activation_add.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm_bias_relu_add/device_gemm_xdl_c_shuffle_bias_relu_add_f16_f16_f16_km_nk_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm_bias_relu_add/device_gemm_xdl_c_shuffle_bias_relu_add_f16_f16_f16_km_nk_mn_instance.cpp index c0950666b1..dcf0e911f5 100644 --- a/library/src/tensor_operation_instance/gpu/gemm_bias_relu_add/device_gemm_xdl_c_shuffle_bias_relu_add_f16_f16_f16_km_nk_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm_bias_relu_add/device_gemm_xdl_c_shuffle_bias_relu_add_f16_f16_f16_km_nk_mn_instance.cpp @@ -1,8 +1,12 @@ -#include -#include "config.hpp" -#include "device_gemm_xdl_c_shuffle_bias_activation_add.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/utility/reduction_operator.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_xdl_c_shuffle_bias_activation_add.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm_bias_relu_add/device_gemm_xdl_c_shuffle_bias_relu_add_f16_f16_f16_mk_kn_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm_bias_relu_add/device_gemm_xdl_c_shuffle_bias_relu_add_f16_f16_f16_mk_kn_mn_instance.cpp index 42c1f72d6e..0cce3e293c 100644 --- a/library/src/tensor_operation_instance/gpu/gemm_bias_relu_add/device_gemm_xdl_c_shuffle_bias_relu_add_f16_f16_f16_mk_kn_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm_bias_relu_add/device_gemm_xdl_c_shuffle_bias_relu_add_f16_f16_f16_mk_kn_mn_instance.cpp @@ -1,8 +1,12 @@ -#include -#include "config.hpp" -#include "device_gemm_xdl_c_shuffle_bias_activation_add.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/utility/reduction_operator.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_xdl_c_shuffle_bias_activation_add.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm_bias_relu_add/device_gemm_xdl_c_shuffle_bias_relu_add_f16_f16_f16_mk_nk_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm_bias_relu_add/device_gemm_xdl_c_shuffle_bias_relu_add_f16_f16_f16_mk_nk_mn_instance.cpp index 3961def81d..aa812b428c 100644 --- a/library/src/tensor_operation_instance/gpu/gemm_bias_relu_add/device_gemm_xdl_c_shuffle_bias_relu_add_f16_f16_f16_mk_nk_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm_bias_relu_add/device_gemm_xdl_c_shuffle_bias_relu_add_f16_f16_f16_mk_nk_mn_instance.cpp @@ -1,8 +1,12 @@ -#include -#include "config.hpp" -#include "device_gemm_xdl_c_shuffle_bias_activation_add.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/utility/reduction_operator.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_xdl_c_shuffle_bias_activation_add.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm_reduce/device_gemm_reduce_xdl_cshuffle_f16_f16_f16_f32_f32_km_kn_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm_reduce/device_gemm_reduce_xdl_cshuffle_f16_f16_f16_f32_f32_km_kn_mn_instance.cpp index e1d2f2f6ff..2958cc28b4 100644 --- a/library/src/tensor_operation_instance/gpu/gemm_reduce/device_gemm_reduce_xdl_cshuffle_f16_f16_f16_f32_f32_km_kn_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm_reduce/device_gemm_reduce_xdl_cshuffle_f16_f16_f16_f32_f32_km_kn_mn_instance.cpp @@ -1,9 +1,12 @@ -#include -#include "config.hpp" -#include "device_gemm_reduce_xdl_cshuffle.hpp" -#include "element_wise_operation.hpp" -#include "reduction_operator.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/utility/reduction_operator.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_reduce_xdl_cshuffle.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm_reduce/device_gemm_reduce_xdl_cshuffle_f16_f16_f16_f32_f32_km_nk_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm_reduce/device_gemm_reduce_xdl_cshuffle_f16_f16_f16_f32_f32_km_nk_mn_instance.cpp index 81509a3fc5..d685798dc9 100644 --- a/library/src/tensor_operation_instance/gpu/gemm_reduce/device_gemm_reduce_xdl_cshuffle_f16_f16_f16_f32_f32_km_nk_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm_reduce/device_gemm_reduce_xdl_cshuffle_f16_f16_f16_f32_f32_km_nk_mn_instance.cpp @@ -1,9 +1,12 @@ -#include -#include "config.hpp" -#include "device_gemm_reduce_xdl_cshuffle.hpp" -#include "element_wise_operation.hpp" -#include "reduction_operator.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/utility/reduction_operator.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_reduce_xdl_cshuffle.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm_reduce/device_gemm_reduce_xdl_cshuffle_f16_f16_f16_f32_f32_mk_kn_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm_reduce/device_gemm_reduce_xdl_cshuffle_f16_f16_f16_f32_f32_mk_kn_mn_instance.cpp index 4d13381d45..bbecb31ef5 100644 --- a/library/src/tensor_operation_instance/gpu/gemm_reduce/device_gemm_reduce_xdl_cshuffle_f16_f16_f16_f32_f32_mk_kn_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm_reduce/device_gemm_reduce_xdl_cshuffle_f16_f16_f16_f32_f32_mk_kn_mn_instance.cpp @@ -1,9 +1,12 @@ -#include -#include "config.hpp" -#include "device_gemm_reduce_xdl_cshuffle.hpp" -#include "element_wise_operation.hpp" -#include "reduction_operator.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/utility/reduction_operator.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_reduce_xdl_cshuffle.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/gemm_reduce/device_gemm_reduce_xdl_cshuffle_f16_f16_f16_f32_f32_mk_nk_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/gemm_reduce/device_gemm_reduce_xdl_cshuffle_f16_f16_f16_f32_f32_mk_nk_mn_instance.cpp index 459d0cd473..281c63fe1a 100644 --- a/library/src/tensor_operation_instance/gpu/gemm_reduce/device_gemm_reduce_xdl_cshuffle_f16_f16_f16_f32_f32_mk_nk_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/gemm_reduce/device_gemm_reduce_xdl_cshuffle_f16_f16_f16_f32_f32_mk_nk_mn_instance.cpp @@ -1,9 +1,12 @@ -#include -#include "config.hpp" -#include "device_gemm_reduce_xdl_cshuffle.hpp" -#include "element_wise_operation.hpp" -#include "reduction_operator.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/utility/reduction_operator.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_reduce_xdl_cshuffle.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/grouped_gemm/device_grouped_gemm_xdl_f16_f16_f16_km_kn_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/grouped_gemm/device_grouped_gemm_xdl_f16_f16_f16_km_kn_mn_instance.cpp index 19f1011c3f..db635fdb80 100644 --- a/library/src/tensor_operation_instance/gpu/grouped_gemm/device_grouped_gemm_xdl_f16_f16_f16_km_kn_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/grouped_gemm/device_grouped_gemm_xdl_f16_f16_f16_km_kn_mn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_grouped_gemm_xdl.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_grouped_gemm_xdl.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/grouped_gemm/device_grouped_gemm_xdl_f16_f16_f16_km_nk_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/grouped_gemm/device_grouped_gemm_xdl_f16_f16_f16_km_nk_mn_instance.cpp index 59e0d24055..d402085f0b 100644 --- a/library/src/tensor_operation_instance/gpu/grouped_gemm/device_grouped_gemm_xdl_f16_f16_f16_km_nk_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/grouped_gemm/device_grouped_gemm_xdl_f16_f16_f16_km_nk_mn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_grouped_gemm_xdl.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_grouped_gemm_xdl.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/grouped_gemm/device_grouped_gemm_xdl_f16_f16_f16_mk_kn_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/grouped_gemm/device_grouped_gemm_xdl_f16_f16_f16_mk_kn_mn_instance.cpp index 35052ae8a9..04ab002d54 100644 --- a/library/src/tensor_operation_instance/gpu/grouped_gemm/device_grouped_gemm_xdl_f16_f16_f16_mk_kn_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/grouped_gemm/device_grouped_gemm_xdl_f16_f16_f16_mk_kn_mn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_grouped_gemm_xdl.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_grouped_gemm_xdl.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/grouped_gemm/device_grouped_gemm_xdl_f16_f16_f16_mk_nk_mn_instance.cpp b/library/src/tensor_operation_instance/gpu/grouped_gemm/device_grouped_gemm_xdl_f16_f16_f16_mk_nk_mn_instance.cpp index cb41d2724c..cb70e56804 100644 --- a/library/src/tensor_operation_instance/gpu/grouped_gemm/device_grouped_gemm_xdl_f16_f16_f16_mk_nk_mn_instance.cpp +++ b/library/src/tensor_operation_instance/gpu/grouped_gemm/device_grouped_gemm_xdl_f16_f16_f16_mk_nk_mn_instance.cpp @@ -1,8 +1,10 @@ -#include -#include "config.hpp" -#include "device_grouped_gemm_xdl.hpp" -#include "element_wise_operation.hpp" -#include "device_operation_instance.hpp" +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_grouped_gemm_xdl.hpp" +#include "ck/library/tensor_operation_instance/device_operation_instance.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_b16_f32_b16.cpp b/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_b16_f32_b16.cpp index 0274d89fc9..12586dbf5f 100644 --- a/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_b16_f32_b16.cpp +++ b/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_b16_f32_b16.cpp @@ -1,4 +1,4 @@ -#include "device_reduce_instance_blockwise.hpp" +#include "ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_f16_f16_f16.cpp b/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_f16_f16_f16.cpp index 8a43d860ea..e22fac910c 100644 --- a/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_f16_f16_f16.cpp +++ b/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_f16_f16_f16.cpp @@ -1,4 +1,4 @@ -#include "device_reduce_instance_blockwise.hpp" +#include "ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_f16_f32_f16.cpp b/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_f16_f32_f16.cpp index 3e0b8ba59c..008c742bf0 100644 --- a/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_f16_f32_f16.cpp +++ b/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_f16_f32_f16.cpp @@ -1,4 +1,4 @@ -#include "device_reduce_instance_blockwise.hpp" +#include "ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_f32_f32_f32.cpp b/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_f32_f32_f32.cpp index ee96311f8c..f85e9b830b 100644 --- a/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_f32_f32_f32.cpp +++ b/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_f32_f32_f32.cpp @@ -1,4 +1,4 @@ -#include "device_reduce_instance_blockwise.hpp" +#include "ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_f32_f64_f32.cpp b/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_f32_f64_f32.cpp index b0ae95e82d..4c2a16c2f2 100644 --- a/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_f32_f64_f32.cpp +++ b/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_f32_f64_f32.cpp @@ -1,4 +1,4 @@ -#include "device_reduce_instance_blockwise.hpp" +#include "ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise.hpp" namespace ck { namespace tensor_operation { @@ -24,5 +24,4 @@ ADD_BLOCKWISE_INST_BY_ID(float, double, float, 7, 0, 0, 2, 1); } // namespace device_reduce_instance } // namespace device } // namespace tensor_operation - } // namespace ck diff --git a/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_f64_f64_f64.cpp b/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_f64_f64_f64.cpp index 9cca2dbbeb..7c72d5e709 100644 --- a/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_f64_f64_f64.cpp +++ b/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_f64_f64_f64.cpp @@ -1,4 +1,4 @@ -#include "device_reduce_instance_blockwise.hpp" +#include "ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_i8_i32_i8.cpp b/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_i8_i32_i8.cpp index 05cd1921ee..bbc673a7eb 100644 --- a/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_i8_i32_i8.cpp +++ b/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_i8_i32_i8.cpp @@ -1,4 +1,4 @@ -#include "device_reduce_instance_blockwise.hpp" +#include "ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_i8_i8_i8.cpp b/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_i8_i8_i8.cpp index 66ef017864..83ad412ef5 100644 --- a/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_i8_i8_i8.cpp +++ b/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise_i8_i8_i8.cpp @@ -1,4 +1,4 @@ -#include "device_reduce_instance_blockwise.hpp" +#include "ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_blockwise.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_multiblock_atomic_add_b16_f32_f32.cpp b/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_multiblock_atomic_add_b16_f32_f32.cpp index 9b2b7f5d8c..ff3c67ead8 100644 --- a/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_multiblock_atomic_add_b16_f32_f32.cpp +++ b/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_multiblock_atomic_add_b16_f32_f32.cpp @@ -1,4 +1,4 @@ -#include "device_reduce_instance_multiblock_atomic_add.hpp" +#include "ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_multiblock_atomic_add.hpp" namespace ck { namespace tensor_operation { @@ -20,5 +20,4 @@ ADD_MULTIBLOCK_ATOMIC_ADD_INST_BY_ID(bhalf_t, float, float, 5, 0, 0, 2, 1); } // namespace device_reduce_instance } // namespace device } // namespace tensor_operation - } // namespace ck diff --git a/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_multiblock_atomic_add_f16_f32_f32.cpp b/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_multiblock_atomic_add_f16_f32_f32.cpp index fc956aa04b..0c163841f2 100644 --- a/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_multiblock_atomic_add_f16_f32_f32.cpp +++ b/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_multiblock_atomic_add_f16_f32_f32.cpp @@ -1,4 +1,4 @@ -#include "device_reduce_instance_multiblock_atomic_add.hpp" +#include "ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_multiblock_atomic_add.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_multiblock_atomic_add_f32_f32_f32.cpp b/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_multiblock_atomic_add_f32_f32_f32.cpp index e5ffd9f976..444a48ad20 100644 --- a/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_multiblock_atomic_add_f32_f32_f32.cpp +++ b/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_multiblock_atomic_add_f32_f32_f32.cpp @@ -1,4 +1,4 @@ -#include "device_reduce_instance_multiblock_atomic_add.hpp" +#include "ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_multiblock_atomic_add.hpp" namespace ck { namespace tensor_operation { @@ -20,5 +20,4 @@ ADD_MULTIBLOCK_ATOMIC_ADD_INST_BY_ID(float, float, float, 5, 0, 0, 2, 1); } // namespace device_reduce_instance } // namespace device } // namespace tensor_operation - } // namespace ck diff --git a/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_multiblock_atomic_add_f32_f64_f32.cpp b/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_multiblock_atomic_add_f32_f64_f32.cpp index 229829b889..40e244d5f9 100644 --- a/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_multiblock_atomic_add_f32_f64_f32.cpp +++ b/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_multiblock_atomic_add_f32_f64_f32.cpp @@ -1,4 +1,4 @@ -#include "device_reduce_instance_multiblock_atomic_add.hpp" +#include "ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_multiblock_atomic_add.hpp" namespace ck { namespace tensor_operation { @@ -20,5 +20,4 @@ ADD_MULTIBLOCK_ATOMIC_ADD_INST_BY_ID(float, double, float, 5, 0, 0, 2, 1); } // namespace device_reduce_instance } // namespace device } // namespace tensor_operation - } // namespace ck diff --git a/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_multiblock_atomic_add_f64_f64_f64.cpp b/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_multiblock_atomic_add_f64_f64_f64.cpp index 497f2695be..43fef2bccd 100644 --- a/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_multiblock_atomic_add_f64_f64_f64.cpp +++ b/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_multiblock_atomic_add_f64_f64_f64.cpp @@ -1,4 +1,4 @@ -#include "device_reduce_instance_multiblock_atomic_add.hpp" +#include "ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_multiblock_atomic_add.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_b16_f32_b16.cpp b/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_b16_f32_b16.cpp index 02fc4b4c01..9189b9e73f 100644 --- a/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_b16_f32_b16.cpp +++ b/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_b16_f32_b16.cpp @@ -1,4 +1,4 @@ -#include "device_reduce_instance_threadwise.hpp" +#include "ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_f16_f16_f16.cpp b/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_f16_f16_f16.cpp index 0984cdc46b..c689eb402b 100644 --- a/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_f16_f16_f16.cpp +++ b/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_f16_f16_f16.cpp @@ -1,4 +1,4 @@ -#include "device_reduce_instance_threadwise.hpp" +#include "ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_f16_f32_f16.cpp b/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_f16_f32_f16.cpp index 64f14bd4e7..80ae9c55dd 100644 --- a/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_f16_f32_f16.cpp +++ b/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_f16_f32_f16.cpp @@ -1,4 +1,4 @@ -#include "device_reduce_instance_threadwise.hpp" +#include "ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise.hpp" namespace ck { namespace tensor_operation { @@ -24,5 +24,4 @@ ADD_THREADWISE_INST_BY_ID(half_t, float, half_t, 7, 0, 0, 2, 1); } // namespace device_reduce_instance } // namespace device } // namespace tensor_operation - } // namespace ck diff --git a/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_f32_f32_f32.cpp b/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_f32_f32_f32.cpp index 69ed303b17..b9435964e0 100644 --- a/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_f32_f32_f32.cpp +++ b/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_f32_f32_f32.cpp @@ -1,4 +1,4 @@ -#include "device_reduce_instance_threadwise.hpp" +#include "ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_f32_f64_f32.cpp b/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_f32_f64_f32.cpp index 5d791cec41..005d268d99 100644 --- a/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_f32_f64_f32.cpp +++ b/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_f32_f64_f32.cpp @@ -1,4 +1,4 @@ -#include "device_reduce_instance_threadwise.hpp" +#include "ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_f64_f64_f64.cpp b/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_f64_f64_f64.cpp index 16c0409134..7f1922c9e6 100644 --- a/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_f64_f64_f64.cpp +++ b/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_f64_f64_f64.cpp @@ -1,4 +1,4 @@ -#include "device_reduce_instance_threadwise.hpp" +#include "ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise.hpp" namespace ck { namespace tensor_operation { @@ -48,5 +48,4 @@ ADD_THREADWISE_INST_BY_ID(double, double, double, 4, 0, 1, 2, 1); } // namespace device_reduce_instance } // namespace device } // namespace tensor_operation - } // namespace ck diff --git a/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_i8_i32_i8.cpp b/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_i8_i32_i8.cpp index 7af7bc03f2..ac81ee5944 100644 --- a/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_i8_i32_i8.cpp +++ b/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_i8_i32_i8.cpp @@ -1,4 +1,4 @@ -#include "device_reduce_instance_threadwise.hpp" +#include "ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_i8_i8_i8.cpp b/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_i8_i8_i8.cpp index 9580aae057..d27e1bc5f2 100644 --- a/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_i8_i8_i8.cpp +++ b/library/src/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise_i8_i8_i8.cpp @@ -1,4 +1,4 @@ -#include "device_reduce_instance_threadwise.hpp" +#include "ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance_threadwise.hpp" namespace ck { namespace tensor_operation { diff --git a/library/src/utility/CMakeLists.txt b/library/src/utility/CMakeLists.txt index 0914855d59..afa6de5119 100644 --- a/library/src/utility/CMakeLists.txt +++ b/library/src/utility/CMakeLists.txt @@ -1,13 +1,3 @@ -include_directories(BEFORE - ${PROJECT_SOURCE_DIR}/include/ck - ${PROJECT_SOURCE_DIR}/include/ck/tensor_operation/gpu/device - ${PROJECT_SOURCE_DIR}/include/ck/tensor_operation/gpu/element - ${PROJECT_SOURCE_DIR}/include/ck/utility - ${PROJECT_SOURCE_DIR}/library/include/ck/library/host_tensor - ${PROJECT_SOURCE_DIR}/library/include/ck/library/reference_tensor_operation/cpu - ${PROJECT_SOURCE_DIR}/library/include/ck/library/utility -) - set(CONV_UTIL_SOURCE conv_util.cpp ) diff --git a/library/src/utility/conv_util.cpp b/library/src/utility/conv_util.cpp index a60d1a3495..bc23f0c911 100644 --- a/library/src/utility/conv_util.cpp +++ b/library/src/utility/conv_util.cpp @@ -1,5 +1,5 @@ -#include "conv_util.hpp" +#include "ck/library/utility/conv_util.hpp" namespace ck { namespace utils { diff --git a/profiler/CMakeLists.txt b/profiler/CMakeLists.txt index ed75f1e1e1..b48f28a23a 100644 --- a/profiler/CMakeLists.txt +++ b/profiler/CMakeLists.txt @@ -1,24 +1,5 @@ include_directories(BEFORE - ${PROJECT_SOURCE_DIR}/include/ck - ${PROJECT_SOURCE_DIR}/include/ck/utility - ${PROJECT_SOURCE_DIR}/include/ck/host_utility - ${PROJECT_SOURCE_DIR}/include/ck/tensor_description - ${PROJECT_SOURCE_DIR}/include/ck/tensor - ${PROJECT_SOURCE_DIR}/include/ck/problem_transform - ${PROJECT_SOURCE_DIR}/include/ck/tensor_operation/gpu/device - ${PROJECT_SOURCE_DIR}/include/ck/tensor_operation/gpu/grid - ${PROJECT_SOURCE_DIR}/include/ck/tensor_operation/gpu/block - ${PROJECT_SOURCE_DIR}/include/ck/tensor_operation/gpu/warp - ${PROJECT_SOURCE_DIR}/include/ck/tensor_operation/gpu/thread - ${PROJECT_SOURCE_DIR}/include/ck/tensor_operation/gpu/element - ${PROJECT_SOURCE_DIR}/library/include/ck/library/host_tensor - ${PROJECT_SOURCE_DIR}/library/include/ck/library/tensor_operation_instance - ${PROJECT_SOURCE_DIR}/library/include/ck/library/tensor_operation_instance/gpu/reduce - ${PROJECT_SOURCE_DIR}/library/include/ck/library/reference_tensor_operation/cpu - ${PROJECT_SOURCE_DIR}/library/include/ck/library/reference_tensor_operation/gpu - ${PROJECT_SOURCE_DIR}/library/include/ck/library/utility - ${PROJECT_SOURCE_DIR}/profiler/include - ${PROJECT_SOURCE_DIR}/external/include/half + ${PROJECT_SOURCE_DIR}/ ) # ck_profiler @@ -33,7 +14,6 @@ set(PROFILER_SOURCE src/profile_batched_gemm.cpp src/profile_conv_fwd_bias_relu.cpp src/profile_conv_fwd_bias_relu_add.cpp - src/profile_conv_fwd_bias_relu_atomic_add.cpp src/profile_convnd_fwd.cpp src/profile_convnd_bwd_data.cpp src/profile_reduce.cpp @@ -59,7 +39,6 @@ target_link_libraries(ckProfiler PRIVATE device_conv2d_fwd_instance) target_link_libraries(ckProfiler PRIVATE device_conv3d_fwd_instance) target_link_libraries(ckProfiler PRIVATE device_conv2d_fwd_bias_relu_instance) target_link_libraries(ckProfiler PRIVATE device_conv2d_fwd_bias_relu_add_instance) -target_link_libraries(ckProfiler PRIVATE device_conv2d_fwd_bias_relu_atomic_add_instance) target_link_libraries(ckProfiler PRIVATE device_convnd_bwd_data_instance) target_link_libraries(ckProfiler PRIVATE device_reduce_instance) target_link_libraries(ckProfiler PRIVATE device_grouped_gemm_instance) diff --git a/include/ck/utility/data_type_enum.hpp b/profiler/include/data_type_enum.hpp similarity index 75% rename from include/ck/utility/data_type_enum.hpp rename to profiler/include/data_type_enum.hpp index fda6a2b05c..e6509af703 100644 --- a/include/ck/utility/data_type_enum.hpp +++ b/profiler/include/data_type_enum.hpp @@ -1,5 +1,4 @@ -#ifndef CK_DATA_TYPE_ENUM_HPP -#define CK_DATA_TYPE_ENUM_HPP +#pragma once namespace ck { @@ -16,4 +15,3 @@ enum struct DataTypeEnum }; } // namespace ck -#endif diff --git a/include/ck/utility/data_type_enum_helper.hpp b/profiler/include/data_type_enum_helper.hpp similarity index 90% rename from include/ck/utility/data_type_enum_helper.hpp rename to profiler/include/data_type_enum_helper.hpp index 9c8e01a7e3..d190a4555d 100644 --- a/include/ck/utility/data_type_enum_helper.hpp +++ b/profiler/include/data_type_enum_helper.hpp @@ -1,8 +1,7 @@ -#ifndef CK_DATA_TYPE_ENUM_HELPER_HPP -#define CK_DATA_TYPE_ENUM_HELPER_HPP +#pragma -#include "data_type.hpp" -#include "data_type_enum.hpp" +#include "ck/utility/data_type.hpp" +#include "profiler/include/data_type_enum.hpp" namespace ck { @@ -73,4 +72,3 @@ struct get_datatype_enum_from_type }; } // namespace ck -#endif diff --git a/profiler/include/profile_batched_gemm_impl.hpp b/profiler/include/profile_batched_gemm_impl.hpp index 3393110c33..6db4ffe84a 100644 --- a/profiler/include/profile_batched_gemm_impl.hpp +++ b/profiler/include/profile_batched_gemm_impl.hpp @@ -2,14 +2,17 @@ #include -#include "check_err.hpp" -#include "config.hpp" -#include "element_wise_operation.hpp" -#include "tensor_layout.hpp" -#include "device.hpp" -#include "host_tensor_generator.hpp" -#include "device_gemm.hpp" -#include "reference_batched_gemm.hpp" +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" + +#include "ck/library/utility/check_err.hpp" +#include "ck/library/utility/conv_util.hpp" +#include "ck/library/host_tensor/device_memory.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" +#include "ck/library/host_tensor/host_tensor_generator.hpp" +#include "ck/library/reference_tensor_operation/cpu/reference_batched_gemm.hpp" namespace ck { namespace tensor_operation { diff --git a/profiler/include/profile_batched_gemm_reduce_impl.hpp b/profiler/include/profile_batched_gemm_reduce_impl.hpp index d1737f588a..5109e91f03 100644 --- a/profiler/include/profile_batched_gemm_reduce_impl.hpp +++ b/profiler/include/profile_batched_gemm_reduce_impl.hpp @@ -1,16 +1,17 @@ #pragma once -#include "config.hpp" -#include "device.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" -#include "host_conv.hpp" -#include "tensor_layout.hpp" -#include "device_tensor.hpp" -#include "element_wise_operation.hpp" -#include "reduction_operator.hpp" -#include "device_gemm_reduce.hpp" -#include "reference_batched_gemm.hpp" +#include "ck/ck.hpp" +#include "ck/utility/reduction_operator.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_reduce.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" + +#include "ck/library/utility/check_err.hpp" +#include "ck/library/utility/conv_util.hpp" +#include "ck/library/host_tensor/device_memory.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" +#include "ck/library/host_tensor/host_tensor_generator.hpp" +#include "ck/library/reference_tensor_operation/cpu/reference_batched_gemm.hpp" namespace ck { namespace tensor_operation { diff --git a/profiler/include/profile_conv_bwd_weight_impl.hpp b/profiler/include/profile_conv_bwd_weight_impl.hpp index 8e3a4074b0..958d264bdb 100644 --- a/profiler/include/profile_conv_bwd_weight_impl.hpp +++ b/profiler/include/profile_conv_bwd_weight_impl.hpp @@ -1,15 +1,16 @@ #pragma once -#include "stream_config.hpp" -#include "config.hpp" -#include "device.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" -#include "tensor_layout.hpp" -#include "device_tensor.hpp" -#include "device_conv_backward_weight.hpp" -#include "element_wise_operation.hpp" -#include "reference_conv_backward_weight.hpp" +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_conv_backward_weight.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" + +#include "ck/library/utility/check_err.hpp" +#include "ck/library/utility/conv_util.hpp" +#include "ck/library/host_tensor/device_memory.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" +#include "ck/library/host_tensor/host_tensor_generator.hpp" +#include "ck/library/reference_tensor_operation/cpu/reference_conv_backward_weight.hpp" namespace ck { namespace tensor_operation { diff --git a/profiler/include/profile_conv_fwd_bias_relu_add_impl.hpp b/profiler/include/profile_conv_fwd_bias_relu_add_impl.hpp index 5ea35cd72f..cefabd3a58 100644 --- a/profiler/include/profile_conv_fwd_bias_relu_add_impl.hpp +++ b/profiler/include/profile_conv_fwd_bias_relu_add_impl.hpp @@ -1,15 +1,15 @@ #pragma once -#include "check_err.hpp" -#include "config.hpp" -#include "device.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" -#include "tensor_layout.hpp" -#include "device_tensor.hpp" -#include "element_wise_operation.hpp" -#include "device_conv_fwd_bias_activation_add.hpp" -#include "reference_conv_fwd_bias_activation_add.hpp" +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_conv_fwd_bias_activation_add.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" + +#include "ck/library/utility/check_err.hpp" +#include "ck/library/host_tensor/device_memory.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" +#include "ck/library/host_tensor/host_tensor_generator.hpp" +#include "ck/library/reference_tensor_operation/cpu/reference_conv_fwd_bias_activation_add.hpp" namespace ck { namespace tensor_operation { diff --git a/profiler/include/profile_conv_fwd_bias_relu_atomic_add_impl.hpp b/profiler/include/profile_conv_fwd_bias_relu_atomic_add_impl.hpp deleted file mode 100644 index f1c2fd300a..0000000000 --- a/profiler/include/profile_conv_fwd_bias_relu_atomic_add_impl.hpp +++ /dev/null @@ -1,331 +0,0 @@ -#pragma once -#include "check_err.hpp" -#include "config.hpp" -#include "device.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" -#include "host_conv.hpp" -#include "tensor_layout.hpp" -#include "device_tensor.hpp" -#include "device_conv_fwd_bias_activation.hpp" -#include "element_wise_operation.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace device_conv2d_fwd_bias_activation_atomic_add_instance { - -using DeviceConvFwdBiasReluPtr = - DeviceConvFwdBiasActivationPtr; - -void add_device_conv2d_fwd_xdl_c_shuffle_bias_relu_atomic_add_nhwc_kyxc_nhwk_f16_instances( - std::vector&); - -} // namespace device_conv2d_fwd_bias_activation_atomic_add_instance -} // namespace device -} // namespace tensor_operation -} // namespace ck - -namespace ck { -namespace profiler { - -void cpu_conv_bias_relu_atomic_add(ck::half_t* in_ptr, - ck::half_t* weight_ptr, - ck::half_t* output_ptr, - ck::half_t* bias_ptr, - const ck::index_t N, - const ck::index_t K, - const ck::index_t C, - const ck::index_t Y, - const ck::index_t X, - const ck::index_t Hi, - const ck::index_t Wi, - const ck::index_t Ho, - const ck::index_t Wo, - const ck::index_t Stride, - const ck::index_t Dilation, - const ck::index_t Pad) -{ - - const auto in_desc = - HostTensorDescriptor(std::vector{static_cast(N), - static_cast(Hi), - static_cast(Wi), - static_cast(C)}); - const auto wei_desc = - HostTensorDescriptor(std::vector{static_cast(K), - static_cast(Y), - static_cast(X), - static_cast(C)}); - const auto out_desc = - HostTensorDescriptor(std::vector{static_cast(N), - static_cast(Ho), - static_cast(Wo), - static_cast(K)}); - const auto bias_desc = - HostTensorDescriptor(std::vector{static_cast(K)}); - - auto f_k = [&](auto k) { - for(int n = 0; n < N; ++n) - { - for(int ho = 0; ho < Ho; ++ho) - { - for(int wo = 0; wo < Wo; ++wo) - { - double v = 0; - for(int c = 0; c < C; ++c) - { - for(int y = 0; y < Y; ++y) - { - int hi = ho * Stride + y * Dilation - Pad; - for(int x = 0; x < X; ++x) - { - int wi = wo * Stride + x * Dilation - Pad; - if(hi >= 0 && hi < Hi && wi >= 0 && wi < Wi) - { - double in = - in_ptr[in_desc.GetOffsetFromMultiIndex(n, hi, wi, c)]; - double wei = - weight_ptr[wei_desc.GetOffsetFromMultiIndex(k, y, x, c)]; - - v += in * wei; - } - } - } - } - - v += bias_ptr[bias_desc.GetOffsetFromMultiIndex(k)]; - - v = v > 0 ? v : 0; - - output_ptr[out_desc.GetOffsetFromMultiIndex(n, ho, wo, k)] = v; - } - } - } - }; - - make_ParallelTensorFunctor(f_k, K)(std::thread::hardware_concurrency()); -} - -template -void profile_conv_fwd_bias_relu_atomic_add_impl(int do_verification, - int init_method, - bool do_log, - bool time_kernel, - ck::index_t N, - ck::index_t K, - ck::index_t C, - std::vector input_spatial_lengths, - std::vector filter_spatial_lengths, - std::vector output_spatial_lengths, - std::vector conv_filter_strides, - std::vector conv_filter_dilations, - std::vector input_left_pads, - std::vector input_right_pads) -{ - const ck::index_t Y = filter_spatial_lengths[0]; - const ck::index_t X = filter_spatial_lengths[1]; - - const ck::index_t Hi = input_spatial_lengths[0]; - const ck::index_t Wi = input_spatial_lengths[1]; - - const ck::index_t Ho = output_spatial_lengths[0]; - const ck::index_t Wo = output_spatial_lengths[1]; - - auto f_host_tensor_descriptor = - [](std::size_t N_, std::size_t C_, std::size_t H, std::size_t W, auto layout) { - if constexpr(is_same::value || - is_same::value || - is_same::value) - { - return HostTensorDescriptor(std::vector({N_, C_, H, W}), - std::vector({C_ * H * W, H * W, W, 1})); - } - else if constexpr(is_same::value || - is_same::value || - is_same::value) - { - return HostTensorDescriptor(std::vector({N_, C_, H, W}), - std::vector({C_ * H * W, 1, W * C_, C_})); - } - }; - - Tensor in_n_c_hi_wi(f_host_tensor_descriptor(N, C, Hi, Wi, InLayout{})); - Tensor wei_k_c_y_x(f_host_tensor_descriptor(K, C, Y, X, WeiLayout{})); - Tensor out_n_k_ho_wo_host_result( - f_host_tensor_descriptor(N, K, Ho, Wo, OutLayout{})); - Tensor out_n_k_ho_wo_device_result( - f_host_tensor_descriptor(N, K, Ho, Wo, OutLayout{})); - - // bias: assume contiguous 1d vector - Tensor bias_k( - HostTensorDescriptor(std::vector({static_cast(K)}))); - - std::cout << "in_n_c_hi_wi: " << in_n_c_hi_wi.mDesc << std::endl; - std::cout << "wei_k_c_y_x: " << wei_k_c_y_x.mDesc << std::endl; - std::cout << "out_n_k_ho_wo: " << out_n_k_ho_wo_host_result.mDesc << std::endl; - std::cout << "bias_k: " << bias_k.mDesc << std::endl; - - switch(init_method) - { - case 0: break; - case 1: - in_n_c_hi_wi.GenerateTensorValue(GeneratorTensor_2{-5, 5}); - wei_k_c_y_x.GenerateTensorValue(GeneratorTensor_2{-5, 5}); - bias_k.GenerateTensorValue(GeneratorTensor_2{-5, 5}); - break; - default: - in_n_c_hi_wi.GenerateTensorValue(GeneratorTensor_3{0.0, 1.0}); - wei_k_c_y_x.GenerateTensorValue(GeneratorTensor_3{-0.5, 0.5}); - bias_k.GenerateTensorValue(GeneratorTensor_3{0.0, 1.0}); - } - - using InElementOp = ck::tensor_operation::element_wise::PassThrough; - using WeiElementOp = ck::tensor_operation::element_wise::PassThrough; - using OutElementOp = ck::tensor_operation::element_wise::AddRelu; - - if(do_verification) - { - cpu_conv_bias_relu_atomic_add(in_n_c_hi_wi.mData.data(), - wei_k_c_y_x.mData.data(), - out_n_k_ho_wo_host_result.mData.data(), - bias_k.mData.data(), - N, - K, - C, - Y, - X, - Hi, - Wi, - Ho, - Wo, - conv_filter_strides[0], - conv_filter_dilations[0], - input_left_pads[0]); - } - - DeviceMem in_device_buf(sizeof(InDataType) * in_n_c_hi_wi.mDesc.GetElementSpace()); - DeviceMem wei_device_buf(sizeof(WeiDataType) * wei_k_c_y_x.mDesc.GetElementSpace()); - DeviceMem out_device_buf(sizeof(OutDataType) * - out_n_k_ho_wo_device_result.mDesc.GetElementSpace()); - DeviceMem bias_device_buf(sizeof(OutDataType) * bias_k.mDesc.GetElementSpace()); - - in_device_buf.ToDevice(in_n_c_hi_wi.mData.data()); - wei_device_buf.ToDevice(wei_k_c_y_x.mData.data()); - bias_device_buf.ToDevice(bias_k.mData.data()); - - using DeviceConvFwdBiasReluPtr = ck::tensor_operation::device:: - DeviceConvFwdBiasActivationPtr; - - // add device operator instances - std::vector op_ptrs; - - if constexpr(ck::is_same_v, ck::half_t> && - ck::is_same_v, ck::half_t> && - ck::is_same_v, ck::half_t>) - { - ck::tensor_operation::device::device_conv2d_fwd_bias_activation_atomic_add_instance:: - add_device_conv2d_fwd_xdl_c_shuffle_bias_relu_atomic_add_nhwc_kyxc_nhwk_f16_instances( - op_ptrs); - } - - if(op_ptrs.size() <= 0) - { - throw std::runtime_error("wrong! no device Conv instance found"); - } - - std::string best_conv_name; - float best_ave_time = 0; - float best_tflops = 0; - float best_gb_per_sec = 0; - - // profile device Conv instances - for(auto& op_ptr : op_ptrs) - { - auto argument_ptr = op_ptr->MakeArgumentPointer( - static_cast(in_device_buf.GetDeviceBuffer()), - static_cast(wei_device_buf.GetDeviceBuffer()), - static_cast(out_device_buf.GetDeviceBuffer()), - static_cast(bias_device_buf.GetDeviceBuffer()), - N, - K, - C, - input_spatial_lengths, - filter_spatial_lengths, - output_spatial_lengths, - conv_filter_strides, - conv_filter_dilations, - input_left_pads, - input_right_pads, - InElementOp{}, - WeiElementOp{}, - OutElementOp{}); - - auto invoker_ptr = op_ptr->MakeInvokerPointer(); - - if(op_ptr->IsSupportedArgument(argument_ptr.get())) - { - std::string conv_name = op_ptr->GetTypeString(); - - float ave_time = - invoker_ptr->Run(argument_ptr.get(), StreamConfig{nullptr, time_kernel}); - - std::size_t flop = std::size_t(2) * N * K * Ho * Wo * C * Y * X; - - std::size_t num_btype = - sizeof(InDataType) * (N * C * Hi * Wi) + sizeof(WeiDataType) * (K * C * Y * X) + - sizeof(OutDataType) * (N * K * Ho * Wo) + sizeof(OutDataType) * (K); - - float tflops = static_cast(flop) / 1.E9 / ave_time; - - float gb_per_sec = num_btype / 1.E6 / ave_time; - - std::cout << "Perf: " << ave_time << " ms, " << tflops << " TFlops, " << gb_per_sec - << " GB/s, " << conv_name << std::endl; - - if(tflops > best_tflops) - { - best_conv_name = conv_name; - best_tflops = tflops; - best_ave_time = ave_time; - best_gb_per_sec = gb_per_sec; - } - - if(do_verification) - { - out_device_buf.FromDevice(out_n_k_ho_wo_device_result.mData.data()); - - ck::utils::check_err(out_n_k_ho_wo_device_result.mData, - out_n_k_ho_wo_host_result.mData); - - if(do_log) - { - LogRangeAsType(std::cout << "in : ", in_n_c_hi_wi.mData, ",") - << std::endl; - LogRangeAsType(std::cout << "wei: ", wei_k_c_y_x.mData, ",") - << std::endl; - LogRangeAsType( - std::cout << "out_host : ", out_n_k_ho_wo_host_result.mData, ",") - << std::endl; - LogRangeAsType( - std::cout << "out_device: ", out_n_k_ho_wo_device_result.mData, ",") - << std::endl; - } - } - } - } - - std::cout << "Best Perf: " << best_ave_time << " ms, " << best_tflops << " TFlops, " - << best_gb_per_sec << " GB/s, " << best_conv_name << std::endl; -} - -} // namespace profiler -} // namespace ck diff --git a/profiler/include/profile_conv_fwd_bias_relu_impl.hpp b/profiler/include/profile_conv_fwd_bias_relu_impl.hpp index eeb2b93e4e..4d32f36f03 100644 --- a/profiler/include/profile_conv_fwd_bias_relu_impl.hpp +++ b/profiler/include/profile_conv_fwd_bias_relu_impl.hpp @@ -1,14 +1,15 @@ #pragma once -#include "check_err.hpp" -#include "config.hpp" -#include "device.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" -#include "tensor_layout.hpp" -#include "device_tensor.hpp" -#include "element_wise_operation.hpp" -#include "device_conv_fwd_bias_activation.hpp" -#include "reference_conv_fwd_bias_activation.hpp" + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_conv_fwd_bias_activation.hpp" + +#include "ck/library/utility/check_err.hpp" +#include "ck/library/host_tensor/device_memory.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" +#include "ck/library/host_tensor/host_tensor_generator.hpp" +#include "ck/library/reference_tensor_operation/cpu/reference_conv_fwd_bias_activation.hpp" namespace ck { namespace tensor_operation { diff --git a/profiler/include/profile_convnd_bwd_data_impl.hpp b/profiler/include/profile_convnd_bwd_data_impl.hpp index 291bf2abc0..4e6e626be1 100644 --- a/profiler/include/profile_convnd_bwd_data_impl.hpp +++ b/profiler/include/profile_convnd_bwd_data_impl.hpp @@ -1,19 +1,21 @@ #pragma once -#include "config.hpp" -#include "device.hpp" -#include "conv_util.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" -#include "tensor_layout.hpp" -#include "device_tensor.hpp" -#include "device_conv_bwd_data.hpp" -#include "element_wise_operation.hpp" -#include "reference_conv_bwd_data.hpp" + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_conv_bwd_data.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" + +#include "ck/library/utility/conv_util.hpp" +#include "ck/library/host_tensor/device_memory.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" +#include "ck/library/host_tensor/host_tensor_generator.hpp" +#include "ck/library/reference_tensor_operation/cpu/reference_conv_bwd_data.hpp" using F16 = ck::half_t; using F32 = float; using BF16 = ck::bhalf_t; using INT8 = int8_t; + namespace ck { namespace tensor_operation { namespace device { diff --git a/profiler/include/profile_gemm_add_add_fastgelu_impl.hpp b/profiler/include/profile_gemm_add_add_fastgelu_impl.hpp index 748c9ada80..864f3474c1 100644 --- a/profiler/include/profile_gemm_add_add_fastgelu_impl.hpp +++ b/profiler/include/profile_gemm_add_add_fastgelu_impl.hpp @@ -2,17 +2,16 @@ #include -#include "check_err.hpp" -#include "config.hpp" -#include "device.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" -#include "host_conv.hpp" -#include "tensor_layout.hpp" -#include "device_tensor.hpp" -#include "element_wise_operation.hpp" -#include "reference_gemm.hpp" -#include "device_gemm_multiple_d.hpp" +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_multiple_d.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" +#include "ck/library/utility/check_err.hpp" +#include "ck/library/host_tensor/device_memory.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" +#include "ck/library/host_tensor/host_tensor_generator.hpp" +#include "ck/library/host_tensor/host_conv.hpp" +#include "ck/library/reference_tensor_operation/cpu/reference_gemm.hpp" namespace ck { namespace tensor_operation { diff --git a/profiler/include/profile_gemm_bias_2d_impl.hpp b/profiler/include/profile_gemm_bias_2d_impl.hpp index 8565f9637c..f9b519388d 100644 --- a/profiler/include/profile_gemm_bias_2d_impl.hpp +++ b/profiler/include/profile_gemm_bias_2d_impl.hpp @@ -1,16 +1,15 @@ #pragma once -#include "check_err.hpp" -#include "config.hpp" -#include "device.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" -#include "host_conv.hpp" -#include "tensor_layout.hpp" -#include "device_tensor.hpp" -#include "element_wise_operation.hpp" -#include "device_gemm_bias.hpp" -#include "reference_gemm_bias_2d.hpp" +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_bias.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" + +#include "ck/library/utility/check_err.hpp" +#include "ck/library/host_tensor/device_memory.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" +#include "ck/library/host_tensor/host_tensor_generator.hpp" +#include "ck/library/reference_tensor_operation/cpu/reference_gemm_bias_2d.hpp" namespace ck { namespace tensor_operation { diff --git a/profiler/include/profile_gemm_bias_add_reduce_impl.hpp b/profiler/include/profile_gemm_bias_add_reduce_impl.hpp index 5b792219c0..dc42dca5dd 100644 --- a/profiler/include/profile_gemm_bias_add_reduce_impl.hpp +++ b/profiler/include/profile_gemm_bias_add_reduce_impl.hpp @@ -1,16 +1,17 @@ #pragma once -#include "check_err.hpp" -#include "config.hpp" -#include "device.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" -#include "host_conv.hpp" -#include "tensor_layout.hpp" -#include "device_tensor.hpp" -#include "element_wise_operation.hpp" -#include "reduction_operator.hpp" -#include "device_gemm_reduce.hpp" -#include "reference_gemm.hpp" + +#include "ck/ck.hpp" +#include "ck/utility/reduction_operator.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_reduce.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" + +#include "ck/library/utility/check_err.hpp" +#include "ck/library/utility/conv_util.hpp" +#include "ck/library/host_tensor/device_memory.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" +#include "ck/library/host_tensor/host_tensor_generator.hpp" +#include "ck/library/reference_tensor_operation/cpu/reference_gemm.hpp" namespace ck { namespace tensor_operation { diff --git a/profiler/include/profile_gemm_bias_relu_add_impl.hpp b/profiler/include/profile_gemm_bias_relu_add_impl.hpp index 6fec17c199..be2fc45f90 100644 --- a/profiler/include/profile_gemm_bias_relu_add_impl.hpp +++ b/profiler/include/profile_gemm_bias_relu_add_impl.hpp @@ -1,16 +1,16 @@ #pragma once -#include "check_err.hpp" -#include "config.hpp" -#include "device.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" -#include "host_conv.hpp" -#include "tensor_layout.hpp" -#include "device_tensor.hpp" -#include "element_wise_operation.hpp" -#include "device_gemm_bias_activation_add.hpp" -#include "reference_gemm_bias_activation_add.hpp" +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_bias_activation_add.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" + +#include "ck/library/utility/check_err.hpp" +#include "ck/library/utility/conv_util.hpp" +#include "ck/library/host_tensor/device_memory.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" +#include "ck/library/host_tensor/host_tensor_generator.hpp" +#include "ck/library/reference_tensor_operation/cpu/reference_gemm_bias_activation_add.hpp" namespace ck { namespace tensor_operation { diff --git a/profiler/include/profile_gemm_bias_relu_impl.hpp b/profiler/include/profile_gemm_bias_relu_impl.hpp index 69010becc5..6eabc17c77 100644 --- a/profiler/include/profile_gemm_bias_relu_impl.hpp +++ b/profiler/include/profile_gemm_bias_relu_impl.hpp @@ -1,16 +1,16 @@ #pragma once -#include "check_err.hpp" -#include "config.hpp" -#include "device.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" -#include "host_conv.hpp" -#include "tensor_layout.hpp" -#include "device_tensor.hpp" -#include "element_wise_operation.hpp" -#include "device_gemm_bias_activation.hpp" -#include "reference_gemm_bias_activation.hpp" +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_bias_activation.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" + +#include "ck/library/utility/check_err.hpp" +#include "ck/library/utility/conv_util.hpp" +#include "ck/library/host_tensor/device_memory.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" +#include "ck/library/host_tensor/host_tensor_generator.hpp" +#include "ck/library/reference_tensor_operation/cpu/reference_gemm_bias_activation.hpp" namespace ck { namespace tensor_operation { diff --git a/profiler/include/profile_gemm_impl.hpp b/profiler/include/profile_gemm_impl.hpp index a3400f89b3..add8fbe8b3 100644 --- a/profiler/include/profile_gemm_impl.hpp +++ b/profiler/include/profile_gemm_impl.hpp @@ -1,19 +1,20 @@ #pragma once + #include #include #include -#include "check_err.hpp" -#include "config.hpp" -#include "device.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" -#include "host_conv.hpp" -#include "tensor_layout.hpp" -#include "device_tensor.hpp" -#include "element_wise_operation.hpp" -#include "device_gemm.hpp" -#include "reference_gemm.hpp" +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" + +#include "ck/library/utility/check_err.hpp" +#include "ck/library/utility/conv_util.hpp" +#include "ck/library/host_tensor/device_memory.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" +#include "ck/library/host_tensor/host_tensor_generator.hpp" +#include "ck/library/reference_tensor_operation/cpu/reference_gemm.hpp" namespace ck { namespace tensor_operation { diff --git a/profiler/include/profile_gemm_reduce_impl.hpp b/profiler/include/profile_gemm_reduce_impl.hpp index 97c23defe0..41dded9410 100644 --- a/profiler/include/profile_gemm_reduce_impl.hpp +++ b/profiler/include/profile_gemm_reduce_impl.hpp @@ -1,16 +1,17 @@ #pragma once -#include "check_err.hpp" -#include "config.hpp" -#include "device.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" -#include "host_conv.hpp" -#include "tensor_layout.hpp" -#include "device_tensor.hpp" -#include "element_wise_operation.hpp" -#include "reduction_operator.hpp" -#include "device_gemm_reduce.hpp" -#include "reference_gemm.hpp" + +#include "ck/ck.hpp" +#include "ck/utility/reduction_operator.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_reduce.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" + +#include "ck/library/utility/check_err.hpp" +#include "ck/library/utility/conv_util.hpp" +#include "ck/library/host_tensor/device_memory.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" +#include "ck/library/host_tensor/host_tensor_generator.hpp" +#include "ck/library/reference_tensor_operation/cpu/reference_gemm.hpp" namespace ck { namespace tensor_operation { diff --git a/profiler/include/profile_grouped_gemm_impl.hpp b/profiler/include/profile_grouped_gemm_impl.hpp index 8806e8ff43..27827d72e7 100644 --- a/profiler/include/profile_grouped_gemm_impl.hpp +++ b/profiler/include/profile_grouped_gemm_impl.hpp @@ -1,17 +1,18 @@ #pragma once + #include -#include "check_err.hpp" -#include "config.hpp" -#include "device.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" -#include "host_conv.hpp" -#include "tensor_layout.hpp" -#include "device_tensor.hpp" -#include "element_wise_operation.hpp" -#include "device_gemm.hpp" -#include "reference_gemm.hpp" +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" + +#include "ck/library/utility/check_err.hpp" +#include "ck/library/utility/conv_util.hpp" +#include "ck/library/host_tensor/device_memory.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" +#include "ck/library/host_tensor/host_tensor_generator.hpp" +#include "ck/library/reference_tensor_operation/cpu/reference_gemm.hpp" namespace ck { namespace tensor_operation { diff --git a/profiler/include/profile_reduce_impl.hpp b/profiler/include/profile_reduce_impl.hpp index 5e192aa1bc..2ff9a09ebc 100644 --- a/profiler/include/profile_reduce_impl.hpp +++ b/profiler/include/profile_reduce_impl.hpp @@ -1,12 +1,14 @@ #pragma once -#include "check_err.hpp" -#include "device_reduce.hpp" -#include "device_reduce_instance.hpp" -#include "reduction_enums.hpp" -#include "host_reduction.hpp" -#include "host_common_util.hpp" -#include "host_tensor_generator.hpp" +#include "ck/utility/reduction_enums.hpp" +#include "ck/tensor_operation/gpu/device/device_reduce.hpp" + +#include "ck/library/utility/check_err.hpp" +#include "ck/library/tensor_operation_instance/gpu/reduce/device_reduce_instance.hpp" +#include "ck/library/host_tensor/device_memory.hpp" +#include "ck/library/host_tensor/host_reduction.hpp" +#include "ck/library/host_tensor/host_common_util.hpp" +#include "ck/library/host_tensor/host_tensor_generator.hpp" namespace ck { namespace tensor_operation { diff --git a/profiler/src/profile_batched_gemm.cpp b/profiler/src/profile_batched_gemm.cpp index fbdc07c3da..386ac216cf 100644 --- a/profiler/src/profile_batched_gemm.cpp +++ b/profiler/src/profile_batched_gemm.cpp @@ -3,18 +3,8 @@ #include #include #include -#include -#include -#include "config.hpp" -#include "print.hpp" -#include "device.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" -#include "host_gemm.hpp" -#include "device_tensor.hpp" -#include "device_base.hpp" -#include "device_batched_gemm_xdl.hpp" -#include "profile_batched_gemm_impl.hpp" + +#include "profiler/include/profile_batched_gemm_impl.hpp" enum struct GemmMatrixLayout { diff --git a/profiler/src/profile_batched_gemm_reduce.cpp b/profiler/src/profile_batched_gemm_reduce.cpp index 594fc6bedb..53a7e513b6 100644 --- a/profiler/src/profile_batched_gemm_reduce.cpp +++ b/profiler/src/profile_batched_gemm_reduce.cpp @@ -2,10 +2,8 @@ #include #include #include -#include -#include -#include "profile_batched_gemm_reduce_impl.hpp" +#include "profiler/include/profile_batched_gemm_reduce_impl.hpp" int profile_batched_gemm_reduce(int argc, char* argv[]) { diff --git a/profiler/src/profile_conv_bwd_weight.cpp b/profiler/src/profile_conv_bwd_weight.cpp index 80413322b3..477bf0d90f 100644 --- a/profiler/src/profile_conv_bwd_weight.cpp +++ b/profiler/src/profile_conv_bwd_weight.cpp @@ -2,9 +2,8 @@ #include #include #include -#include -#include -#include "profile_conv_bwd_weight_impl.hpp" + +#include "profiler/include/profile_conv_bwd_weight_impl.hpp" enum struct ConvDataType { diff --git a/profiler/src/profile_conv_fwd_bias_relu.cpp b/profiler/src/profile_conv_fwd_bias_relu.cpp index ca7dc1935a..fc76e5b125 100644 --- a/profiler/src/profile_conv_fwd_bias_relu.cpp +++ b/profiler/src/profile_conv_fwd_bias_relu.cpp @@ -2,9 +2,8 @@ #include #include #include -#include -#include -#include "profile_conv_fwd_bias_relu_impl.hpp" + +#include "profiler/include/profile_conv_fwd_bias_relu_impl.hpp" enum struct ConvDataType { diff --git a/profiler/src/profile_conv_fwd_bias_relu_add.cpp b/profiler/src/profile_conv_fwd_bias_relu_add.cpp index 5d75f5a294..fc522ae3cd 100644 --- a/profiler/src/profile_conv_fwd_bias_relu_add.cpp +++ b/profiler/src/profile_conv_fwd_bias_relu_add.cpp @@ -2,9 +2,8 @@ #include #include #include -#include -#include -#include "profile_conv_fwd_bias_relu_add_impl.hpp" + +#include "profiler/include/profile_conv_fwd_bias_relu_add_impl.hpp" enum struct ConvDataType { diff --git a/profiler/src/profile_conv_fwd_bias_relu_atomic_add.cpp b/profiler/src/profile_conv_fwd_bias_relu_atomic_add.cpp deleted file mode 100644 index 96d3b10ddf..0000000000 --- a/profiler/src/profile_conv_fwd_bias_relu_atomic_add.cpp +++ /dev/null @@ -1,116 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include "profile_conv_fwd_bias_relu_atomic_add_impl.hpp" - -enum struct ConvDataType -{ - F32_F32_F32, // 0 - F16_F16_F16, // 1 -}; - -enum struct ConvInputLayout -{ - NCHW, // 0 - NHWC, // 1 -}; - -enum struct ConvWeightLayout -{ - KCYX, // 0 - KYXC, // 1 -}; - -enum struct ConvOutputLayout -{ - NKHW, // 0 - NHWK, // 1 -}; - -int profile_conv_fwd_bias_relu_atomic_add(int argc, char* argv[]) -{ - if(argc != 25) - { - printf("arg1: tensor operation (conv_fwd_bias_relu_atomic_add: " - "ForwardConvolution+Bias+ReLu+AtomicAdd)\n"); - printf("arg2: data type (0: fp32; 1: fp16)\n"); - printf("arg3: input tensor layout (0: NCHW; 1: NHWC)\n"); - printf("arg4: weight tensor layout (0: KCYX; 1: KYXC)\n"); - printf("arg5: output tensor layout (0: NKHW; 1: NHWK)\n"); - printf("arg6: verification (0: no; 1: yes)\n"); - printf("arg7: initialization (0: no init; 1: integer value; 2: decimal value)\n"); - printf("arg8: print tensor value (0: no; 1: yes)\n"); - printf("arg9: time kernel (0=n0, 1=yes)\n"); - printf("arg10 to 24: N, K, C, Y, X, Hi, Wi, Sy, Sx, Dy, Dx, LeftPy, LeftPx, RightPy, " - "RightPx\n"); - exit(1); - } - - const auto data_type = static_cast(std::stoi(argv[2])); - const auto in_layout = static_cast(std::stoi(argv[3])); - const auto wei_layout = static_cast(std::stoi(argv[4])); - const auto out_layout = static_cast(std::stoi(argv[5])); - const bool do_verification = std::stoi(argv[6]); - const int init_method = std::stoi(argv[7]); - const bool do_log = std::stoi(argv[8]); - const bool time_kernel = std::stoi(argv[9]); - - const ck::index_t N = std::stoi(argv[10]); - const ck::index_t K = std::stoi(argv[11]); - const ck::index_t C = std::stoi(argv[12]); - const ck::index_t Y = std::stoi(argv[13]); - const ck::index_t X = std::stoi(argv[14]); - const ck::index_t Hi = std::stoi(argv[15]); - const ck::index_t Wi = std::stoi(argv[16]); - - const ck::index_t conv_stride_h = std::stoi(argv[17]); - const ck::index_t conv_stride_w = std::stoi(argv[18]); - const ck::index_t conv_dilation_h = std::stoi(argv[19]); - const ck::index_t conv_dilation_w = std::stoi(argv[20]); - const ck::index_t in_left_pad_h = std::stoi(argv[21]); - const ck::index_t in_left_pad_w = std::stoi(argv[22]); - const ck::index_t in_right_pad_h = std::stoi(argv[23]); - const ck::index_t in_right_pad_w = std::stoi(argv[24]); - - const ck::index_t YEff = (Y - 1) * conv_dilation_h + 1; - const ck::index_t XEff = (X - 1) * conv_dilation_w + 1; - - const ck::index_t Ho = (Hi + in_left_pad_h + in_right_pad_h - YEff) / conv_stride_h + 1; - const ck::index_t Wo = (Wi + in_left_pad_w + in_right_pad_w - XEff) / conv_stride_w + 1; - - if(data_type == ConvDataType::F16_F16_F16 && in_layout == ConvInputLayout::NHWC && - wei_layout == ConvWeightLayout::KYXC && out_layout == ConvOutputLayout::NHWK) - { - ck::profiler::profile_conv_fwd_bias_relu_atomic_add_impl< - 2, - ck::half_t, - ck::half_t, - ck::half_t, - ck::tensor_layout::convolution::NHWC, - ck::tensor_layout::convolution::KYXC, - ck::tensor_layout::convolution::NHWK>( - do_verification, - init_method, - do_log, - time_kernel, - N, - K, - C, - std::vector{Hi, Wi}, - std::vector{Y, X}, - std::vector{Ho, Wo}, - std::vector{conv_stride_h, conv_stride_w}, - std::vector{conv_dilation_h, conv_dilation_w}, - std::vector{in_left_pad_h, in_left_pad_w}, - std::vector{in_right_pad_h, in_right_pad_w}); - } - else - { - throw std::runtime_error("wrong! data_type & layout for this operator is not implemented"); - } - - return 0; -} diff --git a/profiler/src/profile_convnd_bwd_data.cpp b/profiler/src/profile_convnd_bwd_data.cpp index 5d0e6a34c7..e37bef8ec1 100644 --- a/profiler/src/profile_convnd_bwd_data.cpp +++ b/profiler/src/profile_convnd_bwd_data.cpp @@ -2,10 +2,8 @@ #include #include #include -#include -#include -#include "profile_convnd_bwd_data_impl.hpp" +#include "profiler/include/profile_convnd_bwd_data_impl.hpp" namespace { diff --git a/profiler/src/profile_convnd_fwd.cpp b/profiler/src/profile_convnd_fwd.cpp index cb92587897..7ad8ad1b21 100644 --- a/profiler/src/profile_convnd_fwd.cpp +++ b/profiler/src/profile_convnd_fwd.cpp @@ -4,13 +4,13 @@ #include #include #include -#include -#include "conv_util.hpp" -#include "element_wise_operation.hpp" -#include "fill.hpp" -#include "profile_convnd_fwd.hpp" -#include "tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" +#include "ck/library/utility/conv_util.hpp" +#include "ck/library/utility/fill.hpp" + +#include "profiler/include/profile_convnd_fwd.hpp" namespace { diff --git a/profiler/src/profile_gemm.cpp b/profiler/src/profile_gemm.cpp index 0684e18322..b021f1ad71 100644 --- a/profiler/src/profile_gemm.cpp +++ b/profiler/src/profile_gemm.cpp @@ -2,9 +2,8 @@ #include #include #include -#include -#include -#include "profile_gemm_impl.hpp" + +#include "profiler/include/profile_gemm_impl.hpp" enum struct GemmMatrixLayout { diff --git a/profiler/src/profile_gemm_add_add_fastgelu.cpp b/profiler/src/profile_gemm_add_add_fastgelu.cpp index 602f14a78a..da813fff3c 100644 --- a/profiler/src/profile_gemm_add_add_fastgelu.cpp +++ b/profiler/src/profile_gemm_add_add_fastgelu.cpp @@ -2,9 +2,8 @@ #include #include #include -#include -#include "profile_gemm_add_add_fastgelu_impl.hpp" +#include "profiler/include/profile_gemm_add_add_fastgelu_impl.hpp" int profile_gemm_add_add_fastgelu(int argc, char* argv[]) { diff --git a/profiler/src/profile_gemm_bias_2d.cpp b/profiler/src/profile_gemm_bias_2d.cpp index 51dba85f32..8898d5878c 100644 --- a/profiler/src/profile_gemm_bias_2d.cpp +++ b/profiler/src/profile_gemm_bias_2d.cpp @@ -2,9 +2,8 @@ #include #include #include -#include -#include -#include "profile_gemm_bias_2d_impl.hpp" + +#include "profiler/include/profile_gemm_bias_2d_impl.hpp" enum struct GemmMatrixLayout { diff --git a/profiler/src/profile_gemm_bias_add_reduce.cpp b/profiler/src/profile_gemm_bias_add_reduce.cpp index d36e5f1c83..ea07d033f2 100644 --- a/profiler/src/profile_gemm_bias_add_reduce.cpp +++ b/profiler/src/profile_gemm_bias_add_reduce.cpp @@ -2,9 +2,8 @@ #include #include #include -#include -#include -#include "profile_gemm_bias_add_reduce_impl.hpp" + +#include "profiler/include/profile_gemm_bias_add_reduce_impl.hpp" int profile_gemm_bias_add_reduce(int argc, char* argv[]) { diff --git a/profiler/src/profile_gemm_bias_relu.cpp b/profiler/src/profile_gemm_bias_relu.cpp index bf035d9ad9..9b8dbed31a 100644 --- a/profiler/src/profile_gemm_bias_relu.cpp +++ b/profiler/src/profile_gemm_bias_relu.cpp @@ -2,9 +2,8 @@ #include #include #include -#include -#include -#include "profile_gemm_bias_relu_impl.hpp" + +#include "profiler/include/profile_gemm_bias_relu_impl.hpp" enum struct GemmMatrixLayout { diff --git a/profiler/src/profile_gemm_bias_relu_add.cpp b/profiler/src/profile_gemm_bias_relu_add.cpp index 9c324f6cf9..cd1eb7ae52 100644 --- a/profiler/src/profile_gemm_bias_relu_add.cpp +++ b/profiler/src/profile_gemm_bias_relu_add.cpp @@ -2,9 +2,8 @@ #include #include #include -#include -#include -#include "profile_gemm_bias_relu_add_impl.hpp" + +#include "profiler/include/profile_gemm_bias_relu_add_impl.hpp" enum struct GemmMatrixLayout { diff --git a/profiler/src/profile_gemm_reduce.cpp b/profiler/src/profile_gemm_reduce.cpp index a23967acd7..5d186e0754 100644 --- a/profiler/src/profile_gemm_reduce.cpp +++ b/profiler/src/profile_gemm_reduce.cpp @@ -2,9 +2,8 @@ #include #include #include -#include -#include -#include "profile_gemm_reduce_impl.hpp" + +#include "profiler/include/profile_gemm_reduce_impl.hpp" int profile_gemm_reduce(int argc, char* argv[]) { diff --git a/profiler/src/profile_grouped_gemm.cpp b/profiler/src/profile_grouped_gemm.cpp index ea73d446e3..0f2c118f59 100644 --- a/profiler/src/profile_grouped_gemm.cpp +++ b/profiler/src/profile_grouped_gemm.cpp @@ -2,9 +2,8 @@ #include #include #include -#include -#include -#include "profile_grouped_gemm_impl.hpp" + +#include "profiler/include/profile_grouped_gemm_impl.hpp" enum struct GemmMatrixLayout { diff --git a/profiler/src/profile_reduce.cpp b/profiler/src/profile_reduce.cpp index bdbac4fab4..3d94703e11 100644 --- a/profiler/src/profile_reduce.cpp +++ b/profiler/src/profile_reduce.cpp @@ -6,11 +6,12 @@ #include #include -#include "data_type_enum.hpp" -#include "reduction_enums.hpp" +#include "ck/utility/reduction_enums.hpp" -#include "host_common_util.hpp" -#include "profile_reduce_impl.hpp" +#include "ck/library/host_tensor/host_common_util.hpp" + +#include "profiler/include/profile_reduce_impl.hpp" +#include "profiler/include/data_type_enum.hpp" using namespace std; diff --git a/profiler/src/profiler.cpp b/profiler/src/profiler.cpp index ceaebf2c7c..50c3faadef 100644 --- a/profiler/src/profiler.cpp +++ b/profiler/src/profiler.cpp @@ -4,7 +4,7 @@ #include #include -#include "profile_convnd_fwd.hpp" +#include "profiler/include/profile_convnd_fwd.hpp" int profile_gemm(int, char*[]); int profile_gemm_bias_2d(int, char*[]); @@ -17,7 +17,6 @@ int profile_grouped_gemm(int, char*[]); int profile_conv_fwd(int, char*[]); int profile_conv_fwd_bias_relu(int, char*[]); int profile_conv_fwd_bias_relu_add(int, char*[]); -int profile_conv_fwd_bias_relu_atomic_add(int, char*[]); int profile_convnd_bwd_data(int, char*[], int); int profile_reduce(int, char*[]); int profile_conv_bwd_weight(int, char*[]); @@ -36,7 +35,6 @@ static void print_helper_message() " conv_fwd: ForwardConvolution\n" " conv_fwd_bias_relu: ForwardConvolution+Bias+ReLU\n" " conv_fwd_bias_relu_add: ForwardConvolution+Bias+ReLU+Add\n" - " conv_fwd_bias_relu_atomic_add: ForwardConvolution+Bias+ReLU+AtomicAdd\n" " conv1d_bwd_data: BackwardConvolution data 1 dim\n" " conv2d_bwd_data: BackwardConvolution data 2 dim\n" " conv3d_bwd_data: BackwardConvolution data 3 dim\n" @@ -103,10 +101,6 @@ int main(int argc, char* argv[]) { return profile_conv_fwd_bias_relu_add(argc, argv); } - else if(strcmp(argv[1], "conv_fwd_bias_relu_atomic_add") == 0) - { - return profile_conv_fwd_bias_relu_atomic_add(argc, argv); - } else if(strcmp(argv[1], "conv1d_bwd_data") == 0) { return profile_convnd_bwd_data(argc, argv, 1); diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 47ca0b663d..47c13d33e0 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -1,26 +1,5 @@ include_directories(BEFORE ${PROJECT_SOURCE_DIR}/ - ${PROJECT_SOURCE_DIR}/include/ck - ${PROJECT_SOURCE_DIR}/include/ck/utility - ${PROJECT_SOURCE_DIR}/include/ck/host_utility - ${PROJECT_SOURCE_DIR}/include/ck/tensor_description - ${PROJECT_SOURCE_DIR}/include/ck/tensor - ${PROJECT_SOURCE_DIR}/include/ck/problem_transform - ${PROJECT_SOURCE_DIR}/include/ck/tensor_operation/gpu/device - ${PROJECT_SOURCE_DIR}/include/ck/tensor_operation/gpu/grid - ${PROJECT_SOURCE_DIR}/include/ck/tensor_operation/gpu/block - ${PROJECT_SOURCE_DIR}/include/ck/tensor_operation/gpu/warp - ${PROJECT_SOURCE_DIR}/include/ck/tensor_operation/gpu/thread - ${PROJECT_SOURCE_DIR}/include/ck/tensor_operation/gpu/element - ${PROJECT_SOURCE_DIR}/library/include/ck/library/host_tensor - ${PROJECT_SOURCE_DIR}/library/include/ck/library/tensor_operation_instance - ${PROJECT_SOURCE_DIR}/library/include/ck/library/tensor_operation_instance/gpu/reduce - ${PROJECT_SOURCE_DIR}/library/include/ck/library/reference_tensor_operation/cpu - ${PROJECT_SOURCE_DIR}/library/include/ck/library/reference_tensor_operation/gpu - ${PROJECT_SOURCE_DIR}/library/include/ck/library/utility - ${PROJECT_SOURCE_DIR}/test/include - ${PROJECT_SOURCE_DIR}/profiler/include - ${PROJECT_SOURCE_DIR}/external/include/half ) include(googletest) @@ -66,4 +45,3 @@ add_subdirectory(conv2d_bwd_weight) add_subdirectory(convnd_bwd_data) add_subdirectory(block_to_ctile_map) add_subdirectory(softmax) -# DONOT add client_app, that is tested via CI independently diff --git a/test/batched_gemm/batched_gemm_fp16.cpp b/test/batched_gemm/batched_gemm_fp16.cpp index c039e344d2..0d3ee9e488 100644 --- a/test/batched_gemm/batched_gemm_fp16.cpp +++ b/test/batched_gemm/batched_gemm_fp16.cpp @@ -1,6 +1,6 @@ #include -#include "profile_batched_gemm_impl.hpp" +#include "profiler/include/profile_batched_gemm_impl.hpp" namespace { using ADataType = ck::half_t; diff --git a/test/batched_gemm_reduce/CMakeLists.txt b/test/batched_gemm_reduce/CMakeLists.txt index 3ecf19491b..fa1a2bf87f 100644 --- a/test/batched_gemm_reduce/CMakeLists.txt +++ b/test/batched_gemm_reduce/CMakeLists.txt @@ -1,9 +1,3 @@ -include_directories(BEFORE - ${PROJECT_SOURCE_DIR}/profiler/include - ${PROJECT_SOURCE_DIR}/test/include - ${PROJECT_SOURCE_DIR}/external/include/half -) - add_test_executable(test_batched_gemm_reduce_fp16 batched_gemm_reduce_fp16.cpp) target_link_libraries(test_batched_gemm_reduce_fp16 PRIVATE host_tensor) target_link_libraries(test_batched_gemm_reduce_fp16 PRIVATE device_batched_gemm_reduce_instance) diff --git a/test/batched_gemm_reduce/batched_gemm_reduce_fp16.cpp b/test/batched_gemm_reduce/batched_gemm_reduce_fp16.cpp index 7b311cff17..08bfa990ea 100644 --- a/test/batched_gemm_reduce/batched_gemm_reduce_fp16.cpp +++ b/test/batched_gemm_reduce/batched_gemm_reduce_fp16.cpp @@ -1,6 +1,6 @@ #include -#include "profile_batched_gemm_reduce_impl.hpp" +#include "profiler/include/profile_batched_gemm_reduce_impl.hpp" int main() { diff --git a/test/block_to_ctile_map/test_block_to_ctile_map.cpp b/test/block_to_ctile_map/test_block_to_ctile_map.cpp index 662d2a0fa5..f8062730e2 100644 --- a/test/block_to_ctile_map/test_block_to_ctile_map.cpp +++ b/test/block_to_ctile_map/test_block_to_ctile_map.cpp @@ -1,8 +1,9 @@ -#include -#include "ck/tensor_operation/gpu/grid/block_to_ctile_map.hpp" -#include "gtest/gtest.h" #include #include +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/grid/block_to_ctile_map.hpp" using namespace ck; diff --git a/test/client_app/CMakeLists.txt b/test/client_app/CMakeLists.txt deleted file mode 100644 index f8dd8c4e0a..0000000000 --- a/test/client_app/CMakeLists.txt +++ /dev/null @@ -1,11 +0,0 @@ -cmake_minimum_required(VERSION 3.15) -project(ck_app) -add_compile_options(-std=c++14) - -find_package(composable_kernel 1.0.0 COMPONENTS device_operations host_tensor) -find_package(hip REQUIRED PATHS /opt/rocm) -message(STATUS "Build with HIP ${hip_VERSION}") - -add_executable(test_client_app client_app.cpp) - -target_link_libraries(test_client_app PRIVATE composable_kernel::device_operations composable_kernel::host_tensor hip::host) diff --git a/test/client_app/client_app.cpp b/test/client_app/client_app.cpp deleted file mode 100644 index 665a103f70..0000000000 --- a/test/client_app/client_app.cpp +++ /dev/null @@ -1,77 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include - -#include "client_app_impl.hpp" - -int main(int argc, char* argv[]) -{ - if(argc != 25) - { - printf("arg1: tensor operation (conv_fwd: ForwardConvolution)\n"); - printf("arg2: data type (0: fp32; 1: fp16)\n"); - printf("arg3: input tensor layout (0: NCHW; 1: NHWC)\n"); - printf("arg4: weight tensor layout (0: KCYX; 1: KYXC)\n"); - printf("arg5: output tensor layout (0: NKHW; 1: NHWK)\n"); - printf("arg6: verification (0: no; 1: yes)\n"); - printf("arg7: initialization (0: no init; 1: integer value; 2: decimal value)\n"); - printf("arg8: print tensor value (0: no; 1: yes)\n"); - printf("arg9: time kernel (0=n0, 1=yes)\n"); - printf("arg10 to 24: N, K, C, Y, X, Hi, Wi, Sy, Sx, Dy, Dx, LeftPy, LeftPx, RightPy, " - "RightPx\n"); - exit(1); - } - - const ConvDataType data_type = static_cast(std::stoi(argv[2])); - const int in_layout = static_cast(std::stoi(argv[3])); - const int wei_layout = static_cast(std::stoi(argv[4])); - const int out_layout = static_cast(std::stoi(argv[5])); - const bool do_verification = std::stoi(argv[6]); - const int init_method = std::stoi(argv[7]); - const bool do_log = std::stoi(argv[8]); - const bool time_kernel = std::stoi(argv[9]); - - const ck::index_t N = std::stoi(argv[10]); - const ck::index_t K = std::stoi(argv[11]); - const ck::index_t C = std::stoi(argv[12]); - const ck::index_t Y = std::stoi(argv[13]); - const ck::index_t X = std::stoi(argv[14]); - const ck::index_t Hi = std::stoi(argv[15]); - const ck::index_t Wi = std::stoi(argv[16]); - - const ck::index_t conv_stride_h = std::stoi(argv[17]); - const ck::index_t conv_stride_w = std::stoi(argv[18]); - const ck::index_t conv_dilation_h = std::stoi(argv[19]); - const ck::index_t conv_dilation_w = std::stoi(argv[20]); - const ck::index_t in_left_pad_h = std::stoi(argv[21]); - const ck::index_t in_left_pad_w = std::stoi(argv[22]); - const ck::index_t in_right_pad_h = std::stoi(argv[23]); - const ck::index_t in_right_pad_w = std::stoi(argv[24]); - - const ck::index_t YEff = (Y - 1) * conv_dilation_h + 1; - const ck::index_t XEff = (X - 1) * conv_dilation_w + 1; - - const ck::index_t Ho = (Hi + in_left_pad_h + in_right_pad_h - YEff) / conv_stride_h + 1; - const ck::index_t Wo = (Wi + in_left_pad_w + in_right_pad_w - XEff) / conv_stride_w + 1; - - ck::app::profile_conv_fwd_impl(do_verification, - init_method, - do_log, - time_kernel, - data_type, - N, - K, - C, - std::vector{Hi, Wi}, - std::vector{Y, X}, - std::vector{Ho, Wo}, - std::vector{conv_stride_h, conv_stride_w}, - std::vector{conv_dilation_h, conv_dilation_w}, - std::vector{in_left_pad_h, in_left_pad_w}, - std::vector{in_right_pad_h, in_right_pad_w}); - return 1; -} diff --git a/test/client_app/client_app_impl.hpp b/test/client_app/client_app_impl.hpp deleted file mode 100644 index f9e4145ba0..0000000000 --- a/test/client_app/client_app_impl.hpp +++ /dev/null @@ -1,214 +0,0 @@ -#pragma once - -#include "host_interface.hpp" - -enum ConvDataType -{ - F32_F32_F32, // 0 - F16_F16_F16, // 1 - BF16_BF16_BF16, // 2 - INT8_INT8_INT8, // 3 -}; - -enum ConvInputLayout -{ - NCHW, // 0 - NHWC, // 1 -}; - -enum ConvWeightLayout -{ - KCYX, // 0 - KYXC, // 1 -}; - -enum ConvOutputLayout -{ - NKHW, // 0 - NHWK, // 1 -}; - -void check_hip_error(void) -{ - hipError_t err = hipGetLastError(); - if(err != hipSuccess) - { - std::cerr << "Error: " << hipGetErrorString(err) << std::endl; - exit(err); - } -} -std::string getDeviceName(int device) -{ - struct hipDeviceProp_t prop; - hipGetDeviceProperties(&prop, device); - check_hip_error(); - return std::string(prop.name); -} - -int getDriver(void) -{ - int driver; - hipDriverGetVersion(&driver); - check_hip_error(); - return driver; -} - -namespace ck { -namespace app { -struct DeviceMem -{ - DeviceMem() = delete; - DeviceMem(std::size_t mem_size); - void* GetDeviceBuffer(); - void ToDevice(const void* p); - void FromDevice(void* p); - ~DeviceMem(); - - void* mpDeviceBuf; - std::size_t mMemSize; -}; - -DeviceMem::DeviceMem(std::size_t mem_size) : mMemSize(mem_size) -{ - hipGetErrorString(hipMalloc(static_cast(&mpDeviceBuf), mMemSize)); -} - -void* DeviceMem::GetDeviceBuffer() { return mpDeviceBuf; } - -void DeviceMem::ToDevice(const void* p) -{ - hipGetErrorString( - hipMemcpy(mpDeviceBuf, const_cast(p), mMemSize, hipMemcpyHostToDevice)); -} - -void DeviceMem::FromDevice(void* p) -{ - hipGetErrorString(hipMemcpy(p, mpDeviceBuf, mMemSize, hipMemcpyDeviceToHost)); -} - -DeviceMem::~DeviceMem() { hipGetErrorString(hipFree(mpDeviceBuf)); } - -void profile_conv_fwd_impl(int do_verification, - int init_method, - bool do_log, - bool time_kernel, - ConvDataType data_type, - ck::index_t N, - ck::index_t K, - ck::index_t C, - std::vector input_spatial_lengths, - std::vector filter_spatial_lengths, - std::vector output_spatial_lengths, - std::vector conv_filter_strides, - std::vector conv_filter_dilations, - std::vector input_left_pads, - std::vector input_right_pads) -{ - const ck::index_t Y = filter_spatial_lengths[0]; - const ck::index_t X = filter_spatial_lengths[1]; - - const ck::index_t Hi = input_spatial_lengths[0]; - const ck::index_t Wi = input_spatial_lengths[1]; - - const ck::index_t Ho = output_spatial_lengths[0]; - const ck::index_t Wo = output_spatial_lengths[1]; - - const auto in_sz = N * C * Hi * Wi; - const auto wei_sz = K * C * Y * X; - const auto out_sz = N * K * Ho * Wo; - - using WeiDataType = float; - using InDataType = float; - using OutDataType = float; - - app::DeviceMem in_device_buf(sizeof(InDataType) * in_sz); - app::DeviceMem wei_device_buf(sizeof(WeiDataType) * wei_sz); - app::DeviceMem out_device_buf(sizeof(OutDataType) * out_sz); - // data is already on device! - - // add device Conv instances - std::vector conv_ptrs; - if(data_type == F16_F16_F16) - { - add_device_conv2d_fwd_xdl_c_shuffle_nhwc_kyxc_nhwk_f16_instances_t(conv_ptrs); - add_device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_f16_instances_t(conv_ptrs); - } - else if(data_type == BF16_BF16_BF16) - add_device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_bf16_instances_t(conv_ptrs); - else if(data_type == F32_F32_F32) - add_device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_f32_instances_t(conv_ptrs); - else if(data_type == INT8_INT8_INT8) - add_device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_int8_instances_t(conv_ptrs); - else - throw std::runtime_error("wrong! Invalid data type"); - if(conv_ptrs.empty()) - { - throw std::runtime_error("wrong! no device Conv instance found"); - } - - std::string best_conv_name; - float best_ave_time = 0; - float best_tflops = 0; - float best_gb_per_sec = 0; - int deviceIndex = 0; - hipSetDevice(deviceIndex); - check_hip_error(); - - StreamConfig stream_config{nullptr, time_kernel}; - hipStreamCreate(&stream_config.stream_id_); - check_hip_error(); - - // profile device Conv instances - for(auto& conv_ptr : conv_ptrs) - { - auto argument_ptr = - conv_ptr.MakeArgumentPointer(static_cast(in_device_buf.GetDeviceBuffer()), - static_cast(wei_device_buf.GetDeviceBuffer()), - static_cast(out_device_buf.GetDeviceBuffer()), - N, - K, - C, - input_spatial_lengths, - filter_spatial_lengths, - output_spatial_lengths, - conv_filter_strides, - conv_filter_dilations, - input_left_pads, - input_right_pads); - - auto invoker_ptr = conv_ptr.MakeInvokerPointer(); - - if(conv_ptr.IsSupportedArgument(argument_ptr.get())) - { - std::string conv_name = conv_ptr.GetTypeString(); - float ave_time = invoker_ptr->Run(argument_ptr.get(), stream_config); - - std::size_t flop = std::size_t(2) * N * K * Ho * Wo * C * Y * X; - - std::size_t num_btype = sizeof(InDataType) * (N * C * Hi * Wi) + - sizeof(WeiDataType) * (K * C * Y * X) + - sizeof(OutDataType) * (N * K * Ho * Wo); - - float tflops = static_cast(flop) / 1.E9 / ave_time; - - float gb_per_sec = num_btype / 1.E6 / ave_time; - - std::cout << "Perf: " << ave_time << " ms, " << tflops << " TFlops, " << gb_per_sec - << " GB/s, " << conv_name << std::endl; - - if(tflops > best_tflops) - { - best_conv_name = conv_name; - best_tflops = tflops; - best_ave_time = ave_time; - best_gb_per_sec = gb_per_sec; - } - } - } - - std::cout << "Best Perf: " << best_ave_time << " ms, " << best_tflops << " TFlops, " - << best_gb_per_sec << " GB/s, " << best_conv_name << std::endl; -} - -} // namespace app -} // namespace ck diff --git a/test/conv2d_bwd_weight/CMakeLists.txt b/test/conv2d_bwd_weight/CMakeLists.txt index ecd5336c1f..e61c9299c8 100644 --- a/test/conv2d_bwd_weight/CMakeLists.txt +++ b/test/conv2d_bwd_weight/CMakeLists.txt @@ -1,7 +1,2 @@ -include_directories(BEFORE - ${PROJECT_SOURCE_DIR}/profiler/include - ${PROJECT_SOURCE_DIR}/external/include/half -) - add_test_executable(test_conv2d_bwd_weight conv2d_bwd_weight.cpp) target_link_libraries(test_conv2d_bwd_weight PRIVATE host_tensor device_conv2d_bwd_weight_instance conv_util) diff --git a/test/conv2d_bwd_weight/conv2d_bwd_weight.cpp b/test/conv2d_bwd_weight/conv2d_bwd_weight.cpp index 671980f49e..c268136d18 100644 --- a/test/conv2d_bwd_weight/conv2d_bwd_weight.cpp +++ b/test/conv2d_bwd_weight/conv2d_bwd_weight.cpp @@ -2,12 +2,10 @@ #include #include #include -#include -#include #include -#include "conv_util.hpp" -#include "profile_conv_bwd_weight_impl.hpp" +#include "test/convnd_fwd/conv_util.hpp" +#include "profiler/include/profile_conv_bwd_weight_impl.hpp" int test_self() { diff --git a/test/conv_util/conv_util.cpp b/test/conv_util/conv_util.cpp index 98f55b872e..eb6f0d6e53 100644 --- a/test/conv_util/conv_util.cpp +++ b/test/conv_util/conv_util.cpp @@ -3,10 +3,11 @@ #include #include -#include "config.hpp" -#include "conv_util.hpp" -#include "tensor_layout.hpp" -#include "check_err.hpp" +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" + +#include "ck/library/utility/check_err.hpp" +#include "ck/library/utility/conv_util.hpp" namespace { diff --git a/test/convnd_bwd_data/CMakeLists.txt b/test/convnd_bwd_data/CMakeLists.txt index 55d71a41d3..554bcd18fb 100644 --- a/test/convnd_bwd_data/CMakeLists.txt +++ b/test/convnd_bwd_data/CMakeLists.txt @@ -1,7 +1,2 @@ -include_directories(BEFORE - ${PROJECT_SOURCE_DIR}/profiler/include - ${PROJECT_SOURCE_DIR}/external/include/half -) - add_test_executable(test_convnd_bwd_data convnd_bwd_data.cpp) target_link_libraries(test_convnd_bwd_data PRIVATE host_tensor device_convnd_bwd_data_instance conv_util) diff --git a/test/convnd_bwd_data/convnd_bwd_data.cpp b/test/convnd_bwd_data/convnd_bwd_data.cpp index 7284680e0e..a8c780030b 100644 --- a/test/convnd_bwd_data/convnd_bwd_data.cpp +++ b/test/convnd_bwd_data/convnd_bwd_data.cpp @@ -2,11 +2,9 @@ #include #include #include -#include -#include #include -#include "profile_convnd_bwd_data_impl.hpp" +#include "profiler/include/profile_convnd_bwd_data_impl.hpp" int main() { diff --git a/test/convnd_fwd/conv1d_fwd.cpp b/test/convnd_fwd/conv1d_fwd.cpp index 9b4708e94b..69b43ce252 100644 --- a/test/convnd_fwd/conv1d_fwd.cpp +++ b/test/convnd_fwd/conv1d_fwd.cpp @@ -1,12 +1,12 @@ #include #include #include -#include "gtest/gtest.h" +#include -#include "data_type.hpp" -#include "element_wise_operation.hpp" -#include "library/include/ck/library/utility/conv_util.hpp" -#include "conv_util.hpp" +#include "ck/utility/data_type.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" +#include "ck/library/utility/conv_util.hpp" +#include "test/convnd_fwd/conv_util.hpp" namespace { diff --git a/test/convnd_fwd/conv2d_fwd.cpp b/test/convnd_fwd/conv2d_fwd.cpp index 4e0238cc4f..c08909167d 100644 --- a/test/convnd_fwd/conv2d_fwd.cpp +++ b/test/convnd_fwd/conv2d_fwd.cpp @@ -1,13 +1,11 @@ #include #include -#include "gtest/gtest.h" +#include +#include "ck/utility/data_type.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" #include "ck/library/utility/conv_util.hpp" -#include "config.hpp" -#include "conv_util.hpp" -#include "data_type.hpp" -#include "element_wise_operation.hpp" -#include "fill.hpp" +#include "test/convnd_fwd/conv_util.hpp" namespace { diff --git a/test/convnd_fwd/conv3d_fwd.cpp b/test/convnd_fwd/conv3d_fwd.cpp index 2470727fd7..8d09b49f9c 100644 --- a/test/convnd_fwd/conv3d_fwd.cpp +++ b/test/convnd_fwd/conv3d_fwd.cpp @@ -1,14 +1,15 @@ -#include #include #include #include #include -#include "gtest/gtest.h" +#include -#include "data_type.hpp" -#include "element_wise_operation.hpp" -#include "library/include/ck/library/utility/conv_util.hpp" -#include "conv_util.hpp" +#include "ck/utility/data_type.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" + +#include "ck/library/utility/conv_util.hpp" + +#include "test/convnd_fwd/conv_util.hpp" namespace { diff --git a/test/convnd_fwd/conv_util.hpp b/test/convnd_fwd/conv_util.hpp index 1ec83bd118..2d6a847056 100644 --- a/test/convnd_fwd/conv_util.hpp +++ b/test/convnd_fwd/conv_util.hpp @@ -2,12 +2,12 @@ #include -#include "config.hpp" -#include "data_type.hpp" -#include "device_convnd_fwd_xdl_nhwc_kyxc_nhwk.hpp" -#include "element_wise_operation.hpp" -#include "host_tensor.hpp" -#include "sequence.hpp" +#include "ck/ck.hpp" +#include "ck/utility/sequence.hpp" +#include "ck/utility/data_type.hpp" +#include "ck/tensor_operation/gpu/device/device_convnd_fwd_xdl_nhwc_kyxc_nhwk.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" namespace ck { namespace tensor_operation { diff --git a/test/gemm/gemm_dl_fp16.cpp b/test/gemm/gemm_dl_fp16.cpp index 8a539372ba..fa174a80f7 100644 --- a/test/gemm/gemm_dl_fp16.cpp +++ b/test/gemm/gemm_dl_fp16.cpp @@ -1,23 +1,22 @@ #include #include -#include #include #include #include #include -#include "../gemm/gemm_util.hpp" -#include "config.hpp" -#include "print.hpp" -#include "device.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" -#include "host_gemm.hpp" -#include "device_tensor.hpp" -#include "device_gemm_dl.hpp" -#include "element_wise_operation.hpp" -#include "reference_gemm.hpp" -#include "gemm_specialization.hpp" +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_dl.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" + +#include "ck/library/utility/check_err.hpp" +#include "ck/library/host_tensor/device_memory.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" +#include "ck/library/host_tensor/host_tensor_generator.hpp" +#include "ck/library/reference_tensor_operation/cpu/reference_gemm.hpp" + +#include "test/gemm/gemm_util.hpp" using PassThrough = ck::tensor_operation::element_wise::PassThrough; diff --git a/test/gemm/gemm_dl_fp32.cpp b/test/gemm/gemm_dl_fp32.cpp index 3484458042..f3aa9183e7 100644 --- a/test/gemm/gemm_dl_fp32.cpp +++ b/test/gemm/gemm_dl_fp32.cpp @@ -1,23 +1,22 @@ #include #include -#include #include #include #include #include -#include "../gemm/gemm_util.hpp" -#include "config.hpp" -#include "print.hpp" -#include "device.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" -#include "host_gemm.hpp" -#include "device_tensor.hpp" -#include "device_gemm_dl.hpp" -#include "element_wise_operation.hpp" -#include "reference_gemm.hpp" -#include "gemm_specialization.hpp" +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" + +#include "ck/library/utility/check_err.hpp" +#include "ck/library/host_tensor/device_memory.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" +#include "ck/library/host_tensor/host_tensor_generator.hpp" +#include "ck/library/reference_tensor_operation/cpu/reference_gemm.hpp" + +#include "test/gemm/gemm_util.hpp" using PassThrough = ck::tensor_operation::element_wise::PassThrough; diff --git a/test/gemm/gemm_dl_int8.cpp b/test/gemm/gemm_dl_int8.cpp index 5dfb7221cb..aaae865318 100644 --- a/test/gemm/gemm_dl_int8.cpp +++ b/test/gemm/gemm_dl_int8.cpp @@ -1,23 +1,22 @@ #include #include -#include #include #include #include #include -#include "../gemm/gemm_util.hpp" -#include "config.hpp" -#include "print.hpp" -#include "device.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" -#include "host_gemm.hpp" -#include "device_tensor.hpp" -#include "device_gemm_dl.hpp" -#include "element_wise_operation.hpp" -#include "reference_gemm.hpp" -#include "gemm_specialization.hpp" +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_dl.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" + +#include "ck/library/utility/check_err.hpp" +#include "ck/library/host_tensor/device_memory.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" +#include "ck/library/host_tensor/host_tensor_generator.hpp" +#include "ck/library/reference_tensor_operation/cpu/reference_gemm.hpp" + +#include "test/gemm/gemm_util.hpp" using PassThrough = ck::tensor_operation::element_wise::PassThrough; diff --git a/test/gemm/gemm_util.hpp b/test/gemm/gemm_util.hpp index a3cafa6df1..0e7046004f 100644 --- a/test/gemm/gemm_util.hpp +++ b/test/gemm/gemm_util.hpp @@ -1,13 +1,12 @@ -#ifndef GEMM_UTILS_HPP -#define GEMM_UTILS_HPP +#pragma once -#include "check_err.hpp" -#include "config.hpp" -#include "device.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" -#include "reference_gemm.hpp" -#include "tensor_layout.hpp" +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/library/utility/check_err.hpp" +#include "ck/library/host_tensor/device_memory.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" +#include "ck/library/host_tensor/host_tensor_generator.hpp" +#include "ck/library/reference_tensor_operation/cpu/reference_gemm.hpp" namespace ck { namespace gemm_util { @@ -350,4 +349,3 @@ struct TestGemmBF16 } // namespace gemm_util } // namespace ck -#endif diff --git a/test/gemm/gemm_xdl_bf16.cpp b/test/gemm/gemm_xdl_bf16.cpp index 5461088b02..38378fbda8 100644 --- a/test/gemm/gemm_xdl_bf16.cpp +++ b/test/gemm/gemm_xdl_bf16.cpp @@ -1,24 +1,22 @@ #include #include -#include #include #include #include #include -#include "gemm_util.hpp" -#include "config.hpp" -#include "print.hpp" -#include "device.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" -#include "host_gemm.hpp" -#include "device_tensor.hpp" -#include "device_gemm_xdl.hpp" -#include "device_gemm_xdl_cshuffle.hpp" -#include "element_wise_operation.hpp" -#include "reference_gemm.hpp" -#include "gemm_specialization.hpp" +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" + +#include "ck/library/utility/check_err.hpp" +#include "ck/library/host_tensor/device_memory.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" +#include "ck/library/host_tensor/host_tensor_generator.hpp" +#include "ck/library/reference_tensor_operation/cpu/reference_gemm.hpp" + +#include "test/gemm/gemm_util.hpp" using PassThrough = ck::tensor_operation::element_wise::PassThrough; diff --git a/test/gemm/gemm_xdl_fp16.cpp b/test/gemm/gemm_xdl_fp16.cpp index 6fe3f83d1c..5e4ef2f6a1 100644 --- a/test/gemm/gemm_xdl_fp16.cpp +++ b/test/gemm/gemm_xdl_fp16.cpp @@ -1,21 +1,23 @@ #include #include -#include #include #include #include #include -#include "gemm_util.hpp" -#include "config.hpp" -#include "print.hpp" -#include "device.hpp" -#include "host_gemm.hpp" -#include "device_tensor.hpp" -#include "device_gemm_xdl.hpp" -#include "device_gemm_xdl_cshuffle.hpp" -#include "element_wise_operation.hpp" -#include "gemm_specialization.hpp" +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" + +#include "ck/library/utility/check_err.hpp" +#include "ck/library/host_tensor/device_memory.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" +#include "ck/library/host_tensor/host_tensor_generator.hpp" +#include "ck/library/reference_tensor_operation/cpu/reference_gemm.hpp" + +#include "test/gemm/gemm_util.hpp" using PassThrough = ck::tensor_operation::element_wise::PassThrough; diff --git a/test/gemm/gemm_xdl_fp32.cpp b/test/gemm/gemm_xdl_fp32.cpp index 4756d1b4d6..dc8d22876d 100644 --- a/test/gemm/gemm_xdl_fp32.cpp +++ b/test/gemm/gemm_xdl_fp32.cpp @@ -1,24 +1,23 @@ #include #include -#include #include #include #include #include -#include "gemm_util.hpp" -#include "config.hpp" -#include "print.hpp" -#include "device.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" -#include "host_gemm.hpp" -#include "device_tensor.hpp" -#include "device_gemm_xdl.hpp" -#include "device_gemm_xdl_cshuffle.hpp" -#include "element_wise_operation.hpp" -#include "reference_gemm.hpp" -#include "gemm_specialization.hpp" +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" + +#include "ck/library/utility/check_err.hpp" +#include "ck/library/host_tensor/device_memory.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" +#include "ck/library/host_tensor/host_tensor_generator.hpp" +#include "ck/library/reference_tensor_operation/cpu/reference_gemm.hpp" + +#include "test/gemm/gemm_util.hpp" using PassThrough = ck::tensor_operation::element_wise::PassThrough; diff --git a/test/gemm/gemm_xdl_fp64.cpp b/test/gemm/gemm_xdl_fp64.cpp index db37211505..4918db2984 100644 --- a/test/gemm/gemm_xdl_fp64.cpp +++ b/test/gemm/gemm_xdl_fp64.cpp @@ -1,23 +1,23 @@ #include #include -#include #include #include #include #include -#include "gemm_util.hpp" -#include "config.hpp" -#include "print.hpp" -#include "device.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" -#include "host_gemm.hpp" -#include "device_tensor.hpp" -#include "device_gemm_xdl.hpp" -#include "element_wise_operation.hpp" -#include "reference_gemm.hpp" -#include "gemm_specialization.hpp" +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" + +#include "ck/library/utility/check_err.hpp" +#include "ck/library/host_tensor/device_memory.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" +#include "ck/library/host_tensor/host_tensor_generator.hpp" +#include "ck/library/reference_tensor_operation/cpu/reference_gemm.hpp" + +#include "test/gemm/gemm_util.hpp" using PassThrough = ck::tensor_operation::element_wise::PassThrough; diff --git a/test/gemm/gemm_xdl_int8.cpp b/test/gemm/gemm_xdl_int8.cpp index 0075b79cf7..06364ddd92 100644 --- a/test/gemm/gemm_xdl_int8.cpp +++ b/test/gemm/gemm_xdl_int8.cpp @@ -1,24 +1,23 @@ #include #include -#include #include #include #include #include -#include "gemm_util.hpp" -#include "config.hpp" -#include "print.hpp" -#include "device.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" -#include "host_gemm.hpp" -#include "device_tensor.hpp" -#include "device_gemm_xdl.hpp" -#include "device_gemm_xdl_cshuffle.hpp" -#include "element_wise_operation.hpp" -#include "reference_gemm.hpp" -#include "gemm_specialization.hpp" +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" + +#include "ck/library/utility/check_err.hpp" +#include "ck/library/host_tensor/device_memory.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" +#include "ck/library/host_tensor/host_tensor_generator.hpp" +#include "ck/library/reference_tensor_operation/cpu/reference_gemm.hpp" + +#include "test/gemm/gemm_util.hpp" using PassThrough = ck::tensor_operation::element_wise::PassThrough; diff --git a/test/gemm_reduce/CMakeLists.txt b/test/gemm_reduce/CMakeLists.txt index e474af3230..74b787ac27 100644 --- a/test/gemm_reduce/CMakeLists.txt +++ b/test/gemm_reduce/CMakeLists.txt @@ -1,9 +1,3 @@ -include_directories(BEFORE - ${PROJECT_SOURCE_DIR}/profiler/include - ${PROJECT_SOURCE_DIR}/test/include - ${PROJECT_SOURCE_DIR}/external/include/half -) - add_test_executable(test_gemm_reduce_fp16 gemm_reduce_fp16.cpp) target_link_libraries(test_gemm_reduce_fp16 PRIVATE host_tensor) target_link_libraries(test_gemm_reduce_fp16 PRIVATE device_gemm_reduce_instance) diff --git a/test/gemm_reduce/gemm_reduce_fp16.cpp b/test/gemm_reduce/gemm_reduce_fp16.cpp index 6c7bb9658f..42fd6c2d16 100644 --- a/test/gemm_reduce/gemm_reduce_fp16.cpp +++ b/test/gemm_reduce/gemm_reduce_fp16.cpp @@ -1,6 +1,6 @@ #include -#include "profile_gemm_reduce_impl.hpp" +#include "profiler/include/profile_gemm_reduce_impl.hpp" int main() { diff --git a/test/gemm_split_k/gemm_split_k.cpp b/test/gemm_split_k/gemm_split_k.cpp index b63361aa1b..ac0f8796b0 100644 --- a/test/gemm_split_k/gemm_split_k.cpp +++ b/test/gemm_split_k/gemm_split_k.cpp @@ -1,16 +1,21 @@ #include #include #include -#include -#include "config.hpp" -#include "print.hpp" -#include "device.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" -#include "device_tensor.hpp" -#include "host_gemm.hpp" -#include "tensor_layout.hpp" -#include "device_gemm_xdl_splitk.hpp" + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_gemm_xdl_splitk.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" + +#include "ck/library/utility/check_err.hpp" +#include "ck/library/host_tensor/device_memory.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" +#include "ck/library/host_tensor/host_tensor_generator.hpp" +#include "ck/library/host_tensor/device_memory.hpp" +#include "ck/library/reference_tensor_operation/cpu/reference_gemm.hpp" + +#include "ck/library/host_tensor/host_gemm.hpp" enum struct GemmMatrixLayout { diff --git a/test/grouped_gemm/grouped_gemm_fp16.cpp b/test/grouped_gemm/grouped_gemm_fp16.cpp index fc8ec66b51..a38c9629f5 100644 --- a/test/grouped_gemm/grouped_gemm_fp16.cpp +++ b/test/grouped_gemm/grouped_gemm_fp16.cpp @@ -2,21 +2,18 @@ #include #include #include -#include -#include -#include "check_err.hpp" -#include "config.hpp" -#include "print.hpp" -#include "device.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" -#include "host_gemm.hpp" -#include "device_tensor.hpp" -#include "device_grouped_gemm_xdl.hpp" -#include "element_wise_operation.hpp" -#include "reference_gemm.hpp" -#include "gemm_specialization.hpp" +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" +#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp" +#include "ck/tensor_operation/gpu/device/device_grouped_gemm_xdl.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" + +#include "ck/library/utility/check_err.hpp" +#include "ck/library/host_tensor/device_memory.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" +#include "ck/library/host_tensor/host_tensor_generator.hpp" +#include "ck/library/reference_tensor_operation/cpu/reference_gemm.hpp" using PassThrough = ck::tensor_operation::element_wise::PassThrough; diff --git a/test/magic_number_division/magic_number_division.cpp b/test/magic_number_division/magic_number_division.cpp index 751a62be19..3aa6b7e94a 100644 --- a/test/magic_number_division/magic_number_division.cpp +++ b/test/magic_number_division/magic_number_division.cpp @@ -2,16 +2,13 @@ #include #include #include -#include -#include -#include "check_err.hpp" -#include "config.hpp" -#include "magic_division.hpp" -#include "device.hpp" -#include "host_tensor.hpp" -#include "host_tensor_generator.hpp" -#include "device_tensor.hpp" +#include "ck/ck.hpp" +#include "ck/utility/magic_division.hpp" +#include "ck/library/utility/check_err.hpp" +#include "ck/library/host_tensor/device_memory.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" +#include "ck/library/host_tensor/host_tensor_generator.hpp" __global__ void gpu_magic_number_division(uint32_t magic_multiplier, uint32_t magic_shift, diff --git a/test/reduce/reduce_no_index.cpp b/test/reduce/reduce_no_index.cpp index 20030392b5..58ac5aa86d 100644 --- a/test/reduce/reduce_no_index.cpp +++ b/test/reduce/reduce_no_index.cpp @@ -1,7 +1,7 @@ -#include "getopt.h" +#include -#include "host_common_util.hpp" -#include "profile_reduce_impl.hpp" +#include "ck/library/host_tensor/host_common_util.hpp" +#include "profiler/include/profile_reduce_impl.hpp" using namespace ck; diff --git a/test/reduce/reduce_with_index.cpp b/test/reduce/reduce_with_index.cpp index c1918bf388..1851cfc4c8 100644 --- a/test/reduce/reduce_with_index.cpp +++ b/test/reduce/reduce_with_index.cpp @@ -1,7 +1,7 @@ -#include "getopt.h" +#include -#include "host_common_util.hpp" -#include "profile_reduce_impl.hpp" +#include "ck/library/host_tensor/host_common_util.hpp" +#include "profiler/include/profile_reduce_impl.hpp" using namespace ck; diff --git a/test/reference_conv_fwd/reference_conv_fwd.cpp b/test/reference_conv_fwd/reference_conv_fwd.cpp index 69b223989f..f6f31974d4 100644 --- a/test/reference_conv_fwd/reference_conv_fwd.cpp +++ b/test/reference_conv_fwd/reference_conv_fwd.cpp @@ -1,19 +1,19 @@ #include #include -#include #include #include #include -#include "gtest/gtest.h" +#include -#include "check_err.hpp" -#include "config.hpp" -#include "conv_util.hpp" -#include "element_wise_operation.hpp" -#include "fill.hpp" -#include "host_tensor.hpp" -#include "reference_conv_fwd.hpp" -#include "tensor_layout.hpp" +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" +#include "ck/tensor_operation/gpu/device/tensor_layout.hpp" + +#include "ck/library/utility/check_err.hpp" +#include "ck/library/utility/conv_util.hpp" +#include "ck/library/utility/fill.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" +#include "ck/library/reference_tensor_operation/cpu/reference_conv_fwd.hpp" namespace { using InElementOp = ck::tensor_operation::element_wise::PassThrough; diff --git a/test/softmax/test_softmax_util.hpp b/test/softmax/test_softmax_util.hpp index 39182c3c11..feb008774b 100644 --- a/test/softmax/test_softmax_util.hpp +++ b/test/softmax/test_softmax_util.hpp @@ -1,13 +1,15 @@ #include #include -#include "gtest/gtest.h" +#include -#include "config.hpp" -#include "host_tensor.hpp" -#include "check_err.hpp" -#include "number.hpp" -#include "reference_softmax.hpp" -#include "device_softmax.hpp" +#include "ck/ck.hpp" +#include "ck/utility/number.hpp" +#include "ck/tensor_operation/gpu/device/device_softmax.hpp" + +#include "ck/library/utility/check_err.hpp" +#include "ck/library/host_tensor/host_tensor.hpp" +#include "ck/library/host_tensor/device_memory.hpp" +#include "ck/library/reference_tensor_operation/cpu/reference_softmax.hpp" namespace ck { diff --git a/test/space_filling_curve/space_filling_curve.cpp b/test/space_filling_curve/space_filling_curve.cpp index 635d31d683..843ac358f1 100644 --- a/test/space_filling_curve/space_filling_curve.cpp +++ b/test/space_filling_curve/space_filling_curve.cpp @@ -3,7 +3,9 @@ #include #include -#include "tensor_space_filling_curve.hpp" +#include "ck/ck.hpp" +#include "ck/utility/common_header.hpp" +#include "ck/tensor_description/tensor_space_filling_curve.hpp" using namespace ck;