mirror of
https://github.com/ROCm/composable_kernel.git
synced 2026-05-13 09:45:56 +00:00
8daf6ea302
* Added bias_bnorm_clamp instances. * fwd_bias_bnorm_clamp comp instances * fwd_bias_bnorm_mem_inter and mem_intra instances * fwd_bias_bnorm_merged_group_instances * fwd_bias_bnorm_clamp_conv3d_bf16 and f16 instances * Device level changes for fwd_bias_bnorm_clamp * Added the test to the regression test list. * Removed the part 2 and 2x instances * Removed the irrelevant checks in wmma * Refactored the instances to adapt to new device implementation * Updated the reference and include files * enabling tests * Added missing profiler * Added missing instance entry , deleted by mistake * Reduce bias bnorm clamp instances to only a single generic one. * Clean up cmakelists file * clang-format * Change bias bnorm clamp tests to use monotone initialization values to avoid tiny off-integer gemm results on RDNA3 from blowing up. * Renaming some instance lists and add functions to be more standardized. * Commented out non default instances. --------- Co-authored-by: kiefer <kiefer.van.teutem@streamhpc.com>
301 lines
14 KiB
CMake
301 lines
14 KiB
CMake
# Copyright (c) Advanced Micro Devices, Inc., or its affiliates.
|
|
# SPDX-License-Identifier: MIT
|
|
|
|
# ckProfiler
|
|
set(CK_PROFILER_OP_FILTER "" CACHE STRING "Filter for the operators to be profiled. Default is to include all")
|
|
set(CK_PROFILER_INSTANCE_FILTER "" CACHE STRING "Filter for the kernels instances to be profiled. Default is to be the same as the operator filter")
|
|
if (CK_PROFILER_OP_FILTER STREQUAL "")
|
|
set(CK_PROFILER_OP_FILTER ".+")
|
|
endif()
|
|
if (CK_PROFILER_INSTANCE_FILTER STREQUAL "")
|
|
set(CK_PROFILER_INSTANCE_FILTER ${CK_PROFILER_OP_FILTER})
|
|
endif()
|
|
message(STATUS "CK_PROFILER_OP_FILTER: ${CK_PROFILER_OP_FILTER}")
|
|
message(STATUS "CK_PROFILER_INSTANCE_FILTER: ${CK_PROFILER_INSTANCE_FILTER}")
|
|
|
|
set(PROFILER_OPS
|
|
profile_gemm.cpp
|
|
profile_reduce.cpp
|
|
profile_groupnorm_bwd_data.cpp
|
|
profile_groupnorm_fwd.cpp
|
|
profile_layernorm_bwd_data.cpp
|
|
profile_layernorm_bwd_gamma_beta.cpp
|
|
profile_groupnorm_bwd_gamma_beta.cpp
|
|
profile_layernorm_fwd.cpp
|
|
profile_max_pool2d_fwd.cpp
|
|
profile_pool3d_fwd.cpp
|
|
profile_avg_pool3d_bwd.cpp
|
|
profile_max_pool3d_bwd.cpp
|
|
profile_avg_pool2d_bwd.cpp
|
|
profile_max_pool2d_bwd.cpp
|
|
profile_softmax.cpp
|
|
profile_batchnorm_fwd.cpp
|
|
profile_batchnorm_bwd.cpp
|
|
profile_batchnorm_infer.cpp
|
|
profile_conv_tensor_rearrange.cpp
|
|
profile_transpose.cpp
|
|
profile_permute_scale.cpp
|
|
profile_gemm_quantization.cpp
|
|
)
|
|
|
|
if(SUPPORTED_GPU_TARGETS MATCHES "gfx9")
|
|
if(DTYPES MATCHES "fp32" OR DTYPES MATCHES "fp64" OR NOT DEFINED DTYPES)
|
|
list(APPEND PROFILER_OPS profile_contraction_bilinear.cpp)
|
|
list(APPEND PROFILER_OPS profile_contraction_scale.cpp)
|
|
endif()
|
|
if(CK_EXPERIMENTAL_BUILDER)
|
|
list(APPEND PROFILER_OPS profile_grouped_conv_fwd_tile.cpp)
|
|
endif()
|
|
endif()
|
|
|
|
if(SUPPORTED_GPU_TARGETS MATCHES "gfx9|gfx1[12]")
|
|
if(DTYPES MATCHES "fp16" OR NOT DEFINED DTYPES)
|
|
list(APPEND PROFILER_OPS profile_gemm_reduce.cpp)
|
|
list(APPEND PROFILER_OPS profile_batched_gemm_add_relu_gemm_add.cpp)
|
|
list(APPEND PROFILER_OPS profile_gemm_add.cpp)
|
|
list(APPEND PROFILER_OPS profile_grouped_gemm.cpp)
|
|
list(APPEND PROFILER_OPS profile_gemm_streamk.cpp)
|
|
list(APPEND PROFILER_OPS profile_gemm_add_relu.cpp)
|
|
list(APPEND PROFILER_OPS profile_gemm_add_relu_add_layernorm.cpp)
|
|
list(APPEND PROFILER_OPS profile_grouped_gemm_fixed_nk.cpp)
|
|
list(APPEND PROFILER_OPS profile_grouped_gemm_fastgelu.cpp)
|
|
list(APPEND PROFILER_OPS profile_grouped_gemm_tile_loop.cpp)
|
|
list(APPEND PROFILER_OPS profile_grouped_gemm_multiply_tile_loop.cpp)
|
|
endif()
|
|
if(SUPPORTED_GPU_TARGETS MATCHES "gfx9[45]|gfx12")
|
|
list(APPEND PROFILER_OPS profile_gemm_multiply_multiply_wp.cpp)
|
|
list(APPEND PROFILER_OPS profile_gemm_ab_scale.cpp)
|
|
list(APPEND PROFILER_OPS profile_gemm_blockscale_wp.cpp)
|
|
list(APPEND PROFILER_OPS profile_gemm_universal_preshuffle.cpp)
|
|
endif()
|
|
if(SUPPORTED_GPU_TARGETS MATCHES "gfx95")
|
|
list(APPEND PROFILER_OPS profile_gemm_mx.cpp)
|
|
endif()
|
|
list(APPEND PROFILER_OPS profile_batched_gemm_reduce.cpp)
|
|
list(APPEND PROFILER_OPS profile_gemm_add_multiply.cpp)
|
|
list(APPEND PROFILER_OPS profile_gemm_add.cpp)
|
|
list(APPEND PROFILER_OPS profile_gemm_bias_add_reduce.cpp)
|
|
list(APPEND PROFILER_OPS profile_gemm_splitk.cpp)
|
|
list(APPEND PROFILER_OPS profile_gemm_universal_batched.cpp)
|
|
list(APPEND PROFILER_OPS profile_gemm_universal_streamk.cpp)
|
|
list(APPEND PROFILER_OPS profile_conv_fwd_bias_relu.cpp)
|
|
list(APPEND PROFILER_OPS profile_conv_fwd_bias_relu_add.cpp)
|
|
list(APPEND PROFILER_OPS profile_conv_bwd_data.cpp)
|
|
list(APPEND PROFILER_OPS profile_conv_fwd.cpp)
|
|
endif()
|
|
|
|
if((SUPPORTED_GPU_TARGETS MATCHES "gfx9" AND (DTYPES MATCHES "fp16" OR NOT DEFINED DTYPES)) OR
|
|
(SUPPORTED_GPU_TARGETS MATCHES "gfx1[12]"))
|
|
list(APPEND PROFILER_OPS profile_gemm_bilinear.cpp)
|
|
endif()
|
|
if(SUPPORTED_GPU_TARGETS MATCHES "gfx(9[45]|1[12])")
|
|
list(APPEND PROFILER_OPS profile_gemm_multiply_multiply.cpp)
|
|
endif()
|
|
|
|
if(SUPPORTED_GPU_TARGETS MATCHES "gfx9|gfx1[12]")
|
|
list(APPEND PROFILER_OPS profile_gemm_universal.cpp)
|
|
list(APPEND PROFILER_OPS profile_batched_gemm.cpp)
|
|
list(APPEND PROFILER_OPS profile_batched_gemm_b_scale.cpp)
|
|
list(APPEND PROFILER_OPS profile_gemm_b_scale.cpp)
|
|
list(APPEND PROFILER_OPS profile_gemm_universal_reduce.cpp)
|
|
list(APPEND PROFILER_OPS profile_grouped_conv_fwd.cpp)
|
|
list(APPEND PROFILER_OPS profile_grouped_conv_fwd_bias_clamp.cpp)
|
|
list(APPEND PROFILER_OPS profile_grouped_conv_fwd_bias_bnorm_clamp.cpp)
|
|
list(APPEND PROFILER_OPS profile_grouped_conv_fwd_clamp.cpp)
|
|
list(APPEND PROFILER_OPS profile_grouped_conv_bwd_data.cpp)
|
|
list(APPEND PROFILER_OPS profile_grouped_conv_fwd_bilinear.cpp)
|
|
list(APPEND PROFILER_OPS profile_grouped_conv_bwd_weight.cpp)
|
|
list(APPEND PROFILER_OPS profile_grouped_conv_fwd_outelementop.cpp)
|
|
list(APPEND PROFILER_OPS profile_gemm_multi_abd.cpp)
|
|
if(DTYPES MATCHES "fp16" OR NOT DEFINED DTYPES)
|
|
list(APPEND PROFILER_OPS profile_gemm_add_multiply.cpp)
|
|
list(APPEND PROFILER_OPS profile_gemm_multiply_add.cpp)
|
|
list(APPEND PROFILER_OPS profile_gemm_add_silu.cpp)
|
|
list(APPEND PROFILER_OPS profile_gemm_fastgelu.cpp)
|
|
list(APPEND PROFILER_OPS profile_gemm_add_fastgelu.cpp)
|
|
list(APPEND PROFILER_OPS profile_gemm_add_add_fastgelu.cpp)
|
|
list(APPEND PROFILER_SOURCES profile_gemm_add.cpp)
|
|
endif()
|
|
list(APPEND PROFILER_OPS profile_batched_gemm_gemm.cpp)
|
|
endif()
|
|
|
|
if(DL_KERNELS)
|
|
list(APPEND PROFILER_OPS profile_batched_gemm_multi_d.cpp)
|
|
list(APPEND PROFILER_OPS profile_grouped_conv_bwd_weight.cpp)
|
|
endif()
|
|
|
|
if(CK_ENABLE_INT8)
|
|
list(APPEND PROFILER_OPS profile_gemm_quantization.cpp)
|
|
endif()
|
|
|
|
set(PROFILER_SOURCES profiler.cpp)
|
|
foreach(SOURCE ${PROFILER_OPS})
|
|
string(REGEX REPLACE "profile_(.+)\.cpp" "\\1" OP_NAME ${SOURCE})
|
|
if (OP_NAME STREQUAL "")
|
|
message(FATAL_ERROR "Unexpected source file name: ${SOURCE}")
|
|
endif()
|
|
if("${OP_NAME}" MATCHES "${CK_PROFILER_OP_FILTER}")
|
|
list(APPEND PROFILER_SOURCES ${SOURCE})
|
|
endif()
|
|
endforeach()
|
|
message(VERBOSE "ckProfiler sources: ${PROFILER_SOURCES}")
|
|
|
|
set(PROFILER_EXECUTABLE ckProfiler)
|
|
|
|
add_executable(${PROFILER_EXECUTABLE} ${PROFILER_SOURCES})
|
|
target_compile_options(${PROFILER_EXECUTABLE} PRIVATE -Wno-global-constructors)
|
|
# flags to compress the library
|
|
if(NOT WIN32 AND ${hip_VERSION_FLAT} GREATER 600241132)
|
|
message(DEBUG "Adding --offload-compress flag for ${PROFILER_EXECUTABLE}")
|
|
target_compile_options(${PROFILER_EXECUTABLE} PRIVATE --offload-compress)
|
|
endif()
|
|
|
|
|
|
set(DEVICE_INSTANCES "")
|
|
list(APPEND DEVICE_INSTANCES device_gemm_instance)
|
|
list(APPEND DEVICE_INSTANCES device_normalization_fwd_instance)
|
|
list(APPEND DEVICE_INSTANCES device_normalization_bwd_data_instance)
|
|
list(APPEND DEVICE_INSTANCES device_normalization_bwd_gamma_beta_instance)
|
|
list(APPEND DEVICE_INSTANCES device_softmax_instance)
|
|
list(APPEND DEVICE_INSTANCES device_reduce_instance)
|
|
list(APPEND DEVICE_INSTANCES device_batchnorm_instance)
|
|
list(APPEND DEVICE_INSTANCES device_pool2d_fwd_instance)
|
|
list(APPEND DEVICE_INSTANCES device_pool3d_fwd_instance)
|
|
list(APPEND DEVICE_INSTANCES device_avg_pool2d_bwd_instance)
|
|
list(APPEND DEVICE_INSTANCES device_avg_pool3d_bwd_instance)
|
|
list(APPEND DEVICE_INSTANCES device_max_pool_bwd_instance)
|
|
list(APPEND DEVICE_INSTANCES device_image_to_column_instance)
|
|
list(APPEND DEVICE_INSTANCES device_column_to_image_instance)
|
|
list(APPEND DEVICE_INSTANCES device_transpose_instance)
|
|
list(APPEND DEVICE_INSTANCES device_permute_scale_instance)
|
|
|
|
if(SUPPORTED_GPU_TARGETS MATCHES "gfx9|gfx1[12]")
|
|
if(DTYPES MATCHES "fp32" OR DTYPES MATCHES "fp64" OR NOT DEFINED DTYPES)
|
|
list(APPEND DEVICE_INSTANCES device_contraction_bilinear_instance)
|
|
list(APPEND DEVICE_INSTANCES device_contraction_scale_instance)
|
|
endif()
|
|
if(DTYPES MATCHES "fp16" OR NOT DEFINED DTYPES)
|
|
list(APPEND DEVICE_INSTANCES device_gemm_add_instance)
|
|
list(APPEND DEVICE_INSTANCES device_batched_gemm_gemm_instance)
|
|
list(APPEND DEVICE_INSTANCES device_gemm_add_add_fastgelu_instance)
|
|
list(APPEND DEVICE_INSTANCES device_gemm_fastgelu_instance)
|
|
list(APPEND DEVICE_INSTANCES device_batched_gemm_add_relu_gemm_add_instance)
|
|
list(APPEND DEVICE_INSTANCES device_grouped_gemm_instance)
|
|
list(APPEND DEVICE_INSTANCES device_gemm_streamk_instance)
|
|
list(APPEND DEVICE_INSTANCES device_gemm_add_relu_instance)
|
|
list(APPEND DEVICE_INSTANCES device_gemm_add_relu_add_layernorm_instance)
|
|
list(APPEND DEVICE_INSTANCES device_grouped_gemm_fixed_nk_instance)
|
|
list(APPEND DEVICE_INSTANCES device_grouped_gemm_fastgelu_instance)
|
|
list(APPEND DEVICE_INSTANCES device_grouped_gemm_tile_loop_instance)
|
|
endif()
|
|
list(APPEND DEVICE_INSTANCES device_batched_gemm_reduce_instance)
|
|
if(SUPPORTED_GPU_TARGETS MATCHES "gfx9[45]|gfx12")
|
|
list(APPEND DEVICE_INSTANCES device_gemm_multiply_multiply_wp_instance)
|
|
list(APPEND DEVICE_INSTANCES device_gemm_universal_preshuffle_instance)
|
|
endif()
|
|
if(SUPPORTED_GPU_TARGETS MATCHES "gfx9[45]|gfx1[12]")
|
|
list(APPEND DEVICE_INSTANCES device_gemm_ab_scale_instance)
|
|
list(APPEND DEVICE_INSTANCES device_gemm_blockscale_wp_instance)
|
|
endif()
|
|
if(SUPPORTED_GPU_TARGETS MATCHES "gfx95")
|
|
list(APPEND DEVICE_INSTANCES device_gemm_mx_instance)
|
|
endif()
|
|
list(APPEND DEVICE_INSTANCES device_gemm_splitk_instance)
|
|
list(APPEND DEVICE_INSTANCES device_gemm_universal_batched_instance)
|
|
list(APPEND DEVICE_INSTANCES device_gemm_universal_streamk_instance)
|
|
list(APPEND DEVICE_INSTANCES device_gemm_add_multiply_instance)
|
|
list(APPEND DEVICE_INSTANCES device_gemm_add_instance)
|
|
list(APPEND DEVICE_INSTANCES device_gemm_reduce_instance)
|
|
list(APPEND DEVICE_INSTANCES device_gemm_bias_add_reduce_instance)
|
|
list(APPEND DEVICE_INSTANCES device_conv2d_fwd_instance)
|
|
list(APPEND DEVICE_INSTANCES device_conv2d_fwd_bias_relu_instance)
|
|
list(APPEND DEVICE_INSTANCES device_conv2d_fwd_bias_relu_add_instance)
|
|
list(APPEND DEVICE_INSTANCES device_conv1d_bwd_data_instance)
|
|
list(APPEND DEVICE_INSTANCES device_conv3d_bwd_data_instance)
|
|
list(APPEND DEVICE_INSTANCES device_conv2d_bwd_data_instance)
|
|
list(APPEND DEVICE_INSTANCES device_grouped_conv3d_fwd_convscale_instance)
|
|
list(APPEND DEVICE_INSTANCES device_grouped_conv3d_fwd_convinvscale_instance)
|
|
endif()
|
|
|
|
if((SUPPORTED_GPU_TARGETS MATCHES "gfx9" AND (DTYPES MATCHES "fp16" OR NOT DEFINED DTYPES)) OR
|
|
(SUPPORTED_GPU_TARGETS MATCHES "gfx1[12]" ))
|
|
list(APPEND DEVICE_INSTANCES device_gemm_bilinear_instance)
|
|
endif()
|
|
if(SUPPORTED_GPU_TARGETS MATCHES "gfx(9[45]|1[12])")
|
|
list(APPEND DEVICE_INSTANCES device_gemm_multiply_multiply_instance)
|
|
endif()
|
|
|
|
if(SUPPORTED_GPU_TARGETS MATCHES "gfx9|gfx1[12]")
|
|
list(APPEND DEVICE_INSTANCES device_gemm_universal_instance)
|
|
list(APPEND DEVICE_INSTANCES device_batched_gemm_instance)
|
|
list(APPEND DEVICE_INSTANCES device_gemm_b_scale_instance)
|
|
list(APPEND DEVICE_INSTANCES device_gemm_universal_reduce_instance)
|
|
list(APPEND DEVICE_INSTANCES device_batched_gemm_b_scale_instance)
|
|
list(APPEND DEVICE_INSTANCES device_grouped_conv2d_bwd_data_instance)
|
|
list(APPEND DEVICE_INSTANCES device_grouped_conv3d_bwd_data_instance)
|
|
list(APPEND DEVICE_INSTANCES device_grouped_conv1d_fwd_instance)
|
|
list(APPEND DEVICE_INSTANCES device_grouped_conv2d_fwd_instance)
|
|
list(APPEND DEVICE_INSTANCES device_grouped_conv3d_fwd_instance)
|
|
list(APPEND DEVICE_INSTANCES device_grouped_conv2d_fwd_clamp_instance)
|
|
list(APPEND DEVICE_INSTANCES device_grouped_conv3d_fwd_clamp_instance)
|
|
list(APPEND DEVICE_INSTANCES device_grouped_conv3d_fwd_scale_instance)
|
|
list(APPEND DEVICE_INSTANCES device_grouped_conv2d_fwd_bias_clamp_instance)
|
|
list(APPEND DEVICE_INSTANCES device_grouped_conv3d_fwd_bias_clamp_instance)
|
|
list(APPEND DEVICE_INSTANCES device_grouped_conv2d_fwd_bias_bnorm_clamp_instance)
|
|
list(APPEND DEVICE_INSTANCES device_grouped_conv3d_fwd_bias_bnorm_clamp_instance)
|
|
list(APPEND DEVICE_INSTANCES device_grouped_conv3d_fwd_bilinear_instance)
|
|
list(APPEND DEVICE_INSTANCES device_gemm_add_relu_instance)
|
|
list(APPEND DEVICE_INSTANCES device_gemm_multi_abd_instance)
|
|
if(DTYPES MATCHES "fp16" OR NOT DEFINED DTYPES)
|
|
list(APPEND DEVICE_INSTANCES device_gemm_add_multiply_instance)
|
|
list(APPEND DEVICE_INSTANCES device_gemm_multiply_add_instance)
|
|
list(APPEND DEVICE_INSTANCES device_gemm_add_instance)
|
|
list(APPEND DEVICE_INSTANCES device_gemm_add_silu_instance)
|
|
list(APPEND DEVICE_INSTANCES device_gemm_fastgelu_instance)
|
|
list(APPEND DEVICE_INSTANCES device_gemm_add_fastgelu_instance)
|
|
list(APPEND DEVICE_INSTANCES device_gemm_add_add_fastgelu_instance)
|
|
endif()
|
|
list(APPEND DEVICE_INSTANCES device_batched_gemm_gemm_instance)
|
|
list(APPEND DEVICE_INSTANCES device_grouped_conv1d_bwd_weight_instance)
|
|
list(APPEND DEVICE_INSTANCES device_grouped_conv2d_bwd_weight_instance)
|
|
list(APPEND DEVICE_INSTANCES device_grouped_convnd_bwd_weight_instance)
|
|
list(APPEND DEVICE_INSTANCES device_grouped_conv3d_bwd_weight_instance)
|
|
endif()
|
|
|
|
if(SUPPORTED_GPU_TARGETS MATCHES "gfx9")
|
|
if(CK_EXPERIMENTAL_BUILDER)
|
|
list(APPEND DEVICE_INSTANCES device_grouped_conv_fwd_tile_instances)
|
|
endif()
|
|
endif()
|
|
|
|
if(DL_KERNELS)
|
|
list(APPEND DEVICE_INSTANCES device_batched_gemm_multi_d_instance)
|
|
list(APPEND DEVICE_INSTANCES device_grouped_conv1d_bwd_weight_instance)
|
|
list(APPEND DEVICE_INSTANCES device_grouped_conv2d_bwd_weight_instance)
|
|
list(APPEND DEVICE_INSTANCES device_grouped_conv3d_bwd_weight_instance)
|
|
endif()
|
|
|
|
if(CK_ENABLE_INT8)
|
|
list(APPEND DEVICE_INSTANCES device_quantization_instance)
|
|
endif()
|
|
|
|
set(PROFILER_LIBS utility getopt::getopt)
|
|
foreach(LIB ${DEVICE_INSTANCES})
|
|
string(REGEX REPLACE "device_(.+)_instance" "\\1" INSTANCE_NAME ${LIB})
|
|
if (INSTANCE_NAME STREQUAL "")
|
|
message(FATAL_ERROR "Unexpected kernel instance name: ${LIB}")
|
|
endif()
|
|
if("${INSTANCE_NAME}" MATCHES "${CK_PROFILER_INSTANCE_FILTER}")
|
|
# Only link if the target was actually created
|
|
if(TARGET ${LIB})
|
|
list(APPEND PROFILER_LIBS ${LIB})
|
|
else()
|
|
message(VERBOSE "Skipping ${LIB} - no instances built for current GPU targets")
|
|
endif()
|
|
endif()
|
|
endforeach()
|
|
message(VERBOSE "ckProfiler libs: ${PROFILER_LIBS}")
|
|
target_link_libraries(${PROFILER_EXECUTABLE} PRIVATE ${PROFILER_LIBS})
|
|
|
|
rocm_install(TARGETS ${PROFILER_EXECUTABLE} COMPONENT profiler)
|