mirror of
https://github.com/ROCm/composable_kernel.git
synced 2026-05-20 21:09:08 +00:00
780cb29a42
* Fixed cmake errors related to gemm_bilinear. Previously, if the above flags are set, cmake build fails: GPU_TARGETS="gfx1100;gfx1201" -D DTYPES="fp16;bf16;fp8"
* Fixed cmake build errors related to test_fp8
* Updates to support mixed precision
* Adding support for RRR, F8xF16xF16 gemm_universal_wmma - wip
* Added support for F8xF16xF16 to gemm_wmma_universal
* Added support for F16xF8xF16 to gemm_wmma_universal
* Added support for BF16xI4xBF16 to gemm_wmma_universal
* Added support for F16xI4xF16 to gemm_wmma_universal
* Fixed IsSupportedArgument to check ComputeTypeA, ComputeTypeB instead of ADataType, BDataType
* Added missing test class for FP16_KM_NK
* Pre-commit hooks fixes
* Added padding instances for f16xf16xf16
* Fixed cmake errors related to gemm_bilinear. Previously, if the above flags are set, cmake build fails: GPU_TARGETS="gfx1100;gfx1201" -D DTYPES="fp16;bf16;fp8"
* Fixed cmake build errors related to test_fp8
* Ammending changes for adding support for padding instances for f16xf16xf16
* Fixes for padding instances for f16xf16xf16
* Added padding instances for bf16xbf16, f8xf8
* Added packed instances for bf16xi4xbf16
* Added padding instances for f8xf16xf16
* Added padding instances for f16xf8xf16, f16xi4xf16
* Fixed typos for bf16xbf16xbf16 padding instances
* Fixed typos for padded instances
* Added tests for fp16, KM_KN and KM_NK
* Padding not supported for when BDataType is pk_i4_t. Added fix for correct check and removed padding instances.
* Fixed typos
* Updated the set of tests for FP16
* Updated the set of tests for FP16
* Fix typo
* Moved f16xi4 test under the correct data layout group
* example for gemm_universal_bf16
* Adding examples for gemm_wmma instances
* Added the missing parameters
* Fixed review comments and added executable to cmakeLists
* Fixing clang format
* Fixing build erros
* Fixed compilation failure.
* Modified some code as per gemm_universal_examples
* Fixed the gemm specialization error
* Fixed the build errors.
* Fix strides of a/b_thread_desc
The descriptors are larger than needed (even though the compiler don't alloc registers for unused values).
* Load in M/NRepeat dims with thread copy's slice instead of a loop
* Clone BlockwiseGemmXdlops_pipeline_v1 for WMMA implementation
* Implement Intrawave and Interwave variants of pipeline v1
* Add instances for Interwave and Intrawave v1
* Add instances with ABlockLdsExtraM and BBlockLdsExtraN = 0
* Remove instances that are too slow (mostly because of register spilling)
* Add a workaround for fp8/bf8->f32 packed conversion issue
* Add instances for Interwave and Intrawave v1
* Enable profiling of mixed precision with f8 and int4 on WMMA
* Fix segfault in profiler when B is pk_i4_t
b_device_buf's size in bytes is larger than b_k_n_permute so b_device_buf.ToDevice reads out-of-bounds.
* Remove instances that are too slow (mostly because of register spilling)
* Add missing add_device_gemm_wmma_universal_f8_f8_bf16 declarations
* Add test case for bf16_i4
* Add missing Regular tests
* Add test_gemm_universal_xdl/wmma_fp16 to REGRESSION_TESTS
They take more than 30 seconds
* Fix a bug that fp16_i4 validation passes only with PermuteB
A permutation required by conversion from pk_i4_t to half_t does not
depend on PermuteB, they can be used independently.
* Use PermuteB with f16_i4 in most instances (as xdl)
Some instances use PermuteB = false for checking correctness.
See also the previous commit.
* Fix cache flushing for pk_i4
* Add mixed precision examples
* Disable all tests and instances with f8 on gfx11
Even though f8_f16 and f16_f8 don't require f8 WMMA instructions,
gfx11 still lacks hardware instructions for fast f8->f32 conversion.
* Add FP16 KM_NK and KM_KN test suites for XDL
These tests were added to common .inc for better testing of WMMA instances
* Fix int8 DTYPES check for gemm_bilinear
---------
Co-authored-by: Anca Hamuraru <anca@streamhpc.com>
Co-authored-by: Apoorva Kalyani <apoorva@streamhpc.com>
[ROCm/composable_kernel commit: 52b4860a30]
285 lines
12 KiB
CMake
Executable File
285 lines
12 KiB
CMake
Executable File
include_directories(BEFORE
|
|
${PROJECT_SOURCE_DIR}/
|
|
${PROJECT_SOURCE_DIR}/profiler/include
|
|
)
|
|
|
|
include(gtest)
|
|
|
|
add_custom_target(tests)
|
|
|
|
# list of tests that are labelled as REGRESSION_TEST for make regression (runtime more than 30 seconds)
|
|
# all other tests are labelled as SMOKE_TEST
|
|
set(REGRESSION_TESTS
|
|
test_gemm_standalone_xdl_fp16
|
|
test_gemm_fp16
|
|
test_gemm_splitk
|
|
test_batched_gemm
|
|
test_gemm_universal_wmma_fp16
|
|
test_gemm_universal_xdl_fp16
|
|
test_gemm_universal_streamk_fp16
|
|
test_gemm_universal_streamk_bf16
|
|
test_gemm_universal_streamk_fp8
|
|
test_batched_gemm_softmax_gemm_fp16
|
|
test_batched_gemm_softmax_gemm_permute_fp16
|
|
test_batched_gemm_bias_softmax_gemm_permute_fp16
|
|
test_batched_gemm_softmax_gemm_permute_bf16
|
|
test_batched_gemm_bias_softmax_gemm_permute_bf16
|
|
test_grouped_gemm_splitk
|
|
test_reduce_no_index
|
|
test_reduce_with_index
|
|
test_convnd_fwd
|
|
test_convnd_bwd_data
|
|
test_grouped_convnd_fwd
|
|
test_grouped_convnd_bwd_weight
|
|
test_softmax_rank3
|
|
test_softmax_rank4
|
|
test_batchnorm_fwd_rank_4
|
|
test_batchnorm_bwd_rank_4
|
|
test_grouped_convnd_bwd_data_xdl
|
|
test_conv_tensor_rearrange
|
|
test_gemm_mx
|
|
)
|
|
|
|
function(add_test_executable TEST_NAME)
|
|
message("adding test ${TEST_NAME}")
|
|
set(result 1)
|
|
if(DEFINED DTYPES)
|
|
foreach(source IN LISTS ARGN)
|
|
set(test 0)
|
|
if((source MATCHES "_fp16" OR source MATCHES "_f16") AND NOT "fp16" IN_LIST DTYPES)
|
|
set(test 1)
|
|
endif()
|
|
if((source MATCHES "_fp32" OR source MATCHES "_f32") AND NOT "fp32" IN_LIST DTYPES)
|
|
set(test 1)
|
|
endif()
|
|
if((source MATCHES "_fp64" OR source MATCHES "_f64") AND NOT "fp64" IN_LIST DTYPES)
|
|
set(test 1)
|
|
endif()
|
|
if((source MATCHES "_fp8" OR source MATCHES "_f8") AND NOT "fp8" IN_LIST DTYPES)
|
|
set(test 1)
|
|
endif()
|
|
if((source MATCHES "_bf8" OR source MATCHES "_bf8") AND NOT "bf8" IN_LIST DTYPES)
|
|
set(test 1)
|
|
endif()
|
|
if((source MATCHES "_bf16" OR source MATCHES "_b16") AND NOT "bf16" IN_LIST DTYPES)
|
|
set(test 1)
|
|
endif()
|
|
if((source MATCHES "_int8" OR source MATCHES "_i8") AND NOT "int8" IN_LIST DTYPES)
|
|
set(test 1)
|
|
endif()
|
|
if(test EQUAL 1)
|
|
message("removing test ${source} ")
|
|
list(REMOVE_ITEM ARGN "${source}")
|
|
endif()
|
|
endforeach()
|
|
endif()
|
|
|
|
set(TEST_TARGETS ${SUPPORTED_GPU_TARGETS})
|
|
|
|
foreach(source IN LISTS ARGN)
|
|
if(NOT DEFINED DPP_KERNELS AND source MATCHES "_dpp")
|
|
message("removing dpp test ${source} ")
|
|
list(REMOVE_ITEM ARGN "${source}")
|
|
endif()
|
|
endforeach()
|
|
foreach(source IN LISTS ARGN)
|
|
if(NOT DEFINED DL_KERNELS AND source MATCHES "_dl")
|
|
message("removing dl test ${source} ")
|
|
list(REMOVE_ITEM ARGN "${source}")
|
|
endif()
|
|
endforeach()
|
|
foreach(source IN LISTS ARGN)
|
|
if(NOT TEST_TARGETS MATCHES "gfx9" AND source MATCHES "xdl")
|
|
message("removing xdl test ${source} ")
|
|
list(REMOVE_ITEM ARGN "${source}")
|
|
endif()
|
|
endforeach()
|
|
foreach(source IN LISTS ARGN)
|
|
if(NOT TEST_TARGETS MATCHES "gfx11" AND NOT TEST_TARGETS MATCHES "gfx12" AND source MATCHES "wmma")
|
|
message("removing wmma test ${source} ")
|
|
list(REMOVE_ITEM ARGN "${source}")
|
|
endif()
|
|
endforeach()
|
|
#only continue if there are some source files left on the list
|
|
if(ARGN)
|
|
if(ARGN MATCHES "_xdl")
|
|
list(REMOVE_ITEM TEST_TARGETS gfx900 gfx906 gfx906:xnack- gfx1030 gfx1100 gfx1101 gfx1102 gfx1103 gfx1150 gfx1151 gfx1152 gfx1200 gfx1201 gfx10-3-generic gfx11-generic gfx12-generic)
|
|
elseif(ARGN MATCHES "_wmma")
|
|
list(REMOVE_ITEM TEST_TARGETS gfx900 gfx906 gfx906:xnack- gfx908:xnack+ gfx908:xnack- gfx90a:xnack+ gfx90a:xnack- gfx908 gfx90a gfx942 gfx1030 gfx950)
|
|
elseif(ARGN MATCHES "_smfmac")
|
|
list(REMOVE_ITEM TEST_TARGETS gfx900 gfx906 gfx906:xnack- gfx1030 gfx1100 gfx1101 gfx1102 gfx1103 gfx1150 gfx1151 gfx1152 gfx908 gfx90a gfx1200 gfx1201 gfx10-3-generic gfx11-generic gfx12-generic)
|
|
endif()
|
|
set_source_files_properties(${ARGN} PROPERTIES LANGUAGE HIP)
|
|
add_executable(${TEST_NAME} ${ARGN})
|
|
set_property(TARGET ${TEST_NAME} PROPERTY HIP_ARCHITECTURES ${TEST_TARGETS} )
|
|
target_link_libraries(${TEST_NAME} PRIVATE getopt::getopt)
|
|
add_test(NAME ${TEST_NAME} COMMAND $<TARGET_FILE:${TEST_NAME}>)
|
|
add_dependencies(tests ${TEST_NAME})
|
|
add_dependencies(check ${TEST_NAME})
|
|
rocm_install(TARGETS ${TEST_NAME} COMPONENT tests)
|
|
set(result 0)
|
|
endif()
|
|
#message("add_test returns ${result}")
|
|
set(result ${result} PARENT_SCOPE)
|
|
if(result EQUAL 0 AND NOT "${TEST_NAME}" IN_LIST REGRESSION_TESTS)
|
|
set_tests_properties(${TEST_NAME} PROPERTIES LABELS "SMOKE_TEST")
|
|
add_dependencies(smoke ${TEST_NAME})
|
|
elseif(result EQUAL 0 AND "${TEST_NAME}" IN_LIST REGRESSION_TESTS)
|
|
set_tests_properties(${TEST_NAME} PROPERTIES LABELS "REGRESSION_TEST")
|
|
add_dependencies(regression ${TEST_NAME})
|
|
endif()
|
|
endfunction()
|
|
|
|
function(add_gtest_executable TEST_NAME)
|
|
message("adding gtest ${TEST_NAME}")
|
|
set(result 1)
|
|
if(DEFINED DTYPES)
|
|
foreach(source IN LISTS ARGN)
|
|
set(test 0)
|
|
if((source MATCHES "_fp16" OR source MATCHES "_f16") AND NOT "fp16" IN_LIST DTYPES)
|
|
set(test 1)
|
|
endif()
|
|
if((source MATCHES "_fp32" OR source MATCHES "_f32") AND NOT "fp32" IN_LIST DTYPES)
|
|
set(test 1)
|
|
endif()
|
|
if((source MATCHES "_fp64" OR source MATCHES "_f64") AND NOT "fp64" IN_LIST DTYPES)
|
|
set(test 1)
|
|
endif()
|
|
if((source MATCHES "_fp8" OR source MATCHES "_f8") AND NOT "fp8" IN_LIST DTYPES)
|
|
set(test 1)
|
|
endif()
|
|
if((source MATCHES "_bf8" OR source MATCHES "_bf8") AND NOT "bf8" IN_LIST DTYPES)
|
|
set(test 1)
|
|
endif()
|
|
if((source MATCHES "_bf16" OR source MATCHES "_b16") AND NOT "bf16" IN_LIST DTYPES)
|
|
set(test 1)
|
|
endif()
|
|
if((source MATCHES "_int8" OR source MATCHES "_i8") AND NOT "int8" IN_LIST DTYPES)
|
|
set(test 1)
|
|
endif()
|
|
if(test EQUAL 1)
|
|
message("removing gtest ${source} ")
|
|
list(REMOVE_ITEM ARGN "${source}")
|
|
endif()
|
|
endforeach()
|
|
endif()
|
|
|
|
set(TEST_TARGETS ${SUPPORTED_GPU_TARGETS})
|
|
|
|
foreach(source IN LISTS ARGN)
|
|
if(NOT DEFINED DL_KERNELS AND source MATCHES "_dl")
|
|
message("removing dl test ${source} ")
|
|
list(REMOVE_ITEM ARGN "${source}")
|
|
endif()
|
|
endforeach()
|
|
|
|
foreach(source IN LISTS ARGN)
|
|
if(NOT TEST_TARGETS MATCHES "gfx9" AND source MATCHES "xdl")
|
|
message("removing xdl test ${source} ")
|
|
list(REMOVE_ITEM ARGN "${source}")
|
|
endif()
|
|
endforeach()
|
|
|
|
foreach(source IN LISTS ARGN)
|
|
if(NOT TEST_TARGETS MATCHES "gfx95" AND source MATCHES "mx_")
|
|
message("removing microscaling test ${source} ")
|
|
list(REMOVE_ITEM ARGN "${source}")
|
|
endif()
|
|
endforeach()
|
|
|
|
foreach(source IN LISTS ARGN)
|
|
if(NOT TEST_TARGETS MATCHES "gfx11" AND NOT TEST_TARGETS MATCHES "gfx12" AND source MATCHES "wmma")
|
|
message("removing wmma test ${source} ")
|
|
list(REMOVE_ITEM ARGN "${source}")
|
|
endif()
|
|
endforeach()
|
|
|
|
#only continue if there are some source files left on the list
|
|
if(ARGN)
|
|
if(ARGN MATCHES "_xdl")
|
|
list(REMOVE_ITEM TEST_TARGETS gfx900 gfx906 gfx906:xnack- gfx1030 gfx1100 gfx1101 gfx1102 gfx1103 gfx1150 gfx1151 gfx1152 gfx1200 gfx1201 gfx10-3-generic gfx11-generic gfx12-generic)
|
|
elseif(ARGN MATCHES "_wmma")
|
|
list(REMOVE_ITEM TEST_TARGETS gfx900 gfx906 gfx906:xnack- gfx908:xnack+ gfx908:xnack- gfx90a:xnack+ gfx90a:xnack- gfx908 gfx90a gfx942 gfx1030 gfx950)
|
|
elseif(ARGN MATCHES "_smfmac")
|
|
list(REMOVE_ITEM TEST_TARGETS gfx900 gfx906 gfx906:xnack- gfx1030 gfx1100 gfx1101 gfx1102 gfx1103 gfx1150 gfx1151 gfx1152 gfx908 gfx90a gfx1200 gfx1201 gfx10-3-generic gfx11-generic gfx12-generic)
|
|
elseif(ARGN MATCHES "_mx") #only build mx example for gfx950
|
|
list(REMOVE_ITEM TEST_TARGETS gfx900 gfx906 gfx906:xnack- gfx908:xnack+ gfx908:xnack- gfx90a:xnack+ gfx90a:xnack- gfx908 gfx90a gfx942 gfx1030 gfx1100 gfx1101 gfx1102 gfx1103 gfx1150 gfx1151 gfx1152 gfx1200 gfx1201 gfx10-3-generic gfx11-generic gfx12-generic)
|
|
endif()
|
|
set_source_files_properties(${ARGN} PROPERTIES LANGUAGE HIP)
|
|
add_executable(${TEST_NAME} ${ARGN})
|
|
set_property(TARGET ${TEST_NAME} PROPERTY HIP_ARCHITECTURES ${TEST_TARGETS} )
|
|
add_dependencies(tests ${TEST_NAME})
|
|
add_dependencies(check ${TEST_NAME})
|
|
|
|
# suppress gtest warnings
|
|
target_compile_options(${TEST_NAME} PRIVATE -Wno-global-constructors -Wno-undef)
|
|
target_link_libraries(${TEST_NAME} PRIVATE gtest_main getopt::getopt)
|
|
add_test(NAME ${TEST_NAME} COMMAND $<TARGET_FILE:${TEST_NAME}>)
|
|
rocm_install(TARGETS ${TEST_NAME} COMPONENT tests)
|
|
set(result 0)
|
|
endif()
|
|
#message("add_gtest returns ${result}")
|
|
set(result ${result} PARENT_SCOPE)
|
|
if(result EQUAL 0 AND NOT "${TEST_NAME}" IN_LIST REGRESSION_TESTS)
|
|
set_tests_properties(${TEST_NAME} PROPERTIES LABELS "SMOKE_TEST")
|
|
add_dependencies(smoke ${TEST_NAME})
|
|
elseif(result EQUAL 0 AND "${TEST_NAME}" IN_LIST REGRESSION_TESTS)
|
|
set_tests_properties(${TEST_NAME} PROPERTIES LABELS "REGRESSION_TEST")
|
|
add_dependencies(regression ${TEST_NAME})
|
|
endif()
|
|
endfunction()
|
|
|
|
add_compile_options(-Wno-c++20-extensions)
|
|
add_subdirectory(ck_tile)
|
|
add_subdirectory(magic_number_division)
|
|
add_subdirectory(space_filling_curve)
|
|
add_subdirectory(conv_util)
|
|
add_subdirectory(reference_conv_fwd)
|
|
add_subdirectory(gemm)
|
|
add_subdirectory(gemm_add)
|
|
add_subdirectory(gemm_layernorm)
|
|
add_subdirectory(gemm_split_k)
|
|
add_subdirectory(gemm_universal)
|
|
add_subdirectory(gemm_universal_streamk)
|
|
add_subdirectory(gemm_reduce)
|
|
add_subdirectory(batched_gemm)
|
|
add_subdirectory(batched_gemm_reduce)
|
|
add_subdirectory(batched_gemm_gemm)
|
|
add_subdirectory(batched_gemm_softmax_gemm)
|
|
add_subdirectory(batched_gemm_softmax_gemm_permute)
|
|
add_subdirectory(grouped_gemm)
|
|
add_subdirectory(reduce)
|
|
add_subdirectory(convnd_fwd)
|
|
add_subdirectory(convnd_bwd_data)
|
|
add_subdirectory(grouped_convnd_fwd)
|
|
add_subdirectory(grouped_convnd_fwd_bias_clamp)
|
|
add_subdirectory(grouped_convnd_bwd_weight)
|
|
add_subdirectory(block_to_ctile_map)
|
|
add_subdirectory(softmax)
|
|
add_subdirectory(normalization_fwd)
|
|
add_subdirectory(normalization_bwd_data)
|
|
add_subdirectory(normalization_bwd_gamma_beta)
|
|
add_subdirectory(data_type)
|
|
add_subdirectory(elementwise_normalization)
|
|
add_subdirectory(batchnorm)
|
|
add_subdirectory(contraction)
|
|
add_subdirectory(pool)
|
|
add_subdirectory(batched_gemm_multi_d)
|
|
add_subdirectory(grouped_convnd_bwd_data)
|
|
add_subdirectory(conv_tensor_rearrange)
|
|
add_subdirectory(transpose)
|
|
add_subdirectory(permute_scale)
|
|
add_subdirectory(wrapper)
|
|
if(SUPPORTED_GPU_TARGETS MATCHES "gfx11")
|
|
add_subdirectory(wmma_op)
|
|
endif()
|
|
if(SUPPORTED_GPU_TARGETS MATCHES "gfx942" OR SUPPORTED_GPU_TARGETS MATCHES "gfx950") # smfmac needs ROCm6.2
|
|
add_subdirectory(smfmac_op)
|
|
endif()
|
|
if(SUPPORTED_GPU_TARGETS MATCHES "gfx950")
|
|
add_subdirectory(mx_mfma_op)
|
|
add_subdirectory(gemm_mx)
|
|
endif()
|
|
add_subdirectory(position_embedding)
|
|
add_subdirectory(scatter_gather)
|