mirror of
https://github.com/ROCm/composable_kernel.git
synced 2026-05-19 20:40:07 +00:00
ceaecc86ca
* Optimize GEMM on MI200/300:
1. Add new blockwise gemm pipeline
2. Add irregular splitk intances
* clang format + typo fix
* Fix a bug
* initial commit
* Add more instances to irregular splitk
* blkgemm pipeline v1~4 prototype
* Sanity Checked. Known issue:
1. Poor performance of splitk
2. Register spill on blkgemmpipeline v3
* Sanity and Performance fix:
1. fix a bug related to sanity in grouped b2c mapping
2. fix a bug related to sanity and performance in splitk offset
* Sanity and API update:
1. Remove prefetch stage
2. Fix valid check bug
3, Add first gemm_universal instance into ckProfiler
* Add NN instances for gemm universal
* 1. Add NT instances for gemm_universal
2. Fix a bug about Kpadding in gemm_universal
* Fix a bug regarding padding Odd K number
* remove kernel print
* Fix KPadding bug...
* Update safety check
* another try to fix kpadding..
* Sanity checked
* new instances..
* clang format+typo fix
* remove clang format script's change
* Add non-hotloop compile option
* 1. Add fp16xfp8 example
2. pull packed convert f8 from pr1150
* Some miscs.. opt and fix
* Add pipeline description docs
* Split universal gemm instance library to cut profiler compiling time
* uncomment cmakefile
* Fix a bug caused by blockwise_gemm_pipe_v2
* reduce default splitk to 1
* Add 224x256x64 tile size
* update, including:
1. Experiment pipeline 5~7
2. Optimization for pipeline 4
3. Organized instance library
* temp save
* temp save
* Permuted lds layout, sanity and function checked
* clang format
* Move OOB check from RunRead to RunWrite, for better software pipeline.
TODO: agpr spill when NN layout
* clangformat
* A/B splitpipe scheduler for v3
* Fix two bugs
* bug fix
* fix a bug in oob check
* Example for mixed fp16_fp8 gemm
* Clean experimental code blocks
* Add mixed precision gemm into profiler
* tempsave
* optimize m/n major lds layout
* Add RRR GEMM mixed precision instances
* Optimize f8 matrix transpose
* Add test_gemm_universal
* A/B spilt schedule for blkpip v5
* Take ds_read2 into iglp scheduling scheme
* format
* fixed cmake
* Add llvm-option into CI cmake flag
---------
Co-authored-by: Jing Zhang <jizhan@amd.com>
[ROCm/composable_kernel commit: f83e9701e9]
184 lines
6.7 KiB
CMake
184 lines
6.7 KiB
CMake
include_directories(BEFORE
|
|
${PROJECT_SOURCE_DIR}/
|
|
${PROJECT_SOURCE_DIR}/profiler/include
|
|
)
|
|
|
|
include(gtest)
|
|
|
|
add_custom_target(tests)
|
|
|
|
function(add_test_executable TEST_NAME)
|
|
message("adding test ${TEST_NAME}")
|
|
set(result 1)
|
|
if(DEFINED DTYPES)
|
|
foreach(source IN LISTS ARGN)
|
|
set(test 0)
|
|
if((source MATCHES "_fp16" OR source MATCHES "_f16") AND NOT "fp16" IN_LIST DTYPES)
|
|
set(test 1)
|
|
endif()
|
|
if((source MATCHES "_fp32" OR source MATCHES "_f32") AND NOT "fp32" IN_LIST DTYPES)
|
|
set(test 1)
|
|
endif()
|
|
if((source MATCHES "_fp64" OR source MATCHES "_f64") AND NOT "fp64" IN_LIST DTYPES)
|
|
set(test 1)
|
|
endif()
|
|
if((source MATCHES "_fp8" OR source MATCHES "_f8") AND NOT "fp8" IN_LIST DTYPES)
|
|
set(test 1)
|
|
endif()
|
|
if((source MATCHES "_bf8" OR source MATCHES "_bf8") AND NOT "bf8" IN_LIST DTYPES)
|
|
set(test 1)
|
|
endif()
|
|
if((source MATCHES "_bf16" OR source MATCHES "_b16") AND NOT "bf16" IN_LIST DTYPES)
|
|
set(test 1)
|
|
endif()
|
|
if((source MATCHES "_int8" OR source MATCHES "_i8") AND NOT "int8" IN_LIST DTYPES)
|
|
set(test 1)
|
|
endif()
|
|
if(test EQUAL 1)
|
|
message("removing test ${source} ")
|
|
list(REMOVE_ITEM ARGN "${source}")
|
|
endif()
|
|
endforeach()
|
|
endif()
|
|
foreach(source IN LISTS ARGN)
|
|
if(NOT DEFINED DL_KERNELS AND source MATCHES "_dl")
|
|
message("removing dl test ${source} ")
|
|
list(REMOVE_ITEM ARGN "${source}")
|
|
endif()
|
|
endforeach()
|
|
foreach(source IN LISTS ARGN)
|
|
if(NOT GPU_TARGETS MATCHES "gfx9" AND source MATCHES "xdl")
|
|
message("removing xdl test ${source} ")
|
|
list(REMOVE_ITEM ARGN "${source}")
|
|
endif()
|
|
endforeach()
|
|
foreach(source IN LISTS ARGN)
|
|
if(NOT GPU_TARGETS MATCHES "gfx11" AND source MATCHES "wmma")
|
|
message("removing wmma test ${source} ")
|
|
list(REMOVE_ITEM ARGN "${source}")
|
|
endif()
|
|
endforeach()
|
|
#only continue if there are some source files left on the list
|
|
if(ARGN)
|
|
add_executable(${TEST_NAME} ${ARGN})
|
|
target_link_libraries(${TEST_NAME} PRIVATE getopt::getopt)
|
|
add_test(NAME ${TEST_NAME} COMMAND $<TARGET_FILE:${TEST_NAME}>)
|
|
add_dependencies(tests ${TEST_NAME})
|
|
add_dependencies(check ${TEST_NAME})
|
|
rocm_install(TARGETS ${TEST_NAME} COMPONENT tests)
|
|
set(result 0)
|
|
endif()
|
|
#message("add_test returns ${result}")
|
|
set(result ${result} PARENT_SCOPE)
|
|
endfunction()
|
|
|
|
function(add_gtest_executable TEST_NAME)
|
|
message("adding gtest ${TEST_NAME}")
|
|
set(result 1)
|
|
if(DEFINED DTYPES)
|
|
foreach(source IN LISTS ARGN)
|
|
set(test 0)
|
|
if((source MATCHES "_fp16" OR source MATCHES "_f16") AND NOT "fp16" IN_LIST DTYPES)
|
|
set(test 1)
|
|
endif()
|
|
if((source MATCHES "_fp32" OR source MATCHES "_f32") AND NOT "fp32" IN_LIST DTYPES)
|
|
set(test 1)
|
|
endif()
|
|
if((source MATCHES "_fp64" OR source MATCHES "_f64") AND NOT "fp64" IN_LIST DTYPES)
|
|
set(test 1)
|
|
endif()
|
|
if((source MATCHES "_fp8" OR source MATCHES "_f8") AND NOT "fp8" IN_LIST DTYPES)
|
|
set(test 1)
|
|
endif()
|
|
if((source MATCHES "_bf8" OR source MATCHES "_bf8") AND NOT "bf8" IN_LIST DTYPES)
|
|
set(test 1)
|
|
endif()
|
|
if((source MATCHES "_bf16" OR source MATCHES "_b16") AND NOT "bf16" IN_LIST DTYPES)
|
|
set(test 1)
|
|
endif()
|
|
if((source MATCHES "_int8" OR source MATCHES "_i8") AND NOT "int8" IN_LIST DTYPES)
|
|
set(test 1)
|
|
endif()
|
|
if(test EQUAL 1)
|
|
message("removing gtest ${source} ")
|
|
list(REMOVE_ITEM ARGN "${source}")
|
|
endif()
|
|
endforeach()
|
|
endif()
|
|
foreach(source IN LISTS ARGN)
|
|
if(NOT DEFINED DL_KERNELS AND source MATCHES "_dl")
|
|
message("removing dl test ${source} ")
|
|
list(REMOVE_ITEM ARGN "${source}")
|
|
endif()
|
|
endforeach()
|
|
foreach(source IN LISTS ARGN)
|
|
if(NOT GPU_TARGETS MATCHES "gfx9" AND source MATCHES "xdl")
|
|
message("removing xdl test ${source} ")
|
|
list(REMOVE_ITEM ARGN "${source}")
|
|
endif()
|
|
endforeach()
|
|
foreach(source IN LISTS ARGN)
|
|
if(NOT GPU_TARGETS MATCHES "gfx11" AND source MATCHES "wmma")
|
|
message("removing wmma test ${source} ")
|
|
list(REMOVE_ITEM ARGN "${source}")
|
|
endif()
|
|
endforeach()
|
|
#only continue if there are some source files left on the list
|
|
if(ARGN)
|
|
add_executable(${TEST_NAME} ${ARGN})
|
|
add_dependencies(tests ${TEST_NAME})
|
|
add_dependencies(check ${TEST_NAME})
|
|
|
|
# suppress gtest warnings
|
|
target_compile_options(${TEST_NAME} PRIVATE -Wno-global-constructors -Wno-undef)
|
|
target_link_libraries(${TEST_NAME} PRIVATE gtest_main getopt::getopt)
|
|
add_test(NAME ${TEST_NAME} COMMAND $<TARGET_FILE:${TEST_NAME}>)
|
|
rocm_install(TARGETS ${TEST_NAME} COMPONENT tests)
|
|
set(result 0)
|
|
endif()
|
|
#message("add_gtest returns ${result}")
|
|
set(result ${result} PARENT_SCOPE)
|
|
endfunction()
|
|
|
|
add_compile_options(-Wno-c++20-extensions)
|
|
add_subdirectory(magic_number_division)
|
|
add_subdirectory(space_filling_curve)
|
|
add_subdirectory(conv_util)
|
|
add_subdirectory(reference_conv_fwd)
|
|
add_subdirectory(gemm)
|
|
add_subdirectory(gemm_add)
|
|
add_subdirectory(gemm_layernorm)
|
|
add_subdirectory(gemm_split_k)
|
|
add_subdirectory(gemm_universal)
|
|
add_subdirectory(gemm_reduce)
|
|
add_subdirectory(batched_gemm)
|
|
add_subdirectory(batched_gemm_reduce)
|
|
add_subdirectory(batched_gemm_gemm)
|
|
add_subdirectory(batched_gemm_softmax_gemm)
|
|
add_subdirectory(batched_gemm_softmax_gemm_permute)
|
|
add_subdirectory(grouped_gemm)
|
|
add_subdirectory(reduce)
|
|
add_subdirectory(convnd_fwd)
|
|
add_subdirectory(convnd_bwd_data)
|
|
add_subdirectory(grouped_convnd_fwd)
|
|
add_subdirectory(grouped_convnd_bwd_weight)
|
|
add_subdirectory(block_to_ctile_map)
|
|
add_subdirectory(softmax)
|
|
add_subdirectory(normalization_fwd)
|
|
add_subdirectory(normalization_bwd_data)
|
|
add_subdirectory(normalization_bwd_gamma_beta)
|
|
add_subdirectory(data_type)
|
|
add_subdirectory(elementwise_normalization)
|
|
add_subdirectory(batchnorm)
|
|
add_subdirectory(contraction)
|
|
add_subdirectory(pool)
|
|
add_subdirectory(batched_gemm_multi_d)
|
|
add_subdirectory(grouped_convnd_bwd_data)
|
|
add_subdirectory(conv_tensor_rearrange)
|
|
add_subdirectory(transpose)
|
|
add_subdirectory(permute_scale)
|
|
add_subdirectory(wrapper)
|
|
if(GPU_TARGETS MATCHES "gfx11")
|
|
add_subdirectory(wmma_op)
|
|
endif()
|