Implementing Test Filters for Smoke and Regression Tests (#1819)

* smoke and regression targets working with tests * test filters work for both examples and test * removed uneccesary comments * added a missing comment * added a missing comment * fixed typo in the comments * updated README * Update PULL_REQUEST_TEMPLATE.md updating the template for future addition of test cases * Update PULL_REQUEST_TEMPLATE.md [ROCm/composable_kernel commit: 54de3e55e1]
2026-05-14 02:02:46 +00:00 · 2025-01-16 18:40:08 -06:00
parent bddf6109f0
commit ebe7a75ab4
5 changed files with 81 additions and 0 deletions
--- a/.github/PULL_REQUEST_TEMPLATE.md
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -7,6 +7,7 @@ Please describe the motivation behind the pull request, whether it enables a new
 Please put an `x` into the boxes that apply. You can also fill these out after creating the PR. If you're not sure, please don't hesitate to ask.

 - [ ] I have added tests relevant to the introduced functionality, and the unit tests are passing locally
+- [ ] I have added the test to REGRESSION_TESTS list defined at the top of CMakeLists.txt in tests/CMakeLists.txt, **IF** the test takes more than 30 seconds to run.
 - [ ] I have added inline documentation which enables the maintainers with understanding the motivation
 - [ ] I have removed the stale documentation which is no longer relevant after this pull request
 - [ ] (If this change is user-facing) I have added release notes which provide the end users with a brief summary of the improvement from this pull request
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -533,7 +533,13 @@ if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERS
    add_compile_options(-fdiagnostics-color=always)
 endif()

+# make check runs the entire set of examples and tests
 add_custom_target(check COMMAND ${CMAKE_CTEST_COMMAND} --output-on-failure -C ${CMAKE_CFG_INTDIR})
+# make smoke runs the tests and examples that runs within 30 seconds on gfx90a
+add_custom_target(smoke COMMAND ${CMAKE_CTEST_COMMAND} --output-on-failure -C ${CMAKE_CFG_INTDIR} -L "SMOKE_TEST")
+# make regression runs the tests and examples that runs for more 30 seconds on gfx90a
+add_custom_target(regression COMMAND ${CMAKE_CTEST_COMMAND} --output-on-failure -C ${CMAKE_CFG_INTDIR} -L "REGRESSION_TEST")
+

 file(GLOB_RECURSE INSTANCE_FILES "${PROJECT_SOURCE_DIR}/*/device_*_instance.cpp")
 file(GLOB dir_list RELATIVE ${PROJECT_SOURCE_DIR}/library/src/tensor_operation_instance/gpu ${PROJECT_SOURCE_DIR}/library/src/tensor_operation_instance/gpu/*)
--- a/README.md
+++ b/README.md
@@ -121,6 +121,15 @@ Docker images are available on [DockerHub](https://hub.docker.com/r/rocm/composa

    You can find instructions for running each individual example in [example](/example).

+* Build and run smoke/regression examples and tests:
+
+    ```bash
+    make -j smoke # tests and examples that run for < 30 seconds each
+    ```
+     ```bash
+    make -j regression # tests and examples that run for >= 30 seconds each
+    ```
+
 * Build ckProfiler:

    ```bash
--- a/example/CMakeLists.txt
+++ b/example/CMakeLists.txt
@@ -5,6 +5,14 @@ include_directories(BEFORE

 add_custom_target(examples)

+
+# list of examples that are labelled as REGRESSION_EXAMPLE for make regression (runtime more than 30 seconds)
+# all other tests are labelled as SMOKE_EXAMPLE
+set(REGRESSION_EXAMPLES
+    example_sparse_embedding3_forward_layernorm
+)
+
+
 function(add_example_dependencies EXAMPLE_NAME FILE_NAME)
    if(FILE_NAME)
        add_dependencies(EXAMPLE_NAME FILE_NAME)
@@ -107,6 +115,15 @@ function(add_example_executable EXAMPLE_NAME FILE_NAME)
        set(result 0)
    endif()
    #message("add_example returns ${result}")
+    if(result EQUAL 0 AND NOT "${EXAMPLE_NAME}" IN_LIST REGRESSION_EXAMPLES)
+        #message("adding to SMOKE EXAMPLE FILTER ${EXAMPLE_NAME}")
+        set_tests_properties(${EXAMPLE_NAME} PROPERTIES LABELS "SMOKE_TEST")
+        add_dependencies(smoke ${EXAMPLE_NAME})
+    elseif(result EQUAL 0 AND "${EXAMPLE_NAME}" IN_LIST REGRESSION_EXAMPLES)
+        #message("Adding to REGRESSION EXAMPLE FILTER ${EXAMPLE_NAME}")
+        set_tests_properties(${EXAMPLE_NAME} PROPERTIES LABELS "REGRESSION_TEST")
+        add_dependencies(regression ${EXAMPLE_NAME})
+    endif()
    set(result ${result} PARENT_SCOPE)
 endfunction(add_example_executable EXAMPLE_NAME)

@@ -188,8 +205,10 @@ function(add_example_executable_no_testing EXAMPLE_NAME FILE_NAME)
        rocm_install(TARGETS ${EXAMPLE_NAME} COMPONENT examples)
        set(result 0)
    endif()
+    
    #message("add_example returns ${result}")
    set(result ${result} PARENT_SCOPE)
+
 endfunction(add_example_executable_no_testing EXAMPLE_NAME)

 # add all example subdir
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -7,6 +7,34 @@ include(gtest)

 add_custom_target(tests)

+# list of tests that are labelled as REGRESSION_TEST for make regression (runtime more than 30 seconds)
+# all other tests are labelled as SMOKE_TEST
+set(REGRESSION_TESTS
+    test_gemm_standalone_xdl_fp16
+    test_gemm_fp16
+    test_gemm_splitk
+    test_batched_gemm
+    test_gemm_universal
+    test_batched_gemm_softmax_gemm_fp16
+    test_batched_gemm_softmax_gemm_permute_fp16
+    test_batched_gemm_bias_softmax_gemm_permute_fp16
+    test_batched_gemm_softmax_gemm_permute_bf16
+    test_batched_gemm_bias_softmax_gemm_permute_bf16
+    test_grouped_gemm_splitk
+    test_reduce_no_index
+    test_reduce_with_index
+    test_convnd_fwd
+    test_convnd_bwd_data
+    test_grouped_convnd_fwd
+    test_grouped_convnd_bwd_weight
+    test_softmax_rank3
+    test_softmax_rank4
+    test_batchnorm_fwd_rank_4
+    test_batchnorm_bwd_rank_4
+    test_grouped_convnd_bwd_data_xdl
+    test_conv_tensor_rearrange
+)
+
 function(add_test_executable TEST_NAME)
    message("adding test ${TEST_NAME}")
    set(result 1)
@@ -88,6 +116,15 @@ function(add_test_executable TEST_NAME)
    endif()
    #message("add_test returns ${result}")
    set(result ${result} PARENT_SCOPE)
+    if(result EQUAL 0 AND NOT "${TEST_NAME}" IN_LIST REGRESSION_TESTS)
+        message("adding to SMOKE TEST FILTER ${TEST_NAME}")
+        set_tests_properties(${TEST_NAME} PROPERTIES LABELS "SMOKE_TEST")
+        add_dependencies(smoke ${TEST_NAME})
+    elseif(result EQUAL 0 AND "${TEST_NAME}" IN_LIST REGRESSION_TESTS)
+        message("Adding to REGRESSION TEST FILTER ${TEST_NAME}")
+        set_tests_properties(${TEST_NAME} PROPERTIES LABELS "REGRESSION_TEST")
+        add_dependencies(regression ${TEST_NAME})
+    endif()
 endfunction()

 function(add_gtest_executable TEST_NAME)
@@ -168,6 +205,15 @@ function(add_gtest_executable TEST_NAME)
    endif()
    #message("add_gtest returns ${result}")
    set(result ${result} PARENT_SCOPE)
+    if(result EQUAL 0 AND NOT "${TEST_NAME}" IN_LIST REGRESSION_TESTS)
+        #message("adding to smoke test FILTER ${TEST_NAME}")
+        set_tests_properties(${TEST_NAME} PROPERTIES LABELS "SMOKE_TEST")
+        add_dependencies(smoke ${TEST_NAME})
+    elseif(result EQUAL 0 AND "${TEST_NAME}" IN_LIST REGRESSION_TESTS)
+        #message("Adding to REGRESSION TEST FILTER ${TEST_NAME}")
+        set_tests_properties(${TEST_NAME} PROPERTIES LABELS "REGRESSION_TEST")
+        add_dependencies(regression ${TEST_NAME})
+    endif()
 endfunction()

 add_compile_options(-Wno-c++20-extensions)