From ebe7a75ab4f9c9c10d35ae51b13a605fa3d13b2f Mon Sep 17 00:00:00 2001 From: Aviral Goel Date: Thu, 16 Jan 2025 18:40:08 -0600 Subject: [PATCH] Implementing Test Filters for Smoke and Regression Tests (#1819) * smoke and regression targets working with tests * test filters work for both examples and test * removed uneccesary comments * added a missing comment * added a missing comment * fixed typo in the comments * updated README * Update PULL_REQUEST_TEMPLATE.md updating the template for future addition of test cases * Update PULL_REQUEST_TEMPLATE.md [ROCm/composable_kernel commit: 54de3e55e1fbd04a7fa218893eb2167d44a9756d] --- .github/PULL_REQUEST_TEMPLATE.md | 1 + CMakeLists.txt | 6 +++++ README.md | 9 +++++++ example/CMakeLists.txt | 19 +++++++++++++ test/CMakeLists.txt | 46 ++++++++++++++++++++++++++++++++ 5 files changed, 81 insertions(+) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index b3fcabec34..8a988ad1c9 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -7,6 +7,7 @@ Please describe the motivation behind the pull request, whether it enables a new Please put an `x` into the boxes that apply. You can also fill these out after creating the PR. If you're not sure, please don't hesitate to ask. - [ ] I have added tests relevant to the introduced functionality, and the unit tests are passing locally +- [ ] I have added the test to REGRESSION_TESTS list defined at the top of CMakeLists.txt in tests/CMakeLists.txt, **IF** the test takes more than 30 seconds to run. - [ ] I have added inline documentation which enables the maintainers with understanding the motivation - [ ] I have removed the stale documentation which is no longer relevant after this pull request - [ ] (If this change is user-facing) I have added release notes which provide the end users with a brief summary of the improvement from this pull request diff --git a/CMakeLists.txt b/CMakeLists.txt index 191aad8721..b4ea875034 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -533,7 +533,13 @@ if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERS add_compile_options(-fdiagnostics-color=always) endif() +# make check runs the entire set of examples and tests add_custom_target(check COMMAND ${CMAKE_CTEST_COMMAND} --output-on-failure -C ${CMAKE_CFG_INTDIR}) +# make smoke runs the tests and examples that runs within 30 seconds on gfx90a +add_custom_target(smoke COMMAND ${CMAKE_CTEST_COMMAND} --output-on-failure -C ${CMAKE_CFG_INTDIR} -L "SMOKE_TEST") +# make regression runs the tests and examples that runs for more 30 seconds on gfx90a +add_custom_target(regression COMMAND ${CMAKE_CTEST_COMMAND} --output-on-failure -C ${CMAKE_CFG_INTDIR} -L "REGRESSION_TEST") + file(GLOB_RECURSE INSTANCE_FILES "${PROJECT_SOURCE_DIR}/*/device_*_instance.cpp") file(GLOB dir_list RELATIVE ${PROJECT_SOURCE_DIR}/library/src/tensor_operation_instance/gpu ${PROJECT_SOURCE_DIR}/library/src/tensor_operation_instance/gpu/*) diff --git a/README.md b/README.md index 719c008c2b..95f44d8872 100644 --- a/README.md +++ b/README.md @@ -121,6 +121,15 @@ Docker images are available on [DockerHub](https://hub.docker.com/r/rocm/composa You can find instructions for running each individual example in [example](/example). +* Build and run smoke/regression examples and tests: + + ```bash + make -j smoke # tests and examples that run for < 30 seconds each + ``` + ```bash + make -j regression # tests and examples that run for >= 30 seconds each + ``` + * Build ckProfiler: ```bash diff --git a/example/CMakeLists.txt b/example/CMakeLists.txt index f5ae4145e7..f26d738625 100644 --- a/example/CMakeLists.txt +++ b/example/CMakeLists.txt @@ -5,6 +5,14 @@ include_directories(BEFORE add_custom_target(examples) + +# list of examples that are labelled as REGRESSION_EXAMPLE for make regression (runtime more than 30 seconds) +# all other tests are labelled as SMOKE_EXAMPLE +set(REGRESSION_EXAMPLES + example_sparse_embedding3_forward_layernorm +) + + function(add_example_dependencies EXAMPLE_NAME FILE_NAME) if(FILE_NAME) add_dependencies(EXAMPLE_NAME FILE_NAME) @@ -107,6 +115,15 @@ function(add_example_executable EXAMPLE_NAME FILE_NAME) set(result 0) endif() #message("add_example returns ${result}") + if(result EQUAL 0 AND NOT "${EXAMPLE_NAME}" IN_LIST REGRESSION_EXAMPLES) + #message("adding to SMOKE EXAMPLE FILTER ${EXAMPLE_NAME}") + set_tests_properties(${EXAMPLE_NAME} PROPERTIES LABELS "SMOKE_TEST") + add_dependencies(smoke ${EXAMPLE_NAME}) + elseif(result EQUAL 0 AND "${EXAMPLE_NAME}" IN_LIST REGRESSION_EXAMPLES) + #message("Adding to REGRESSION EXAMPLE FILTER ${EXAMPLE_NAME}") + set_tests_properties(${EXAMPLE_NAME} PROPERTIES LABELS "REGRESSION_TEST") + add_dependencies(regression ${EXAMPLE_NAME}) + endif() set(result ${result} PARENT_SCOPE) endfunction(add_example_executable EXAMPLE_NAME) @@ -188,8 +205,10 @@ function(add_example_executable_no_testing EXAMPLE_NAME FILE_NAME) rocm_install(TARGETS ${EXAMPLE_NAME} COMPONENT examples) set(result 0) endif() + #message("add_example returns ${result}") set(result ${result} PARENT_SCOPE) + endfunction(add_example_executable_no_testing EXAMPLE_NAME) # add all example subdir diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index c499482bd8..29a216c704 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -7,6 +7,34 @@ include(gtest) add_custom_target(tests) +# list of tests that are labelled as REGRESSION_TEST for make regression (runtime more than 30 seconds) +# all other tests are labelled as SMOKE_TEST +set(REGRESSION_TESTS + test_gemm_standalone_xdl_fp16 + test_gemm_fp16 + test_gemm_splitk + test_batched_gemm + test_gemm_universal + test_batched_gemm_softmax_gemm_fp16 + test_batched_gemm_softmax_gemm_permute_fp16 + test_batched_gemm_bias_softmax_gemm_permute_fp16 + test_batched_gemm_softmax_gemm_permute_bf16 + test_batched_gemm_bias_softmax_gemm_permute_bf16 + test_grouped_gemm_splitk + test_reduce_no_index + test_reduce_with_index + test_convnd_fwd + test_convnd_bwd_data + test_grouped_convnd_fwd + test_grouped_convnd_bwd_weight + test_softmax_rank3 + test_softmax_rank4 + test_batchnorm_fwd_rank_4 + test_batchnorm_bwd_rank_4 + test_grouped_convnd_bwd_data_xdl + test_conv_tensor_rearrange +) + function(add_test_executable TEST_NAME) message("adding test ${TEST_NAME}") set(result 1) @@ -88,6 +116,15 @@ function(add_test_executable TEST_NAME) endif() #message("add_test returns ${result}") set(result ${result} PARENT_SCOPE) + if(result EQUAL 0 AND NOT "${TEST_NAME}" IN_LIST REGRESSION_TESTS) + message("adding to SMOKE TEST FILTER ${TEST_NAME}") + set_tests_properties(${TEST_NAME} PROPERTIES LABELS "SMOKE_TEST") + add_dependencies(smoke ${TEST_NAME}) + elseif(result EQUAL 0 AND "${TEST_NAME}" IN_LIST REGRESSION_TESTS) + message("Adding to REGRESSION TEST FILTER ${TEST_NAME}") + set_tests_properties(${TEST_NAME} PROPERTIES LABELS "REGRESSION_TEST") + add_dependencies(regression ${TEST_NAME}) + endif() endfunction() function(add_gtest_executable TEST_NAME) @@ -168,6 +205,15 @@ function(add_gtest_executable TEST_NAME) endif() #message("add_gtest returns ${result}") set(result ${result} PARENT_SCOPE) + if(result EQUAL 0 AND NOT "${TEST_NAME}" IN_LIST REGRESSION_TESTS) + #message("adding to smoke test FILTER ${TEST_NAME}") + set_tests_properties(${TEST_NAME} PROPERTIES LABELS "SMOKE_TEST") + add_dependencies(smoke ${TEST_NAME}) + elseif(result EQUAL 0 AND "${TEST_NAME}" IN_LIST REGRESSION_TESTS) + #message("Adding to REGRESSION TEST FILTER ${TEST_NAME}") + set_tests_properties(${TEST_NAME} PROPERTIES LABELS "REGRESSION_TEST") + add_dependencies(regression ${TEST_NAME}) + endif() endfunction() add_compile_options(-Wno-c++20-extensions)