mirror of
https://github.com/ROCm/composable_kernel.git
synced 2026-04-20 14:59:17 +00:00
[rocm-libraries] ROCm/rocm-libraries#5249 (commit 2a114bb)
[CK] [CK_TILE] Improve build and test time of CI with smart dependency parser (#5249) ## Motivation Existing dependency parser needs full build of tests to determine which tests are affected by code changes in a PR. This still takes 2-4 hours for building the tests which slows down the CI as the number of tests grow. To resolve this issue we implemented a smart dependency parser which uses CMake Configure to parse dependencies and build only the affected test cases. We have ensured that two approaches are available 1) CMake pre-build analysis for each PR to ensure fast build and test. 2) Ninja post-build analysis to enable full build for nightly tests. ## Technical Details ```bash ### 1. Configure the project with CMake cmake -G Ninja -DCMAKE_EXPORT_COMPILE_COMMANDS=ON .. ### 2. Analyze dependencies (no build required!) python3 ../script/dependency-parser/main.py cmake-parse compile_commands.json build.ninja \ --workspace-root .. --output cmake_dependency_mapping.json --parallel 8 ### 3. Find tests affected by changes python3 ../script/dependency-parser/main.py select cmake_dependency_mapping.json origin/develop \ HEAD --test-prefix --output tests_to_run.json ### 4. Build only affected tests ninja $(jq -r '.executables[]' tests_to_run.json | tr '\n' ' ') ### 5. Run affected tests ctest -R "$(jq -r '.regex' tests_to_run.json)" ``` ### Jenkins Integration - Added `buildMode` to jenkinsfile to integrate both `selective` and `full` build methods ### Known Limitations ### 1. Build-Time Generated Headers (HIGH RISK) **Problem:** Files generated during the build process (e.g., via `add_custom_command`) cannot be analyzed before building. **Example:** ```cmake add_custom_command( OUTPUT ${CMAKE_BINARY_DIR}/generated/config.hpp COMMAND generate_config.sh DEPENDS template.hpp.in ) ``` **Impact:** If a source file includes `generated/config.hpp`, the dependency won't be detected until after building. **Mitigation:** - CK analysis shows **no generated headers** currently used - If generated headers are added in the future, they must be built first - Recommendation: Generate headers in CMake configure phase (not build phase) when possible ## Test Plan **1. Modified Files:** ``` include/ck_tile/ops/common.hpp include/ck_tile/ops/gemm.hpp include/ck_tile/ops/gemm/warp/warp_gemm.hpp ``` **2. Compare tests selected between `build.ninja` and `cmake-parse` methods** ## Test Result - 1. The test completed in 5-6 minutes finding about 8000+ executables that should be built. - 2. We selected a commit 5ccc1387ea which resulted in same 7 tests with both legacy and new methods. - PR | Legacy tests | Smart tests | Notes -- | -- | -- | -- 5261 | 453 | 455 | Only 2 tests (test_amdgcn_mma and test_amdgcn_sparse_mma) 5168 | 0 | 0 | Changes in dispatcher only. No CK tests invoked. 5249 | 0 | 0 | Changes to dependency parser. No CK tests invoked 5260 | 0 | 0 | Changes in dispatcher only. No CK tests invoked. 5174 | 1 | 1 | One test from FMHA affected by this PR in both cases 5383 | 0 | 0 | Changes are only in benchmark files. Did not trigger any tests 5445 | 1 | 1 | Changes are only to tests/ck_tile/gemm_streamk. Only triggered one streamk test in both cases. 5454 | 3 | 3 | Both methods identified same test_grouped_conv_bwd tests 5427 | 234 | 234 | Core infrastructure header changes. Detected exactly same tests 5388 | 85 | 85 | modifies warp-level GEMM operations (warp_gemm.hpp, warp_gemm_dispatcher.hpp). Correctly identified all the streamK gemm tests ## Submission Checklist - [x ] Look over the contributing guidelines at https://github.com/ROCm/ROCm/blob/develop/CONTRIBUTING.md#pull-requests.
This commit is contained in:
Yaswanth Raparti
committed by
assistant-librarian[bot]
assistant-librarian[bot]
parent
345a56c55e
commit
652d3456ca
109
Jenkinsfile
vendored
109
Jenkinsfile
vendored
@@ -1,3 +1,29 @@
|
||||
// Composable Kernel Jenkins Pipeline
|
||||
//
|
||||
// SMART BUILD SYSTEM:
|
||||
// This pipeline uses intelligent dependency analysis to speed up PR builds while
|
||||
// maintaining full validation on nightly runs.
|
||||
//
|
||||
// How it works:
|
||||
// 1. PR Builds (Selective):
|
||||
// - Configure: cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON (~30s)
|
||||
// - Analyze: Parse compile_commands.json + clang -MM for dependencies (~2min)
|
||||
// - Select: git diff to find affected tests (~1s)
|
||||
// - Build: ninja <affected-tests> only (minutes vs hours)
|
||||
// - Test: ctest -R <affected-pattern>
|
||||
//
|
||||
// 2. Nightly Builds (Full):
|
||||
// - FORCE_CI=true from cron triggers full build
|
||||
// - All targets built and tested for validation
|
||||
//
|
||||
// 3. Safety Checks:
|
||||
// - Forces full build if CMake configuration changes
|
||||
// - Forces full build if dependency cache stale (>7 days)
|
||||
// - Manual override: set DISABLE_SMART_BUILD=true
|
||||
//
|
||||
// Benefits: PR builds 5h → 30min (typical), nightly builds unchanged
|
||||
// See: script/dependency-parser/README.md for details
|
||||
//
|
||||
def rocmnode(name) {
|
||||
return '(rocmtest || miopen) && (' + name + ')'
|
||||
}
|
||||
@@ -678,13 +704,21 @@ def cmake_build(Map conf=[:]){
|
||||
}
|
||||
setup_cmd = conf.get(
|
||||
"setup_cmd",
|
||||
"""${cmake_envs} cmake -G Ninja ${setup_args} -DCMAKE_CXX_FLAGS=" -O3 " .. """
|
||||
)
|
||||
build_cmd = conf.get(
|
||||
"build_cmd",
|
||||
"${build_envs} ninja -j${nt} ${config_targets}"
|
||||
"""${cmake_envs} cmake -G Ninja ${setup_args} -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DCMAKE_CXX_FLAGS=" -O3 " .. """
|
||||
)
|
||||
|
||||
// Smart-build: Only build if running all tests or forced
|
||||
// Otherwise, smart-build will determine what to build after cmake configure
|
||||
if (runAllUnitTests) {
|
||||
build_cmd = conf.get(
|
||||
"build_cmd",
|
||||
"${build_envs} ninja -j${nt} ${config_targets}"
|
||||
)
|
||||
} else {
|
||||
// Smart-build enabled: skip full build, only run cmake configure
|
||||
build_cmd = ""
|
||||
}
|
||||
|
||||
cmd = conf.get("cmd", """
|
||||
${setup_cmd}
|
||||
${build_cmd}
|
||||
@@ -741,25 +775,44 @@ def cmake_build(Map conf=[:]){
|
||||
|
||||
//run tests except when NO_CK_BUILD is set
|
||||
if(!setup_args.contains("NO_CK_BUILD")){
|
||||
sh "python3 ../script/ninja_json_converter.py .ninja_log --legacy-format --output ck_build_trace_${arch_name}.json"
|
||||
archiveArtifacts "ck_build_trace_${arch_name}.json"
|
||||
sh "python3 ../script/parse_ninja_trace.py ck_build_trace_${arch_name}.json"
|
||||
if (params.NINJA_BUILD_TRACE || params.BUILD_INSTANCES_ONLY){
|
||||
if (params.NINJA_FTIME_TRACE) {
|
||||
echo "running ClangBuildAnalyzer"
|
||||
sh "/ClangBuildAnalyzer/build/ClangBuildAnalyzer --all . clang_build.log"
|
||||
sh "/ClangBuildAnalyzer/build/ClangBuildAnalyzer --analyze clang_build.log > clang_build_analysis_${arch_name}.log"
|
||||
archiveArtifacts "clang_build_analysis_${arch_name}.log"
|
||||
}
|
||||
|
||||
|
||||
// do not run unit tests when building instances only
|
||||
if(!params.BUILD_INSTANCES_ONLY){
|
||||
if (!runAllUnitTests){
|
||||
sh "../script/launch_tests.sh"
|
||||
// Smart Build: Run smart_build_and_test.sh
|
||||
sh """
|
||||
export WORKSPACE_ROOT=${env.WORKSPACE}
|
||||
export PARALLEL=32
|
||||
export NINJA_JOBS=${nt}
|
||||
export ARCH_NAME=${arch_name}
|
||||
export PROCESS_NINJA_TRACE=true
|
||||
export NINJA_FTIME_TRACE=${params.NINJA_FTIME_TRACE ? 'true' : 'false'}
|
||||
bash ../script/dependency-parser/smart_build_and_test.sh
|
||||
"""
|
||||
|
||||
// Archive artifacts if they were generated
|
||||
if (fileExists("ck_build_trace_${arch_name}.json")) {
|
||||
archiveArtifacts "ck_build_trace_${arch_name}.json"
|
||||
}
|
||||
if (fileExists("clang_build_analysis_${arch_name}.log")) {
|
||||
archiveArtifacts "clang_build_analysis_${arch_name}.log"
|
||||
}
|
||||
}
|
||||
else{
|
||||
sh "ninja check"
|
||||
echo "Full test suite requested (RUN_ALL_UNIT_TESTS=true or develop branch)"
|
||||
sh "ninja -j${nt} check"
|
||||
|
||||
// Process ninja build trace after full build
|
||||
sh "python3 ../script/ninja_json_converter.py .ninja_log --legacy-format --output ck_build_trace_${arch_name}.json"
|
||||
archiveArtifacts "ck_build_trace_${arch_name}.json"
|
||||
sh "python3 ../script/parse_ninja_trace.py ck_build_trace_${arch_name}.json"
|
||||
|
||||
if (params.NINJA_FTIME_TRACE) {
|
||||
echo "running ClangBuildAnalyzer"
|
||||
sh "/ClangBuildAnalyzer/build/ClangBuildAnalyzer --all . clang_build.log"
|
||||
sh "/ClangBuildAnalyzer/build/ClangBuildAnalyzer --analyze clang_build.log > clang_build_analysis_${arch_name}.log"
|
||||
archiveArtifacts "clang_build_analysis_${arch_name}.log"
|
||||
}
|
||||
}
|
||||
if (params.RUN_BUILDER_TESTS && !setup_args.contains("-DCK_CXX_STANDARD=") && !setup_args.contains("gfx10") && !setup_args.contains("gfx11")) {
|
||||
sh 'ninja check-builder'
|
||||
@@ -781,12 +834,24 @@ def cmake_build(Map conf=[:]){
|
||||
}
|
||||
else{
|
||||
// run unit tests unless building library for all targets
|
||||
// Note: This else block is when NINJA_BUILD_TRACE=false and BUILD_INSTANCES_ONLY=false
|
||||
// So no ninja trace processing needed here
|
||||
if (!params.BUILD_INSTANCES_ONLY){
|
||||
if (!runAllUnitTests){
|
||||
sh "../script/launch_tests.sh"
|
||||
// Smart Build: Run smart_build_and_test.sh
|
||||
sh """
|
||||
export WORKSPACE_ROOT=${env.WORKSPACE}
|
||||
export PARALLEL=32
|
||||
export NINJA_JOBS=${nt}
|
||||
export ARCH_NAME=${arch_name}
|
||||
export PROCESS_NINJA_TRACE=false
|
||||
export NINJA_FTIME_TRACE=false
|
||||
bash ../script/dependency-parser/smart_build_and_test.sh
|
||||
"""
|
||||
}
|
||||
else{
|
||||
sh "ninja check"
|
||||
echo "Full test suite requested (RUN_ALL_UNIT_TESTS=true or develop branch)"
|
||||
sh "ninja -j${nt} check"
|
||||
}
|
||||
if (params.RUN_BUILDER_TESTS && !setup_args.contains("-DCK_CXX_STANDARD=") && !setup_args.contains("gfx10") && !setup_args.contains("gfx11")) {
|
||||
sh 'ninja check-builder'
|
||||
@@ -1241,6 +1306,10 @@ pipeline {
|
||||
name: "USE_SCCACHE",
|
||||
defaultValue: true,
|
||||
description: "Use the sccache for building CK (default: ON)")
|
||||
booleanParam(
|
||||
name: "DISABLE_SMART_BUILD",
|
||||
defaultValue: false,
|
||||
description: "Disable smart build system and force full build/test (default: OFF). Smart build uses pre-build dependency analysis for selective testing on PRs, full builds on nightly runs.")
|
||||
booleanParam(
|
||||
name: "RUN_CPPCHECK",
|
||||
defaultValue: false,
|
||||
|
||||
Reference in New Issue
Block a user