mirror of
https://github.com/ROCm/composable_kernel.git
synced 2026-05-04 05:31:24 +00:00
[CK] [CK_TILE] Improve build and test time of CI with smart dependency parser (#5249) ## Motivation Existing dependency parser needs full build of tests to determine which tests are affected by code changes in a PR. This still takes 2-4 hours for building the tests which slows down the CI as the number of tests grow. To resolve this issue we implemented a smart dependency parser which uses CMake Configure to parse dependencies and build only the affected test cases. We have ensured that two approaches are available 1) CMake pre-build analysis for each PR to ensure fast build and test. 2) Ninja post-build analysis to enable full build for nightly tests. ## Technical Details ```bash ### 1. Configure the project with CMake cmake -G Ninja -DCMAKE_EXPORT_COMPILE_COMMANDS=ON .. ### 2. Analyze dependencies (no build required!) python3 ../script/dependency-parser/main.py cmake-parse compile_commands.json build.ninja \ --workspace-root .. --output cmake_dependency_mapping.json --parallel 8 ### 3. Find tests affected by changes python3 ../script/dependency-parser/main.py select cmake_dependency_mapping.json origin/develop \ HEAD --test-prefix --output tests_to_run.json ### 4. Build only affected tests ninja $(jq -r '.executables[]' tests_to_run.json | tr '\n' ' ') ### 5. Run affected tests ctest -R "$(jq -r '.regex' tests_to_run.json)" ``` ### Jenkins Integration - Added `buildMode` to jenkinsfile to integrate both `selective` and `full` build methods ### Known Limitations ### 1. Build-Time Generated Headers (HIGH RISK) **Problem:** Files generated during the build process (e.g., via `add_custom_command`) cannot be analyzed before building. **Example:** ```cmake add_custom_command( OUTPUT ${CMAKE_BINARY_DIR}/generated/config.hpp COMMAND generate_config.sh DEPENDS template.hpp.in ) ``` **Impact:** If a source file includes `generated/config.hpp`, the dependency won't be detected until after building. **Mitigation:** - CK analysis shows **no generated headers** currently used - If generated headers are added in the future, they must be built first - Recommendation: Generate headers in CMake configure phase (not build phase) when possible ## Test Plan **1. Modified Files:** ``` include/ck_tile/ops/common.hpp include/ck_tile/ops/gemm.hpp include/ck_tile/ops/gemm/warp/warp_gemm.hpp ``` **2. Compare tests selected between `build.ninja` and `cmake-parse` methods** ## Test Result - 1. The test completed in 5-6 minutes finding about 8000+ executables that should be built. - 2. We selected a commit 5ccc1387ea which resulted in same 7 tests with both legacy and new methods. - PR | Legacy tests | Smart tests | Notes -- | -- | -- | -- 5261 | 453 | 455 | Only 2 tests (test_amdgcn_mma and test_amdgcn_sparse_mma) 5168 | 0 | 0 | Changes in dispatcher only. No CK tests invoked. 5249 | 0 | 0 | Changes to dependency parser. No CK tests invoked 5260 | 0 | 0 | Changes in dispatcher only. No CK tests invoked. 5174 | 1 | 1 | One test from FMHA affected by this PR in both cases 5383 | 0 | 0 | Changes are only in benchmark files. Did not trigger any tests 5445 | 1 | 1 | Changes are only to tests/ck_tile/gemm_streamk. Only triggered one streamk test in both cases. 5454 | 3 | 3 | Both methods identified same test_grouped_conv_bwd tests 5427 | 234 | 234 | Core infrastructure header changes. Detected exactly same tests 5388 | 85 | 85 | modifies warp-level GEMM operations (warp_gemm.hpp, warp_gemm_dispatcher.hpp). Correctly identified all the streamK gemm tests ## Submission Checklist - [x ] Look over the contributing guidelines at https://github.com/ROCm/ROCm/blob/develop/CONTRIBUTING.md#pull-requests.
108 lines
4.1 KiB
Bash
Executable File
108 lines
4.1 KiB
Bash
Executable File
#!/bin/bash
|
|
# Copyright (c) Advanced Micro Devices, Inc., or its affiliates.
|
|
# SPDX-License-Identifier: MIT
|
|
|
|
# CI Safety Check for Smart Build System
|
|
#
|
|
# This script determines when to force full builds vs selective builds.
|
|
# Integrates with existing Jenkins infrastructure (FORCE_CI, BRANCH_NAME, etc.)
|
|
#
|
|
# Exit codes:
|
|
# 0 = Selective build OK (use smart build)
|
|
# 1 = Full build required
|
|
#
|
|
# Environment variables (set by Jenkins):
|
|
# FORCE_CI - Set to "true" for nightly/scheduled builds
|
|
# BRANCH_NAME - Git branch name
|
|
# CHANGE_ID - PR number (set by Jenkins Multibranch Pipeline for PRs)
|
|
# CHANGE_TARGET - Base branch for PR builds (set by Jenkins Multibranch Pipeline)
|
|
#
|
|
# Note: CHANGE_ID may not be set even for PR builds if Jenkins job is not
|
|
# configured as Multibranch Pipeline. Script uses three-dot git diff syntax
|
|
# to correctly detect PR changes regardless of CHANGE_ID availability.
|
|
#
|
|
# Manual override (set by developer/admin if needed):
|
|
# DISABLE_SMART_BUILD - Set to "true" to force full build
|
|
# BASE_BRANCH - Override base branch (default: "develop")
|
|
|
|
set -e
|
|
|
|
# Configuration
|
|
FORCE_FULL_BUILD=false
|
|
REASON=""
|
|
BASE_BRANCH="${CHANGE_TARGET:-${BASE_BRANCH:-develop}}"
|
|
|
|
# 1. Check if this is a nightly/scheduled build
|
|
# Existing Jenkins infrastructure sets FORCE_CI=true for cron-triggered builds
|
|
if [ "$FORCE_CI" = "true" ]; then
|
|
FORCE_FULL_BUILD=true
|
|
REASON="nightly/scheduled build (FORCE_CI=true from Jenkins cron)"
|
|
fi
|
|
|
|
# 2. Manual override to disable smart build
|
|
# Set DISABLE_SMART_BUILD=true in Jenkins job parameters if you want to force a full build
|
|
if [ "$DISABLE_SMART_BUILD" = "true" ]; then
|
|
FORCE_FULL_BUILD=true
|
|
REASON="manual override (DISABLE_SMART_BUILD=true)"
|
|
fi
|
|
|
|
# 3. Force full build if CMakeLists.txt or cmake/ configuration changed
|
|
# Always compare against base branch (not consecutive commits) to avoid false positives from merge commits
|
|
# Three-dot syntax (...) only shows changes actually made in the PR, not changes from merged develop branch
|
|
if [ -n "$CHANGE_ID" ]; then
|
|
# This is a PR build (CHANGE_ID set by Jenkins Multibranch Pipeline)
|
|
CHANGED_FILES=$(git diff --name-only origin/${BASE_BRANCH}...HEAD 2>/dev/null || echo "")
|
|
else
|
|
# Fallback: Works for both branch builds and PRs without CHANGE_ID
|
|
# Use three-dot syntax to avoid including merge commit changes from develop
|
|
CHANGED_FILES=$(git diff --name-only origin/${BASE_BRANCH}...HEAD 2>/dev/null || echo "")
|
|
fi
|
|
|
|
if echo "$CHANGED_FILES" | grep -qE "(CMakeLists\.txt|cmake/.*\.cmake)"; then
|
|
FORCE_FULL_BUILD=true
|
|
REASON="build system configuration changed (CMakeLists.txt or cmake/*.cmake)"
|
|
fi
|
|
|
|
# 4. Force full build if dependency cache is older than 7 days
|
|
CACHE_FILE="cmake_dependency_mapping.json"
|
|
if [ -f "$CACHE_FILE" ]; then
|
|
# Different stat command for Linux vs macOS
|
|
if [[ "$OSTYPE" == "darwin"* ]]; then
|
|
CACHE_MTIME=$(stat -f %m "$CACHE_FILE")
|
|
else
|
|
CACHE_MTIME=$(stat -c %Y "$CACHE_FILE")
|
|
fi
|
|
CURRENT_TIME=$(date +%s)
|
|
CACHE_AGE_DAYS=$(( ($CURRENT_TIME - $CACHE_MTIME) / 86400 ))
|
|
|
|
if [ $CACHE_AGE_DAYS -gt 7 ]; then
|
|
FORCE_FULL_BUILD=true
|
|
REASON="dependency cache older than 7 days"
|
|
fi
|
|
fi
|
|
|
|
# Output decision
|
|
echo "========================================="
|
|
echo "Smart Build Safety Check"
|
|
echo "========================================="
|
|
echo "FORCE_CI: ${FORCE_CI:-false}"
|
|
echo "BRANCH_NAME: ${BRANCH_NAME:-unknown}"
|
|
echo "BASE_BRANCH: ${BASE_BRANCH}"
|
|
echo "CHANGE_ID: ${CHANGE_ID:-<not a PR>}"
|
|
echo "DISABLE_SMART_BUILD: ${DISABLE_SMART_BUILD:-false}"
|
|
echo "-----------------------------------------"
|
|
|
|
if [ "$FORCE_FULL_BUILD" = true ]; then
|
|
echo "Decision: 🔴 FULL BUILD REQUIRED"
|
|
echo "Reason: $REASON"
|
|
echo "========================================="
|
|
echo "export SMART_BUILD_MODE=full" > build_mode.env
|
|
exit 1 # Exit with error to signal full build needed
|
|
else
|
|
echo "Decision: 🟢 SELECTIVE BUILD ENABLED"
|
|
echo "Using smart build for faster CI"
|
|
echo "========================================="
|
|
echo "export SMART_BUILD_MODE=selective" > build_mode.env
|
|
exit 0 # Exit success to signal selective build OK
|
|
fi
|