From 433b969e7db08988d3253bcb05bf259c373dc94f Mon Sep 17 00:00:00 2001 From: andrew clark Date: Wed, 8 Oct 2025 15:48:08 -0600 Subject: [PATCH] CI Skip and Status Checks Fix (#2952) * Update Jenkinsfile Adding logic to skip CI checks when a commit contains changes to non-relevant files like docs, .md, licenses, and .github workflow files. * Update Jenkinsfile * Update Jenkinsfile * Update Jenkinsfile Testing skip env var * Update Jenkinsfile Fixing syntax * Update Jenkinsfile Simplifying CI check logic * Update Jenkinsfile Testing skipping logic on stages. * Update Jenkinsfile Removing post block. The status for skipped stages are already reported. * Testing Docs Testing modifications to files in the docs folder do not trigger a the build and test stages. * Testing Multifile Trigger Removed Jenkinsfile from the skip patterns. Reversed change to docs file. This test should not skip CI checks. * Clean code Renamed setup stage to be more descriptive. Added pipeline env variable for consistency. Moved performance test results stage conditional up a level so the parent stage appropriate reports the status if it is skipped. * Fixing syntax error * Updated CRON Flags Added the FORCE_CI flag to the CRON instructions. This will ensure CI does not skip the job. * Updating logging Making logs more explicit. * Comment update Cleaning comments. * Update Jenkinsfile Reverting performance reports when condition. * Parallel Test Testing stage status with parallel stages * Update Jenkinsfile * Update Jenkinsfile Removing stages for quick testing * Update Jenkinsfile * Testing skipped parallel stages Testing the addition of a coordination stage to always pass and give an update to skipped parent stages with parallel sub-stages. * Testing parallel stages Adding coordination stage to test if parent check status is correctly updated. * Simplified performance results stage Removed parent stage as there are no other parallel stages to execute (yet). * Testing final clean up stage * Testing check status update Testing - forcing status to update after a stage skip. * Testing results stage skip * Removing test stage * Testing pipeline * Testing post status updates * Process Test Results Post Event Update The stage will report success when it skips or is successful. * Testing non-relevant file change This should skip build and test in CI * Reverting test updating regex file patterns to use strings instead of regex literal syntax. * Fixing file matching regex * Testing docs modification * Fixing default env var value * Correcting env var assignment * Pipeline test Updating docs file. Should skip ci. * Testing Pipeline Setting default run ci state. * Adding debugging * Removing debugging * Pipeline test Should skip pipeline * Pipeline Test Mixed files to trigger a CI run * Adding additional status updates The parent stage sometimes remains in pending even if the child stage completes when skipped. Added an additional status update for the parent stage. * Fixing variable name * Moving stage names Moved the performance stage names to a single location because they are referenced multiple times. This reduces errors with typos in the future. * Revert "Moving stage names" This reverts commit 7cf6743e548c6ebdbf7f26b1647c308ce556e06c. * Update Jenkinsfile Handle both truly empty arrays and arrays containing only empty strings. Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> [ROCm/composable_kernel commit: 0a4c45b4d3d4dff423c0777f5883d3067c65da20] --- Jenkinsfile | 149 +++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 141 insertions(+), 8 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index bb904052bd..11a9d9eb74 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -46,6 +46,58 @@ def runShell(String command){ return (output != "") } +def shouldRunCICheck() { + // Define patterns for files that should not trigger CI + def skipFilePatterns = [ + /^\.github\/.*/, // GitHub workflow files + /^docs\/.*/, // Documentation files + /^LICENSE$/, // License file + /^.*\.gitignore$/, // Git ignore files + /.*\.md$/ // Markdown files + ] + + try { + // Get the list of changed files + def changedFiles = sh( + returnStdout: true, + script: ''' + if [ "$CHANGE_ID" != "" ]; then + # For PR builds, compare against target branch + git diff --name-only origin/$CHANGE_TARGET...HEAD + else + # For regular builds, compare against previous commit + git diff --name-only HEAD~1..HEAD + fi + ''' + ).trim().split('\n') + + if (changedFiles.isEmpty() || (changedFiles.size() == 1 && changedFiles[0].trim().isEmpty())) { + echo "No changed files detected - this might be a manual trigger or merge commit, running CI for safety" + return true + } + + echo "Changed files: ${changedFiles.join(', ')}" + + // Check if any changed files are not in the skip patterns + def hasFilesRequiringCI = changedFiles.any { file -> + !skipFilePatterns.any { pattern -> + file ==~ pattern + } + } + + if (hasFilesRequiringCI) { + echo "Found files that require CI" + return true + } else { + echo "Only non-relevant files changed, skipping CI" + return false + } + } catch (Exception e) { + echo "Error checking changed files: ${e.getMessage()}, running CI by default" + return true + } +} + def getBaseDockerImageName(){ def img if (params.USE_CUSTOM_DOCKER != ""){ @@ -931,14 +983,14 @@ def run_pytorch_tests(Map conf=[:]){ } //launch develop branch daily jobs -CRON_SETTINGS = BRANCH_NAME == "develop" ? '''0 23 * * * % RUN_FULL_QA=true;RUN_CK_TILE_FMHA_TESTS=true;RUN_PERFORMANCE_TESTS=true - 0 22 * * * % RUN_FULL_QA=true;DISABLE_DL_KERNELS=true;RUN_TILE_ENGINE_GEMM_TESTS=true;RUN_PERFORMANCE_TESTS=true;RUN_ALL_UNIT_TESTS=true - 0 21 * * * % RUN_GROUPED_CONV_LARGE_CASES_TESTS=true;hipTensor_test=true;BUILD_GFX908=true;BUILD_GFX942=true;BUILD_GFX950=true;RUN_PERFORMANCE_TESTS=true;RUN_ALL_UNIT_TESTS=true - 0 19 * * * % BUILD_DOCKER=true;COMPILER_VERSION=amd-staging;BUILD_COMPILER=/llvm-project/build/bin/clang++;USE_SCCACHE=false;NINJA_BUILD_TRACE=true;RUN_ALL_UNIT_TESTS=true - 0 17 * * * % BUILD_DOCKER=true;COMPILER_VERSION=amd-mainline;BUILD_COMPILER=/llvm-project/build/bin/clang++;USE_SCCACHE=false;NINJA_BUILD_TRACE=true;RUN_ALL_UNIT_TESTS=true - 0 15 * * * % BUILD_INSTANCES_ONLY=true;USE_SCCACHE=false;NINJA_BUILD_TRACE=true - 0 13 * * * % RUN_AITER_TESTS=true;BUILD_LEGACY_OS=true;USE_SCCACHE=false;RUN_PERFORMANCE_TESTS=false - 0 11 * * * % RUN_PYTORCH_TESTS=true;RUN_CODEGEN_TESTS=false;USE_SCCACHE=false;RUN_PERFORMANCE_TESTS=false;BUILD_GFX10=false;BUILD_GFX11=false;BUILD_GFX12=false;BUILD_GFX90A=false''' : "" +CRON_SETTINGS = BRANCH_NAME == "develop" ? '''0 23 * * * % RUN_FULL_QA=true;RUN_CK_TILE_FMHA_TESTS=true;RUN_PERFORMANCE_TESTS=true;FORCE_CI=true + 0 22 * * * % RUN_FULL_QA=true;DISABLE_DL_KERNELS=true;RUN_TILE_ENGINE_GEMM_TESTS=true;RUN_PERFORMANCE_TESTS=true;RUN_ALL_UNIT_TESTS=true;FORCE_CI=true + 0 21 * * * % RUN_GROUPED_CONV_LARGE_CASES_TESTS=true;hipTensor_test=true;BUILD_GFX908=true;BUILD_GFX942=true;BUILD_GFX950=true;RUN_PERFORMANCE_TESTS=true;RUN_ALL_UNIT_TESTS=true;FORCE_CI=true + 0 19 * * * % BUILD_DOCKER=true;COMPILER_VERSION=amd-staging;BUILD_COMPILER=/llvm-project/build/bin/clang++;USE_SCCACHE=false;NINJA_BUILD_TRACE=true;RUN_ALL_UNIT_TESTS=true;FORCE_CI=true + 0 17 * * * % BUILD_DOCKER=true;COMPILER_VERSION=amd-mainline;BUILD_COMPILER=/llvm-project/build/bin/clang++;USE_SCCACHE=false;NINJA_BUILD_TRACE=true;RUN_ALL_UNIT_TESTS=true;FORCE_CI=true + 0 15 * * * % BUILD_INSTANCES_ONLY=true;USE_SCCACHE=false;NINJA_BUILD_TRACE=true;FORCE_CI=true + 0 13 * * * % RUN_AITER_TESTS=true;BUILD_LEGACY_OS=true;USE_SCCACHE=false;RUN_PERFORMANCE_TESTS=false;FORCE_CI=true + 0 11 * * * % RUN_PYTORCH_TESTS=true;RUN_CODEGEN_TESTS=false;USE_SCCACHE=false;RUN_PERFORMANCE_TESTS=false;BUILD_GFX10=false;BUILD_GFX11=false;BUILD_GFX12=false;BUILD_GFX90A=false;FORCE_CI=true''' : "" pipeline { agent none @@ -1093,6 +1145,10 @@ pipeline { name: 'ck_aiter_branch', defaultValue: 'develop', description: 'Specify which branch of CK to test with AITER (default: develop)') + booleanParam( + name: "FORCE_CI", + defaultValue: false, + description: "Force CI to run even when only non-relevant files are changed (default: OFF)") } environment{ dbuser = "${dbuser}" @@ -1106,7 +1162,20 @@ pipeline { DOCKER_BUILDKIT = "1" } stages{ + stage("Determine CI Execution") { + agent{ label rocmnode("nogpu") } + steps { + script { + env.SHOULD_RUN_CI = String.valueOf(params.FORCE_CI.toBoolean() || shouldRunCICheck()) + echo "SHOULD_RUN_CI: ${env.SHOULD_RUN_CI}" + } + } + } stage("Build Docker"){ + when { + beforeAgent true + expression { env.SHOULD_RUN_CI.toBoolean() } + } parallel{ stage('Docker /opt/rocm'){ agent{ label rocmnode("nogpu") } @@ -1118,6 +1187,11 @@ pipeline { } } stage("Static checks") { + when { + beforeAgent true + expression { env.SHOULD_RUN_CI.toBoolean() } + expression { params.RUN_CPPCHECK.toBoolean() } + } parallel{ stage('Clang Format and Cppcheck') { when { @@ -1178,6 +1252,10 @@ pipeline { } stage("Run Pytorch Tests") { + when { + beforeAgent true + expression { env.SHOULD_RUN_CI.toBoolean() } + } parallel { stage("Run Pytorch Tests on gfx942") @@ -1196,6 +1274,10 @@ pipeline { } stage("Run AITER Tests") { + when { + beforeAgent true + expression { env.SHOULD_RUN_CI.toBoolean() } + } parallel { stage("Run AITER Tests on gfx942") @@ -1226,6 +1308,10 @@ pipeline { } stage("Run Grouped Conv Large Case Tests") { + when { + beforeAgent true + expression { env.SHOULD_RUN_CI.toBoolean() } + } parallel { stage("Run Grouped Conv Large Case Tests on gfx90a") @@ -1250,6 +1336,10 @@ pipeline { } stage("Run Comprehensive Convolution Dataset Tests") { + when { + beforeAgent true + expression { env.SHOULD_RUN_CI.toBoolean() } + } parallel { stage("Run Comprehensive Dataset Tests on gfx90a") @@ -1282,6 +1372,10 @@ pipeline { } stage("Run Codegen Tests") { + when { + beforeAgent true + expression { env.SHOULD_RUN_CI.toBoolean() } + } parallel { stage("Run Codegen Tests on gfx90a") @@ -1305,6 +1399,10 @@ pipeline { } stage("Run CK_TILE_FMHA Tests") { + when { + beforeAgent true + expression { env.SHOULD_RUN_CI.toBoolean() } + } parallel { stage("Run CK_TILE_FMHA Tests on gfx90a") @@ -1368,6 +1466,10 @@ pipeline { } stage("Run TILE_ENGINE_GEMM Tests") { + when { + beforeAgent true + expression { env.SHOULD_RUN_CI.toBoolean() } + } parallel { stage("Run TILE_ENGINE_GEMM Tests on gfx90a") @@ -1485,6 +1587,10 @@ pipeline { stage("Build CK and run Tests") { + when { + beforeAgent true + expression { env.SHOULD_RUN_CI.toBoolean() } + } parallel { stage("Build CK with RHEL8") @@ -1702,6 +1808,17 @@ pipeline { } } } + post { + success { + script { + // Report the parent stage build ck and run tests status + def variant = env.STAGE_NAME + gitStatusWrapper(credentialsId: "${env.ck_git_creds}", gitHubContext: "${variant}", account: 'ROCm', repo: 'composable_kernel') { + echo "Reporting success status for build ck and run tests" + } + } + } + } } stage("Process Performance Test Results") { @@ -1719,6 +1836,22 @@ pipeline { } } } + post { + success { + script { + // Report the skipped parent's stage status + def parentVariant = "Process Performance Test Results" + gitStatusWrapper(credentialsId: "${env.ck_git_creds}", gitHubContext: "${parentVariant}", account: 'ROCm', repo: 'composable_kernel') { + echo "Process Performance Test Results stage skipped." + } + // Report the skipped stage's status + def variant = "Process results" + gitStatusWrapper(credentialsId: "${env.ck_git_creds}", gitHubContext: "${variant}", account: 'ROCm', repo: 'composable_kernel') { + echo "Process Performance Test Results stage skipped." + } + } + } + } } } }