From 4488950d01d8fbb16faf2d05841e263cdafb72c9 Mon Sep 17 00:00:00 2001 From: Illia Silin <98187287+illsilin@users.noreply.github.com> Date: Thu, 11 Jul 2024 13:22:40 -0700 Subject: [PATCH] Add CK_TILE tests to daily CI builds. (#1381) * add ck_tile tests to CI * build and run ck_tile tests on gfx90a and gfx942 in parallel * fix groovy syntax * turn ck_tile tests OFF by default * skip creating the build folder * build ck_tile examples with 64 threads * build ck_tile examples with cmake-ck-dev.sh script * add video group to docker on mi300 * do not retry to rebuild the early CI stages * help prevent jenkins false failure * restore cron trigger [ROCm/composable_kernel commit: 98a01bbc72fabd4a15d1472357104913a35d619e] --- Jenkinsfile | 64 ++++++++++++++++++- .../ck_tile/01_fmha/script/benchmark_bwd.sh | 0 .../ck_tile/01_fmha/script/smoke_test_bwd.sh | 3 +- .../ck_tile/01_fmha/script/smoke_test_fwd.sh | 4 +- 4 files changed, 66 insertions(+), 5 deletions(-) mode change 100644 => 100755 example/ck_tile/01_fmha/script/benchmark_bwd.sh mode change 100644 => 100755 example/ck_tile/01_fmha/script/smoke_test_bwd.sh diff --git a/Jenkinsfile b/Jenkinsfile index f65ddab5c3..e9d55992d8 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -315,6 +315,10 @@ def buildHipClangJob(Map conf=[:]){ if (params.COMPILER_VERSION == "amd-staging" || params.COMPILER_VERSION == "amd-mainline-open" || params.COMPILER_COMMIT != ""){ dockerOpts = dockerOpts + " --env HIP_CLANG_PATH='/llvm-project/build/bin' " } + def video_id = sh(returnStdout: true, script: 'getent group video | cut -d: -f3') + def render_id = sh(returnStdout: true, script: 'getent group render | cut -d: -f3') + dockerOpts = dockerOpts + " --group-add=${video_id} --group-add=${render_id} " + echo "Docker flags: ${dockerOpts}" def variant = env.STAGE_NAME @@ -366,6 +370,11 @@ def runCKProfiler(Map conf=[:]){ if (conf.get("enforce_xnack_on", false)) { dockerOpts = dockerOpts + " --env HSA_XNACK=1 " } + def video_id = sh(returnStdout: true, script: 'getent group video | cut -d: -f3') + def render_id = sh(returnStdout: true, script: 'getent group render | cut -d: -f3') + dockerOpts = dockerOpts + " --group-add=${video_id} --group-add=${render_id} " + echo "Docker flags: ${dockerOpts}" + def dockerArgs = "--build-arg PREFIX=${prefixpath} --build-arg compiler_version='${params.COMPILER_VERSION}' --build-arg compiler_commit='${params.COMPILER_COMMIT}' --build-arg ROCMVERSION='${params.ROCMVERSION}' " def variant = env.STAGE_NAME @@ -653,7 +662,7 @@ def process_results(Map conf=[:]){ } //launch develop branch daily at 23:00 UT in FULL_QA mode and at 19:00 UT with latest staging compiler version -CRON_SETTINGS = BRANCH_NAME == "develop" ? '''0 23 * * * % RUN_FULL_QA=true;ROCMVERSION=6.1; +CRON_SETTINGS = BRANCH_NAME == "develop" ? '''0 23 * * * % RUN_FULL_QA=true;ROCMVERSION=6.1; RUN_CK_TILE_TESTS=true 0 21 * * * % ROCMVERSION=6.1;hipTensor_test=true 0 19 * * * % BUILD_DOCKER=true;DL_KERNELS=true;COMPILER_VERSION=amd-staging;COMPILER_COMMIT=;USE_SCCACHE=false 0 17 * * * % BUILD_DOCKER=true;DL_KERNELS=true;COMPILER_VERSION=amd-mainline-open;COMPILER_COMMIT=;USE_SCCACHE=false @@ -724,6 +733,10 @@ pipeline { name: "RUN_CODEGEN_TESTS", defaultValue: true, description: "Run the codegen tests (default: ON)") + booleanParam( + name: "RUN_CK_TILE_TESTS", + defaultValue: false, + description: "Run the ck_tile tests (default: OFF)") booleanParam( name: "BUILD_INSTANCES_ONLY", defaultValue: false, @@ -816,7 +829,6 @@ pipeline { beforeAgent true expression { params.RUN_CODEGEN_TESTS.toBoolean() } } - options { retry(2) } agent{ label rocmnode("gfx90a")} environment{ setup_args = "NO_CK_BUILD" @@ -833,6 +845,52 @@ pipeline { } } } + } + stage("Run CK_TILE Tests") + { + parallel + { + stage("Run CK_TILE Tests on gfx90a") + { + when { + beforeAgent true + expression { params.RUN_CK_TILE_TESTS.toBoolean() } + } + agent{ label rocmnode("gfx90a") } + environment{ + setup_args = "NO_CK_BUILD" + execute_args = """ ../script/cmake-ck-dev.sh ../ gfx90a && \ + make -j64 tile_example_fmha_fwd tile_example_fmha_bwd && \ + cd ../ && + example/ck_tile/01_fmha/script/smoke_test_fwd.sh && \ + example/ck_tile/01_fmha/script/smoke_test_bwd.sh""" + } + steps{ + buildHipClangJobAndReboot(setup_args:setup_args, no_reboot:true, build_type: 'Release', execute_cmd: execute_args) + cleanWs() + } + } + stage("Run CK_TILE Tests on gfx942") + { + when { + beforeAgent true + expression { params.RUN_CK_TILE_TESTS.toBoolean() } + } + agent{ label rocmnode("gfx942") } + environment{ + setup_args = "NO_CK_BUILD" + execute_args = """ ../script/cmake-ck-dev.sh ../ gfx942 && \ + make -j64 tile_example_fmha_fwd tile_example_fmha_bwd && \ + cd ../ && + example/ck_tile/01_fmha/script/smoke_test_fwd.sh && \ + example/ck_tile/01_fmha/script/smoke_test_bwd.sh""" + } + steps{ + buildHipClangJobAndReboot(setup_args:setup_args, no_reboot:true, build_type: 'Release', execute_cmd: execute_args) + cleanWs() + } + } + } } stage("Build CK and run Tests") { @@ -973,7 +1031,7 @@ pipeline { beforeAgent true expression { params.RUN_PERFORMANCE_TESTS.toBoolean() } } - options { retry(2) } + options { retry(1) } agent{ label rocmnode("gfx90a")} environment{ setup_args = """ -DGPU_TARGETS="gfx90a" -DBUILD_DEV=On """ diff --git a/example/ck_tile/01_fmha/script/benchmark_bwd.sh b/example/ck_tile/01_fmha/script/benchmark_bwd.sh old mode 100644 new mode 100755 diff --git a/example/ck_tile/01_fmha/script/smoke_test_bwd.sh b/example/ck_tile/01_fmha/script/smoke_test_bwd.sh old mode 100644 new mode 100755 index 9fe795471b..d6830aa2ec --- a/example/ck_tile/01_fmha/script/smoke_test_bwd.sh +++ b/example/ck_tile/01_fmha/script/smoke_test_bwd.sh @@ -8,7 +8,7 @@ export CK_WARMUP=0 export CK_REPEAT=1 COMMON_ARGS='-v=1' - +set -x for prec in "fp16" "bf16" ; do for perm in 0 1 ; do for hdim in 32 64 128 ; do @@ -31,3 +31,4 @@ done done done done +set +x diff --git a/example/ck_tile/01_fmha/script/smoke_test_fwd.sh b/example/ck_tile/01_fmha/script/smoke_test_fwd.sh index 63813e079c..779e8d09ee 100755 --- a/example/ck_tile/01_fmha/script/smoke_test_fwd.sh +++ b/example/ck_tile/01_fmha/script/smoke_test_fwd.sh @@ -10,7 +10,7 @@ export CK_REPEAT=1 COMMON_ARGS='-v=1 -warmup=0 -repeat=1' # mode=0 # export HIP_VISIBLE_DEVICES=4 - +set -x for prec in "fp16" "bf16" ; do for mode in 1 0 ; do for perm in 0 1 ; do @@ -40,6 +40,7 @@ done done done + for perm in 0 1 ; do for bias in "n" "e" "a" ; do for b in 1 2 ; do @@ -49,3 +50,4 @@ done done done done +set +x